Add FIPS and CRT-libcrypto dimensitons

WillChilds-Klein · May 17, 2024 · 0eddfe9 · 0eddfe9
1 parent 81ddc56
commit 0eddfe9
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 4 deletions.
diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml
@@ -100,6 +100,15 @@ jobs:
           ./tests/ci/integration/run_socat_integration.sh
   python-main:
     if: github.repository_owner == 'aws'
+    strategy:
+      fail-fast: false
+      matrix:
+        openssl_in_crt:
+          - "0"
+          - "1"
+        fips:
+          - "0"
+          - "1"
     runs-on: ubuntu-latest
     steps:
       - name: Install OS Dependencies
@@ -110,8 +119,20 @@ jobs:
       - name: Build AWS-LC, build python, run tests
         run: |
           ./tests/ci/integration/run_python_integration.sh main
+        env:
+          FIPS: ${{ matrix.fips }}
+          AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: ${{ matrix.openssl_in_crt }}
   python-releases:
     if: github.repository_owner == 'aws'
+    strategy:
+      fail-fast: false
+      matrix:
+        openssl_in_crt:
+          - "0"
+          - "1"
+        fips:
+          - "0"
+          - "1"
     runs-on: ubuntu-latest
     steps:
       - name: Install OS Dependencies
@@ -122,6 +143,9 @@ jobs:
       - name: Build AWS-LC, build python, run tests
         run: |
           ./tests/ci/integration/run_python_integration.sh 3.10 3.11 3.12
+        env:
+          FIPS: ${{ matrix.fips }}
+          AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: ${{ matrix.openssl_in_crt }}
   bind9:
     if: github.repository_owner == 'aws'
     runs-on: ubuntu-latest

diff --git a/tests/ci/integration/run_python_integration.sh b/tests/ci/integration/run_python_integration.sh
@@ -10,6 +10,10 @@ set -exuo pipefail
 
 # Set up environment.
 
+# Default env parameters to "off"
+FIPS=${FIPS:-"0"}
+AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=${AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO:-"0"}
+
 # SYS_ROOT
 #  - SRC_ROOT(aws-lc)
 #    - SCRATCH_FOLDER
@@ -24,6 +28,7 @@ set -exuo pipefail
 
 # Assumes script is executed from the root of aws-lc directory
 SCRATCH_FOLDER="${SRC_ROOT}/PYTHON_BUILD_ROOT"
+CRT_SRC_FOLDER="${SCRATCH_FOLDER}/aws-crt-python"
 PYTHON_SRC_FOLDER="${SCRATCH_FOLDER}/python-src"
 PYTHON_PATCH_FOLDER="${SRC_ROOT}/tests/ci/integration/python_patch"
 PYTHON_INTEG_TEST_FOLDER="${SRC_ROOT}/tests/ci/integration/python_tests"
@@ -43,10 +48,16 @@ function python_build() {
 
 function python_run_tests() {
     local branch=${1}
+    local python='./python'
     pushd ${branch}
     # We statically link, so need to call into python itself to assert that we're
     # correctly built against AWS-LC
-    ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC
+    if [[ ${FIPS} == "1" ]]; then
+        local expected_version_str='AWS-LC'
+    else
+        local expected_version_str='AWS-LC FIPS'
+    fi
+    ${python} -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep ${expected_version_str}
     # see https://github.com/pypa/setuptools/issues/3007
     export SETUPTOOLS_USE_DISTUTILS=stdlib
     # A number of python module tests fail on our public CI images, but they're
@@ -72,6 +83,34 @@ function python_run_tests() {
     popd
 }
 
+function install_crt_python() {
+    mkdir -p ${CRT_SRC_FOLDER}
+    pushd ${CRT_SRC_FOLDER}
+    git clone https://github.com/awslabs/aws-crt-python.git .
+    git submodule update --init
+    popd
+    local python='./python'
+    ${python} -m pip install wheel
+    ${python} -m pip install ${CRT_SRC_FOLDER}
+    # below was adapted from aws-crt-python's CI
+    # https://github.com/awslabs/aws-crt-python/blob/d76c3dacc94c1aa7dfc7346a77be78dc990b5171/.github/workflows/ci.yml#L159
+    local awscrt_path=$(${python} -c "import _awscrt; print(_awscrt.__file__)")
+    echo "AWSCRT_PATH: $awscrt_path"
+    local linked_against=$(ldd $awscrt_path)
+    echo "LINKED AGAINST: $linked_against"
+    local uses_libcrypto_so=$(echo "$linked_against" | grep 'libcrypto*.so' | head -1)
+    echo "USES LIBCRYTPO: $uses_libcrypto_so"
+    # by default, the python CRT bindings bundle their own libcrypto wheel
+    # built from AWS-LC. AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO can be specified
+    # in build environment to tell CRT to link against system libcrypto
+    # (usually OpenSSL) instead.
+    if [[ ${AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO} == "1" ]]; then
+        test -n "$uses_libcrypto_so"
+    else
+        test -z "$uses_libcrypto_so"
+    fi
+}
+
 function python_run_3rd_party_tests() {
     local branch=${1}
     pushd ${branch}
@@ -84,9 +123,13 @@ function python_run_3rd_party_tests() {
     ${python} -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC
     echo installing other OpenSSL-dependent modules...
     ${python} -m ensurepip
-    ${python} -m pip install 'boto3[crt]' 'cryptography' 'pyopenssl'
-    # this appears to be needed by more recent python versions
+    # setupttols not installed by default on more recent python versions
+    # see https://github.com/python/cpython/issues/95299
     ${python} -m pip install setuptools
+    install_crt_python
+    ${python} -m pip install 'boto3[crt]'
+    ${python} -m pip install 'cryptography'
+    ${python} -m pip install 'pyopenssl'
     echo running minor integration test of those dependencies...
     for test in ${PYTHON_INTEG_TEST_FOLDER}/*.py; do
         ${python} ${test}
@@ -150,7 +193,8 @@ mkdir -p ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER}
 
 aws_lc_build ${SRC_ROOT} ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} \
     -DBUILD_TESTING=OFF \
-    -DBUILD_SHARED_LIBS=0
+    -DBUILD_SHARED_LIBS=0 \
+    -DFIPS=${FIPS}
 
 # Some systems install under "lib64" instead of "lib"
 ln -s ${AWS_LC_INSTALL_FOLDER}/lib64 ${AWS_LC_INSTALL_FOLDER}/lib