From 1749bc6aa17bdf3dce756d3931744c871cfd4bd8 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Fri, 26 Jan 2024 08:06:51 +0000 Subject: [PATCH 1/6] Filter selected Python plugins in integration jobs, to prevent deps from one plugin interfering with the build of others --- .github/workflows/integration-cloud.yaml | 7 ++++++- .../python/requirements_plugins.txt | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-cloud.yaml b/.github/workflows/integration-cloud.yaml index ed38782f9..84384fdbc 100644 --- a/.github/workflows/integration-cloud.yaml +++ b/.github/workflows/integration-cloud.yaml @@ -229,12 +229,17 @@ jobs: . ./venv/bin/activate python -m pip install --upgrade pip + - name: Select plugin dependencies + run: | + cd tracdap-runtime/python + sed -n '/BEGIN_PLUGIN ${{ inputs.target }}/, /END_PLUGIN ${{ inputs.target }}/p' requirements_plugins.txt > requirements_selected.txt + - name: Install dependencies run: | . ./venv/bin/activate cd tracdap-runtime/python pip install -r requirements.txt - pip install -r requirements_plugins.txt + pip install -r requirements_selected.txt - name: Protoc code generation run: | diff --git a/tracdap-runtime/python/requirements_plugins.txt b/tracdap-runtime/python/requirements_plugins.txt index 0762828ae..ff6fa1313 100644 --- a/tracdap-runtime/python/requirements_plugins.txt +++ b/tracdap-runtime/python/requirements_plugins.txt @@ -9,11 +9,18 @@ # So e.g. to use the Azure libraries, Python 3.8 is required # We can still support older Python versions, so long as those plugins are not being used +# To control dependencies installed during CI, the BEGIN_PLUGIN and END_PLUGIN markers are used +# Running CI for a particular plugin will only install the dependencies between those markers +# This prevents dependency issues with one plugin breaking CI for all other plugins +# It also cuts down the setup time of integration jobs + # ---------------------------------------------------------------------------------------------------------------------- # AWS # ---------------------------------------------------------------------------------------------------------------------- +# BEGIN_PLUGIN aws + # AWS client libraries botocore == 1.29.156 @@ -23,10 +30,14 @@ boto3 == 1.26.156 # This is a separate library that depends on a very old version of botocore # It is not really possible to use them together and maintain compliance +# END_PLUGIN aws + # ---------------------------------------------------------------------------------------------------------------------- # GCP # ---------------------------------------------------------------------------------------------------------------------- +# BEGIN_PLUGIN gcp + # GCP client libraries google-auth == 2.20.0 @@ -37,12 +48,16 @@ google-cloud-storage == 2.9.0 gcsfs == 2023.6.0 +# END_PLUGIN gcp + # ---------------------------------------------------------------------------------------------------------------------- # Azure # ---------------------------------------------------------------------------------------------------------------------- -# AWS client libraries +# BEGIN_PLUGIN azure + +# Azure client libraries azure-core == 1.29.6 azure-identity == 1.15.0 @@ -55,3 +70,5 @@ adlfs == 2023.4.0 # azure-identity is still pulling in a compromised version of the cryptography package # force a compatible version with the required fix cryptography~=41.0.7 + +# END_PLUGIN azure From 527cfb9b6827dc137606fea29e562aa33f8511e5 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Fri, 26 Jan 2024 08:13:53 +0000 Subject: [PATCH 2/6] Remove Azure login workaround after fix from Azure --- .github/workflows/integration-cloud.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration-cloud.yaml b/.github/workflows/integration-cloud.yaml index 84384fdbc..8feb87582 100644 --- a/.github/workflows/integration-cloud.yaml +++ b/.github/workflows/integration-cloud.yaml @@ -204,9 +204,7 @@ jobs: - name: Authenticate to Azure if: ${{ inputs.target == 'azure' }} - # There was a breaking change in azure/login, so we need to force the version - # https://github.com/Azure/login/issues/403 - uses: azure/login@v1.6.1 + uses: azure/login@v1 with: client-id: ${{ secrets.AZURE_CI_CLIENT }} tenant-id: ${{ secrets.AZURE_CI_TENANT }} From 9293fe5fcb080ccd1ac7fbac543497cb2d6c8eff Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Fri, 26 Jan 2024 08:15:58 +0000 Subject: [PATCH 3/6] Add a comment about plugin dependency filtering --- .github/workflows/integration-cloud.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-cloud.yaml b/.github/workflows/integration-cloud.yaml index 8feb87582..2af954281 100644 --- a/.github/workflows/integration-cloud.yaml +++ b/.github/workflows/integration-cloud.yaml @@ -227,6 +227,8 @@ jobs: . ./venv/bin/activate python -m pip install --upgrade pip + # Filter plugin dependencies, only install for the plugin being tested + # This prevents dependency issues in one plugin affecting all the others - name: Select plugin dependencies run: | cd tracdap-runtime/python From 0672385325c921f3a55ed47eb1020726de5a1a61 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Fri, 26 Jan 2024 10:56:02 +0000 Subject: [PATCH 4/6] Keep GCP on Python 3.11, due to dependencies in the cloud libraries --- .github/workflows/integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8f64c95a8..67bf9130d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -366,6 +366,8 @@ jobs: PYTHON_TESTS: int_storage_aws*.py, JAVA_TESTS: int-storage } + # GCP does not support Python 3.12 yet, so run integration tests on 3.11 + # See https://github.com/googleapis/python-crc32c/issues/178 - { SERVICE: storage, TARGET: gcp, PROTOCOL: GCS, @@ -376,6 +378,7 @@ jobs: TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml', TRAC_SECRET_KEY: short-lived-secret, PYTHON_TESTS: int_storage_gcp*.py, + PYTHON_VERSION: "3.11", JAVA_TESTS: int-storage } - { SERVICE: storage, From 4b8e5963482a7cf1ad015133c71aa0c1ad44ae16 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Fri, 26 Jan 2024 11:05:35 +0000 Subject: [PATCH 5/6] Keep Azure on Python 3.11, due to dependencies in the cloud libraries --- .github/workflows/integration.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 67bf9130d..3fff5cc02 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -367,7 +367,7 @@ jobs: JAVA_TESTS: int-storage } # GCP does not support Python 3.12 yet, so run integration tests on 3.11 - # See https://github.com/googleapis/python-crc32c/issues/178 + # See https://github.com/googleapis/python-crc32c/issues/178 (there may be more dependencies affected) - { SERVICE: storage, TARGET: gcp, PROTOCOL: GCS, @@ -381,6 +381,8 @@ jobs: PYTHON_VERSION: "3.11", JAVA_TESTS: int-storage } + # Azure does not support Python 3.12 yet, so run integration tests on 3.11 + # See https://github.com/aio-libs/multidict/issues/887 (there may be more dependencies affected) - { SERVICE: storage, TARGET: azure, PROTOCOL: BLOB, @@ -391,6 +393,7 @@ jobs: TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml', TRAC_SECRET_KEY: short-lived-secret, PYTHON_TESTS: int_storage_azure*.py, + PYTHON_VERSION: "3.11", JAVA_TESTS: int-storage } uses: ./.github/workflows/integration-cloud.yaml From c48966f9db0f119024e71303e6b01152da6a1c1e Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Fri, 26 Jan 2024 14:22:54 +0000 Subject: [PATCH 6/6] Add a note on Python 3.12 support for GCP and Azure to the Python runtime readme --- tracdap-runtime/python/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracdap-runtime/python/README.md b/tracdap-runtime/python/README.md index c677d13db..026433302 100644 --- a/tracdap-runtime/python/README.md +++ b/tracdap-runtime/python/README.md @@ -21,7 +21,7 @@ The TRAC runtime for Python has these requirements: * PySpark 3.0 up to 3.5 (optional) 3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark. - +As of January 2024, the Python libraries for GCP and Azure do not yet support Python 3.12. ## Installing the runtime