Merge remote-tracking branch 'origin/dev' into CLIENT-3146-free-threa…

…ding

.github/actions/run-ee-server/action.yml

@@ -1,4 +1,4 @@
-name: 'Run EE Server'
+name: 'Run EE Server in a Docker container'
 description: 'Run EE server. Returns once server is ready. Only tested on Linux and macOS'
 # NOTE: do not share this server container with others
 # since it's using the default admin / admin credentials
@@ -20,37 +20,15 @@ inputs:
   docker-hub-password:
     description: Required for using release candidates
     required: false
+  where-is-client-connecting-from:
+    required: false
+    description: 'docker-host, separate-docker-container, "remote-connection" via DOCKER_HOST'
+    default: 'docker-host'
 
 runs:
   using: "composite"
   steps:
-  - name: Install crudini to manipulate config.conf
-    # This will only work on the Github hosted runners.
-    run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt"
-    working-directory: .github/workflows
-    shell: bash
-
-  - name: Create config.conf
-    run: cp config.conf.template config.conf
-    working-directory: test
-    shell: bash
-
-  - name: Use enterprise edition instead of community edition in config.conf
-    run: |
-      crudini --existing=param --set config.conf enterprise-edition hosts ''
-      crudini --existing=param --set config.conf enterprise-edition hosts 127.0.0.1:3000
-    working-directory: test
-    shell: bash
-
-  - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV
-    shell: bash
-
-  - name: Set credentials in config file
-    run: |
-      crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
-      crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
-    working-directory: test
-    shell: bash
+  # Start up server
 
   - name: Log into Docker Hub to get server RC
     if: ${{ inputs.use-server-rc == 'true' }}
@@ -60,7 +38,7 @@ runs:
   - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV
     shell: bash
 
-  - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-security-and-sc >> $GITHUB_ENV
+  - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-python-client-testing >> $GITHUB_ENV
     shell: bash
 
   # macOS Github runners and Windows self-hosted runners don't have buildx installed by default
@@ -92,6 +70,9 @@ runs:
       is-security-enabled: true
       is-strong-consistency-enabled: true
 
+  - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV
+    shell: bash
+
   - run: echo ASADM_AUTH_FLAGS="--user=${{ env.SUPERUSER_NAME_AND_PASSWORD }} --password=${{ env.SUPERUSER_NAME_AND_PASSWORD }}" >> $GITHUB_ENV
     shell: bash
 
@@ -106,3 +87,56 @@ runs:
   # For debugging
   - run: docker logs aerospike
     shell: bash
+
+  # Configure tests
+
+  - name: Install crudini to manipulate config.conf
+    run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt"
+    working-directory: .github/workflows
+    shell: bash
+
+  - name: Create config.conf
+    run: cp config.conf.template config.conf
+    working-directory: test
+    shell: bash
+
+  - name: Disable community edition connection
+    run: crudini --existing=param --set config.conf community-edition hosts ''
+    working-directory: test
+    shell: bash
+
+  - name: Set credentials in config file
+    run: |
+      crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
+      crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
+    working-directory: test
+    shell: bash
+
+  - name: Set IP address to localhost
+    if: ${{ inputs.where-is-client-connecting-from == 'docker-host' }}
+    run: echo SERVER_IP=127.0.0.1 >> $GITHUB_ENV
+    working-directory: test
+    shell: bash
+
+  - name: Set IP address to remote machine running the Docker daemon
+    if: ${{ inputs.where-is-client-connecting-from == 'remote-connection' }}
+    run: |
+      SERVER_IP=${DOCKER_HOST/tcp:\/\//}
+      echo SERVER_IP=${SERVER_IP/:2375/} >> $GITHUB_ENV
+    working-directory: test
+    shell: bash
+
+  - name: Set IP address to Docker container for the server
+    if: ${{ inputs.where-is-client-connecting-from == 'separate-docker-container' }}
+    run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) >> $GITHUB_ENV
+    shell: bash
+
+  - name: Invalid input
+    if: ${{ env.SERVER_IP == '' }}
+    run: exit 1
+    shell: bash
+
+  - name: Set EE server's IP address
+    run: crudini --existing=param --set config.conf enterprise-edition hosts ${{ env.SERVER_IP }}:3000
+    working-directory: test
+    shell: bash

.github/workflows/build-wheels.yml

@@ -205,24 +205,15 @@ jobs:
       if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }}
       uses: ./.github/actions/setup-docker-on-macos
 
-    - name: 'macOS x86: run Aerospike server in Docker container and connect via localhost'
-      if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }}
+    - name: 'Run Aerospike server in Docker container and configure tests accordingly'
+      if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
       uses: ./.github/actions/run-ee-server
       with:
         use-server-rc: ${{ inputs.use-server-rc }}
         server-tag: ${{ inputs.server-tag }}
         docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
-
-    # TODO: combine this composite action and the above into one
-    - name: "Linux: run Aerospike server in Docker container and configure config.conf to connect to the server container's Docker IP address"
-      if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && startsWith(inputs.platform-tag, 'manylinux') }}
-      uses: ./.github/actions/run-ee-server-for-ext-container
-      with:
-        use-server-rc: ${{ inputs.use-server-rc }}
-        server-tag: ${{ inputs.server-tag }}
-        docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
-        docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
+        where-is-client-connecting-from: ${{ inputs.platform-tag == 'macosx_x86_64' && 'docker-host' || 'separate-docker-container' }}
 
     - name: If not running tests against server, only run basic import test
       if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'false' }}
@@ -258,7 +249,7 @@ jobs:
           yum install python-devel -y &&
           yum install python-setuptools -y
         # delvewheel is not enabled by default but we do need to repair the wheel
-        CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel"
+        CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*"
         CIBW_FREE_THREADED_SUPPORT: ${{ matrix.python-tag == 'cp313t' && '1' || '0' }}
         CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}"
         CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }}
@@ -320,6 +311,7 @@ jobs:
           server-tag: ${{ inputs.server-tag }}
           docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
           docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
+          where-is-client-connecting-from: ${{ inputs.platform-tag == 'win_amd64' && 'remote-connection' || 'docker-host' }}
 
       - name: Download wheel
         uses: actions/download-artifact@v4
@@ -359,14 +351,6 @@ jobs:
         run: ${{ env.PIP_COMMAND }} install aerospike --force-reinstall --no-index --find-links=./
         shell: bash
 
-      - name: Connect to Docker container on remote machine with Docker daemon
-        if: ${{ inputs.platform-tag == 'win_amd64' }}
-        # DOCKER_HOST contains the IP address of the remote machine
-        run: |
-          $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")}
-          crudini --set config.conf enterprise-edition hosts ${env:DOCKER_HOST_IP}:3000
-        working-directory: test
-
       - run: ${{ env.PIP_COMMAND }} install pytest -c requirements.txt
         working-directory: test
         shell: bash

.github/workflows/stage-tests.yml

@@ -102,6 +102,7 @@ jobs:
         server-tag: ${{ inputs.server-tag }}
         docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
+        where-is-client-connecting-from: 'separate-docker-container'
 
     - name: Run distro container
       # Run distro container on host network to access the Aerospike server using localhost (without having to change config.conf)
@@ -194,6 +195,7 @@ jobs:
         server-tag: ${{ inputs.server-tag }}
         docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
+        where-is-client-connecting-from: 'docker-host'
 
     - name: Install wheel
       run: python3 -m pip install *.whl

.github/workflows/test-server-rc.yml

@@ -34,12 +34,13 @@ jobs:
 
     - run: docker run -d --name manylinux quay.io/pypa/manylinux2014_${{ matrix.platform[0] }} tail -f /dev/null
 
-    - uses: ./.github/actions/run-ee-server-for-ext-container
+    - uses: ./.github/actions/run-ee-server
       with:
         use-server-rc: true
         server-tag: latest
         docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
+        where-is-client-connecting-from: 'docker-container'
 
     - uses: actions/download-artifact@v4
       with:

.github/workflows/valgrind.yml

@@ -21,34 +21,122 @@ on:
         required: false
         default: false
 
+env:
+  PYTHON_TAG: cp38
+
 jobs:
+  look-for-wheel-in-jfrog:
+    outputs:
+      num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }}
+      # So we can pass the python tag to a reusable workflow
+      python-tag: ${{ env.PYTHON_TAG }}
+    runs-on: ubuntu-22.04
+    env:
+      JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt
+    steps:
+    - uses: jfrog/setup-jfrog-cli@v4
+      env:
+        JF_URL: ${{ secrets.JFROG_PLATFORM_URL }}
+        JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }}
+
+    - name: Get shortened commit hash of this workflow run
+      # versioningit commit sha is always 8 chars long it seems
+      run: echo SHORT_GITHUB_SHA=$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV
+
+    - name: Look for wheel built with default settings in JFrog
+      # AQL has the option to exclude patterns from search results
+      # but it doesn't allow regex, so we can't filter out any type of label in a wheel name
+      # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex
+      # to handle those new labels without updating the regex.
+      run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}
+
+    - name: Show unfiltered results
+      run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}
+
+    - name: Install sponge
+      run: sudo apt install -y moreutils
+
+    - name: Filter out wheels with labels in results
+      run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}
+      shell: bash
+
+    - name: Check if artifacts with labels were filtered out
+      run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}
+
+    - name: Count artifacts
+      id: count_num_artifacts_found
+      run: echo num_artifacts=$(jq length ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_OUTPUT
+
+    - name: Multiple artifacts found, not sure which one to use. Fail out
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }}
+      run: exit 1
+
+    - name: Found the exact artifact in JFrog. Get the artifact name
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }}
+      run: echo ARTIFACT_PATH=$(jq -r .[0].path ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_ENV
+
+    - name: Then download artifact from JFrog
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }}
+      run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }}
+
+    - name: Pass to valgrind job
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }}
+      uses: actions/upload-artifact@v4
+      with:
+        # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel
+        if-no-files-found: error
+        path: './*.whl'
+
   build-manylinux-wheel:
+    needs: look-for-wheel-in-jfrog
+    if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }}
     uses: ./.github/workflows/build-wheels.yml
     with:
-      python-tags: '["cp38"]'
+      python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]'
       platform-tag: manylinux_x86_64
       sha-to-build-and-test: ${{ github.sha }}
     secrets: inherit
 
+  upload-built-wheel-to-jfrog:
+    needs: build-manylinux-wheel
+    # TODO: this job should skip when this workflow is run on central branches
+    # We already have artifacts available for central branches in the PyPI-type JFrog repo
+    # The problem is we have to conditionally skip this job, but using the github context to get the branch name
+    # doesn't work for some reason. Just leave this alone for now.
+    uses: ./.github/workflows/upload-to-jfrog.yml
+    with:
+      jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }}
+    secrets: inherit
+
   valgrind:
     env:
       MASSIF_REPORT_FILE_NAME: massif.out
-    needs: build-manylinux-wheel
+    needs: [
+      look-for-wheel-in-jfrog,
+      build-manylinux-wheel
+    ]
+    # Case 1: Found artifact in JFrog
+    # Case 2: Did not find artifact in JFrog, had to build it in GHA
+    if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' && (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 1) || (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 && needs.build-manylinux-wheel.result == 'success')) }}
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: recursive
         fetch-depth: 0
 
+    - name: Convert Python tag to Python version
+      run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV
+      shell: bash
+
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: '${{ env.PYTHON_VERSION }}'
         architecture: 'x64'
 
     - uses: actions/download-artifact@v4
       with:
-        name: cp38-manylinux_x86_64.build
+        merge-multiple: true
 
     - name: Install client
       run: pip install ./*.whl
@@ -76,6 +164,18 @@ jobs:
     - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }}
       working-directory: test
 
+    # TODO: upload report as artifact
     - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }}
       if: ${{ !cancelled() && inputs.massif }}
       working-directory: test
+
+  # See reason for deleting artifacts in dev-workflow-p2.yml
+  delete-artifacts:
+    needs: [
+      # These jobs must have downloaded the artifact from Github before we can delete it
+      upload-built-wheel-to-jfrog,
+      valgrind
+    ]
+    # Workflow run must clean up after itself even if cancelled
+    if: ${{ always() }}
+    uses: ./.github/workflows/delete-artifacts.yml