[CLIENT-2258] Backport 6.*: Remove auto-serialization and auto-deseri…

…alization (#488)

* Return a client error when writing unsupported Python types to the server
* Convert AS_BYTES_PYTHON server types to bytearrays when reading data from the server
* Remove support for serializing booleans as the AS_BYTES_PYTHON server type
* Remove aerospike.SERIALIZER_PYTHON
* client.put(): set serializer parameter's default value to aerospike.SERIALIZER_NONE
* client.put(): fix bug where Python bytes bin values can't be used if serializer parameter is set to SERIALIZER_NONE

Extra changes:
* Docs: data mapping: add headers
* Change send_bool_as default value to AS_BOOL
* Add extra tests to verify client.operate() and expression behavior with instance-level serializers and deserializers
* Add build wheels workflow
* Only run CE tests since EE tests are broken on client version 6.1.2
* Remove C client code that breaks with later openssl versions

Co-authored-by: dwelch-spike <[email protected]>

.github/actions/run-ee-server/action.yml

@@ -0,0 +1,62 @@
+name: 'Run EE Server'
+description: 'Run EE server'
+inputs:
+  use-server-rc:
+    required: true
+    default: false
+  server-tag:
+    required: true
+    default: false
+
+runs:
+  using: "composite"
+  steps:
+  - name: Install crudini to manipulate config.conf
+    run: pip install crudini
+    shell: bash
+
+  - name: Add enterprise edition config to config.conf
+    run: |
+      crudini --set config.conf enterprise-edition hosts 127.0.0.1:3000
+      crudini --set config.conf enterprise-edition user superuser
+      crudini --set config.conf enterprise-edition password superuser
+    working-directory: test
+    shell: bash
+
+  - name: Create config folder to store configs in
+    run: mkdir configs
+    shell: bash
+
+  - name: Use release server
+    if: ${{ inputs.use-server-rc == 'false' }}
+    run: echo "SERVER_IMAGE=aerospike/aerospike-server-enterprise:${{ inputs.server-tag }}" >> $GITHUB_ENV
+    shell: bash
+
+  - name: Use release candidate server
+    if: ${{ inputs.use-server-rc == 'true' }}
+    run: echo "SERVER_IMAGE=aerospike.jfrog.io/docker/aerospike/aerospike-server-enterprise-rc:${{ inputs.server-tag }}" >> $GITHUB_ENV
+    shell: bash
+
+  - name: Get default aerospike.conf from Docker server EE container
+    run: |
+      docker run -d --name aerospike -p 3000-3002:3000-3002 $SERVER_IMAGE
+      sleep 5
+      docker cp aerospike:/etc/aerospike/aerospike.conf ./configs/aerospike.conf
+      docker container stop aerospike
+      docker container rm aerospike
+    shell: bash
+
+  - name: Enable security features using aerospike.conf
+    # Security stanza
+    run: echo -e "security {\n\tenable-quotas true\n}\n" >> ./aerospike.conf
+    working-directory: ./configs
+    shell: bash
+
+  - name: Run enterprise edition server
+    run: docker run -tid -v $(pwd)/configs:/opt/aerospike/etc -p 3000:3000 --name aerospike $SERVER_IMAGE asd --config-file /opt/aerospike/etc/aerospike.conf
+    shell: bash
+
+  - name: Create user in database for tests
+    # Use default admin user to create another user for testing
+    run: docker exec aerospike asadm --user admin --password admin --enable -e "manage acl create user superuser password superuser roles read-write-udf sys-admin user-admin data-admin"
+    shell: bash

.github/workflows/build-wheels.yml

@@ -0,0 +1,169 @@
+name: Build wheels
+
+# Builds wheels and sends to QE and Aerospike artifactory
+on:
+  workflow_dispatch:
+    inputs:
+      use-server-rc:
+        type: boolean
+        required: true
+        default: false
+      # Test against a server version
+      # This is helpful if you need to create a backport from an older client major version
+      # And that older client major version does not accomodate for breaking changes in the latest server version
+      server-tag:
+        required: true
+        default: 'latest'
+
+jobs:
+  test-sdist:
+    name: Build and install sdist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+        architecture: 'x64'
+
+    - run: sudo apt update
+    - name: Install build dependencies (apt packages)
+      run: sudo apt install python3-dev libssl-dev -y
+    - name: Install build dependencies (pip packages)
+      run: python3 -m pip install build
+
+    - name: Build source distribution
+      run: python3 -m build --sdist
+
+    - name: Upload sdist to GitHub
+      uses: actions/upload-artifact@v3
+      with:
+        path: ./dist/*.tar.gz
+        # Artifact name, not the file name
+        name: sdist
+
+  manylinux_x86_64:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Python versions to build wheels on
+        python: [
+          ["cp36", "3.6"],
+          ["cp37", "3.7"],
+          ["cp38", "3.8"],
+          ["cp39", "3.9"],
+        ]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Use release server
+      if: ${{ inputs.use-server-rc == false }}
+      run: echo "SERVER_IMAGE=aerospike/aerospike-server:${{ inputs.server-tag }}" >> $GITHUB_ENV
+
+    - name: Use release candidate server
+      if: ${{ inputs.use-server-rc == true }}
+      run: echo "SERVER_IMAGE=aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:${{ inputs.server-tag }}" >> $GITHUB_ENV
+
+    - name: Run server
+      run: |
+        docker run -d --name aerospike -p 3000-3002:3000-3002 $SERVER_IMAGE
+
+    - name: Wait for server to start
+      run: sleep 5
+
+    - name: Set config.conf to use Docker IP address of Aerospike server
+      # config.conf should be copied into the cibuildwheel Docker container
+      run: |
+        export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike)
+        pip install crudini
+        crudini --set config.conf community-edition hosts ${SERVER_DOCKER_IP}:3000
+      working-directory: test
+
+    - name: Build wheel
+      uses: pypa/[email protected]
+      env:
+        CIBW_BUILD: ${{ matrix.python[0] }}-manylinux_x86_64
+        CIBW_BUILD_FRONTEND: build
+        CIBW_BEFORE_ALL_LINUX: >
+          yum install openssl-devel -y &&
+          yum install python-devel -y &&
+          yum install python-setuptools -y
+        CIBW_ARCHS: "x86_64"
+        CIBW_TEST_COMMAND: >
+          cd {project}/test/ &&
+          pip install -r requirements.txt &&
+          python -m pytest new_tests/
+
+    - name: Upload wheels to GitHub
+      uses: actions/upload-artifact@v3
+      if: ${{ always() }}
+      with:
+        path: ./wheelhouse/*.whl
+        # Artifact name, not the file name
+        name: manylinux-x86_64-${{ matrix.python[0] }}
+
+  macOS-x86:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [
+          ["cp36", "3.6"],
+          ["cp37", "3.7"],
+          ["cp38", "3.8"],
+          ["cp39", "3.9"],
+        ]
+        os: [macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - name: Set up Python ${{ matrix.python-version[1] }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version[1] }}
+
+    - name: Build wheel
+      uses: pypa/[email protected]
+      env:
+        CIBW_BUILD: ${{ matrix.python-version[0] }}-macosx_x86_64
+        CIBW_BUILD_FRONTEND: build
+        CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix [email protected])/lib/" CPATH="$(brew --prefix [email protected])/include/" STATIC_SSL=1
+        CIBW_ARCHS: "x86_64"
+        CIBW_BEFORE_TEST: >
+          export USE_SERVER_RC=${{ inputs.use-server-rc }} &&
+          SERVER_TAG=${{ inputs.server-tag }} vagrant up &&
+          sleep 3 &&
+          pip install -r test/requirements.txt
+        CIBW_TEST_COMMAND: >
+          cd {project}/test/ &&
+          python -m pytest new_tests/
+
+    - name: Save macOS wheel
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.os }}-x86_64-${{ matrix.python-version[0] }}-wheel
+        path: wheelhouse/*.whl
+
+  send-to-qe:
+    needs: [manylinux_x86_64]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - run: echo "Upload to QE"
+      # - uses: shallwefootball/s3-upload-action@master
+      #   with:
+      #     aws_key_id: ${{ secrets.AWS_KEY_ID }}
+      #     aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+      #     aws_bucket: ${{ secrets.AWS_BUCKET }}
+      #     # Send all distributions to QE build system
+      #     source_dir: './wheelhouse'

VERSION

@@ -1 +1 @@
-6.1.2
+6.2.0

Vagrantfile

@@ -0,0 +1,81 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+    # The most common configuration options are documented and commented below.
+    # For a complete reference, please see the online documentation at
+    # https://docs.vagrantup.com.
+
+    # Every Vagrant development environment requires a box. You can search for
+    # boxes at https://vagrantcloud.com/search.
+    config.vm.box = "ubuntu/jammy64"
+
+    # Disable automatic box update checking. If you disable this, then
+    # boxes will only be checked for updates when the user runs
+    # `vagrant box outdated`. This is not recommended.
+    # config.vm.box_check_update = false
+
+    # Create a forwarded port mapping which allows access to a specific port
+    # within the machine from a port on the host machine. In the example below,
+    # accessing "localhost:8080" will access port 80 on the guest machine.
+    # NOTE: This will enable public access to the opened port
+    # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+    # Create a forwarded port mapping which allows access to a specific port
+    # within the machine from a port on the host machine and only allow access
+    # via 127.0.0.1 to disable public access
+    config.vm.network "forwarded_port", guest: 3000, host: 3000, host_ip: "127.0.0.1"
+
+    # Create a private network, which allows host-only access to the machine
+    # using a specific IP.
+    # config.vm.network "private_network", ip: "192.168.33.10"
+
+    # Create a public network, which generally matched to bridged network.
+    # Bridged networks make the machine appear as another physical device on
+    # your network.
+    # config.vm.network "public_network"
+
+    # Share an additional folder to the guest VM. The first argument is
+    # the path on the host to the actual folder. The second argument is
+    # the path on the guest to mount the folder. And the optional third
+    # argument is a set of non-required options.
+    # config.vm.synced_folder "../data", "/vagrant_data"
+
+    # Provider-specific configuration so you can fine-tune various
+    # backing providers for Vagrant. These expose provider-specific options.
+    # Example for VirtualBox:
+    #
+    # config.vm.provider "virtualbox" do |vb|
+    #   # Display the VirtualBox GUI when booting the machine
+    #   vb.gui = true
+    #
+    #   # Customize the amount of memory on the VM:
+    #   vb.memory = "1024"
+    # end
+    #
+    # View the documentation for the provider you are using for more
+    # information on available options.
+
+    # Enable provisioning with a shell script. Additional provisioners such as
+    # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the
+    # documentation for more information about their specific syntax and use.
+    # config.vm.provision "shell", inline: <<-SHELL
+    #   apt-get update
+    #   apt-get install -y apache2
+    # SHELL
+
+    config.vm.provision "docker" do |d|
+      server_tag = ENV["SERVER_TAG"] || "latest"
+      if ENV["USE_SERVER_RC"] == "true"
+        $image_name = "aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:#{server_tag}"
+      else
+        $image_name = "aerospike/aerospike-server:#{server_tag}"
+      end
+      d.run $image_name,
+        args: "-p 3000:3000"
+    end
+  end

doc/aerospike.rst

@@ -850,18 +850,14 @@ Job Statuses
 Serialization Constants
 -----------------------
 
-.. data:: SERIALIZER_PYTHON
-
-    Use the cPickle serializer to handle unsupported types (default)
-
 .. data:: SERIALIZER_USER
 
     Use a user-defined serializer to handle unsupported types. Must have \
     been registered for the aerospike class or configured for the Client object
 
 .. data:: SERIALIZER_NONE
 
-    Do not serialize bins whose data type is unsupported
+    Do not serialize bins whose data type is unsupported (default)
 
 .. versionadded:: 1.0.47
 
@@ -872,10 +868,6 @@ Send Bool Constants
 
 Specifies how the Python client will write Python booleans.
 
-.. data:: PY_BYTES
-
-    Write Python Booleans as PY_BYTES_BLOBs.
-
 .. data:: INTEGER
 
     Write Python Booleans as integers.

doc/client.rst

@@ -179,7 +179,7 @@ Record Operations
 -----------------
 
 
-    .. method:: put(key, bins: dict[, meta: dict[, policy: dict[, serializer]]])
+    .. method:: put(key, bins: dict[, meta: dict[, policy: dict[, serializer=aerospike.SERIALIZER_NONE]]])
 
         Write a record with a given *key* to the cluster.