From c41a23cb28c9871b7f3842e32833e821cce15886 Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:59:47 -0700 Subject: [PATCH] [CLIENT-2258] Backport 6.*: Remove auto-serialization and auto-deserialization (#488) * Return a client error when writing unsupported Python types to the server * Convert AS_BYTES_PYTHON server types to bytearrays when reading data from the server * Remove support for serializing booleans as the AS_BYTES_PYTHON server type * Remove aerospike.SERIALIZER_PYTHON * client.put(): set serializer parameter's default value to aerospike.SERIALIZER_NONE * client.put(): fix bug where Python bytes bin values can't be used if serializer parameter is set to SERIALIZER_NONE Extra changes: * Docs: data mapping: add headers * Change send_bool_as default value to AS_BOOL * Add extra tests to verify client.operate() and expression behavior with instance-level serializers and deserializers * Add build wheels workflow * Only run CE tests since EE tests are broken on client version 6.1.2 * Remove C client code that breaks with later openssl versions Co-authored-by: dwelch-spike <53876192+dwelch-spike@users.noreply.github.com> --- .github/actions/run-ee-server/action.yml | 62 +++++ .github/workflows/build-wheels.yml | 169 ++++++++++++++ VERSION | 2 +- Vagrantfile | 81 +++++++ aerospike-client-c | 2 +- doc/aerospike.rst | 10 +- doc/client.rst | 2 +- doc/data_mapping.rst | 65 +++--- doc/query.rst | 2 +- doc/scan.rst | 2 +- src/include/policy.h | 7 +- src/main/aerospike.c | 2 +- src/main/client/put.c | 2 +- src/main/client/type.c | 4 +- src/main/conversions.c | 94 +++----- src/main/policy.c | 2 - src/main/serializer.c | 114 ++------- test/config.conf | 15 +- test/new_tests/test_arithmetic_expressions.py | 1 - test/new_tests/test_bool_config.py | 2 - test/new_tests/test_data.py | 13 -- test/new_tests/test_expressions_base.py | 4 +- test/new_tests/test_expressions_list.py | 35 ++- test/new_tests/test_expressions_map.py | 14 +- test/new_tests/test_map_operation_helpers.py | 29 ++- test/new_tests/test_nested_cdt_ctx.py | 118 +++++----- test/new_tests/test_operate.py | 12 +- test/new_tests/test_operate_helpers.py | 13 +- test/new_tests/test_operate_ordered.py | 10 +- test/new_tests/test_select.py | 5 +- test/new_tests/test_zserializers.py | 216 ++++++++++-------- test/requirements.txt | 2 +- 32 files changed, 632 insertions(+), 479 deletions(-) create mode 100644 .github/actions/run-ee-server/action.yml create mode 100644 .github/workflows/build-wheels.yml create mode 100644 Vagrantfile diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml new file mode 100644 index 000000000..41eec0cda --- /dev/null +++ b/.github/actions/run-ee-server/action.yml @@ -0,0 +1,62 @@ +name: 'Run EE Server' +description: 'Run EE server' +inputs: + use-server-rc: + required: true + default: false + server-tag: + required: true + default: false + +runs: + using: "composite" + steps: + - name: Install crudini to manipulate config.conf + run: pip install crudini + shell: bash + + - name: Add enterprise edition config to config.conf + run: | + crudini --set config.conf enterprise-edition hosts 127.0.0.1:3000 + crudini --set config.conf enterprise-edition user superuser + crudini --set config.conf enterprise-edition password superuser + working-directory: test + shell: bash + + - name: Create config folder to store configs in + run: mkdir configs + shell: bash + + - name: Use release server + if: ${{ inputs.use-server-rc == 'false' }} + run: echo "SERVER_IMAGE=aerospike/aerospike-server-enterprise:${{ inputs.server-tag }}" >> $GITHUB_ENV + shell: bash + + - name: Use release candidate server + if: ${{ inputs.use-server-rc == 'true' }} + run: echo "SERVER_IMAGE=aerospike.jfrog.io/docker/aerospike/aerospike-server-enterprise-rc:${{ inputs.server-tag }}" >> $GITHUB_ENV + shell: bash + + - name: Get default aerospike.conf from Docker server EE container + run: | + docker run -d --name aerospike -p 3000-3002:3000-3002 $SERVER_IMAGE + sleep 5 + docker cp aerospike:/etc/aerospike/aerospike.conf ./configs/aerospike.conf + docker container stop aerospike + docker container rm aerospike + shell: bash + + - name: Enable security features using aerospike.conf + # Security stanza + run: echo -e "security {\n\tenable-quotas true\n}\n" >> ./aerospike.conf + working-directory: ./configs + shell: bash + + - name: Run enterprise edition server + run: docker run -tid -v $(pwd)/configs:/opt/aerospike/etc -p 3000:3000 --name aerospike $SERVER_IMAGE asd --config-file /opt/aerospike/etc/aerospike.conf + shell: bash + + - name: Create user in database for tests + # Use default admin user to create another user for testing + run: docker exec aerospike asadm --user admin --password admin --enable -e "manage acl create user superuser password superuser roles read-write-udf sys-admin user-admin data-admin" + shell: bash diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml new file mode 100644 index 000000000..5a66fe288 --- /dev/null +++ b/.github/workflows/build-wheels.yml @@ -0,0 +1,169 @@ +name: Build wheels + +# Builds wheels and sends to QE and Aerospike artifactory +on: + workflow_dispatch: + inputs: + use-server-rc: + type: boolean + required: true + default: false + # Test against a server version + # This is helpful if you need to create a backport from an older client major version + # And that older client major version does not accomodate for breaking changes in the latest server version + server-tag: + required: true + default: 'latest' + +jobs: + test-sdist: + name: Build and install sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + architecture: 'x64' + + - run: sudo apt update + - name: Install build dependencies (apt packages) + run: sudo apt install python3-dev libssl-dev -y + - name: Install build dependencies (pip packages) + run: python3 -m pip install build + + - name: Build source distribution + run: python3 -m build --sdist + + - name: Upload sdist to GitHub + uses: actions/upload-artifact@v3 + with: + path: ./dist/*.tar.gz + # Artifact name, not the file name + name: sdist + + manylinux_x86_64: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Python versions to build wheels on + python: [ + ["cp36", "3.6"], + ["cp37", "3.7"], + ["cp38", "3.8"], + ["cp39", "3.9"], + ] + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Use release server + if: ${{ inputs.use-server-rc == false }} + run: echo "SERVER_IMAGE=aerospike/aerospike-server:${{ inputs.server-tag }}" >> $GITHUB_ENV + + - name: Use release candidate server + if: ${{ inputs.use-server-rc == true }} + run: echo "SERVER_IMAGE=aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:${{ inputs.server-tag }}" >> $GITHUB_ENV + + - name: Run server + run: | + docker run -d --name aerospike -p 3000-3002:3000-3002 $SERVER_IMAGE + + - name: Wait for server to start + run: sleep 5 + + - name: Set config.conf to use Docker IP address of Aerospike server + # config.conf should be copied into the cibuildwheel Docker container + run: | + export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) + pip install crudini + crudini --set config.conf community-edition hosts ${SERVER_DOCKER_IP}:3000 + working-directory: test + + - name: Build wheel + uses: pypa/cibuildwheel@v2.11.2 + env: + CIBW_BUILD: ${{ matrix.python[0] }}-manylinux_x86_64 + CIBW_BUILD_FRONTEND: build + CIBW_BEFORE_ALL_LINUX: > + yum install openssl-devel -y && + yum install python-devel -y && + yum install python-setuptools -y + CIBW_ARCHS: "x86_64" + CIBW_TEST_COMMAND: > + cd {project}/test/ && + pip install -r requirements.txt && + python -m pytest new_tests/ + + - name: Upload wheels to GitHub + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + path: ./wheelhouse/*.whl + # Artifact name, not the file name + name: manylinux-x86_64-${{ matrix.python[0] }} + + macOS-x86: + strategy: + fail-fast: false + matrix: + python-version: [ + ["cp36", "3.6"], + ["cp37", "3.7"], + ["cp38", "3.8"], + ["cp39", "3.9"], + ] + os: [macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Set up Python ${{ matrix.python-version[1] }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version[1] }} + + - name: Build wheel + uses: pypa/cibuildwheel@v2.11.2 + env: + CIBW_BUILD: ${{ matrix.python-version[0] }}-macosx_x86_64 + CIBW_BUILD_FRONTEND: build + CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 + CIBW_ARCHS: "x86_64" + CIBW_BEFORE_TEST: > + export USE_SERVER_RC=${{ inputs.use-server-rc }} && + SERVER_TAG=${{ inputs.server-tag }} vagrant up && + sleep 3 && + pip install -r test/requirements.txt + CIBW_TEST_COMMAND: > + cd {project}/test/ && + python -m pytest new_tests/ + + - name: Save macOS wheel + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.os }}-x86_64-${{ matrix.python-version[0] }}-wheel + path: wheelhouse/*.whl + + send-to-qe: + needs: [manylinux_x86_64] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - run: echo "Upload to QE" + # - uses: shallwefootball/s3-upload-action@master + # with: + # aws_key_id: ${{ secrets.AWS_KEY_ID }} + # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} + # aws_bucket: ${{ secrets.AWS_BUCKET }} + # # Send all distributions to QE build system + # source_dir: './wheelhouse' diff --git a/VERSION b/VERSION index 5e3254243..6abaeb2f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.1.2 +6.2.0 diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..dc47f1f3a --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,81 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "ubuntu/jammy64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + config.vm.network "forwarded_port", guest: 3000, host: 3000, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + # vb.memory = "1024" + # end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # apt-get update + # apt-get install -y apache2 + # SHELL + + config.vm.provision "docker" do |d| + server_tag = ENV["SERVER_TAG"] || "latest" + if ENV["USE_SERVER_RC"] == "true" + $image_name = "aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:#{server_tag}" + else + $image_name = "aerospike/aerospike-server:#{server_tag}" + end + d.run $image_name, + args: "-p 3000:3000" + end + end diff --git a/aerospike-client-c b/aerospike-client-c index fcde89179..34a94004b 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit fcde8917920fde779b5be073ed7568f8c62689d0 +Subproject commit 34a94004b7e5a3b9c78cbdc93c74a87a58de26a8 diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 4d59cfb98..8e403accc 100755 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -850,10 +850,6 @@ Job Statuses Serialization Constants ----------------------- -.. data:: SERIALIZER_PYTHON - - Use the cPickle serializer to handle unsupported types (default) - .. data:: SERIALIZER_USER Use a user-defined serializer to handle unsupported types. Must have \ @@ -861,7 +857,7 @@ Serialization Constants .. data:: SERIALIZER_NONE - Do not serialize bins whose data type is unsupported + Do not serialize bins whose data type is unsupported (default) .. versionadded:: 1.0.47 @@ -872,10 +868,6 @@ Send Bool Constants Specifies how the Python client will write Python booleans. -.. data:: PY_BYTES - - Write Python Booleans as PY_BYTES_BLOBs. - .. data:: INTEGER Write Python Booleans as integers. diff --git a/doc/client.rst b/doc/client.rst index bc44944cf..deacbaa9d 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -179,7 +179,7 @@ Record Operations ----------------- - .. method:: put(key, bins: dict[, meta: dict[, policy: dict[, serializer]]]) + .. method:: put(key, bins: dict[, meta: dict[, policy: dict[, serializer=aerospike.SERIALIZER_NONE]]]) Write a record with a given *key* to the cluster. diff --git a/doc/data_mapping.rst b/doc/data_mapping.rst index 693efd4d1..c34630d19 100644 --- a/doc/data_mapping.rst +++ b/doc/data_mapping.rst @@ -6,29 +6,27 @@ .. rubric:: How Python types map to server types -.. note:: +Default Behavior +---------------- + +By default, the :py:class:`~aerospike.Client` maps the supported Python types to Aerospike server \ +`types `_. \ +When an unsupported type is encountered by the module: + +1. When sending data to the server, it does not serialize the type and will throw an error. +2. When reading `AS_BYTES_PYTHON` types from the server, it returns the raw bytes as a :class:`bytearray`. + To deserialize this data, the application must use cPickle instead of relying on the client to do it automatically. - By default, the :py:class:`~aerospike.Client` maps the supported types \ - :py:class:`int`, :py:class:`bool`, :py:class:`str`, :py:class:`float`, :py:class:`bytearray`, \ - :py:class:`list`, :py:class:`dict` to matching aerospike server \ - `types `_ \ - (int, string, double, blob, list, map). When an unsupported type is \ - encountered, the module uses \ - `cPickle `_ \ - to serialize and deserialize the data, storing it into a blob of type \ - `'Python' `_ \ - (`AS_BYTES_PYTHON `_). - - The functions :func:`~aerospike.set_serializer` and :func:`~aerospike.set_deserializer` \ - allow for user-defined functions to handle serialization, instead. The user provided function will be run instead of cPickle. \ - The serialized data is stored as \ - type (\ - `AS_BYTES_BLOB `_). \ - This type allows the storage of binary data readable by Aerospike Clients in other languages. \ - The *serialization* config param of :func:`aerospike.client` registers an \ - instance-level pair of functions that handle serialization. - - Unless a user specified serializer has been provided, all other types will be stored as Python specific bytes. Python specific bytes may not be readable by Aerospike Clients for other languages. +Serializers +----------- + +However, the functions :func:`~aerospike.set_serializer` and :func:`~aerospike.set_deserializer` \ +allow for user-defined functions to handle serialization. +The serialized data is stored in the server with generic encoding \ +(`AS_BYTES_BLOB `_). +This type allows the storage of binary data readable by Aerospike Clients in other languages. \ +The *serialization* config parameter of :func:`aerospike.client` registers an \ +instance-level pair of functions that handle serialization. .. warning:: @@ -36,9 +34,14 @@ In order to support cross client compatibility and rolling upgrades, Python client version 6.x comes with a new client config, send_bool_as. Send_bool_as configures how the client writes Python booleans and allows for opting into using the new boolean type. It is important to consider how other clients connected to the Aerospike database write booleans in order to maintain cross client compatibility. - If a client reads and writes booleans as integers then the Python client should too, if they work with the same data. - Send_bool_as can be set so the client writes Python booleans as AS_BYTES_PYTHON, integer, or the new server boolean type. - All versions before 6.x wrote Python booleans as AS_BYTES_PYTHON. + For example, if there is a client that reads and writes booleans as integers, then another Python client working with the same data should do the same thing. + + ``send_bool_as`` can be set so the client writes Python booleans as integers or the Aerospike native boolean type. + + All versions before ``6.x`` wrote Python booleans as ``AS_BYTES_PYTHON``. + +Data Mappings +------------- The following table shows which Python types map directly to Aerospike server types. @@ -72,7 +75,11 @@ The following table shows which Python types map directly to Aerospike server ty It is possible to nest these datatypes. For example a list may contain a dictionary, or a dictionary may contain a list as a value. -.. note:: - - Unless a user specified serializer has been provided, all other types will be stored as Python specific bytes. Python specific bytes may not be readable by Aerospike Clients for other languages. - +.. _integer: https://docs.aerospike.com/server/guide/data-types/scalar-data-types#integer +.. _string: https://docs.aerospike.com/server/guide/data-types/scalar-data-types#string +.. _double: https://docs.aerospike.com/server/guide/data-types/scalar-data-types#double +.. _map: https://docs.aerospike.com/server/guide/data-types/cdt-map +.. _key ordered map: https://docs.aerospike.com/server/guide/data-types/cdt-map +.. _list: https://docs.aerospike.com/server/guide/data-types/cdt-list +.. _blob: https://docs.aerospike.com/server/guide/data-types/blob +.. _GeoJSON: https://docs.aerospike.com/server/guide/data-types/geospatial diff --git a/doc/query.rst b/doc/query.rst index 544d1a3b5..f73a70d48 100755 --- a/doc/query.rst +++ b/doc/query.rst @@ -239,7 +239,7 @@ Query Methods :param str module: the name of the Lua module. :param str function: the name of the Lua function within the *module*. :param list arguments: optional arguments to pass to the *function*. NOTE: these arguments must be types supported by Aerospike See: `supported data types `_. - If you need to use an unsuported type, (e.g. set or tuple) you can use a serializer like pickle first. + If you need to use an unsupported type, (e.g. set or tuple) you must use your own serializer. :return: one of the supported types, :class:`int`, :class:`str`, :class:`float` (double), :class:`list`, :class:`dict` (map), :class:`bytearray` (bytes), :class:`bool`. .. seealso:: `Developing Stream UDFs `_ diff --git a/doc/scan.rst b/doc/scan.rst index 8ca7af33c..59ba4c7bf 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -40,7 +40,7 @@ Scan Methods :param str module: the name of the Lua module. :param str function: the name of the Lua function within the *module*. :param list arguments: optional arguments to pass to the *function*. NOTE: these arguments must be types supported by Aerospike See: `supported data types `_. - If you need to use an unsuported type, (e.g. set or tuple) you can use a serializer such as pickle first. + If you need to use an unsupported type, (e.g. set or tuple) you must use your own serializer. :return: one of the supported types, :class:`int`, :class:`str`, :class:`float` (double), :class:`list`, :class:`dict` (map), :class:`bytearray` (bytes), :class:`bool`. .. seealso:: `Developing Record UDFs `_ diff --git a/src/include/policy.h b/src/include/policy.h index eebb5bb6d..68884ba89 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -37,16 +37,15 @@ */ enum Aerospike_serializer_values { - SERIALIZER_NONE, - SERIALIZER_PYTHON, /* default handler for serializer type */ + SERIALIZER_NONE, /* default handler for serializer type */ + SERIALIZER_PYTHON, SERIALIZER_JSON, SERIALIZER_USER, }; enum Aerospike_send_bool_as_values { - SEND_BOOL_AS_PY_BYTES, /* default for writing Python bools */ SEND_BOOL_AS_INTEGER, - SEND_BOOL_AS_AS_BOOL, + SEND_BOOL_AS_AS_BOOL, /* default for writing Python bools */ }; enum Aerospike_list_operations { diff --git a/src/main/aerospike.c b/src/main/aerospike.c index c1647ca0d..01145cc6f 100644 --- a/src/main/aerospike.c +++ b/src/main/aerospike.c @@ -155,7 +155,7 @@ static int Aerospike_Clear(PyObject *aerospike) MOD_INIT(aerospike) { - const char version[8] = "6.1.2"; + const char version[8] = "6.2.0"; // Makes things "thread-safe" PyEval_InitThreads(); int i = 0; diff --git a/src/main/client/put.c b/src/main/client/put.c index abbdb1cec..1e413ddf5 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -181,7 +181,7 @@ PyObject *AerospikeClient_Put(AerospikeClient *self, PyObject *args, PyObject *py_meta = NULL; PyObject *py_policy = NULL; PyObject *py_serializer_option = NULL; - long serializer_option = SERIALIZER_PYTHON; + long serializer_option = SERIALIZER_NONE; // Python Function Keyword Arguments static char *kwlist[] = {"key", "bins", "meta", diff --git a/src/main/client/type.c b/src/main/client/type.c index 827347a6d..48c6f76ca 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -844,7 +844,7 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, self->has_connected = false; self->use_shared_connection = false; self->as = NULL; - self->send_bool_as = SEND_BOOL_AS_PY_BYTES; + self->send_bool_as = SEND_BOOL_AS_AS_BOOL; if (PyArg_ParseTupleAndKeywords(args, kwds, "O:client", kwlist, &py_config) == false) { @@ -1270,7 +1270,7 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, PyObject *py_send_bool_as = PyDict_GetItemString(py_config, "send_bool_as"); if (py_send_bool_as != NULL && PyLong_Check(py_send_bool_as)) { int send_bool_as_temp = PyLong_AsLong(py_send_bool_as); - if (send_bool_as_temp >= SEND_BOOL_AS_PY_BYTES && + if (send_bool_as_temp >= SEND_BOOL_AS_INTEGER && send_bool_as_temp <= SEND_BOOL_AS_AS_BOOL) { self->send_bool_as = send_bool_as_temp; } diff --git a/src/main/conversions.c b/src/main/conversions.c index 81992bf01..e02e156c3 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -64,10 +64,6 @@ static bool requires_int(uint64_t op); -static as_status py_bool_to_py_bytes_blob(AerospikeClient *self, as_error *err, - as_static_pool *static_pool, - PyObject *py_bool, as_bytes **target, - int serializer_type); static as_status py_bool_to_as_integer(as_error *err, PyObject *py_bool, as_integer **target); static as_status py_bool_to_as_bool(as_error *err, PyObject *py_bool, @@ -695,15 +691,6 @@ as_status pyobject_to_val(AerospikeClient *self, as_error *err, PyBool_Check( py_obj)) { //TODO Change to true bool support post jump version. switch (self->send_bool_as) { - case SEND_BOOL_AS_PY_BYTES:; - as_bytes *bool_bytes = NULL; - if (py_bool_to_py_bytes_blob(self, err, static_pool, py_obj, - &bool_bytes, - serializer_type) != AEROSPIKE_OK) { - return err->code; - } - *val = (as_val *)bool_bytes; - break; case SEND_BOOL_AS_AS_BOOL:; as_boolean *converted_bool = NULL; if (py_bool_to_as_bool(err, py_obj, &converted_bool) != @@ -770,17 +757,14 @@ as_status pyobject_to_val(AerospikeClient *self, as_error *err, *val = (as_val *)as_geojson_new(geo_value, false); } else if (PyByteArray_Check(py_obj)) { - as_bytes *bytes; - GET_BYTES_POOL(bytes, static_pool, err); - if (err->code == AEROSPIKE_OK) { - if (serialize_based_on_serializer_policy(self, serializer_type, - &bytes, py_obj, - err) != AEROSPIKE_OK) { - return err->code; - } - *val = (as_val *)bytes; - } - } + Py_ssize_t str_len = PyByteArray_Size(py_obj); + as_bytes *bytes = as_bytes_new(str_len); + + char *str = PyByteArray_AsString(py_obj); + as_bytes_set(bytes, 0, (const uint8_t *)str, str_len); + + *val = (as_val *)bytes; + } else if (PyList_Check(py_obj)) { as_list *list = NULL; pyobject_to_list(self, err, py_obj, &list, static_pool, @@ -901,15 +885,6 @@ as_status pyobject_to_record(AerospikeClient *self, as_error *err, PyBool_Check( value)) { //TODO Change to true bool support post jump version. switch (self->send_bool_as) { - case SEND_BOOL_AS_PY_BYTES:; - as_bytes *bool_bytes = NULL; - if (py_bool_to_py_bytes_blob( - self, err, static_pool, value, &bool_bytes, - serializer_type) != AEROSPIKE_OK) { - return err->code; - } - ret_val = as_record_set_bytes(rec, name, bool_bytes); - break; case SEND_BOOL_AS_AS_BOOL:; as_boolean *converted_bool = NULL; if (py_bool_to_as_bool(err, value, &converted_bool) != @@ -1000,18 +975,24 @@ as_status pyobject_to_record(AerospikeClient *self, as_error *err, char *val = PyString_AsString(value); ret_val = as_record_set_strp(rec, name, val, false); } - else if (PyByteArray_Check(value)) { - as_bytes *bytes; - GET_BYTES_POOL(bytes, static_pool, err); - if (err->code == AEROSPIKE_OK) { - if (serialize_based_on_serializer_policy( - self, serializer_type, &bytes, value, err) != - AEROSPIKE_OK) { - return err->code; - } - ret_val = as_record_set_bytes(rec, name, bytes); - } - } + else if (PyBytes_Check(value)) { + Py_ssize_t str_len = PyBytes_Size(value); + as_bytes *bytes = as_bytes_new(str_len); + + char *str = PyBytes_AsString(value); + as_bytes_set(bytes, 0, (const uint8_t *)str, str_len); + + ret_val = as_record_set_bytes(rec, name, bytes); + } + else if (PyByteArray_Check(value)) { + Py_ssize_t str_len = PyByteArray_Size(value); + as_bytes *bytes = as_bytes_new(str_len); + + char *str = PyByteArray_AsString(value); + as_bytes_set(bytes, 0, (const uint8_t *)str, str_len); + + ret_val = as_record_set_bytes(rec, name, bytes); + } else if (PyList_Check(value)) { // as_list as_list *list = NULL; @@ -2552,29 +2533,6 @@ static bool requires_int(uint64_t op) op == CDT_CTX_LIST_INDEX_CREATE; } -/* - * py_bool_to_py_bytes_blob serializes py_bool. - * Target should be a NULL pointer to an as_integer. py_bool_to_py_bytes_blob will get memory for target - * from the static pool, static_pool. The pool should be destroyed after use, by the caller. - */ -static as_status py_bool_to_py_bytes_blob(AerospikeClient *self, as_error *err, - as_static_pool *static_pool, - PyObject *py_bool, as_bytes **target, - int serializer_type) -{ - GET_BYTES_POOL(*target, static_pool, err); - if (err->code != AEROSPIKE_OK) { - return err->code; - } - - if (serialize_based_on_serializer_policy(self, serializer_type, target, - py_bool, err) != AEROSPIKE_OK) { - return err->code; - } - - return AEROSPIKE_OK; -} - /* * py_bool_to_as_integer converts a python object to an as_integer based on its truth value. * Target should be a NULL pointer to an as_integer. py_bool_to_as_integer will allocate a new diff --git a/src/main/policy.c b/src/main/policy.c index 3dcb19986..f50261009 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -175,11 +175,9 @@ static AerospikeConstants aerospike_constants[] = { {AS_POLICY_REPLICA_PREFER_RACK, "POLICY_REPLICA_PREFER_RACK"}, {AS_POLICY_COMMIT_LEVEL_ALL, "POLICY_COMMIT_LEVEL_ALL"}, {AS_POLICY_COMMIT_LEVEL_MASTER, "POLICY_COMMIT_LEVEL_MASTER"}, - {SERIALIZER_PYTHON, "SERIALIZER_PYTHON"}, {SERIALIZER_USER, "SERIALIZER_USER"}, {SERIALIZER_JSON, "SERIALIZER_JSON"}, {SERIALIZER_NONE, "SERIALIZER_NONE"}, - {SEND_BOOL_AS_PY_BYTES, "PY_BYTES"}, {SEND_BOOL_AS_INTEGER, "INTEGER"}, {SEND_BOOL_AS_AS_BOOL, "AS_BOOL"}, {AS_INDEX_STRING, "INDEX_STRING"}, diff --git a/src/main/serializer.c b/src/main/serializer.c index ee1440e40..d7158dbe2 100644 --- a/src/main/serializer.c +++ b/src/main/serializer.c @@ -352,50 +352,8 @@ extern as_status serialize_based_on_serializer_policy(AerospikeClient *self, set_as_bytes(bytes, my_bytes, my_bytes_len, AS_BYTES_BLOB, error_p); } else { - - /* get the sys.modules dictionary */ - PyObject *sysmodules = PyImport_GetModuleDict(); - PyObject *cpickle_module = NULL; - if (PyMapping_HasKeyString(sysmodules, "pickle")) { - cpickle_module = PyMapping_GetItemString(sysmodules, "pickle"); - } - else { - cpickle_module = PyImport_ImportModule("pickle"); - } - - if (!cpickle_module) { - /* insert error handling here! and exit this function */ - as_error_update(error_p, AEROSPIKE_ERR_CLIENT, - "Unable to load pickle module"); - goto CLEANUP; - } - else { - PyObject *py_funcname = PyString_FromString("dumps"); - - Py_INCREF(cpickle_module); - initresult = PyObject_CallMethodObjArgs( - cpickle_module, py_funcname, value, NULL); - Py_DECREF(cpickle_module); - Py_DECREF(py_funcname); - - if (!initresult) { - /* more error handling &c */ - as_error_update(error_p, AEROSPIKE_ERR_CLIENT, - "Unable to call dumps function"); - goto CLEANUP; - } - else { - Py_INCREF(initresult); - - char *return_value; - Py_ssize_t len; - PyBytes_AsStringAndSize(initresult, &return_value, &len); - set_as_bytes(bytes, (uint8_t *)return_value, len, - AS_BYTES_PYTHON, error_p); - Py_DECREF(initresult); - } - } - Py_XDECREF(cpickle_module); + as_error_update(error_p, AEROSPIKE_ERR_CLIENT, + "Unable to serialize unknown Python native type."); } } break; case SERIALIZER_JSON: @@ -476,57 +434,23 @@ extern as_status deserialize_based_on_as_bytes_type(AerospikeClient *self, as_error *error_p) { switch (as_bytes_get_type(bytes)) { - case AS_BYTES_PYTHON: { - PyObject *sysmodules = PyImport_GetModuleDict(); - PyObject *cpickle_module = NULL; - if (PyMapping_HasKeyString(sysmodules, "pickle")) { - cpickle_module = PyMapping_GetItemString(sysmodules, "pickle"); - } - else { - cpickle_module = PyImport_ImportModule("pickle"); - } - - PyObject *initresult = NULL; - if (!cpickle_module) { - /* insert error handling here! and exit this function */ - as_error_update(error_p, AEROSPIKE_ERR_CLIENT, - "Unable to load pickle module"); - goto CLEANUP; - } - else { - char *bytes_val_p = (char *)bytes->value; - PyObject *py_value = - PyBytes_FromStringAndSize(bytes_val_p, as_bytes_size(bytes)); - PyObject *py_funcname = PyString_FromString("loads"); - - Py_INCREF(cpickle_module); - initresult = PyObject_CallMethodObjArgs(cpickle_module, py_funcname, - py_value, NULL); - Py_DECREF(cpickle_module); - Py_DECREF(py_funcname); - Py_DECREF(py_value); - if (!initresult) { - // At this point we want to try to fallback to returning a byte array - uint32_t bval_size = as_bytes_size(bytes); - initresult = PyByteArray_FromStringAndSize( - (char *)as_bytes_get(bytes), bval_size); - // We couldn't convert the value into a byte array - if (!initresult) { - as_error_update(error_p, AEROSPIKE_ERR_CLIENT, - "Unable to deserialize bytes"); - Py_XDECREF(cpickle_module); - goto CLEANUP; - } - // The fallback deserialization succeeded - *retval = initresult; - as_error_update(error_p, AEROSPIKE_OK, NULL); - } - else { - *retval = initresult; - } - } - Py_XDECREF(cpickle_module); - } break; + case AS_BYTES_PYTHON:; + // Automatically convert AS_BYTES_PYTHON server types to bytearrays. + // This prevents the client from throwing an exception and + // breaking applications that don't handle the exception + // in case it still fetches AS_BYTES_PYTHON types stored in the server. + // Applications using this client must deserialize the bytearrays + // manually with cPickle. + uint32_t bval_size = as_bytes_size(bytes); + PyObject *py_val = PyByteArray_FromStringAndSize( + (char *)as_bytes_get(bytes), bval_size); + if (!py_val) { + as_error_update(error_p, AEROSPIKE_ERR_CLIENT, + "Unable to deserialize AS_BYTES_PYTHON bytes"); + goto CLEANUP; + } + *retval = py_val; + as_error_update(error_p, AEROSPIKE_OK, NULL); case AS_BYTES_BLOB: { if (self->user_deserializer_call_info.callback) { execute_user_callback(&self->user_deserializer_call_info, &bytes, diff --git a/test/config.conf b/test/config.conf index e07ae2302..e9fd3924a 100644 --- a/test/config.conf +++ b/test/config.conf @@ -2,15 +2,6 @@ hosts: 127.0.0.1:3000 [enterprise-edition] -hosts:bob-cluster-a:4333 -user:generic_client -password:generic_client - -[policies] -auth_mode:3 - -[tls] -enable: 1 -cafile: /code/src/aerospike/enterprise/as-dev-infra/certs/Platinum/cacert.pem -keyfile: /code/src/aerospike/enterprise/as-dev-infra/certs/Client-Chainless/key.pem -certfile: /code/src/aerospike/enterprise/as-dev-infra/certs/Client-Chainless/cert.pem +hosts : +user : +password : diff --git a/test/new_tests/test_arithmetic_expressions.py b/test/new_tests/test_arithmetic_expressions.py index bda51160f..e792fadcf 100644 --- a/test/new_tests/test_arithmetic_expressions.py +++ b/test/new_tests/test_arithmetic_expressions.py @@ -366,7 +366,6 @@ def test_mod_pos(self, bin, val, check): @pytest.mark.parametrize("bin, val, check, expected", [ (FloatBin("fbin"), [4.0], [5.0], e.InvalidRequest), - (FloatBin("fbin"), [10.0], [FloatBin("fbin")], e.InvalidRequest), (IntBin("ibin"), [2], 0, e.FilteredOut), (IntBin("ibin"), [5.0], 25, e.InvalidRequest), (FloatBin("fbin"), [3], 15.0, e.InvalidRequest), diff --git a/test/new_tests/test_bool_config.py b/test/new_tests/test_bool_config.py index c9e522a6c..c8af47745 100644 --- a/test/new_tests/test_bool_config.py +++ b/test/new_tests/test_bool_config.py @@ -45,11 +45,9 @@ def setup(self, request, as_connection): pass @pytest.mark.parametrize("send_bool_as, expected_true, expected_false", [ - (aerospike.PY_BYTES, True, False), (aerospike.INTEGER, 1, 0), (aerospike.AS_BOOL, True, False), (100, True, False), - (0, True, False), (-1, True, False) ]) def test_bool_read_write_pos(self, send_bool_as, expected_true, expected_false): diff --git a/test/new_tests/test_data.py b/test/new_tests/test_data.py index 593d29540..45830ac35 100644 --- a/test/new_tests/test_data.py +++ b/test/new_tests/test_data.py @@ -15,7 +15,6 @@ class SomeClass(object): (('test', 'demo', 'string'), {'place': "New York", 'name': 'John'}), (('test', 'demo', u"bb"), {'a': [u'aa', 2, u'aa', 4, u'cc', 3, 2, 1]}), (('test', u'demo', 1), {'age': 1, 'name': 'name1'}), - (('test', 'demo', 1), {"is_present": None}), (('test', 'unknown_set', 1), { 'a': {'k': [bytearray("askluy3oijs", "utf-8")]}}), @@ -44,15 +43,6 @@ class SomeClass(object): # (('test', 'demo', 1), # {'odict': OrderedDict(sorted({'banana': 3, 'apple': 4, 'pear': 1, 'orange': 2}.items(), # key=lambda t: t[0]))}), - - # Tuple Data - (('test', 'demo', 'tuple_key'), {'tuple_seq': tuple('abc')}), - - # Set Data - (('test', 'demo', 'set_key'), {"set_data": set([1, 2])}), - (('test', 'demo', 'fset_key'), { - "fset_data": frozenset(["Frankfurt", "Basel", "Freiburg"])}), - # Hybrid (('test', 'demo', 'multiple_bins'), { 'i': ["nanslkdl", 1, bytearray("asd;as[d'as;d", "utf-8")], @@ -66,9 +56,6 @@ class SomeClass(object): 'age': 24}, {'name': 'Marlen', 'age': 25}] }), - (('test', 'demo', 'map_tuple_key'), { - 'seq': {'bb': tuple('abc')} - }), ] key_neg = [ diff --git a/test/new_tests/test_expressions_base.py b/test/new_tests/test_expressions_base.py index e8e798af3..a55473d97 100644 --- a/test/new_tests/test_expressions_base.py +++ b/test/new_tests/test_expressions_base.py @@ -92,6 +92,8 @@ def setup(self, request, as_connection): 'balance': i * 10, 'key': i, 'alt_name': 'name%s' % (str(i)), 'list_bin': [ + # TODO: None should not be encoded + # because it's not documented to represent a server null type None, i, "string_test" + str(i), @@ -100,7 +102,7 @@ def setup(self, request, as_connection): bytearray("bytearray_test" + str(i), "utf8"), ("bytes_test" + str(i)).encode("utf8"), i % 2 == 1, - aerospike.null, + aerospike.null(), float(i) ], 'ilist_bin': [ diff --git a/test/new_tests/test_expressions_list.py b/test/new_tests/test_expressions_list.py index f4988c143..73f1969fc 100644 --- a/test/new_tests/test_expressions_list.py +++ b/test/new_tests/test_expressions_list.py @@ -79,8 +79,7 @@ def __init__(self, i): bytearray("bytearray_test" + str(8), "utf8"), ("bytes_test" + str(8)).encode("utf8"), 8 % 2 == 1, - aerospike.null, - TestUsrDefinedClass(8), + aerospike.null(), float(8), GEO_POLY ] @@ -108,8 +107,7 @@ def setup(self, request, as_connection): bytearray("bytearray_test" + str(i), "utf8"), ("bytes_test" + str(i)).encode("utf8"), i % 2 == 1, - aerospike.null, - TestUsrDefinedClass(i), + aerospike.null(), float(i), GEO_POLY ], @@ -145,13 +143,8 @@ def setup(self, request, as_connection): ], 'nlist_bin': [ None, - aerospike.null, - aerospike.null - ], - 'bllist_bin': [ - TestUsrDefinedClass(1), - TestUsrDefinedClass(3), - TestUsrDefinedClass(4) + aerospike.null(), + aerospike.null() ], 'flist_bin': [ 1.0, @@ -197,13 +190,12 @@ def test_list_get_by_index_pos(self, ctx_types, ctx_indexes, bin_type, index, re (None, None, "bytes_test3".encode("utf8"), aerospike.LIST_RETURN_VALUE, ["bytes_test3".encode("utf8")], 1), (None, None, bytearray("bytearray_test3", "utf8"), aerospike.LIST_RETURN_VALUE, [bytearray("bytearray_test3", "utf8")], 1), #(None, None, True, aerospike.LIST_RETURN_VALUE, [True], 9), NOTE: this won't work because booleans are not serialized by default in expressions. - (None, None, None, aerospike.LIST_RETURN_VALUE, [None], _NUM_RECORDS), + (None, None, None, aerospike.LIST_RETURN_VALUE, [None, None], _NUM_RECORDS), (None, None, [26, 27, 28, 6], aerospike.LIST_RETURN_VALUE, [[26, 27, 28, 6]], 1), ([list_index], [3], 6, aerospike.LIST_RETURN_VALUE, [6], 1), (None, None, {31: 31, 32: 32, 33: 33, 8: 8}, aerospike.LIST_RETURN_VALUE, [{31: 31, 32: 32, 33: 33, 8: 8}], 1), - (None, None, aerospike.null, aerospike.LIST_RETURN_VALUE, [aerospike.null], _NUM_RECORDS), + (None, None, aerospike.null(), aerospike.LIST_RETURN_VALUE, [aerospike.null(), aerospike.null()], _NUM_RECORDS), (None, None, GEO_POLY, aerospike.LIST_RETURN_VALUE, [GEO_POLY], _NUM_RECORDS), - (None, None, TestUsrDefinedClass(4), aerospike.LIST_RETURN_VALUE, [TestUsrDefinedClass(4)], 1) ]) def test_list_get_by_value_pos(self, ctx_types, ctx_indexes, value, return_type, check, expected): """ @@ -228,11 +220,10 @@ def test_list_get_by_value_pos(self, ctx_types, ctx_indexes, value, return_type, # (None, None, 4, 7, aerospike.LIST_RETURN_RANK, [[1], [1], [1]], 3), temporarily failing because of bool jump rank (None, None, "string_test3","string_test6", aerospike.LIST_RETURN_INDEX, [[2], [2], [2]], 3), (None, None, "bytes_test6".encode("utf8"), "bytes_test9".encode("utf8"), aerospike.LIST_RETURN_COUNT, [1, 1, 1], 3), - (None, None, bytearray("bytearray_test3", "utf8"), bytearray("bytearray_test6", "utf8"), aerospike.LIST_RETURN_REVERSE_INDEX, [[6], [6], [6]], 3), + (None, None, bytearray("bytearray_test3", "utf8"), bytearray("bytearray_test6", "utf8"), aerospike.LIST_RETURN_REVERSE_INDEX, [[5] for _ in range(3)], 3), (None, None, [26, 27, 28, 6], [26, 27, 28, 9], aerospike.LIST_RETURN_VALUE, [[[26, 27, 28, 6]], [[26, 27, 28, 7]], [[26, 27, 28, 8]]], 3), ([list_index], [3], 5, 9, aerospike.LIST_RETURN_REVERSE_RANK, [[3], [3], [3]], 4), (None, None, GEO_POLY, aerospike.CDTInfinite(), aerospike.LIST_RETURN_VALUE, [[GEO_POLY], [GEO_POLY], [GEO_POLY]], _NUM_RECORDS), - (None, None, TestUsrDefinedClass(4), TestUsrDefinedClass(7), aerospike.LIST_RETURN_VALUE, [[TestUsrDefinedClass(4)], [TestUsrDefinedClass(5)], [TestUsrDefinedClass(6)]], 3) #NOTE py_bytes cannot be compard directly server side ]) def test_list_get_by_value_range_pos(self, ctx_types, ctx_indexes, begin, end, return_type, check, expected): """ @@ -278,9 +269,9 @@ def test_list_get_by_value_range_neg(self, ctx, begin, end, return_type, check, (None, None, ["string_test3", 3], aerospike.LIST_RETURN_VALUE, ["string_test3", 3], 0), (None, None, ["bytes_test8".encode("utf8"), 8, GEO_POLY], aerospike.LIST_RETURN_VALUE, [8, "bytes_test8".encode("utf8"), GEO_POLY], 1), (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_VALUE, LIST_BIN_EXAMPLE, 1), - (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_INDEX, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ,10, 11], 1), - (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_REVERSE_INDEX, [11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0], 1), - (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_COUNT, 12, 1), + (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_INDEX, [i for i in range(len(LIST_BIN_EXAMPLE))], 1), + (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_REVERSE_INDEX, [i for i in range(len(LIST_BIN_EXAMPLE) - 1, -1, -1)], 1), + (None, None, LIST_BIN_EXAMPLE, aerospike.LIST_RETURN_COUNT, len(LIST_BIN_EXAMPLE), 1), # (None, None, [8], aerospike.LIST_RETURN_RANK, [1], 1), temporarily failing because of bool jump rank ([list_index], [3], [26, 6], aerospike.LIST_RETURN_INDEX, [0, 3], 1), ]) @@ -299,9 +290,9 @@ def test_list_get_by_value_list_pos(self, ctx_types, ctx_indexes, value, return_ expr = Eq(ListGetByValueList(ctx, return_type, value, 'list_bin'), check) verify_multiple_expression_result(self.as_connection, self.test_ns, self.test_set, expr.compile(), 'list_bin', expected) - @pytest.mark.parametrize("ctx_types, ctx_indexes, value, return_type, check, expected", [ - (None, None, [10, [26, 27, 28, 10]], aerospike.LIST_RETURN_VALUE, (10, [26, 27, 28, 10]), e.InvalidRequest) - ]) + @pytest.mark.parametrize("ctx_types, ctx_indexes, value, return_type, check, expected", + [(None, None, [10, [26, 27, 28, 10]], aerospike.LIST_RETURN_VALUE, "a", e.InvalidRequest)] + ) def test_list_get_by_value_list_neg(self, ctx_types, ctx_indexes, value, return_type, check, expected): """ Invoke ListGetByValueList() with expected failures. diff --git a/test/new_tests/test_expressions_map.py b/test/new_tests/test_expressions_map.py index 45aedb136..f75879060 100644 --- a/test/new_tests/test_expressions_map.py +++ b/test/new_tests/test_expressions_map.py @@ -124,8 +124,7 @@ def setup(self, request, as_connection): bytearray("bytearray_test" + str(i), "utf8"), ("bytes_test" + str(i)).encode("utf8"), i % 2 == 1, - aerospike.null, - TestUsrDefinedClass(i), + aerospike.null(), float(i), GEO_POLY ], @@ -166,13 +165,8 @@ def setup(self, request, as_connection): }, 'nmap_bin': { 1: None, - 2: aerospike.null, - 3: aerospike.null - }, - 'blmap_bin': { - 1: TestUsrDefinedClass(1), - 2: TestUsrDefinedClass(3), - 3: TestUsrDefinedClass(4) + 2: aerospike.null(), + 3: aerospike.null() }, 'fmap_bin': { 1.0: 1.0, @@ -317,7 +311,7 @@ def test_map_increment_pos(self, bin, bin_name, ctx, policy, key, value, expecte "imap_bin", None, {'map_write_flags': aerospike.MAP_WRITE_FLAGS_NO_FAIL}, - [aerospike.CDTInfinite, 10, 1, 1, 3, 6], + [4, 10, 1, 1, 3, 6], ), ( "smap_bin", diff --git a/test/new_tests/test_map_operation_helpers.py b/test/new_tests/test_map_operation_helpers.py index 232ddd2d4..6e768991c 100644 --- a/test/new_tests/test_map_operation_helpers.py +++ b/test/new_tests/test_map_operation_helpers.py @@ -57,15 +57,9 @@ def setup(self, request, as_connection): Setup Method """ self.keys = [] - self.test_map = { - "a" : 5, - "b" : 4, - "c" : 3, - "d" : 2, - "e" : 1, - "f" : True, - "g" : False - } + # In a CDT, different types are also ordered from each other + # boolean < integer + self.test_map = {"a": 5, "b": 4, "c": 3, "d": 2, "e": 1, "f": True, "g": False} self.test_key = 'test', 'demo', 'new_map_op' self.test_bin = 'map' @@ -232,17 +226,20 @@ def test_map_remove_by_index_range(self): def test_map_remove_by_rank(self): operations = [map_ops.map_remove_by_rank(self.test_bin, 1, return_type=aerospike.MAP_RETURN_KEY)] ret_vals = get_map_result_from_operation(self.as_connection, self.test_key, operations, self.test_bin) - assert ret_vals == "d" + # True > False + assert ret_vals == "f" res_map = self.as_connection.get(self.test_key)[2][self.test_bin] - assert "d" not in res_map + assert "f" not in res_map def test_map_remove_by_rank_range(self): operations = [map_ops.map_remove_by_rank_range(self.test_bin, 1, 2, return_type=aerospike.MAP_RETURN_KEY)] ret_vals = get_map_result_from_operation(self.as_connection, self.test_key, operations, self.test_bin) - assert set(ret_vals) == set(["d", "c"]) + # f: True, e: 1 + # Removes 2 items from rank 1 and going up the rank + assert set(ret_vals) == set(["e", "f"]) res_map = self.as_connection.get(self.test_key)[2][self.test_bin] - assert "d" not in res_map - assert "c" not in res_map + assert "e" not in res_map + assert "f" not in res_map def test_map_get_by_key(self): @@ -306,7 +303,7 @@ def test_map_get_by_index_range(self): def test_map_get_by_rank(self): operations = [map_ops.map_get_by_rank(self.test_bin, 1, return_type=aerospike.MAP_RETURN_KEY)] ret_vals = get_map_result_from_operation(self.as_connection, self.test_key, operations, self.test_bin) - assert ret_vals == "d" + assert ret_vals == "f" def test_map_get_by_rank_range(self): sort_map(self.as_connection, self.test_key, self.test_bin) @@ -314,4 +311,4 @@ def test_map_get_by_rank_range(self): map_ops.map_get_by_rank_range(self.test_bin, 1, 2, return_type=aerospike.MAP_RETURN_KEY) ] ret_vals = get_map_result_from_operation(self.as_connection, self.test_key, operations, self.test_bin) - assert ret_vals == ["d", "c"] \ No newline at end of file + assert ret_vals == ["f", "e"] diff --git a/test/new_tests/test_nested_cdt_ctx.py b/test/new_tests/test_nested_cdt_ctx.py index 5569dfd37..10ff30c6a 100644 --- a/test/new_tests/test_nested_cdt_ctx.py +++ b/test/new_tests/test_nested_cdt_ctx.py @@ -55,7 +55,7 @@ def setup(self, request, as_connection): random.seed(datetime.now()) self.nested_list_order = [[4, 2, 5],[1, 4, 2, 3],[[2,2,2]]] self.nested_map = { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}} + 'second': {'nested': {"a": 4, "b": 5, "c": 6}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}} self.layered_map = {'first': {'one': {1: {'g': 'layer', 'l': 'done'} } }, 'second': {'two': {2: {'g': 'layer', 'l': 'bye'} } } } self.num_map = {1: {1: 'v1', 2: 'v2', 3: 'v3'}, 2: {4: 'v4', 5: 'v5', 6: 'v6'}, 3: {7: 'v7', 8: 'v8', 9: 'v9', 10: {11: 'v11'}}} @@ -1527,7 +1527,7 @@ def test_ctx_list_sort_negative(self, sort_flags, list_indexes, expected): @pytest.mark.parametrize("ctx_types, key, return_type, list_indexes, expected", [ ([map_index], 'greet', aerospike.MAP_RETURN_VALUE, [0], 'hi'), ([map_index], 3, aerospike.MAP_RETURN_VALUE, [0], 'hello'), - ([map_index], 'nested', aerospike.MAP_RETURN_VALUE, [1], {4,5,6}), + ([map_index], 'nested', aerospike.MAP_RETURN_VALUE, [1], {"a": 4, "b": 5, "c": 6}), ([map_index], 'dog', aerospike.MAP_RETURN_VALUE, [1], None), ([map_index, map_index, map_index], 'fish', aerospike.MAP_RETURN_VALUE, [2,0,0], 'pond'), # why does this fail? ([map_key], 'nested', aerospike.MAP_RETURN_INDEX, ['second'], 1) @@ -1610,7 +1610,7 @@ def test_ctx_map_get_by_key_range_negative(self, ctx_types, key_start, key_stop, @pytest.mark.parametrize("ctx_types, key, return_type, inverted, list_indexes, expected", [ ([map_index], ['greet'], aerospike.MAP_RETURN_VALUE, False, [0], ['hi']), ([map_index], ['numbers', 3], aerospike.MAP_RETURN_VALUE, False, [0], ['hello', [3,1,2]]), - ([map_index], ['nested', 'hundred'], aerospike.MAP_RETURN_VALUE, False, [1], [100, {4,5,6}]), + ([map_index], ['nested', 'hundred'], aerospike.MAP_RETURN_VALUE, False, [1], [100, {"a": 4,"b": 5,"c": 6}]), ([map_index], ['dog'], aerospike.MAP_RETURN_VALUE, False, [1], []), ([map_index, map_index, map_index], ['horse', 'fish'], aerospike.MAP_RETURN_VALUE, False, [2,0,0], ['pond', 'shoe']), ([map_key], ['nested'], aerospike.MAP_RETURN_INDEX, True, ['second'], [0]) @@ -1696,7 +1696,7 @@ def test_ctx_map_get_by_index_negative(self, index, return_type, list_indexes, e @pytest.mark.parametrize("ctx_types, index, rmv_count, return_type, inverted, list_indexes, expected", [ ([map_index], 1, 1, aerospike.MAP_RETURN_VALUE, False, [0], ['hi']), ([map_index], 0, 3, aerospike.MAP_RETURN_VALUE, False, [0], ['hello', 'hi', [3,1,2]]), - ([map_index], 0, 2, aerospike.MAP_RETURN_VALUE, False, [1], [100, {4,5,6}]), + ([map_index], 0, 2, aerospike.MAP_RETURN_VALUE, False, [1], [100, {"a": 4,"b": 5,"c": 6}]), ([map_index, map_index, map_index], 0, 2, aerospike.MAP_RETURN_VALUE, False, [2,0,0], ['pond', 'shoe']), ([map_key], 1, 2, aerospike.MAP_RETURN_INDEX, True, ['second'], [0]), ([map_rank, map_value], 0, 3, aerospike.MAP_RETURN_INDEX, False, [1, {'cat': 'dog', @@ -1824,7 +1824,7 @@ def test_ctx_map_get_by_value_range_negative(self, value_start, value_end, retur @pytest.mark.parametrize("ctx_types, values, return_type, inverted, list_indexes, expected", [ ([map_index], ['hi', 'hello'], aerospike.MAP_RETURN_VALUE, False, [0], ['hello', 'hi']), ([map_index], ['hello'], aerospike.MAP_RETURN_VALUE, False, [0], ['hello']), - ([map_value], [{4,5,6}, 100], aerospike.MAP_RETURN_VALUE, False, [{'nested': {4,5,6,}, 'hundred': 100}], [100, {4,5,6}]), + ([map_value], [{"a": 4,"b": 5,"c": 6}, 100], aerospike.MAP_RETURN_VALUE, False, [{'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}], [100, {"a": 4,"b": 5,"c": 6}]), ([map_index], ['dog'], aerospike.MAP_RETURN_VALUE, False, [1], []), ([map_index, map_key], ['dog', ['bird']], aerospike.MAP_RETURN_VALUE, True, [2,'one'], [{'horse': 'shoe', 'fish': 'pond'}]), ]) @@ -1868,7 +1868,7 @@ def test_ctx_map_get_by_value_list_negative(self, values, return_type, inverted, @pytest.mark.parametrize("ctx_types, rank, return_type, list_indexes, expected", [ ([map_index], 1, aerospike.MAP_RETURN_VALUE, [0], 'hi'), ([map_index], 0, aerospike.MAP_RETURN_VALUE, [0], 'hello'), - ([map_index], 1, aerospike.MAP_RETURN_VALUE, [1], {4,5,6}), + ([map_index], 1, aerospike.MAP_RETURN_VALUE, [1], {"a": 4,"b": 5,"c": 6}), ([map_index, map_index, map_index], 0, aerospike.MAP_RETURN_VALUE, [2,0,0], 'pond'), ([map_key], 1, aerospike.MAP_RETURN_INDEX, ['second'], 1) ]) @@ -2131,13 +2131,13 @@ def test_ctx_map_increment_negative(self, ctx_types, key, amount, map_policy, ma @pytest.mark.parametrize("ctx_types, key, amount, map_policy, map_indexes, expected_bin", [ ([map_index], 'hundred', 27, None, [1], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 73}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 73}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_rank, map_key], 'new', 10, None, [2,1,'barn'], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond', 'new': -10}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_key], 2, 50, None, [2,'one'], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird'], 2: -50}, 'two': []}}), ]) def test_ctx_map_decrement(self, ctx_types, key, amount, map_policy, map_indexes, expected_bin): @@ -2224,11 +2224,11 @@ def test_ctx_map_size_negative(self, ctx_types, map_indexes, expected): 'second': {}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_key], [2,'one'], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {}, 'two': []}}), ([map_index, map_key, map_value], [2,'one', {'horse': 'shoe', 'fish': 'pond'}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {}, 'cage': ['bird']}, 'two': []}}), ]) def test_ctx_map_clear(self, ctx_types, map_indexes, expected): @@ -2269,19 +2269,19 @@ def test_ctx_map_clear_negative(self, ctx_types, map_indexes, expected): @pytest.mark.parametrize("ctx_types, key, return_type, list_indexes, expected_val, expected_bin", [ ([map_index], 'greet', aerospike.MAP_RETURN_VALUE, [0], 'hi', { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 3, aerospike.MAP_RETURN_VALUE, [0], 'hello', { 'first': {'greet': 'hi', 'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_index], 'nested', aerospike.MAP_RETURN_VALUE, [1], {4,5,6}, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_index], 'nested', aerospike.MAP_RETURN_VALUE, [1], {"a": 4,"b": 5,"c": 6,}, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 'dog', aerospike.MAP_RETURN_VALUE, [1], None, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_index, map_index], 'fish', aerospike.MAP_RETURN_VALUE, [2,0,0], 'pond', - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe'}, 'cage': ['bird']}, 'two': []}}), ([map_key], 'nested', aerospike.MAP_RETURN_INDEX, ['second'], 1, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': { 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, @@ -2326,22 +2326,22 @@ def test_ctx_map_remove_by_key_negative(self, key, return_type, list_indexes, ex @pytest.mark.parametrize("ctx_types, key, return_type, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], ['greet'], aerospike.MAP_RETURN_VALUE, False, [0], ['hi'], { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], ['numbers', 3], aerospike.MAP_RETURN_VALUE, False, [0], ['hello', [3,1,2]], { 'first': {'greet': 'hi',}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_index], ['nested', 'hundred'], aerospike.MAP_RETURN_VALUE, False, [1], [100, {4,5,6}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_index], ['nested', 'hundred'], aerospike.MAP_RETURN_VALUE, False, [1], [100, {"a": 4,"b": 5,"c": 6,}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], ['dog'], aerospike.MAP_RETURN_VALUE, False, [1], [], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_index, map_index], ['horse', 'fish'], aerospike.MAP_RETURN_VALUE, False, [2,0,0], ['pond', 'shoe'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {}, 'cage': ['bird']}, 'two': []}}), ([map_key], ['nested'], aerospike.MAP_RETURN_INDEX, True, ['second'], [0], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': { 'nested': {4,5,6,}}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': { 'nested': {"a": 4,"b": 5,"c": 6,}}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}) ]) def test_ctx_map_remove_by_key_list(self, ctx_types, key, return_type, inverted, list_indexes, expected_val, expected_bin): @@ -2432,22 +2432,22 @@ def test_ctx_map_remove_by_key_range_negative(self, key_start, key_end, return_t @pytest.mark.parametrize("ctx_types, value, return_type, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], 'hi', aerospike.MAP_RETURN_VALUE, False, [0], ['hi'], { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 'hello', aerospike.MAP_RETURN_VALUE, False, [0], ['hello'], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_index], {4,5,6}, aerospike.MAP_RETURN_VALUE, False, [1], [{4,5,6}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_index], {"a": 4,"b": 5,"c": 6,}, aerospike.MAP_RETURN_VALUE, False, [1], [{"a": 4,"b": 5,"c": 6,}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 'dog', aerospike.MAP_RETURN_VALUE, False, [1], [], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_index, map_index], 'pond', aerospike.MAP_RETURN_VALUE, True, [2,0,0], ['shoe'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_key], 100, aerospike.MAP_RETURN_INDEX, False, ['second'], [0], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6}}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}) ]) def test_ctx_map_remove_by_value(self, ctx_types, value, return_type, inverted, list_indexes, expected_val, expected_bin): @@ -2489,19 +2489,19 @@ def test_ctx_map_remove_by_value_negative(self, value, return_type, inverted, li @pytest.mark.parametrize("ctx_types, values, return_type, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], ['hi', 'hello'], aerospike.MAP_RETURN_VALUE, False, [0], ['hello', 'hi'], { 'first': {'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], ['hello'], aerospike.MAP_RETURN_VALUE, False, [0], ['hello'], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_value], [{4,5,6}, 100], aerospike.MAP_RETURN_VALUE, False, [{'nested': {4,5,6,}, 'hundred': 100}], [100, {4,5,6}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_value], [{"a": 4,"b": 5,"c": 6,}, 100], aerospike.MAP_RETURN_VALUE, False, [{'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}], [100, {"a": 4,"b": 5,"c": 6,}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], ['dog'], aerospike.MAP_RETURN_VALUE, False, [1], [], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_key], ['dog', ['bird']], aerospike.MAP_RETURN_VALUE, True, [2,'one'], [{'horse': 'shoe', 'fish': 'pond'}], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'cage': ['bird']}, 'two': []}}), ]) def test_ctx_map_remove_by_value_list(self, ctx_types, values, return_type, inverted, list_indexes, expected_val, expected_bin): @@ -2591,16 +2591,16 @@ def test_ctx_map_remove_by_value_range_negative(self, value_start, value_end, re @pytest.mark.parametrize("ctx_types, index, return_type, list_indexes, expected_val, expected_bin", [ ([map_index], 1, aerospike.MAP_RETURN_VALUE, [0], 'hi', { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 0, aerospike.MAP_RETURN_VALUE, [0], 'hello', { 'first': {'greet': 'hi', 'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_index], 1, aerospike.MAP_RETURN_VALUE, [1], {4,5,6}, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_index], 1, aerospike.MAP_RETURN_VALUE, [1], {"a": 4,"b": 5,"c": 6,}, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_index, map_index], 0, aerospike.MAP_RETURN_VALUE, [2,0,0], 'pond', - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe'}, 'cage': ['bird']}, 'two': []}}), ([map_key], 1, aerospike.MAP_RETURN_INDEX, ['second'], 1, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': { 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, @@ -2646,22 +2646,22 @@ def test_ctx_map_remove_by_index_negative(self, index, return_type, list_indexes @pytest.mark.parametrize("ctx_types, index, rmv_count, return_type, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], 1, 1, aerospike.MAP_RETURN_VALUE, False, [0], ['hi'], { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 0, 3, aerospike.MAP_RETURN_VALUE, False, [0], ['hello', 'hi', [3,1,2]], { 'first': {}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_index], 0, 2, aerospike.MAP_RETURN_VALUE, False, [1], [100, {4,5,6}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_index], 0, 2, aerospike.MAP_RETURN_VALUE, False, [1], [100, {"a": 4,"b": 5,"c": 6,}], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_index, map_index], 0, 2, aerospike.MAP_RETURN_VALUE, False, [2,0,0], ['pond', 'shoe'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {}, 'cage': ['bird']}, 'two': []}}), ([map_key], 1, 2, aerospike.MAP_RETURN_INDEX, True, ['second'], [0], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6}}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_value], 0, 3, aerospike.MAP_RETURN_INDEX, False, ['third', {'cat': 'dog', 'barn': {'fish': 'pond', 'horse': 'shoe'}, 'cage': ['bird']}], - [0,1,2], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + [0,1,2], { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {}, 'two': []}}) ]) def test_ctx_map_remove_by_index_range(self, ctx_types, index, rmv_count, return_type, inverted, list_indexes, expected_val, expected_bin): @@ -2705,16 +2705,16 @@ def test_ctx_map_remove_by_index_range_negative(self, index, rmv_count, return_t @pytest.mark.parametrize("ctx_types, rank, return_type, list_indexes, expected_val, expected_bin", [ ([map_index], 1, aerospike.MAP_RETURN_VALUE, [0], 'hi', { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 0, aerospike.MAP_RETURN_VALUE, [0], 'hello', { 'first': {'greet': 'hi', 'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), - ([map_index], 1, aerospike.MAP_RETURN_VALUE, [1], {4,5,6}, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, + ([map_index], 1, aerospike.MAP_RETURN_VALUE, [1], {"a": 4,"b": 5,"c": 6,}, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index, map_index, map_index], 0, aerospike.MAP_RETURN_VALUE, [2,0,0], 'pond', - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6,}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe'}, 'cage': ['bird']}, 'two': []}}), ([map_key], 1, aerospike.MAP_RETURN_INDEX, ['second'], 1, { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': { 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, @@ -2809,16 +2809,16 @@ def test_ctx_map_remove_by_rank_range_negative(self, rank, rmv_count, return_typ @pytest.mark.parametrize("ctx_types, value, offset, return_type, count, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], 'hi', 0, aerospike.MAP_RETURN_VALUE, 1, False, [0], ['hi'], { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 'hi', 1, aerospike.MAP_RETURN_VALUE, 3, True, [0], ['hello', 'hi'], { 'first': {'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_index, map_value], 'pond', 0, aerospike.MAP_RETURN_VALUE, 2, False, ['third',0,{'horse': 'shoe', 'fish': 'pond'}], ['pond', 'shoe'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_rank], {'horse': 'shoe', 'fish': 'pond'}, 0, aerospike.MAP_RETURN_VALUE, 2, True, ['third',1], ['dog',['bird']], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'barn': {'horse': 'shoe', 'fish': 'pond'}}, 'two': []}}), ]) def test_ctx_map_remove_by_value_rank_range_relative(self, ctx_types, value, offset, return_type, count, inverted, list_indexes, expected_val, expected_bin): @@ -2905,16 +2905,16 @@ def test_ctx_map_get_by_value_rank_range_relative_negative(self, value, offset, @pytest.mark.parametrize("ctx_types, key, offset, return_type, count, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], 'greet', 0, aerospike.MAP_RETURN_VALUE, 1, False, [0], ['hi'], { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 'greet', 1, aerospike.MAP_RETURN_VALUE, 3, True, [0], ['hello', 'hi'], { 'first': {'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_index, map_value], 'fish', 0, aerospike.MAP_RETURN_VALUE, 2, False, ['third',0,{'horse': 'shoe', 'fish': 'pond'}], ['pond', 'shoe'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_rank], 'barn', 0, aerospike.MAP_RETURN_VALUE, 2, True, ['third',1], ['dog'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ]) def test_ctx_map_remove_by_key_index_range_relative(self, ctx_types, key, offset, return_type, count, inverted, list_indexes, expected_val, expected_bin): @@ -2957,16 +2957,16 @@ def test_ctx_map_remove_by_key_index_range_relative_negative(self, key, offset, @pytest.mark.parametrize("ctx_types, key, offset, return_type, count, inverted, list_indexes, expected_val, expected_bin", [ ([map_index], 'greet', 0, aerospike.MAP_RETURN_VALUE, 1, False, [0], ['hi'], { 'first': {'numbers': [3, 1, 2], 3: 'hello'}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_index], 'greet', 1, aerospike.MAP_RETURN_VALUE, 3, True, [0], ['hello', 'hi'], { 'first': {'numbers': [3, 1, 2]}, - 'second': {'nested': {4,5,6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, + 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_index, map_value], 'fish', 0, aerospike.MAP_RETURN_VALUE, 2, False, ['third',0,{'horse': 'shoe', 'fish': 'pond'}], ['pond', 'shoe'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'cat': 'dog', 'barn': {}, 'cage': ['bird']}, 'two': []}}), ([map_key, map_rank], 'barn', 0, aerospike.MAP_RETURN_VALUE, 2, True, ['third',1], ['dog'], - { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {4,5,6}, 'hundred': 100}, + { 'first': {'greet': 'hi', 'numbers': [3, 1, 2], 3: 'hello'}, 'second': {'nested': {"a": 4,"b": 5,"c": 6,}, 'hundred': 100}, 'third': {'one': {'barn': {'horse': 'shoe', 'fish': 'pond'}, 'cage': ['bird']}, 'two': []}}), ]) def test_ctx_map_remove_by_key_index_range_relative(self, ctx_types, key, offset, return_type, count, inverted, list_indexes, expected_val, expected_bin): diff --git a/test/new_tests/test_operate.py b/test/new_tests/test_operate.py index 7ccbca871..6474bc87a 100644 --- a/test/new_tests/test_operate.py +++ b/test/new_tests/test_operate.py @@ -148,12 +148,6 @@ def teardown(): "val": {"no": 89}}, {"op": aerospike.OPERATOR_READ, "bin": "write_bin"}], {'write_bin': {u'no': 89}}), - (('test', 'demo', 1), # write_tuple_positive - [{"op": aerospike.OPERATOR_WRITE, - "bin": "write_bin", - "val": tuple('abc')}, - {"op": aerospike.OPERATOR_READ, "bin": "write_bin"}], - {'write_bin': ('a', 'b', 'c')}), (('test', 'demo', 1), # with_bin_bytearray [{"op": aerospike.OPERATOR_PREPEND, "bin": bytearray("asd[;asjk", "utf-8"), @@ -1086,13 +1080,13 @@ def test_pos_operate_with_list_insert_index_negative(self): ([ {"op": aerospike.OP_LIST_APPEND_ITEMS, "bin": "string_bin", - "val": [['z', 'x'], ('y', 'w')]}, + "val": [['z', 'x'], ['y', 'w']]}, {"op": aerospike.OP_LIST_GET_RANGE, "bin": "string_bin", "index": 3, "val": 3} - ], {"string_bin": ['d', ['z', 'x'], ('y', 'w')]}, "string_bin", ['a', 'b', 'c', - 'd', ['z', 'x'], ('y', 'w')]), + ], {"string_bin": ['d', ['z', 'x'], ['y', 'w']]}, "string_bin", ['a', 'b', 'c', + 'd', ['z', 'x'], ['y', 'w']]), ([ {"op": aerospike.OP_LIST_INSERT, "bin": "string_bin", diff --git a/test/new_tests/test_operate_helpers.py b/test/new_tests/test_operate_helpers.py index 2633d37d3..16b59b907 100644 --- a/test/new_tests/test_operate_helpers.py +++ b/test/new_tests/test_operate_helpers.py @@ -154,13 +154,6 @@ def teardown(): operations.read("write_bin") ], {'write_bin': {u'no': 89}}), - ( - ('test', 'demo', 1), # write_tuple_positive - [ - operations.write("write_bin", ('a', 'b', 'c')), - operations.read("write_bin") - ], - {'write_bin': ('a', 'b', 'c')}), ( ('test', 'demo', 1), # with_bin_bytearray [ @@ -897,10 +890,10 @@ def test_pos_operate_with_list_insert_index_negative(self): ], {"string_bin": {"new_val": 1}}, "string_bin", ['a', 'b', 'c', 'd', {'new_val': 1}]), ([ - list_operations.list_append_items("string_bin", [['z', 'x'], ('y', 'w')]), + list_operations.list_append_items("string_bin", [['z', 'x'], ['y', 'w']]), list_operations.list_get_range("string_bin", 3, 3) - ], {"string_bin": ['d', ['z', 'x'], ('y', 'w')]}, "string_bin", ['a', 'b', 'c', - 'd', ['z', 'x'], ('y', 'w')]), + ], {"string_bin": ['d', ['z', 'x'], ['y', 'w']]}, "string_bin", ['a', 'b', 'c', + 'd', ['z', 'x'], ['y', 'w']]), ([ list_operations.list_insert("string_bin", 2, True), list_operations.list_pop("string_bin", 2) diff --git a/test/new_tests/test_operate_ordered.py b/test/new_tests/test_operate_ordered.py index ba6ce4473..cad436b52 100644 --- a/test/new_tests/test_operate_ordered.py +++ b/test/new_tests/test_operate_ordered.py @@ -102,12 +102,6 @@ def teardown(): "val": {"no": 89}}, {"op": aerospike.OPERATOR_READ, "bin": "write_bin"}], [('write_bin', {u'no': 89})]), - (('test', 'demo', 1), # write_tuple_positive - [{"op": aerospike.OPERATOR_WRITE, - "bin": "write_bin", - "val": tuple('abc')}, - {"op": aerospike.OPERATOR_READ, "bin": "write_bin"}], - [('write_bin', ('a', 'b', 'c'))]), (('test', 'demo', 1), # with_bin_bytearray [{"op": aerospike.OPERATOR_PREPEND, "bin": bytearray("asd[;asjk", "utf-8"), @@ -506,12 +500,12 @@ def test_pos_operate_ordered_with_list_insert_index_negative(self): ([ {"op": aerospike.OP_LIST_APPEND_ITEMS, "bin": "string_bin", - "val": [['z', 'x'], ('y', 'w')]}, + "val": [['z', 'x'], ['y', 'w']]}, {"op": aerospike.OP_LIST_GET_RANGE, "bin": "string_bin", "index": 3, "val": 3} - ], [('string_bin', 6), ('string_bin', ['d', ['z', 'x'], ('y', 'w')])]), + ], [('string_bin', 6), ('string_bin', ['d', ['z', 'x'], ['y', 'w']])]), ([ {"op": aerospike.OP_LIST_INSERT, "bin": "string_bin", diff --git a/test/new_tests/test_select.py b/test/new_tests/test_select.py index 9eacfaaf3..efd9331fa 100644 --- a/test/new_tests/test_select.py +++ b/test/new_tests/test_select.py @@ -29,7 +29,6 @@ def setup(self, request, as_connection): 'b': {"key": "asd';q;'1';"}, 'c': 1234, 'd': '!@#@#$QSDAsd;as', - 'n': None, ('a' * 14): 'long_bin_14', } @@ -156,12 +155,12 @@ def test_select_with_key_with_existent_and_non_existent_bins_to_select( self ): - bins_to_select = ['c', 'd', 'n', 'fake'] + bins_to_select = ['c', 'd', 'fake'] _, meta, bins = self.as_connection.select( self.test_key, bins_to_select) - assert bins == {'c': 1234, 'd': '!@#@#$QSDAsd;as', 'n': None} + assert bins == {'c': 1234, 'd': '!@#@#$QSDAsd;as'} assert meta is not None def test_select_with_key_and_non_existent_bin_in_middle(self): diff --git a/test/new_tests/test_zserializers.py b/test/new_tests/test_zserializers.py index 2c1534954..7fa152e54 100644 --- a/test/new_tests/test_zserializers.py +++ b/test/new_tests/test_zserializers.py @@ -3,6 +3,8 @@ from .test_base_class import TestBaseClass from .as_status_codes import AerospikeStatus from aerospike import exception as e +from aerospike_helpers.operations import operations +from aerospike_helpers.expressions import base as expr import pytest import sys @@ -128,56 +130,24 @@ def test_put_with_no_serializer_arg_and_class_serializer_set(self): Test that when no serializer arg is passed in, and no instance serializer has been set, and a class serializer has been set, - the default language serializer is used. + the object fails to be serialized. ''' aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) - response = self.as_connection.put( - self.test_key, self.mixed_record) - - _, _, bins = self.as_connection.get(self.test_key) - - # The class serializer would have altered our record, so we - # We check that the data is the same as what we stored - assert bins == {'normal': 1234, - 'tuple': (1, 2, 3)} + with pytest.raises(e.ClientError): + self.as_connection.put(self.test_key, self.mixed_record) - def test_builtin_with_class_serializer(self): + def test_put_with_serializer_arg_none_and_instance_serializer_set(self): """ - Invoke put() for mixed data record with builtin serializer + Invoke put() for mixed data record with serializer set to NONE """ method_config = { 'serialization': (instance_serializer, instance_deserializer)} client = TestBaseClass.get_new_connection(method_config) - response = client.put( - self.test_key, self.mixed_record, - serializer=aerospike.SERIALIZER_PYTHON) - - _, _, bins = client.get(self.test_key) - - assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} - client.close() - - def test_builtin_with_class_serializer(self): - """ - Invoke put() for mixed data record with builtin serializer - """ - aerospike.set_serializer(class_serializer) - aerospike.set_deserializer(class_deserializer) - - response = self.as_connection.put( - self.test_key, self.mixed_record, - serializer=aerospike.SERIALIZER_PYTHON) - - assert response == 0 - - _, _, bins = self.as_connection.get(self.test_key) - - # The class serializer would have altered our record, so we - # We check that the data is the same as what we stored - assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} + with pytest.raises(e.ClientError): + client.put(self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_NONE) def test_with_class_serializer(self): """ @@ -204,9 +174,8 @@ def test_with_class_serializer(self): def test_builtin_with_class_serializer_and_instance_serializer(self): """ - Invoke put() passing SERIALIZER_PYTHON and verify that it - causes the language serializer to override instance and class - serializers + Invoke put() passing SERIALIZER_NONE and verify that it + causes the instance and class serializers to be ignored """ aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) @@ -214,24 +183,13 @@ def test_builtin_with_class_serializer_and_instance_serializer(self): 'serialization': (instance_serializer, instance_deserializer)} client = TestBaseClass.get_new_connection(method_config) - rec = {'normal': 1234, 'tuple': (1, 2, 3)} - response = client.put( - self.test_key, self.mixed_record, - serializer=aerospike.SERIALIZER_PYTHON) - - assert response == 0 - - _, _, bins = client.get(self.test_key) - - # The instance and class serializers would have mutated this, - # So getting back what we put in means the language serializer ran - assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} - client.close() + with pytest.raises(e.ClientError): + client.put(self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_NONE) def test_with_class_serializer_and_instance_serializer(self): """ Invoke put() for mixed data record with class and instance serializer. - Verify that the instance serializer takes precedence + Verify that the instance serializer takes precedence over the class serializer """ aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) @@ -252,27 +210,6 @@ def test_with_class_serializer_and_instance_serializer(self): } client.close() - def test_with_unset_serializer_python_serializer(self): - """ - Invoke put() for mixed data record with python serializer and - calling unset_serializers - """ - aerospike.set_serializer(json.dumps) - aerospike.set_deserializer(json.loads) - method_config = { - } - client = TestBaseClass.get_new_connection(method_config) - - aerospike.unset_serializers() - response = client.put( - self.test_key, self.mixed_record, - serializer=aerospike.SERIALIZER_PYTHON) - - _, _, bins = client.get(self.test_key) - - assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} - client.close() - def test_unset_instance_serializer(self): """ Verify that calling unset serializers does not remove an instance level @@ -307,15 +244,8 @@ def test_setting_serializer_is_a_per_rec_setting(self): self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) - self.as_connection.put( - ('test', 'demo', 'test_record_2'), self.mixed_record) - - _, _, record = self.as_connection.get( - ('test', 'demo', 'test_record_2')) - - self.as_connection.remove(('test', 'demo', 'test_record_2')) - # this should not have been serialized with the class serializer - assert record == self.mixed_record + with pytest.raises(e.ClientError): + self.as_connection.put(("test", "demo", "test_record_2"), self.mixed_record) def test_unsetting_serializers_after_a_record_put(self): aerospike.set_serializer(class_serializer) @@ -328,10 +258,10 @@ def test_unsetting_serializers_after_a_record_put(self): aerospike.unset_serializers() _, _, record = self.as_connection.get(self.test_key) - # this should not have been deserialized with the class serializer - # it will have been deserialized by the class deserializer, - # so this should be a deserialization of 'class serialized' - assert record['tuple'] != ('class serialized', 'class deserialized') + # it will have been serialized by the class serializer, + # so this should be a deserialization of b'class serialized' with the bytes type + # since server blob type is deserialized into Python types + assert record["tuple"] == b'class serialized' def test_changing_deserializer_after_a_record_put(self): aerospike.set_serializer(class_serializer) @@ -356,9 +286,8 @@ def test_only_setting_a_serializer(self): serializer=aerospike.SERIALIZER_USER) _, _, record = self.as_connection.get(self.test_key) - # this should not have been deserialized with the class serializer - # it should now have used the language default serializer - assert record['tuple'] != ('class serialized', 'class deserialized') + # fetching the record doesn't require it's value to be deserialized + assert record["tuple"] == b'class serialized' def test_only_setting_a_deserializer(self): # This should raise an error @@ -457,9 +386,8 @@ def test_put_with_invalid_serializer_constant(self): def test_put_with_invalid_serializer_arg_type(self): - self.as_connection.put( - self.test_key, self.mixed_record, - serializer="") + with pytest.raises(e.ClientError): + self.as_connection.put(self.test_key, self.mixed_record, serializer="") def test_serializer_raises_error(self): @@ -485,3 +413,99 @@ def test_deserializer_raises_error(self): assert response['normal'] == 1234 assert isinstance(response['tuple'], bytearray) + + # Operations and expressions + + def test_operate_with_no_serializer(self): + ops = [ + operations.write("tuple", self.mixed_record["tuple"]) + ] + with pytest.raises(e.ClientError): + self.as_connection.operate(self.test_key, ops) + + def test_instance_serializer_with_operate(self): + method_config = {"serialization": (instance_serializer, instance_deserializer)} + client = TestBaseClass.get_new_connection(method_config) + + ops = [operations.write(bin_name, bin_value) for bin_name, bin_value in self.mixed_record.items()] + client.operate(self.test_key, ops) + + _, _, bins = client.get(self.test_key) + + # tuples JSON-encode to a list, and we use this fact to check which + # serializer ran: + assert bins == {"normal": 1234, "tuple": ("instance serialized", "instance deserialized")} + client.close() + + def test_instance_deserializer_with_operate(self): + method_config = {"serialization": (instance_serializer, instance_deserializer)} + client = TestBaseClass.get_new_connection(method_config) + + client.put(self.test_key, self.mixed_record) + + ops = [ + operations.read("tuple") + ] + _, _, bins = client.operate(self.test_key, ops) + + # tuples JSON-encode to a list, and we use this fact to check which + # serializer ran: + assert bins == {"tuple": ("instance serialized", "instance deserialized")} + client.close() + + def test_instance_serializer_with_expressions(self): + # We have to use a different serializer and deserializer for this test case + # since the instance serializer and deserializer in this module don't deserialize the server value + # back to the original Python value. + method_config = {"serialization": (json.dumps, json.loads)} + client = TestBaseClass.get_new_connection(method_config) + + client.put(self.test_key, self.mixed_record) + + # The tuple bin should be serialized as a server blob type + # (1, 2, 3) will be serialized by the json.dumps serializer into a server blob type + # If it wasn't able to be serialized, client would throw a filtered out error + exp = expr.Eq(expr.BlobBin("tuple"), (1, 2, 3)).compile() + client.get(self.test_key, {"expressions": exp}) + + client.close() + + def test_module_serializer_with_operate_neg(self): + # The module-level serializer will not work with operate() + aerospike.set_serializer(class_serializer) + + ops = [operations.write("tuple", self.mixed_record["tuple"])] + with pytest.raises(e.ClientError): + self.as_connection.operate(self.test_key, ops) + + def test_module_deserializer_with_operate(self): + # However, the module-level de-serializer will work with operate() + aerospike.set_deserializer(class_deserializer) + + # Serialize the tuple and put it in the server + method_config = {"serialization": (instance_serializer, instance_deserializer)} + client = TestBaseClass.get_new_connection(method_config) + client.put(self.test_key, self.mixed_record) + + # Try to fetch and deserialize the tuple with operate() + ops = [ + operations.read("tuple") + ] + _, _, rec = self.as_connection.operate(self.test_key, ops) + + assert rec["tuple"] == ("instance serialized", "class deserialized") + + def test_module_serializer_with_expressions_neg(self): + # The module-level serializer will not work with expressions + method_config = {"serialization": (instance_serializer, instance_deserializer)} + client = TestBaseClass.get_new_connection(method_config) + + client.put(self.test_key, self.mixed_record) + + aerospike.set_serializer(class_serializer) + + exp = expr.Eq(expr.BlobBin("tuple"), (1, 2, 3)).compile() + with pytest.raises(e.ClientError): + self.as_connection.get(self.test_key, {"expressions": exp}) + + client.close() diff --git a/test/requirements.txt b/test/requirements.txt index 4ad3332f5..7f3b0c70d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,3 +1,3 @@ -pytest +pytest==6.2.5 psutil pytest-asyncio