[CLIENT-2645] Add ttl option for default write policies in client con…

…fig (#536)

* client.batch_operate() now takes in an optional ttl parameter instead of taking ttl through a batch policy.
* Apply policy now takes in a ttl option.
* Add ttl option to aerospike.Scan class.
* Docs: batch apply policies section now references TTL constants section instead of copying information from the latter.

---------

Co-authored-by: dwelch-spike <[email protected]>

aerospike-stubs/aerospike.pyi

@@ -284,6 +284,7 @@ SERIALIZER_USER: Literal[3]
 TTL_DONT_UPDATE: Literal[0xFFFFFFFE]
 TTL_NAMESPACE_DEFAULT: Literal[0]
 TTL_NEVER_EXPIRE: Literal[0xFFFFFFFF]
+TTL_CLIENT_DEFAULT: Literal[0xFFFFFFFD]
 UDF_TYPE_LUA: Literal[0]
 
 @final
@@ -320,7 +321,7 @@ class Client:
     def apply(self, key: tuple, module: str, function: str, args: list, policy: dict = ...) -> Union[str, int, float, bytearray, list, dict]: ...
     def batch_apply(self, keys: list, module: str, function: str, args: list, policy_batch: dict = ..., policy_batch_apply: dict = ...) -> BatchRecords: ...
     def batch_get_ops(self, keys: list, ops: list, policy: dict) -> list: ...
-    def batch_operate(self, keys: list, ops: list, policy_batch: dict = ..., policy_batch_write: dict = ...) -> BatchRecords: ...
+    def batch_operate(self, keys: list, ops: list, policy_batch: dict = ..., policy_batch_write: dict = ..., ttl: int = ...) -> BatchRecords: ...
     def batch_remove(self, keys: list, policy_batch: dict = ..., policy_batch_remove: dict = ...) -> BatchRecords: ...
     def batch_read(self, keys: list, bins: list[str] = ..., policy_batch: dict = ...) -> BatchRecords: ...
     def batch_write(self, batch_records: BatchRecords, policy_batch: dict = ...) -> BatchRecords: ...
@@ -442,6 +443,7 @@ class Query:
     def where(self, predicate: tuple, ctx: list = ...) -> None: ...
 
 class Scan:
+    ttl: int
     def __init__(self, *args, **kwargs) -> None: ...
     def add_ops(self, ops: list) -> None: ...
     def apply(self, module: str, function: str, arguments: list = ...) -> Any: ...

doc/aerospike.rst

@@ -858,6 +858,13 @@ Specifies the TTL constants.
 
     Do not change the current TTL of the record.
 
+.. data:: TTL_CLIENT_DEFAULT
+
+    NOTE: only applies to the policies mentioned below.
+
+    Use the applicable policy ttl in write, operate, batch write, and scan policies.
+    If the policy is not defined for the transaction, use the default client-level policy's ttl.
+
 .. _auth_mode:
 
 Auth Mode Constants

doc/client.rst

@@ -358,14 +358,15 @@ Batch Operations
 
         .. note:: Requires server version >= 6.0.0.
 
-    .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict]) -> BatchRecords
+    .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict], [ttl: int]) -> BatchRecords
 
         Perform the same read/write transactions on multiple keys.
 
         :param list keys: The keys to operate on.
         :param list ops: List of operations to apply.
         :param dict policy_batch: See :ref:`aerospike_batch_policies`.
         :param dict policy_batch_write: See :ref:`aerospike_batch_write_policies`.
+        :param int ttl: The time-to-live (expiration) of each record in seconds.
 
         :return: an instance of :class:`BatchRecords <aerospike_helpers.batch.records>`.
 
@@ -1508,7 +1509,7 @@ Metadata Dictionary
 
 The metadata dictionary has the following key-value pairs:
 
-    * ``"ttl"`` (:class:`int`): record time to live in seconds. See :ref:`TTL_CONSTANTS`.
+    * ``"ttl"`` (:class:`int`): record time to live in seconds. See :ref:`TTL_CONSTANTS` for possible special values.
     * ``"gen"`` (:class:`int`): record generation
 
 .. _aerospike_policies:
@@ -1575,6 +1576,14 @@ Write Policies
             | One of the :ref:`POLICY_EXISTS` values such as :data:`aerospike.POLICY_EXISTS_CREATE`
             |
             | Default: :data:`aerospike.POLICY_EXISTS_IGNORE`
+        * **ttl**
+            The default time-to-live (expiration) of the record in seconds. This field will only be used if
+            the write transaction:
+
+            1. Doesn't contain a metadata dictionary with a ``ttl`` value.
+            2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`.
+
+            There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`.
         * **gen**
             | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE`
             |
@@ -1740,6 +1749,14 @@ Operate Policies
             | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE`
             |
             | Default: :data:`aerospike.POLICY_GEN_IGNORE`
+        * **ttl** (:class:`int`)
+            The default time-to-live (expiration) of the record in seconds. This field will only be used if an
+            operate transaction:
+
+            1. Doesn't contain a metadata dictionary with a ``ttl`` value.
+            2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`.
+
+            There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`.
         * **replica**
             | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER`
             |
@@ -1843,6 +1860,11 @@ Apply Policies
             | One of the :ref:`POLICY_COMMIT_LEVEL` values such as :data:`aerospike.POLICY_COMMIT_LEVEL_ALL`
             |
             | Default: :data:`aerospike.POLICY_COMMIT_LEVEL_ALL`
+        * **ttl** (:class:`int`)
+            The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply
+            transaction doesn't have an apply policy with a ``ttl`` value that overrides this field.
+
+            There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`.
         * **durable_delete** (:class:`bool`)
             | Perform durable delete
             |
@@ -2086,11 +2108,21 @@ Batch Write Policies
             |
             | Default: None
         * **ttl** :class:`int`
-            | The time-to-live (expiration) in seconds to apply to every record in the batch.
-            |
-            | The ttl must be a 32-bit unsigned integer, or a :exc:`~aerospike.exception.ParamError` will be raised.
-            |
-            | Default: ``0``
+            The time-to-live (expiration) in seconds to apply to every record in the batch. This field will only be
+            used if:
+            1. A :meth:`~aerospike.Client.batch_write` call contains a :class:`~aerospike_helpers.batch.records.Write` that:
+
+               a. Doesn't contain a metadata dictionary with a ``ttl`` value.
+               b. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`.
+
+            2. A :meth:`~aerospike.Client.batch_operate` call:
+
+               a. Doesn't pass in a `ttl` argument.
+               b. Passes in `aerospike.TTL_CLIENT_DEFAULT` to the `ttl` parameter.
+
+            There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`.
+
+            Default: ``0``
 
 .. _aerospike_batch_apply_policies:
 
@@ -2115,20 +2147,11 @@ Batch Apply Policies
         * **ttl** int
             | Time to live (expiration) of the record in seconds.
             |
-            | 0 which means that the
-            | record will adopt the default TTL value from the namespace.
+            | See :ref:`TTL_CONSTANTS` for possible special values.
             |
-            | 0xFFFFFFFF (also, -1 in a signed 32 bit int)
-            | which means that the record
-            | will get an internal "void_time" of zero, and thus will never expire.
-            |
-            | 0xFFFFFFFE (also, -2 in a signed 32 bit int)
-            | which means that the record
-            |
-            | ttl will not change when the record is updated.
             | Note that the TTL value will be employed ONLY on write/update calls.
             |
-            | Default: 0
+            | Default: ``0``
         * **durable_delete** :class:`bool`
             | If the transaction results in a record deletion, leave a tombstone for the record. This prevents deleted records from reappearing after node failures. Valid for Aerospike Server Enterprise Edition only.
             |

doc/query.rst

@@ -80,20 +80,10 @@ Fields
         Default: ``0`` (no limit)
 
     ttl (:class:`int`)
-        The time-to-live (expiration) of the record in seconds.
+        The time-to-live (expiration) of the record in seconds. If set to :data:`aerospike.TTL_CLIENT_DEFAULT`, use the
+        client's default write policy ttl.
 
-        There are also special values that can be set in the record TTL:
-
-            ``0`` (``TTL_NAMESPACE_DEFAULT``)
-                Which means that the record will adopt the default TTL value from the namespace.
-
-            ``0xFFFFFFFF`` (``TTL_NEVER_EXPIRE``)
-                (also, ``-1`` in a signed 32 bit int) Which means that the record will never expire.
-
-            ``0xFFFFFFFE`` (``TTL_DONT_UPDATE``)
-                (also, ``-2`` in a signed 32 bit int)
-                Which means that the record ttl will not change when the record is
-                updated.
+        See :ref:`TTL_CONSTANTS` for more possible special values.
 
         .. note::
             Note that the TTL value will be employed ONLY on background query writes.

doc/scan.rst

@@ -22,10 +22,30 @@ bins returned can be filtered using :meth:`select`.
     `Scans <http://www.aerospike.com/docs/guide/scan.html>`_ and \
     `Managing Scans <http://www.aerospike.com/docs/operations/manage/scans/>`_.
 
+Fields
+======
+
+.. class:: Scan
+
+    ttl (:class:`int`)
+        The time-to-live (expiration) of the record in seconds. Note that ttl
+        is only used on background scan writes.
+
+        If this is set to :data:`aerospike.TTL_CLIENT_DEFAULT`, the scan will use the
+        client's default scan policy ttl.
+
+        See :ref:`TTL_CONSTANTS` for special values that can be set in the record ttl.
+
+        Default: ``0`` (no limit)
+
+        .. note::
+            Requires server version >= 6.0.0
+
 Methods
 =======
 
 .. class:: Scan
+    :noindex:
 
     .. deprecated:: 7.0.0 :class:`aerospike.Query` should be used instead.
 
@@ -556,6 +576,12 @@ Policies
             | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER`
             |
             | Default: ``aerospike.POLICY_REPLICA_SEQUENCE``
+        * **ttl** (:class:`int`)
+            The default time-to-live (expiration) of the record in seconds. This field will only be used on
+            background scan writes if :py:attr:`aerospike.Scan.ttl` is set to
+            :data:`aerospike.TTL_CLIENT_DEFAULT`.
+
+            There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`.
 
 .. _aerospike_scan_options:
 

src/main/client/batch_operate.c

@@ -108,11 +108,13 @@ static bool batch_operate_cb(const as_batch_result *results, uint32_t n,
  * @param py_ops                    The list containing op dictionaries.
  * @param py_policy_batch      		Python dict used to populate policy_batch.
  * @param py_policy_batch_write     Python dict used to populate policy_batch_write.
+ * @param py_ttl                    TTL value to set for each record.
  *******************************************************************************************************
  */
 static PyObject *AerospikeClient_Batch_Operate_Invoke(
     AerospikeClient *self, as_error *err, PyObject *py_keys, PyObject *py_ops,
-    PyObject *py_policy_batch, PyObject *py_policy_batch_write)
+    PyObject *py_policy_batch, PyObject *py_policy_batch_write,
+    PyObject *py_ttl)
 {
     long operation;
     long return_type = -1;
@@ -220,26 +222,15 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke(
                 &batch_write_exp_list_p) != AEROSPIKE_OK) {
             goto CLEANUP;
         }
+    }
 
-        // The C client's batch write policy doesn't have a ttl option
-        // The correct way is to set the ttl inside the as_operations object
-        PyObject *py_ttl = PyDict_GetItemString(py_policy_batch_write, "ttl");
-        Py_XINCREF(py_ttl);
-        // Default ttl
-        if (py_ttl != NULL) {
-            if (PyLong_Check(py_ttl)) {
-                long ttl = PyLong_AsLong(py_ttl);
-                if (ttl > UINT32_MAX || ttl < 0) {
-                    as_error_update(err, AEROSPIKE_ERR_PARAM,
-                                    "ttl is out of range. It must be a 32 bit "
-                                    "unsigned integer.");
-                    Py_DECREF(py_ttl);
-                    goto CLEANUP;
-                }
-                ops.ttl = ttl;
-            }
-        }
-        Py_XDECREF(py_ttl);
+    if (py_ttl == NULL || py_ttl == Py_None) {
+        // If ttl in this transaction's batch write policy isn't set, use the client config's default batch write
+        // policy ttl
+        ops.ttl = AS_RECORD_CLIENT_DEFAULT_TTL;
+    }
+    else {
+        ops.ttl = (uint32_t)PyLong_AsLong(py_ttl);
     }
 
     // import batch_records helper
@@ -355,15 +346,16 @@ PyObject *AerospikeClient_Batch_Operate(AerospikeClient *self, PyObject *args,
     PyObject *py_keys = NULL;
     PyObject *py_ops = NULL;
     PyObject *py_results = NULL;
+    PyObject *py_ttl = NULL;
 
     as_error_init(&err);
 
     // Python Function Keyword Arguments
-    static char *kwlist[] = {"keys", "ops", "policy_batch",
-                             "policy_batch_write", NULL};
-    if (PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:batch_Operate", kwlist,
+    static char *kwlist[] = {
+        "keys", "ops", "policy_batch", "policy_batch_write", "ttl", NULL};
+    if (PyArg_ParseTupleAndKeywords(args, kwds, "OO|OOO:batch_Operate", kwlist,
                                     &py_keys, &py_ops, &py_policy_batch,
-                                    &py_policy_batch_write) == false) {
+                                    &py_policy_batch_write, &py_ttl) == false) {
         return NULL;
     }
 
@@ -381,8 +373,14 @@ PyObject *AerospikeClient_Batch_Operate(AerospikeClient *self, PyObject *args,
         goto ERROR;
     }
 
+    if (py_ttl && py_ttl != Py_None && !PyLong_Check(py_ttl)) {
+        as_error_update(&err, AEROSPIKE_ERR_PARAM, "ttl should be an integer");
+        goto ERROR;
+    }
+
     py_results = AerospikeClient_Batch_Operate_Invoke(
-        self, &err, py_keys, py_ops, py_policy_batch, py_policy_batch_write);
+        self, &err, py_keys, py_ops, py_policy_batch, py_policy_batch_write,
+        py_ttl);
 
     return py_results;
 

src/main/client/batch_write.c

@@ -277,10 +277,8 @@ static PyObject *AerospikeClient_BatchWriteInvoke(AerospikeClient *self,
             ops = as_operations_new(py_ops_size);
             garb->ops_to_free = ops;
 
-            if (py_meta) {
-                if (check_and_set_meta(py_meta, ops, err) != AEROSPIKE_OK) {
-                    goto CLEANUP0;
-                }
+            if (check_and_set_meta(py_meta, ops, err) != AEROSPIKE_OK) {
+                goto CLEANUP0;
             }
 
             for (Py_ssize_t i = 0; i < py_ops_size; i++) {

src/main/client/operate.c

@@ -868,10 +868,8 @@ static PyObject *AerospikeClient_Operate_Invoke(AerospikeClient *self,
     memset(&static_pool, 0, sizeof(static_pool));
     CHECK_CONNECTED(err);
 
-    if (py_meta) {
-        if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) {
-            goto CLEANUP;
-        }
+    if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) {
+        goto CLEANUP;
     }
 
     for (i = 0; i < size; i++) {
@@ -1041,10 +1039,8 @@ AerospikeClient_OperateOrdered_Invoke(AerospikeClient *self, as_error *err,
         }
     }
 
-    if (py_meta) {
-        if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) {
-            goto CLEANUP;
-        }
+    if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) {
+        goto CLEANUP;
     }
 
     for (Py_ssize_t i = 0; i < ops_list_size; i++) {

src/main/conversions.c

@@ -1118,6 +1118,9 @@ as_status pyobject_to_record(AerospikeClient *self, as_error *err,
                                         "TTL should be an int or long");
                     }
                 }
+                else {
+                    rec->ttl = AS_RECORD_CLIENT_DEFAULT_TTL;
+                }
 
                 if (py_gen) {
                     if (PyLong_Check(py_gen)) {
@@ -1137,6 +1140,9 @@ as_status pyobject_to_record(AerospikeClient *self, as_error *err,
                 }
             }
         }
+        else {
+            rec->ttl = AS_RECORD_CLIENT_DEFAULT_TTL;
+        }
 
         if (err->code != AEROSPIKE_OK) {
             as_record_destroy(rec);
@@ -2183,6 +2189,10 @@ as_status check_and_set_meta(PyObject *py_meta, as_operations *ops,
             }
             ops->ttl = ttl;
         }
+        else {
+            // Metadata dict was present, but ttl field did not exist
+            ops->ttl = AS_RECORD_CLIENT_DEFAULT_TTL;
+        }
 
         if (py_gen) {
             if (PyLong_Check(py_gen)) {
@@ -2205,6 +2215,10 @@ as_status check_and_set_meta(PyObject *py_meta, as_operations *ops,
         return as_error_update(err, AEROSPIKE_ERR_PARAM,
                                "Metadata should be of type dictionary");
     }
+    else {
+        // Metadata dict was not set by user
+        ops->ttl = AS_RECORD_CLIENT_DEFAULT_TTL;
+    }
     return err->code;
 }