Stabilising changes for version 3.0.0

CHANGELOG.rst

@@ -16,6 +16,10 @@ Features
 * Array Data Type Support. Adds native support for NumPy arrays
   (currently only for ``np.float64`` element type and up to 32 dimensions).
 
+.. note::
+    **Server Requirement**: This feature requires QuestDB server version 8.4.0 or higher.
+    Ensure your server is upgraded before ingesting array types, otherwise data ingestion will fail.
+
 .. code-block:: python
 
         import numpy as np

README.rst

@@ -5,7 +5,7 @@ QuestDB Client Library for Python
 This is the official Python client library for `QuestDB <https://questdb.io>`_.
 
 This client library implements QuestDB's variant of the
-`Ingestion Line Protocol <https://questdb.io/docs/reference/api/ilp/overview/>`_
+`InfluxDB Line Protocol <https://questdb.io/docs/reference/api/ilp/overview/>`_
 (ILP) over HTTP and TCP.
 
 ILP provides the fastest way to insert data into QuestDB.
@@ -53,6 +53,7 @@ The most common way to insert data is from a Pandas dataframe.
         'amount': [0.00044, 0.001],
 
         # NumPy float64 arrays are supported from v3.0.0rc1 onwards.
+        # Note that requires QuestDB server >= 8.4.0 for array support
         'ord_book_bids': [
             np.array([2615.54, 2618.63]),
             np.array([39269.98, 39270.00])
@@ -82,6 +83,7 @@ You can also send individual rows. This only requires a more minimal installatio
                 'amount': 0.00044,
 
                 # NumPy float64 arrays are supported from v3.0.0rc1 onwards.
+                # Note that requires QuestDB server >= 8.4.0 for array support
                 'ord_book_bids': np.array([2615.54, 2618.63]),
             },
             at=TimestampNanos.now())

ci/run_tests_pipeline.yaml

@@ -68,3 +68,51 @@ stages:
               JAVA_HOME: $(JAVA_HOME_11_X64)
               QDB_REPO_PATH: './questdb'
             condition: eq(variables.vsQuestDbMaster, true)
+      - job: TestsAgainstVariousNumpyVersion1x
+        pool:
+          name: "Azure Pipelines"
+          vmImage: "ubuntu-latest"
+        timeoutInMinutes: 45
+        steps:
+          - checkout: self
+            fetchDepth: 1
+            lfs: false
+            submodules: true
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.9'
+          - script: |
+              python3 --version
+              python3 -m pip install uv
+              sudo apt-get install -y libopenblas-dev pkg-config
+            displayName: "Install uv"
+          - script: uv run --with 'numpy==1.21.0' test/test.py -v TestBufferProtocolVersionV2
+            displayName: "Test vs numpy 1.21"
+          - script: uv run --with 'numpy==1.24.0' test/test.py -v TestBufferProtocolVersionV2
+            displayName: "Test vs numpy 1.24"
+          - script: uv run --with 'numpy==1.26.0' test/test.py -v TestBufferProtocolVersionV2
+            displayName: "Test vs numpy 1.26"
+      - job: TestsAgainstVariousNumpyVersion2x
+        pool:
+          name: "Azure Pipelines"
+          vmImage: "ubuntu-latest"
+        timeoutInMinutes: 45
+        steps:
+          - checkout: self
+            fetchDepth: 1
+            lfs: false
+            submodules: true
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.11'
+          - script: |
+              python3 --version
+              python3 -m pip install uv
+              sudo apt-get install -y libopenblas-dev pkg-config
+            displayName: "Install uv"
+          - script: uv run --with 'numpy==2.0.0' test/test.py -v TestBufferProtocolVersionV2
+            displayName: "Test vs numpy 2.0"
+          - script: uv run --with 'numpy==2.2.0' test/test.py -v TestBufferProtocolVersionV2
+            displayName: "Test vs numpy 2.2"
+          - script: uv run --with 'numpy==2.3.0' test/test.py -v TestBufferProtocolVersionV2
+            displayName: "Test vs numpy 2.3"

dev_requirements.txt

@@ -7,6 +7,6 @@ sphinx-rtd-theme>=1.0.0
 twine>=4.0.1
 bump2version>=1.0.1
 pandas>=1.3.5
-numpy>=1.21.6
+numpy>=1.21.0
 pyarrow>=10.0.1
 fastparquet>=2023.10.1

docs/conf.py

@@ -5,6 +5,11 @@
 
 from questdb.ingress import *
 
+autodoc_mock_imports = ["_cython"]
+autodoc_type_aliases = {
+    'datetime': 'datetime.datetime',
+}
+
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.autosummary',

docs/conf.rst

@@ -240,6 +240,9 @@ Specifies the version of InfluxDB Line Protocol to use. Valid options are:
 
   TCP/TCPS: Defaults to version 1 for compatibility
 
+.. note::
+    Protocol version ``2`` requires QuestDB server version 8.4.0 or higher.
+
 .. _sender_conf_buffer:
 
 Buffer

docs/sender.rst

@@ -302,6 +302,9 @@ Here is a configuration string with ``protocol_version=2`` for ``TCP``:
 
 See the :ref:`sender_conf_protocol_version` section for more details.
 
+.. note::
+    Protocol version ``2`` requires QuestDB server version 8.4.0 or higher.
+
 Error Reporting
 ===============
 

pyproject.toml

@@ -19,7 +19,7 @@ classifiers = [
     "Topic :: Database :: Front-Ends",
     "Topic :: Scientific/Engineering"
 ]
-dependencies = ["numpy>=1.26.0"]  # Keep in sync with build-system.requires
+dependencies = ["numpy>=1.21.0"]  # Keep in sync with build-system.requires
 [project.license]
 text = "Apache License 2.0"
 
@@ -46,7 +46,7 @@ requires = [
     "setuptools>=45.2.0",
     "wheel>=0.34.2",
     "cython>=0.29.24",
-    "numpy>=1.26.0",  # keep in sync with project.dependencies
+    "numpy>=1.21.0",  # keep in sync with project.dependencies
 ]
 
 

src/questdb/dataframe.pxi

@@ -2065,17 +2065,29 @@ cdef void_int _dataframe_serialize_cell_column_arr_f64__arr_f64_numpyobj(
     cdef:
         size_t rank = PyArray_NDIM(arr)
         const uint8_t* data_ptr = <const uint8_t *> PyArray_DATA(arr)
-        line_sender_error * err = NULL
-    if not line_sender_buffer_column_f64_arr_byte_strides(
-            ls_buf,
-            col.name,
-            rank,
-            <const size_t*> PyArray_DIMS(arr),
-            <const ssize_t*> PyArray_STRIDES(arr), # N.B.: Strides expressed as byte jumps
-            data_ptr,
-            PyArray_NBYTES(arr),
-            &err):
-        raise c_err_to_py(err)
+        line_sender_error * err = NULL\
+
+    if PyArray_FLAGS(arr) & NPY_ARRAY_C_CONTIGUOUS != 0:
+        if not line_sender_buffer_column_f64_arr_c_major(
+                ls_buf,
+                col.name,
+                rank,
+                <const size_t *> PyArray_DIMS(arr),
+                data_ptr,
+                PyArray_NBYTES(arr),
+                &err):
+            raise c_err_to_py(err)
+    else:
+        if not line_sender_buffer_column_f64_arr_byte_strides(
+                ls_buf,
+                col.name,
+                rank,
+                <const size_t*> PyArray_DIMS(arr),
+                <const ssize_t*> PyArray_STRIDES(arr), # N.B.: Strides expressed as byte jumps
+                data_ptr,
+                PyArray_NBYTES(arr),
+                &err):
+            raise c_err_to_py(err)
 
 cdef void_int _dataframe_serialize_cell_column_ts__dt64ns_tz_arrow(
         line_sender_buffer* ls_buf,

src/questdb/extra_numpy.pxd

@@ -8,6 +8,7 @@ from cpython.object cimport PyObject
 from numpy cimport (
     # Constants
     NPY_DOUBLE,  # N.B.: From `#include <numpy/npy_common.h>`: `#define NPY_FLOAT64 NPY_DOUBLE`
+    NPY_ARRAY_C_CONTIGUOUS,
 
     # Types
     PyArrayObject,
@@ -31,3 +32,4 @@ cdef extern from "numpy/arrayobject.h":
     npy_intp PyArray_DIM(PyArrayObject*, size_t) nogil
     npy_intp PyArray_STRIDE(PyArrayObject*, size_t) nogil
     int PyArray_NDIM(PyArrayObject*) nogil
+    int PyArray_FLAGS(PyArrayObject*)

src/questdb/ingress.pyi

@@ -55,9 +55,7 @@ class IngressErrorCode(Enum):
     HttpNotSupported = ...
     ServerFlushError = ...
     ConfigError = ...
-    ArrayLargeDimError = ...
-    ArrayInternalError = ...
-    ArrayWriteToBufferError = ...
+    ArrayError = ...
     ProtocolVersionError = ...
     BadDataFrame = ...
 
@@ -189,7 +187,7 @@ class SenderTransaction:
 
     To create a transaction:
 
-    .. code_block:: python
+    .. code-block:: python
 
         with sender.transaction('table_name') as txn:
             txn.row(..)
@@ -212,6 +210,8 @@ class SenderTransaction:
         Write a row for the table in the transaction.
 
         The table name is taken from the transaction.
+
+        **Note**: Support for NumPy arrays (``np.array``) requires QuestDB server version 8.4.0 or higher.
         """
 
     def dataframe(
@@ -295,6 +295,8 @@ class Buffer:
         This should match the ``cairo.max.file.name.length`` setting of the
         QuestDB instance you're connecting to.
 
+    **Note**: Protocol version ``2`` requires QuestDB server version 8.4.0 or higher.
+
     .. code-block:: python
 
         # These two buffer constructions are equivalent.
@@ -450,6 +452,8 @@ class Buffer:
             * - ``None``
               - *Column is skipped and not serialized.*
 
+        **Note**: Support for NumPy arrays (``np.array``) requires QuestDB server version 8.4.0 or higher.
+
         If the destination table was already created, then the columns types
         will be cast to the types of the existing columns whenever possible
         (Refer to the QuestDB documentation pages linked above).
@@ -727,6 +731,9 @@ class Buffer:
               interpreted as the current QuestDB server time set on receipt of
               message.
 
+            * **η**: Support for NumPy arrays (``np.array``) requires QuestDB
+              server version 8.4.0 or higher.
+
         **Error Handling and Recovery**
 
         In case an exception is raised during dataframe serialization, the
@@ -834,6 +841,7 @@ class Sender:
         auto_flush_rows: Optional[int] = None,
         auto_flush_bytes: bool = False,
         auto_flush_interval: int = 1000,
+        protocol_version=None,
         init_buf_size: int = 65536,
         max_name_len: int = 127,
     ): ...
@@ -859,6 +867,7 @@ class Sender:
         auto_flush_rows: Optional[int] = None,
         auto_flush_bytes: bool = False,
         auto_flush_interval: int = 1000,
+        protocol_version=None,
         init_buf_size: int = 65536,
         max_name_len: int = 127,
     ) -> Sender:
@@ -894,6 +903,7 @@ class Sender:
         auto_flush_rows: Optional[int] = None,
         auto_flush_bytes: bool = False,
         auto_flush_interval: int = 1000,
+        protocol_version=None,
         init_buf_size: int = 65536,
         max_name_len: int = 127,
     ) -> Sender:
@@ -956,7 +966,13 @@ class Sender:
     @property
     def protocol_version(self) -> int:
         """
-        Returns the QuestDB server's recommended default line protocol version.
+        The protocol version used by the sender.
+
+        Protocol version 1 is retained for backwards compatibility with
+        older QuestDB versions.
+
+        Protocol version 2 introduces binary floating point support and
+        the array datatype.
         """
 
     @property
@@ -1020,6 +1036,8 @@ class Sender:
         in the constructor.
 
         Refer to the :func:`Buffer.row` documentation for details on arguments.
+
+        **Note**: Support for NumPy arrays (``np.array``) requires QuestDB server version 8.4.0 or higher.
         """
 
     def dataframe(