From 6bbe28899600b8f286827b5e90ac1bb9a491b8e4 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 8 Oct 2024 20:08:53 +0300 Subject: [PATCH 001/125] [fix] Rework code according to Valgrind's recommendations --- pyreindexer/lib/include/pyobjtools.cc | 148 ++++++++-------- pyreindexer/lib/include/pyobjtools.h | 11 +- pyreindexer/lib/src/rawpyreindexer.cc | 216 ++++++++++------------- pyreindexer/lib/src/rawpyreindexer.h | 60 +++---- pyreindexer/lib/src/reindexerinterface.h | 4 +- pyreindexer/query_results.py | 6 +- pyreindexer/tests/tests/test_sql.py | 4 +- setup.py | 2 +- 8 files changed, 208 insertions(+), 243 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 39e63da..975d017 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -4,57 +4,30 @@ namespace pyreindexer { -void pyValueSerialize(PyObject **value, reindexer::WrSerializer &wrSer) { - if (*value == Py_None) { - wrSer << "null"; - } else if (PyBool_Check(*value)) { - bool v = PyLong_AsLong(*value) != 0; - wrSer << v; - } else if (PyFloat_Check(*value)) { - double v = PyFloat_AsDouble(*value); - double intpart; - if (std::modf(v, &intpart) == 0.0) { - wrSer << int64_t(v); - } else { - wrSer << v; - } - } else if (PyLong_Check(*value)) { - long v = PyLong_AsLong(*value); - wrSer << v; - } else if (PyUnicode_Check(*value)) { - const char *v = PyUnicode_AsUTF8(*value); - wrSer.PrintJsonString(v); - } else if (PyList_Check(*value)) { - pyListSerialize(value, wrSer); - } else if (PyDict_Check(*value)) { - pyDictSerialize(value, wrSer); - } else { - throw reindexer::Error(errParseJson, std::string("Unable to parse value of type ") + Py_TYPE(*value)->tp_name); - } -} +void pyValueSerialize(PyObject** value, reindexer::WrSerializer& wrSer); -void pyListSerialize(PyObject **list, reindexer::WrSerializer &wrSer) { +void pyListSerialize(PyObject** list, reindexer::WrSerializer& wrSer) { if (!PyList_Check(*list)) { throw reindexer::Error(errParseJson, std::string("List expected, got ") + Py_TYPE(*list)->tp_name); } wrSer << '['; - unsigned sz = PyList_Size(*list); - for (unsigned i = 0; i < sz; i++) { - PyObject *value = PyList_GetItem(*list, i); - - pyValueSerialize(&value, wrSer); - - if (i < sz - 1) { + Py_ssize_t sz = PyList_Size(*list); + for (Py_ssize_t i = 0; i < sz; ++i) { + if (i > 0) { wrSer << ','; } + + PyObject* value = PyList_GetItem(*list, i); + + pyValueSerialize(&value, wrSer); } wrSer << ']'; } -void pyDictSerialize(PyObject **dict, reindexer::WrSerializer &wrSer) { +void pyDictSerialize(PyObject** dict, reindexer::WrSerializer& wrSer) { if (!PyDict_Check(*dict)) { throw reindexer::Error(errParseJson, std::string("Dictionary expected, got ") + Py_TYPE(*dict)->tp_name); } @@ -62,47 +35,72 @@ void pyDictSerialize(PyObject **dict, reindexer::WrSerializer &wrSer) { wrSer << '{'; Py_ssize_t sz = PyDict_Size(*dict); - if (!sz) { - wrSer << '}'; + if (sz) { + PyObject *key = nullptr, *value = nullptr; + Py_ssize_t pos = 0; - return; - } - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(*dict, &pos, &key, &value)) { - const char *k = PyUnicode_AsUTF8(key); - wrSer.PrintJsonString(k); - wrSer << ':'; + while (PyDict_Next(*dict, &pos, &key, &value)) { + if (pos > 1) { + wrSer << ','; + } - pyValueSerialize(&value, wrSer); + const char* k = PyUnicode_AsUTF8(key); + wrSer.PrintJsonString(k); + wrSer << ':'; - if (pos != sz) { - wrSer << ','; + pyValueSerialize(&value, wrSer); } } wrSer << '}'; } -void PyObjectToJson(PyObject **obj, reindexer::WrSerializer &wrSer) { +void pyValueSerialize(PyObject** value, reindexer::WrSerializer& wrSer) { + if (*value == Py_None) { + wrSer << "null"; + } else if (PyBool_Check(*value)) { + bool v = PyLong_AsLong(*value) != 0; + wrSer << v; + } else if (PyFloat_Check(*value)) { + double v = PyFloat_AsDouble(*value); + double intpart; + if (std::modf(v, &intpart) == 0.0) { + wrSer << int64_t(v); + } else { + wrSer << v; + } + } else if (PyLong_Check(*value)) { + long v = PyLong_AsLong(*value); + wrSer << v; + } else if (PyUnicode_Check(*value)) { + const char* v = PyUnicode_AsUTF8(*value); + wrSer.PrintJsonString(v); + } else if (PyList_Check(*value)) { + pyListSerialize(value, wrSer); + } else if (PyDict_Check(*value)) { + pyDictSerialize(value, wrSer); + } else { + throw reindexer::Error(errParseJson, std::string("Unable to parse value of type ") + Py_TYPE(*value)->tp_name); + } +} + +void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer) { if (PyDict_Check(*obj)) { pyDictSerialize(obj, wrSer); } else if (PyList_Check(*obj) ) { pyListSerialize(obj, wrSer); } else { - throw reindexer::Error(errParseJson, + throw reindexer::Error(errParseJson, std::string("PyObject must be a dictionary or a list for JSON serializing, got ") + Py_TYPE(*obj)->tp_name); } } -std::vector ParseListToStrVec(PyObject **list) { +std::vector ParseListToStrVec(PyObject** list) { std::vector vec; Py_ssize_t sz = PyList_Size(*list); for (Py_ssize_t i = 0; i < sz; i++) { - PyObject *item = PyList_GetItem(*list, i); + PyObject* item = PyList_GetItem(*list, i); if (!PyUnicode_Check(item)) { throw reindexer::Error(errParseJson, std::string("String expected, got ") + Py_TYPE(item)->tp_name); @@ -114,40 +112,49 @@ std::vector ParseListToStrVec(PyObject **list) { return vec; } -PyObject *pyValueFromJsonValue(const gason::JsonValue &value) { - PyObject *pyValue = nullptr; +PyObject* pyValueFromJsonValue(const gason::JsonValue& value) { + PyObject* pyValue = nullptr; switch (value.getTag()) { case gason::JSON_NUMBER: - pyValue = PyLong_FromSize_t(value.toNumber()); + pyValue = PyLong_FromSize_t(value.toNumber()); // new ref break; case gason::JSON_DOUBLE: - pyValue = PyFloat_FromDouble(value.toDouble()); + pyValue = PyFloat_FromDouble(value.toDouble()); // new ref break; case gason::JSON_STRING: { auto sv = value.toString(); - pyValue = PyUnicode_FromStringAndSize(sv.data(), sv.size()); + pyValue = PyUnicode_FromStringAndSize(sv.data(), sv.size()); // new ref break; } case gason::JSON_NULL: pyValue = Py_None; + Py_INCREF(pyValue); // new ref break; case gason::JSON_TRUE: pyValue = Py_True; + Py_INCREF(pyValue); // new ref break; case gason::JSON_FALSE: pyValue = Py_False; + Py_INCREF(pyValue); // new ref break; case gason::JSON_ARRAY: - pyValue = PyList_New(0); - for (const auto &v : value) { - PyList_Append(pyValue, pyValueFromJsonValue(v.value)); + pyValue = PyList_New(0); // new ref + for (const auto& v : value) { + PyObject* dictFromJson = pyValueFromJsonValue(v.value); // stolen ref + PyList_Append(pyValue, dictFromJson); // new ref + Py_XDECREF(dictFromJson); } break; case gason::JSON_OBJECT: - pyValue = PyDict_New(); - for (const auto &v : value) { - PyDict_SetItem(pyValue, PyUnicode_FromStringAndSize(v.key.data(), v.key.size()), pyValueFromJsonValue(v.value)); + pyValue = PyDict_New(); // new ref + for (const auto& v : value) { + PyObject* dictFromJson = pyValueFromJsonValue(v.value); // stolen ref + PyObject* pyKey = PyUnicode_FromStringAndSize(v.key.data(), v.key.size()); // new ref + PyDict_SetItem(pyValue, pyKey, dictFromJson); // new refs + Py_XDECREF(pyKey); + Py_XDECREF(dictFromJson); } break; } @@ -155,13 +162,12 @@ PyObject *pyValueFromJsonValue(const gason::JsonValue &value) { return pyValue; } -PyObject *PyObjectFromJson(reindexer::span json) { +PyObject* PyObjectFromJson(reindexer::span json) { try { gason::JsonParser parser; auto root = parser.Parse(json); - // new ref - return pyValueFromJsonValue(root.value); - } catch (const gason::Exception &ex) { + return pyValueFromJsonValue(root.value); // stolen ref + } catch (const gason::Exception& ex) { throw reindexer::Error(errParseJson, std::string("PyObjectFromJson: ") + ex.what()); } } diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index d45bbfe..89e98e2 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -8,14 +8,9 @@ namespace pyreindexer { -void pyValueSerialize(PyObject **item, reindexer::WrSerializer &wrSer); -void pyListSerialize(PyObject **list, reindexer::WrSerializer &wrSer); -void pyDictSerialize(PyObject **dict, reindexer::WrSerializer &wrSer); -PyObject *pyValueFromJsonValue(const gason::JsonValue &value); +std::vector ParseListToStrVec(PyObject** dict); -std::vector ParseListToStrVec(PyObject **dict); - -void PyObjectToJson(PyObject **dict, reindexer::WrSerializer &wrSer); -PyObject *PyObjectFromJson(reindexer::span json); +void PyObjectToJson(PyObject** dict, reindexer::WrSerializer& wrSer); +PyObject* PyObjectFromJson(reindexer::span json); } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index b20d87c..9f4295a 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -3,33 +3,34 @@ namespace pyreindexer { static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr) { - QueryResultsWrapper *qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); - + QueryResultsWrapper* qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); delete qresWrapperPtr; } -static PyObject *queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { - QueryResultsWrapper *qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); +static PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { + QueryResultsWrapper* qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); WrSerializer wrSer; qresWrapperPtr->GetItemJSON(wrSer, false); qresWrapperPtr->Next(); - PyObject *dictFromJson = nullptr; + PyObject* dictFromJson = nullptr; try { dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref - } catch (const Error &err) { + } catch (const Error& err) { Py_XDECREF(dictFromJson); return Py_BuildValue("is{}", err.code(), err.what().c_str()); } - return Py_BuildValue("isO", errOK, "", dictFromJson); + PyObject* res = Py_BuildValue("isO", errOK, "", dictFromJson); // new ref + Py_DECREF(dictFromJson); + + return res; } -static PyObject *Init(PyObject *self, PyObject *args) { +static PyObject* Init(PyObject* self, PyObject* args) { uintptr_t rx = initReindexer(); - if (rx == 0) { PyErr_SetString(PyExc_RuntimeError, "Initialization error"); @@ -39,9 +40,8 @@ static PyObject *Init(PyObject *self, PyObject *args) { return Py_BuildValue("k", rx); } -static PyObject *Destroy(PyObject *self, PyObject *args) { +static PyObject* Destroy(PyObject* self, PyObject* args) { uintptr_t rx = 0; - if (!PyArg_ParseTuple(args, "k", &rx)) { return nullptr; } @@ -51,63 +51,54 @@ static PyObject *Destroy(PyObject *self, PyObject *args) { Py_RETURN_NONE; } -static PyObject *Connect(PyObject *self, PyObject *args) { +static PyObject* Connect(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *dsn = nullptr; - + char* dsn = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &dsn)) { return nullptr; } Error err = getDB(rx)->Connect(dsn); - return pyErr(err); } -static PyObject *NamespaceOpen(PyObject *self, PyObject *args) { +static PyObject* NamespaceOpen(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - + char* ns = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { return nullptr; } Error err = getDB(rx)->OpenNamespace(ns); - return pyErr(err); } -static PyObject *NamespaceClose(PyObject *self, PyObject *args) { +static PyObject* NamespaceClose(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - + char* ns = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { return nullptr; } Error err = getDB(rx)->CloseNamespace(ns); - return pyErr(err); } -static PyObject *NamespaceDrop(PyObject *self, PyObject *args) { +static PyObject* NamespaceDrop(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - + char* ns = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { return nullptr; } Error err = getDB(rx)->DropNamespace(ns); - return pyErr(err); } -static PyObject *IndexAdd(PyObject *self, PyObject *args) { +static PyObject* IndexAdd(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - PyObject *indexDefDict = nullptr; // borrowed ref after ParseTuple - + char* ns = nullptr; + PyObject* indexDefDict = nullptr; // borrowed ref after ParseTuple if (!PyArg_ParseTuple(args, "ksO!", &rx, &ns, &PyDict_Type, &indexDefDict)) { return nullptr; } @@ -118,7 +109,7 @@ static PyObject *IndexAdd(PyObject *self, PyObject *args) { try { PyObjectToJson(&indexDefDict, wrSer); - } catch (const Error &err) { + } catch (const Error& err) { Py_DECREF(indexDefDict); return pyErr(err); @@ -128,20 +119,16 @@ static PyObject *IndexAdd(PyObject *self, PyObject *args) { IndexDef indexDef; Error err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); - if (!err.ok()) { - return pyErr(err); + if (err.ok()) { + err = getDB(rx)->AddIndex(ns, indexDef); } - - err = getDB(rx)->AddIndex(ns, indexDef); - return pyErr(err); } -static PyObject *IndexUpdate(PyObject *self, PyObject *args) { +static PyObject* IndexUpdate(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - PyObject *indexDefDict = nullptr; // borrowed ref after ParseTuple - + char* ns = nullptr; + PyObject* indexDefDict = nullptr; // borrowed ref after ParseTuple if (!PyArg_ParseTuple(args, "ksO!", &rx, &ns, &PyDict_Type, &indexDefDict)) { return nullptr; } @@ -152,7 +139,7 @@ static PyObject *IndexUpdate(PyObject *self, PyObject *args) { try { PyObjectToJson(&indexDefDict, wrSer); - } catch (const Error &err) { + } catch (const Error& err) { Py_DECREF(indexDefDict); return pyErr(err); @@ -162,35 +149,28 @@ static PyObject *IndexUpdate(PyObject *self, PyObject *args) { IndexDef indexDef; Error err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); - if (!err.ok()) { - return pyErr(err); + if (err.ok()) { + err = getDB(rx)->UpdateIndex(ns, indexDef); } - - err = getDB(rx)->UpdateIndex(ns, indexDef); - return pyErr(err); } -static PyObject *IndexDrop(PyObject *self, PyObject *args) { +static PyObject* IndexDrop(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - char *indexName = nullptr; - + char *ns = nullptr, *indexName = nullptr; if (!PyArg_ParseTuple(args, "kss", &rx, &ns, &indexName)) { return nullptr; } Error err = getDB(rx)->DropIndex(ns, IndexDef(indexName)); - return pyErr(err); } -static PyObject *itemModify(PyObject *self, PyObject *args, ItemModifyMode mode) { +static PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t rx = 0; - char *ns = nullptr; - PyObject *itemDefDict = nullptr; // borrowed ref after ParseTuple - PyObject *preceptsList = nullptr; // borrowed ref after ParseTuple if passed - + char* ns = nullptr; + PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "ksO!|O!", &rx, &ns, &PyDict_Type, &itemDefDict, &PyList_Type, &preceptsList)) { return nullptr; } @@ -208,7 +188,7 @@ static PyObject *itemModify(PyObject *self, PyObject *args, ItemModifyMode mode) try { PyObjectToJson(&itemDefDict, wrSer); - } catch (const Error &err) { + } catch (const Error& err) { Py_DECREF(itemDefDict); Py_XDECREF(preceptsList); @@ -217,11 +197,10 @@ static PyObject *itemModify(PyObject *self, PyObject *args, ItemModifyMode mode) Py_DECREF(itemDefDict); - char *json = const_cast(wrSer.c_str()); - + char* json = const_cast(wrSer.c_str()); err = item.Unsafe().FromJSON(json, 0, mode == ModeDelete); if (!err.ok()) { - Py_XDECREF(preceptsList); + Py_XDECREF(preceptsList); return pyErr(err); } @@ -231,7 +210,7 @@ static PyObject *itemModify(PyObject *self, PyObject *args, ItemModifyMode mode) try { itemPrecepts = ParseListToStrVec(&preceptsList); - } catch (const Error &err) { + } catch (const Error& err) { Py_DECREF(preceptsList); return pyErr(err); @@ -263,78 +242,67 @@ static PyObject *itemModify(PyObject *self, PyObject *args, ItemModifyMode mode) return pyErr(err); } -static PyObject *ItemInsert(PyObject *self, PyObject *args) { return itemModify(self, args, ModeInsert); } - -static PyObject *ItemUpdate(PyObject *self, PyObject *args) { return itemModify(self, args, ModeUpdate); } - -static PyObject *ItemUpsert(PyObject *self, PyObject *args) { return itemModify(self, args, ModeUpsert); } - -static PyObject *ItemDelete(PyObject *self, PyObject *args) { return itemModify(self, args, ModeDelete); } +static PyObject* ItemInsert(PyObject* self, PyObject* args) { return itemModify(self, args, ModeInsert); } +static PyObject* ItemUpdate(PyObject* self, PyObject* args) { return itemModify(self, args, ModeUpdate); } +static PyObject* ItemUpsert(PyObject* self, PyObject* args) { return itemModify(self, args, ModeUpsert); } +static PyObject* ItemDelete(PyObject* self, PyObject* args) { return itemModify(self, args, ModeDelete); } -static PyObject *PutMeta(PyObject *self, PyObject *args) { +static PyObject* PutMeta(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - char *key = nullptr; - char *value = nullptr; - + char *ns = nullptr, *key = nullptr, *value = nullptr; if (!PyArg_ParseTuple(args, "ksss", &rx, &ns, &key, &value)) { return nullptr; } Error err = getDB(rx)->PutMeta(ns, key, value); - return pyErr(err); } -static PyObject *GetMeta(PyObject *self, PyObject *args) { +static PyObject* GetMeta(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - char *key = nullptr; - + char *ns = nullptr, *key = nullptr; if (!PyArg_ParseTuple(args, "kss", &rx, &ns, &key)) { return nullptr; } std::string value; Error err = getDB(rx)->GetMeta(ns, key, value); - return Py_BuildValue("iss", err.code(), err.what().c_str(), value.c_str()); } -static PyObject *DeleteMeta(PyObject *self, PyObject *args) { +static PyObject* DeleteMeta(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - char *key = nullptr; - + char *ns = nullptr, *key = nullptr; if (!PyArg_ParseTuple(args, "kss", &rx, &ns, &key)) { return nullptr; } Error err = getDB(rx)->DeleteMeta(ns, key); - return pyErr(err); } -static PyObject *Select(PyObject *self, PyObject *args) { +static PyObject* Select(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *query = nullptr; - + char* query = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &query)) { return nullptr; } - auto db = getDB(rx); - QueryResultsWrapper *qresWrapper = new QueryResultsWrapper(); + auto qresWrapper = new QueryResultsWrapper(); + Error err = getDB(rx)->Select(query, *qresWrapper); + + if (!err.ok()) { + delete qresWrapper; - Error err = db->Select(query, *qresWrapper); + return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); + } return Py_BuildValue("iskI", err.code(), err.what().c_str(), reinterpret_cast(qresWrapper), qresWrapper->Count()); } -static PyObject *EnumMeta(PyObject *self, PyObject *args) { +static PyObject* EnumMeta(PyObject* self, PyObject* args) { uintptr_t rx = 0; - char *ns = nullptr; - + char* ns = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { return nullptr; } @@ -345,27 +313,26 @@ static PyObject *EnumMeta(PyObject *self, PyObject *args) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); } - PyObject *list = PyList_New(keys.size()); // new ref + PyObject* list = PyList_New(keys.size()); // new ref if (!list) { return nullptr; } - for (auto it = keys.begin(); it != keys.end(); it++) { - unsigned pos = std::distance(keys.begin(), it); - - PyList_SetItem(list, pos, Py_BuildValue("s", it->c_str())); // stolen ref + Py_ssize_t pos = 0; + for (const auto& key : keys) { + PyList_SetItem(list, pos, Py_BuildValue("s", key.c_str())); // stolen ref + ++pos; } - PyObject *res = Py_BuildValue("isO", err.code(), err.what().c_str(), list); + PyObject* res = Py_BuildValue("isO", err.code(), err.what().c_str(), list); Py_DECREF(list); return res; } -static PyObject *EnumNamespaces(PyObject *self, PyObject *args) { +static PyObject* EnumNamespaces(PyObject* self, PyObject* args) { uintptr_t rx = 0; unsigned enumAll = 0; - if (!PyArg_ParseTuple(args, "kI", &rx, &enumAll)) { return nullptr; } @@ -376,22 +343,21 @@ static PyObject *EnumNamespaces(PyObject *self, PyObject *args) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); } - PyObject *list = PyList_New(nsDefs.size()); // new ref + PyObject* list = PyList_New(nsDefs.size()); // new ref if (!list) { return nullptr; } WrSerializer wrSer; - for (auto it = nsDefs.begin(); it != nsDefs.end(); it++) { - unsigned pos = std::distance(nsDefs.begin(), it); - + Py_ssize_t pos = 0; + for (const auto& ns : nsDefs) { wrSer.Reset(); - it->GetJSON(wrSer, false); + ns.GetJSON(wrSer, false); - PyObject *dictFromJson = nullptr; + PyObject* dictFromJson = nullptr; try { dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref - } catch (const Error &err) { + } catch (const Error& err) { Py_XDECREF(dictFromJson); Py_DECREF(list); @@ -399,17 +365,17 @@ static PyObject *EnumNamespaces(PyObject *self, PyObject *args) { } PyList_SetItem(list, pos, dictFromJson); // stolen ref + ++pos; } - PyObject *res = Py_BuildValue("isO", err.code(), err.what().c_str(), list); + PyObject* res = Py_BuildValue("isO", err.code(), err.what().c_str(), list); Py_DECREF(list); return res; } -static PyObject *QueryResultsWrapperIterate(PyObject *self, PyObject *args) { +static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args) { uintptr_t qresWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &qresWrapperAddr)) { return nullptr; } @@ -417,9 +383,8 @@ static PyObject *QueryResultsWrapperIterate(PyObject *self, PyObject *args) { return queryResultsWrapperIterate(qresWrapperAddr); } -static PyObject *QueryResultsWrapperDelete(PyObject *self, PyObject *args) { +static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args) { uintptr_t qresWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &qresWrapperAddr)) { return nullptr; } @@ -429,36 +394,39 @@ static PyObject *QueryResultsWrapperDelete(PyObject *self, PyObject *args) { Py_RETURN_NONE; } -static PyObject *GetAggregationResults(PyObject *self, PyObject *args) { - uintptr_t qresWrapperAddr; - +static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { + uintptr_t qresWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &qresWrapperAddr)) { - return NULL; + return nullptr; } - QueryResultsWrapper *qresWrapper = getQueryResultsWrapper(qresWrapperAddr); + QueryResultsWrapper* qresWrapper = getQueryResultsWrapper(qresWrapperAddr); - const auto &aggResults = qresWrapper->GetAggregationResults(); + const auto& aggResults = qresWrapper->GetAggregationResults(); WrSerializer wrSer; wrSer << "["; for (size_t i = 0; i < aggResults.size(); ++i) { if (i > 0) { wrSer << ','; } + aggResults[i].GetJSON(wrSer); } wrSer << "]"; - PyObject *dictFromJson = nullptr; + PyObject* dictFromJson = nullptr; try { - dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref - } catch (const Error &err) { + dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref + } catch (const Error& err) { Py_XDECREF(dictFromJson); return Py_BuildValue("is{}", err.code(), err.what().c_str()); } - return Py_BuildValue("isO", errOK, "", dictFromJson); + PyObject* res = Py_BuildValue("isO", errOK, "", dictFromJson); // new ref + Py_DECREF(dictFromJson); + + return res; } } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 6b8e676..83ab10f 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -30,50 +30,48 @@ using reindexer::NamespaceDef; using reindexer::WrSerializer; inline static uintptr_t initReindexer() { - DBInterface *db = new DBInterface(); - + DBInterface* db = new DBInterface(); return reinterpret_cast(db); } -inline static DBInterface *getDB(uintptr_t rx) { return reinterpret_cast(rx); } +inline static DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } inline static void destroyReindexer(uintptr_t rx) { - DBInterface *db = getDB(rx); - + DBInterface* db = getDB(rx); delete db; } -inline static PyObject *pyErr(const Error &err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } +inline static PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } -inline static QueryResultsWrapper *getQueryResultsWrapper(uintptr_t qresWrapperAddr) { - return reinterpret_cast(qresWrapperAddr); +inline static QueryResultsWrapper* getQueryResultsWrapper(uintptr_t qresWrapperAddr) { + return reinterpret_cast(qresWrapperAddr); } static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr); -static PyObject *Init(PyObject *self, PyObject *args); -static PyObject *Destroy(PyObject *self, PyObject *args); -static PyObject *Connect(PyObject *self, PyObject *args); -static PyObject *NamespaceOpen(PyObject *self, PyObject *args); -static PyObject *NamespaceClose(PyObject *self, PyObject *args); -static PyObject *NamespaceDrop(PyObject *self, PyObject *args); -static PyObject *IndexAdd(PyObject *self, PyObject *args); -static PyObject *IndexUpdate(PyObject *self, PyObject *args); -static PyObject *IndexDrop(PyObject *self, PyObject *args); -static PyObject *ItemInsert(PyObject *self, PyObject *args); -static PyObject *ItemUpdate(PyObject *self, PyObject *args); -static PyObject *ItemUpsert(PyObject *self, PyObject *args); -static PyObject *ItemDelete(PyObject *self, PyObject *args); -static PyObject *PutMeta(PyObject *self, PyObject *args); -static PyObject *GetMeta(PyObject *self, PyObject *args); -static PyObject *DeleteMeta(PyObject *self, PyObject *args); -static PyObject *Select(PyObject *self, PyObject *args); -static PyObject *EnumMeta(PyObject *self, PyObject *args); -static PyObject *EnumNamespaces(PyObject *self, PyObject *args); - -static PyObject *QueryResultsWrapperIterate(PyObject *self, PyObject *args); -static PyObject *QueryResultsWrapperDelete(PyObject *self, PyObject *args); -static PyObject *GetAggregationResults(PyObject *self, PyObject *args); +static PyObject* Init(PyObject* self, PyObject* args); +static PyObject* Destroy(PyObject* self, PyObject* args); +static PyObject* Connect(PyObject* self, PyObject* args); +static PyObject* NamespaceOpen(PyObject* self, PyObject* args); +static PyObject* NamespaceClose(PyObject* self, PyObject* args); +static PyObject* NamespaceDrop(PyObject* self, PyObject* args); +static PyObject* IndexAdd(PyObject* self, PyObject* args); +static PyObject* IndexUpdate(PyObject* self, PyObject* args); +static PyObject* IndexDrop(PyObject* self, PyObject* args); +static PyObject* ItemInsert(PyObject* self, PyObject* args); +static PyObject* ItemUpdate(PyObject* self, PyObject* args); +static PyObject* ItemUpsert(PyObject* self, PyObject* args); +static PyObject* ItemDelete(PyObject* self, PyObject* args); +static PyObject* PutMeta(PyObject* self, PyObject* args); +static PyObject* GetMeta(PyObject* self, PyObject* args); +static PyObject* DeleteMeta(PyObject* self, PyObject* args); +static PyObject* Select(PyObject* self, PyObject* args); +static PyObject* EnumMeta(PyObject* self, PyObject* args); +static PyObject* EnumNamespaces(PyObject* self, PyObject* args); + +static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args); +static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); +static PyObject* GetAggregationResults(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 435d857..1e31e58 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -104,7 +104,7 @@ class ReindexerInterface { Error EnumMeta(std::string_view ns, std::vector& keys) { return execute([this, ns, &keys] { return enumMeta(ns, keys); }); } - Error Select(const std::string &query, QueryResultsWrapper& result); + Error Select(const std::string& query, QueryResultsWrapper& result); Error EnumNamespaces(std::vector& defs, EnumNamespacesOpts opts) { return execute([this, &defs, &opts] { return enumNamespaces(defs, opts); }); } @@ -129,7 +129,7 @@ class ReindexerInterface { Error getMeta(std::string_view ns, const std::string& key, std::string& data) { return db_.GetMeta({ns.data(), ns.size()}, key, data); } Error deleteMeta(std::string_view ns, const std::string& key) { return db_.DeleteMeta({ns.data(), ns.size()}, key); } Error enumMeta(std::string_view ns, std::vector& keys) { return db_.EnumMeta({ns.data(), ns.size()}, keys); } - Error select(const std::string &query, typename DBT::QueryResultsT& result) { return db_.Select(query, result); } + Error select(const std::string& query, typename DBT::QueryResultsT& result) { return db_.Select(query, result); } Error enumNamespaces(std::vector& defs, EnumNamespacesOpts opts) { return db_.EnumNamespaces(defs, opts); } Error stop(); diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index be70d93..decb257 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -77,14 +77,12 @@ def _close_iterator(self): self.qres_iter_count = 0 self.api.query_results_delete(self.qres_wrapper_ptr) - def get_agg_results(self): """Returns aggregation results for the current query """ - self.err_code, self.err_msg, res = self.api.get_agg_results( - self.qres_wrapper_ptr) + self.err_code, self.err_msg, res = self.api.get_agg_results(self.qres_wrapper_ptr) if self.err_code: raise Exception(self.err_msg) - return res \ No newline at end of file + return res diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index 08c96ce..cf9d0bf 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -58,7 +58,7 @@ def test_sql_delete(self, namespace, index, item): def test_sql_select_with_syntax_error(self, namespace, index, item): # Given("Create namespace with item") # When ("Execute SQL query SELECT with incorrect syntax") - query = f'SELECT *' + query = 'SELECT *' # Then ("Check that selected item is in result") assert_that(calling(sql_query).with_args(namespace, query), raises(Exception, matching=has_string(string_contains_in_order( @@ -79,4 +79,4 @@ def test_sql_select_with_aggregations(self, namespace, index, items): # Then ("Check that returned agg results are correct") for agg in select_result: assert_that(agg['value'], equal_to(expected_values[agg['type']]), - f"Incorrect aggregation result for {agg['type']}") \ No newline at end of file + f"Incorrect aggregation result for {agg['type']}") diff --git a/setup.py b/setup.py index f2cae2b..adff914 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.2.37', + version='0.2.38', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', author_email='igtulm@gmail.com', From d27da93e797ff8009a2de6d8937fe2692c6ef012 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 9 Oct 2024 10:07:52 +0300 Subject: [PATCH 002/125] Clear code --- pyreindexer/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyreindexer/__init__.py b/pyreindexer/__init__.py index 8e894e8..d83b82b 100644 --- a/pyreindexer/__init__.py +++ b/pyreindexer/__init__.py @@ -3,4 +3,3 @@ """ from pyreindexer.rx_connector import RxConnector -# from pyreindexer.index_definition import IndexDefinition From 81ca8706ec83203b26e2202df854fa7a25ca5d7d Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 9 Oct 2024 12:33:32 +0300 Subject: [PATCH 003/125] Avoid PEP's warnings --- pyreindexer/example/main.py | 4 ++-- pyreindexer/query_results.py | 2 ++ pyreindexer/raiser_mixin.py | 3 +++ pyreindexer/tests/tests/test_namespace.py | 2 -- pyreindexer/tests/tests/test_sql.py | 5 +++-- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index baf1db8..99371ac 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -19,7 +19,7 @@ def create_index_example(db, namespace): try: db.index_add(namespace, index_definition) - except Exception: + except (Exception,): db.index_drop(namespace, 'id') db.index_add(namespace, index_definition) @@ -58,7 +58,7 @@ def select_item_query_example(db, namespace): def rx_example(): db = RxConnector('builtin:///tmp/pyrx') - #db = RxConnector('cproto://127.0.0.1:6534/pyrx') +# db = RxConnector('cproto://127.0.0.1:6534/pyrx') namespace = 'test_table' diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index decb257..be81ffc 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -25,6 +25,8 @@ def __init__(self, api, qres_wrapper_ptr, qres_iter_count): self.qres_wrapper_ptr = qres_wrapper_ptr self.qres_iter_count = qres_iter_count self.pos = 0 + self.err_code = 0 + self.err_msg = "" def __iter__(self): """Returns the current iteration result. diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index c1c5a0c..e7c62d8 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -2,6 +2,9 @@ class RaiserMixin(object): """ RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity. """ + err_code: int + err_msg: str + rx: int def raise_on_error(self): """Checks if there is an error code and raises with an error message. diff --git a/pyreindexer/tests/tests/test_namespace.py b/pyreindexer/tests/tests/test_namespace.py index d18340d..e46a73b 100644 --- a/pyreindexer/tests/tests/test_namespace.py +++ b/pyreindexer/tests/tests/test_namespace.py @@ -32,5 +32,3 @@ def test_cannot_delete_ns_not_created(self, database): namespace_name = 'test_ns' assert_that(calling(drop_namespace).with_args(database, namespace_name), raises(Exception, matching=has_string( f"Namespace '{namespace_name}' does not exist"))) - - diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index cf9d0bf..c88227d 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -19,7 +19,8 @@ def test_sql_select_with_join(self, namespace, second_namespace_for_join, index, db, namespace_name = namespace second_namespace_name, second_ns_item_definition_join = second_namespace_for_join # When ("Execute SQL query SELECT with JOIN") - query = f'SELECT id FROM {namespace_name} INNER JOIN {second_namespace_name} ON {namespace_name}.id = {second_namespace_name}.id' + query = (f'SELECT id FROM {namespace_name} INNER JOIN {second_namespace_name}' + f' ON {namespace_name}.id = {second_namespace_name}.id') item_list = sql_query(namespace, query) # Then ("Check that selected item is in result") assert_that(item_list, @@ -74,7 +75,7 @@ def test_sql_select_with_aggregations(self, namespace, index, items): select_result = db.select(f'SELECT min(id), max(id), avg(id) FROM {namespace_name}').get_agg_results() assert_that(len(select_result), 3, "The aggregation result must contain 3 elements") - expected_values = {"min":1,"max":10,"avg":5.5} + expected_values = {"min": 1, "max": 10, "avg": 5.5} # Then ("Check that returned agg results are correct") for agg in select_result: From 2f9edd5791943c845b78e691029d3c533e2f29d7 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 9 Oct 2024 12:34:02 +0300 Subject: [PATCH 004/125] Update code --- pyreindexer/index_definition.py | 2 +- pyreindexer/lib/src/reindexerinterface.cc | 2 +- pyreindexer/lib/src/reindexerinterface.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyreindexer/index_definition.py b/pyreindexer/index_definition.py index c1a7664..30e4fad 100644 --- a/pyreindexer/index_definition.py +++ b/pyreindexer/index_definition.py @@ -37,7 +37,7 @@ def __setitem__(self, attr, value): super(IndexDefinition, self).update({attr: value}) return self - def update(self, dict_part=None): + def update(self, *args, **kwargs): raise NotImplementedError( 'Bulk update is not implemented for IndexDefinition instance') diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 5842f2f..ac5d749 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -10,7 +10,7 @@ ReindexerInterface::ReindexerInterface() {} template <> ReindexerInterface::ReindexerInterface() { - std::atomic running{false}; + std::atomic_bool running{false}; executionThr_ = std::thread([this, &running] { cmdAsync_.set(loop_); cmdAsync_.set([this](reindexer::net::ev::async&) { diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 1e31e58..30b4d7f 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -42,7 +42,7 @@ class GenericCommand : public ICommand { private: CallableT command_; Error err_; - std::atomic executed_ = {false}; + std::atomic_bool executed_{false}; }; template From 0ded690c3cf41bbdd75fae2244d9a16811b301b7 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Mon, 14 Oct 2024 19:29:50 +0300 Subject: [PATCH 005/125] Support sync transactions --- pyreindexer/lib/include/transaction_wrapper.h | 65 +++++++++ pyreindexer/lib/src/rawpyreindexer.cc | 129 ++++++++++++++++++ pyreindexer/lib/src/rawpyreindexer.h | 23 ++++ pyreindexer/lib/src/reindexerinterface.cc | 35 ++++- pyreindexer/lib/src/reindexerinterface.h | 15 ++ setup.py | 10 +- 6 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 pyreindexer/lib/include/transaction_wrapper.h diff --git a/pyreindexer/lib/include/transaction_wrapper.h b/pyreindexer/lib/include/transaction_wrapper.h new file mode 100644 index 0000000..e4351b3 --- /dev/null +++ b/pyreindexer/lib/include/transaction_wrapper.h @@ -0,0 +1,65 @@ +#pragma once + +#include "reindexerinterface.h" + +#ifdef PYREINDEXER_CPROTO +#include "client/cororeindexer.h" +#else +#include "core/reindexer.h" +#endif + +namespace pyreindexer { + +#ifdef PYREINDEXER_CPROTO +using DBInterface = ReindexerInterface; +using TransactionT = reindexer::client::CoroTransaction; +using ItemT = reindexer::client::Item; +#else +using DBInterface = ReindexerInterface; +using TransactionT = reindexer::Transaction; +using ItemT = reindexer::Item; +#endif + +class TransactionWrapper { +public: + TransactionWrapper(DBInterface* db) : db_{db} { + assert(db_); + } + + void Init(std::string_view ns, TransactionT&& transaction) { + transaction_ = std::move(transaction); + ns_ = ns; + } + + Error Start(std::string_view ns) { + assert(!ns_.empty()); + return db_->StartTransaction(ns, *this); + } + + Error Commit() { + assert(!ns_.empty()); + return db_->CommitTransaction(transaction_); + } + + Error Rollback() { + assert(!ns_.empty()); + return db_->RollbackTransaction(transaction_); + } + + Error Modify(ItemT&& item, ItemModifyMode mode) { + assert(!ns_.empty()); + return db_->Modify(transaction_, std::move(item), mode); + } + + ItemT NewItem() { + assert(!ns_.empty()); + return db_->NewItem(ns_); + } + +private: + DBInterface* db_ = nullptr; + TransactionT transaction_; + std::string ns_; +}; + +} // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 9f4295a..f724e6d 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -7,6 +7,11 @@ static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr) { delete qresWrapperPtr; } +static void transactionWrapperDelete(uintptr_t transactionWrapperAddr) { + TransactionWrapper* transactionWrapperPtr = getTransactionWrapper(transactionWrapperAddr); + delete transactionWrapperPtr; +} + static PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { QueryResultsWrapper* qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); @@ -429,4 +434,128 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { return res; } +static PyObject* StartTransaction(PyObject* self, PyObject* args) { + uintptr_t rx = 0; + char* ns = nullptr; + if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { + return nullptr; + } + + auto db = getDB(rx); + auto transaction = new TransactionWrapper(db); + Error err = transaction->Start(ns); + if (!err.ok()) { + delete transaction; + + return Py_BuildValue("isk", err.code(), err.what().c_str(), 0); + } + + return Py_BuildValue("isk", err.code(), err.what().c_str(), reinterpret_cast(transaction)); +} + +enum class StopTransactionMode : bool { Rollback = false, Commit = true }; + +template +static PyObject* stopTransaction(PyObject* self, PyObject* args) { + uintptr_t transactionWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { + return nullptr; + } + + auto transaction = getTransactionWrapper(transactionWrapperAddr); + + Error err = (stopMode == StopTransactionMode::Commit) ? transaction->Commit() : transaction->Rollback(); + + transactionWrapperDelete(transactionWrapperAddr); + + return pyErr(err); +} + +static PyObject* CommitTransaction(PyObject* self, PyObject* args) { + return stopTransaction(self, args); +} + +static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { + return stopTransaction(self, args); +} + +static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { + uintptr_t transactionWrapperAddr = 0; + PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kO!|O!", &transactionWrapperAddr, &PyDict_Type, &itemDefDict, &PyList_Type, &preceptsList)) { + return nullptr; + } + + Py_INCREF(itemDefDict); + Py_XINCREF(preceptsList); + + auto transaction = getTransactionWrapper(transactionWrapperAddr); + + auto item = transaction->NewItem(); + Error err = item.Status(); + if (!err.ok()) { + Py_DECREF(itemDefDict); + Py_XDECREF(preceptsList); + + return pyErr(err); + } + + WrSerializer wrSer; + + try { + PyObjectToJson(&itemDefDict, wrSer); + } catch (const Error& err) { + Py_DECREF(itemDefDict); + Py_XDECREF(preceptsList); + + return pyErr(err); + } + + Py_DECREF(itemDefDict); + + char* json = const_cast(wrSer.c_str()); + err = item.Unsafe().FromJSON(json, 0, mode == ModeDelete); + if (!err.ok()) { + Py_XDECREF(preceptsList); + + return pyErr(err); + } + + if (preceptsList != nullptr && mode != ModeDelete) { + std::vector itemPrecepts; + + try { + itemPrecepts = ParseListToStrVec(&preceptsList); + } catch (const Error& err) { + Py_DECREF(preceptsList); + + return pyErr(err); + } + + item.SetPrecepts(itemPrecepts); + } + + Py_XDECREF(preceptsList); + + switch (mode) { + case ModeInsert: + case ModeUpdate: + case ModeUpsert: + case ModeDelete: + err = transaction->Modify(std::move(item), mode); + return pyErr(err); + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown item modify transaction mode"); + return nullptr; + } + + return nullptr; +} + +static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeInsert); } +static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpdate); } +static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpsert); } +static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeDelete); } + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 83ab10f..be98db8 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -14,6 +14,7 @@ #include "pyobjtools.h" #include "queryresults_wrapper.h" +#include "transaction_wrapper.h" #include "tools/serializer.h" #ifdef PYREINDEXER_CPROTO @@ -49,6 +50,12 @@ inline static QueryResultsWrapper* getQueryResultsWrapper(uintptr_t qresWrapperA static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr); +inline static TransactionWrapper* getTransactionWrapper(uintptr_t transactionWrapperAddr) { + return reinterpret_cast(transactionWrapperAddr); +} + +static void transactionWrapperDelete(uintptr_t transactionWrapperAddr); + static PyObject* Init(PyObject* self, PyObject* args); static PyObject* Destroy(PyObject* self, PyObject* args); static PyObject* Connect(PyObject* self, PyObject* args); @@ -73,6 +80,14 @@ static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); static PyObject* GetAggregationResults(PyObject* self, PyObject* args); +static PyObject* StartTransaction(PyObject* self, PyObject* args); +static PyObject* CommitTransaction(PyObject* self, PyObject* args); +static PyObject* RollbackTransaction(PyObject* self, PyObject* args); +static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args); +static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args); +static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args); +static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args); + // clang-format off static PyMethodDef module_methods[] = { {"init", Init, METH_NOARGS, "init reindexer instance"}, @@ -99,6 +114,14 @@ static PyMethodDef module_methods[] = { {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, {"get_agg_results", GetAggregationResults, METH_VARARGS, "get aggregation results"}, + {"start_transaction", StartTransaction, METH_VARARGS, "start transaction"}, + {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction"}, + {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction"}, + {"item_insert_transaction", ItemInsertTransaction, METH_VARARGS, "item insert transaction"}, + {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, + {"item_upsert_transaction", ItemUpsertTransaction, METH_VARARGS, "item upsert transaction"}, + {"item_delete_transaction", ItemDeleteTransaction, METH_VARARGS, "item delete transaction"}, + {nullptr, nullptr, 0, nullptr} }; // clang-format on diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index ac5d749..911907d 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -2,6 +2,7 @@ #include "client/cororeindexer.h" #include "core/reindexer.h" #include "queryresults_wrapper.h" +#include "transaction_wrapper.h" namespace pyreindexer { @@ -53,7 +54,7 @@ ReindexerInterface::~ReindexerInterface() { template Error ReindexerInterface::Select(const std::string& query, QueryResultsWrapper& result) { - return execute([this, &query, &result] { + return execute([this, query, &result] { auto res = select(query, result.qresPtr); result.db_ = this; result.iterInit(); @@ -72,6 +73,38 @@ Error ReindexerInterface::FetchResults(QueryResultsWrapper& result) { }); } +template +Error ReindexerInterface::StartTransaction(std::string_view ns, TransactionWrapper& transactionWrapper) { + return execute([this, ns, &transactionWrapper] { + auto transaction = startTransaction(ns); + auto error = transaction.Status(); + transactionWrapper.Init(ns, std::move(transaction)); + return error; + }); +} + +template <> +Error ReindexerInterface::commitTransaction(reindexer::Transaction& transaction) { + reindexer::QueryResults resultDummy; + return db_.CommitTransaction(transaction, resultDummy); +} +template <> +Error ReindexerInterface::commitTransaction(reindexer::client::CoroTransaction& transaction) { + return db_.CommitTransaction(transaction); +} + +template <> +Error ReindexerInterface::modify(reindexer::Transaction& transaction, + reindexer::Item&& item, ItemModifyMode mode) { + transaction.Modify(std::move(item), mode); + return errOK; +} +template <> +Error ReindexerInterface::modify(reindexer::client::CoroTransaction& transaction, + reindexer::client::Item&& item, ItemModifyMode mode) { + return transaction.Modify(std::move(item), mode); +} + template <> Error ReindexerInterface::execute(std::function f) { return f(); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 30b4d7f..226baa3 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -17,6 +17,7 @@ using reindexer::NamespaceDef; using reindexer::EnumNamespacesOpts; class QueryResultsWrapper; +class TransactionWrapper; struct ICommand { virtual Error Status() const = 0; @@ -109,6 +110,16 @@ class ReindexerInterface { return execute([this, &defs, &opts] { return enumNamespaces(defs, opts); }); } Error FetchResults(QueryResultsWrapper& result); + Error StartTransaction(std::string_view ns, TransactionWrapper& transactionWrapper); + Error CommitTransaction(typename DBT::TransactionT& tr) { + return execute([this, &tr] { return commitTransaction(tr); }); + } + Error RollbackTransaction(typename DBT::TransactionT& tr) { + return execute([this, &tr] { return rollbackTransaction(tr); }); + } + Error Modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode) { + return execute([this, &tr, &item, mode] { return modify(tr, std::move(item), mode); }); + } private: Error execute(std::function f); @@ -131,6 +142,10 @@ class ReindexerInterface { Error enumMeta(std::string_view ns, std::vector& keys) { return db_.EnumMeta({ns.data(), ns.size()}, keys); } Error select(const std::string& query, typename DBT::QueryResultsT& result) { return db_.Select(query, result); } Error enumNamespaces(std::vector& defs, EnumNamespacesOpts opts) { return db_.EnumNamespaces(defs, opts); } + typename DBT::TransactionT startTransaction(std::string_view ns) { return db_.NewTransaction({ns.data(), ns.size()}); } + Error commitTransaction(typename DBT::TransactionT& tr); + Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } + Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); Error stop(); DBT db_; diff --git a/setup.py b/setup.py index adff914..d7be32a 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ def build_cmake(self, ext): 'Documentation': 'https://reindexer.io/', 'Releases': 'https://github.com/Restream/reindexer-py/releases', 'Tracker': 'https://github.com/Restream/reindexer-py/issues', + 'Telegram chat': 'https://t.me/reindexer', }, long_description=open("README.md", encoding="utf-8").read(), long_description_content_type="text/markdown", @@ -76,6 +77,7 @@ def build_cmake(self, ext): 'lib/include/pyobjtools.h', 'lib/include/pyobjtools.cc', 'lib/include/queryresults_wrapper.h', + 'lib/include/transaction_wrapper.h', 'lib/src/rawpyreindexer.h', 'lib/src/rawpyreindexer.cc', 'lib/src/reindexerinterface.h', @@ -100,7 +102,7 @@ def build_cmake(self, ext): 'tests/helpers/__init__.py', 'tests/__init__.py' ]}, - python_requires=">=3.6,<3.13", + python_requires=">=3.6", test_suite='tests', install_requires=['envoy==0.0.3', 'delegator==0.0.3', 'pyhamcrest==2.0.2', 'pytest==6.2.5'], classifiers=[ @@ -112,7 +114,6 @@ def build_cmake(self, ext): _c2('Natural Language', 'Russian'), _c2('Operating System', 'MacOS'), _c2('Operating System', 'POSIX', 'Linux'), - _c2('Operating System', 'Microsoft', 'Windows'), _c2('Programming Language', 'Python'), _c2('Programming Language', 'Python', '3.6'), _c2('Programming Language', 'Python', '3.7'), @@ -121,10 +122,15 @@ def build_cmake(self, ext): _c2('Programming Language', 'Python', '3.10'), _c2('Programming Language', 'Python', '3.11'), _c2('Programming Language', 'Python', '3.12'), + _c2('Programming Language', 'Python', 'Implementation'), + _c2('Programming Language', 'Python', 'Implementation', 'CPython'), + _c2('Programming Language', 'Python', 'Implementation', 'PyPy'), _c2('Topic', 'Database'), _c2('Topic', 'Database', 'Database Engines/Servers'), _c2('Topic', 'Software Development'), + _c2('Topic', 'Software Development', 'Libraries'), _c2('Topic', 'Software Development', 'Libraries', 'Python Modules'), + _c2('Topic', 'Software Development', 'Libraries', 'Application Frameworks'), ], platforms=['ALT Linux', 'RED OS', 'Astra Linux'], ) From de64dacffd919ac7a6808d5752d04b148dddbfc1 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Mon, 14 Oct 2024 20:31:16 +0300 Subject: [PATCH 006/125] Rewrite query results wrapper --- .../lib/include/queryresults_wrapper.h | 40 ++++++++++++----- pyreindexer/lib/include/transaction_wrapper.h | 5 +-- pyreindexer/lib/src/rawpyreindexer.cc | 45 ++++++++++++------- pyreindexer/lib/src/rawpyreindexer.h | 26 ----------- pyreindexer/lib/src/reindexerinterface.cc | 13 +++--- 5 files changed, 65 insertions(+), 64 deletions(-) diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index 37d7214..2d1e554 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -21,25 +21,41 @@ using QueryResultsT = reindexer::QueryResults; class QueryResultsWrapper { public: - QueryResultsWrapper() : qresPtr(kResultsJson) {} - size_t Count() const { return qresPtr.Count(); } - void GetItemJSON(reindexer::WrSerializer& wrser, bool withHdrLen) { itPtr.GetJSON(wrser, withHdrLen); } - void Next() { + QueryResultsWrapper(DBInterface* db) : db_{db}, qres_{kResultsJson} { assert(db_); + } + + void Wrap(QueryResultsT&& qres) { + qres_ = std::move(qres); + it_ = qres_.begin(); + } + + Error Select(const std::string& query) { + return db_->Select(query, *this); + } + + size_t Count() const { return qres_.Count(); } + + void GetItemJSON(reindexer::WrSerializer& wrser, bool withHdrLen) { it_.GetJSON(wrser, withHdrLen); } + + void Next() { db_->FetchResults(*this); } - const std::vector& GetAggregationResults() const& { return qresPtr.GetAggregationResults(); } + void FetchResults() { + ++it_; + if (it_ == qres_.end()) { + it_ = qres_.begin(); + } + } + + const std::vector& GetAggregationResults() const& { return qres_.GetAggregationResults(); } const std::vector& GetAggregationResults() const&& = delete; private: - friend DBInterface; - - void iterInit() { itPtr = qresPtr.begin(); } - - DBInterface* db_ = nullptr; - QueryResultsT qresPtr; - QueryResultsT::Iterator itPtr; + DBInterface* db_{nullptr}; + QueryResultsT qres_; + QueryResultsT::Iterator it_; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/include/transaction_wrapper.h b/pyreindexer/lib/include/transaction_wrapper.h index e4351b3..cc19468 100644 --- a/pyreindexer/lib/include/transaction_wrapper.h +++ b/pyreindexer/lib/include/transaction_wrapper.h @@ -26,13 +26,12 @@ class TransactionWrapper { assert(db_); } - void Init(std::string_view ns, TransactionT&& transaction) { + void Wrap(std::string_view ns, TransactionT&& transaction) { transaction_ = std::move(transaction); ns_ = ns; } Error Start(std::string_view ns) { - assert(!ns_.empty()); return db_->StartTransaction(ns, *this); } @@ -57,7 +56,7 @@ class TransactionWrapper { } private: - DBInterface* db_ = nullptr; + DBInterface* db_{nullptr}; TransactionT transaction_; std::string ns_; }; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index f724e6d..3d80d92 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -2,11 +2,33 @@ namespace pyreindexer { +static uintptr_t initReindexer() { + DBInterface* db = new DBInterface(); + return reinterpret_cast(db); +} + +static DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } + +static void destroyReindexer(uintptr_t rx) { + DBInterface* db = getDB(rx); + delete db; +} + +static PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } + +static QueryResultsWrapper* getQueryResultsWrapper(uintptr_t qresWrapperAddr) { + return reinterpret_cast(qresWrapperAddr); +} + static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr) { QueryResultsWrapper* qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); delete qresWrapperPtr; } +static TransactionWrapper* getTransactionWrapper(uintptr_t transactionWrapperAddr) { + return reinterpret_cast(transactionWrapperAddr); +} + static void transactionWrapperDelete(uintptr_t transactionWrapperAddr) { TransactionWrapper* transactionWrapperPtr = getTransactionWrapper(transactionWrapperAddr); delete transactionWrapperPtr; @@ -293,8 +315,9 @@ static PyObject* Select(PyObject* self, PyObject* args) { return nullptr; } - auto qresWrapper = new QueryResultsWrapper(); - Error err = getDB(rx)->Select(query, *qresWrapper); + auto db = getDB(rx); + auto qresWrapper = new QueryResultsWrapper(db); + Error err = qresWrapper->Select(query); if (!err.ok()) { delete qresWrapper; @@ -454,9 +477,7 @@ static PyObject* StartTransaction(PyObject* self, PyObject* args) { } enum class StopTransactionMode : bool { Rollback = false, Commit = true }; - -template -static PyObject* stopTransaction(PyObject* self, PyObject* args) { +static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransactionMode stopMode) { uintptr_t transactionWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { return nullptr; @@ -464,20 +485,15 @@ static PyObject* stopTransaction(PyObject* self, PyObject* args) { auto transaction = getTransactionWrapper(transactionWrapperAddr); - Error err = (stopMode == StopTransactionMode::Commit) ? transaction->Commit() : transaction->Rollback(); + assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); + Error err = (StopTransactionMode::Commit == stopMode) ? transaction->Commit() : transaction->Rollback(); transactionWrapperDelete(transactionWrapperAddr); return pyErr(err); } - -static PyObject* CommitTransaction(PyObject* self, PyObject* args) { - return stopTransaction(self, args); -} - -static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { - return stopTransaction(self, args); -} +static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Commit); } +static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Rollback); } static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t transactionWrapperAddr = 0; @@ -552,7 +568,6 @@ static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModif return nullptr; } - static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeInsert); } static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpdate); } static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpsert); } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index be98db8..bc89aea 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -30,32 +30,6 @@ using reindexer::IndexDef; using reindexer::NamespaceDef; using reindexer::WrSerializer; -inline static uintptr_t initReindexer() { - DBInterface* db = new DBInterface(); - return reinterpret_cast(db); -} - -inline static DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } - -inline static void destroyReindexer(uintptr_t rx) { - DBInterface* db = getDB(rx); - delete db; -} - -inline static PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } - -inline static QueryResultsWrapper* getQueryResultsWrapper(uintptr_t qresWrapperAddr) { - return reinterpret_cast(qresWrapperAddr); -} - -static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr); - -inline static TransactionWrapper* getTransactionWrapper(uintptr_t transactionWrapperAddr) { - return reinterpret_cast(transactionWrapperAddr); -} - -static void transactionWrapperDelete(uintptr_t transactionWrapperAddr); - static PyObject* Init(PyObject* self, PyObject* args); static PyObject* Destroy(PyObject* self, PyObject* args); static PyObject* Connect(PyObject* self, PyObject* args); diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 911907d..6b3df8d 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -55,9 +55,9 @@ ReindexerInterface::~ReindexerInterface() { template Error ReindexerInterface::Select(const std::string& query, QueryResultsWrapper& result) { return execute([this, query, &result] { - auto res = select(query, result.qresPtr); - result.db_ = this; - result.iterInit(); + typename DBT::QueryResultsT qres; + auto res = select(query, qres); + result.Wrap(std::move(qres)); return res; }); } @@ -65,10 +65,7 @@ Error ReindexerInterface::Select(const std::string& query, QueryResultsWrap template Error ReindexerInterface::FetchResults(QueryResultsWrapper& result) { return execute([&result] { - ++(result.itPtr); - if (result.itPtr == result.qresPtr.end()) { - result.iterInit(); - } + result.FetchResults(); return errOK; }); } @@ -78,7 +75,7 @@ Error ReindexerInterface::StartTransaction(std::string_view ns, Transaction return execute([this, ns, &transactionWrapper] { auto transaction = startTransaction(ns); auto error = transaction.Status(); - transactionWrapper.Init(ns, std::move(transaction)); + transactionWrapper.Wrap(ns, std::move(transaction)); return error; }); } From 0392983204a084aca711791330f6b9b8330656ec Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 15 Oct 2024 10:19:00 +0300 Subject: [PATCH 007/125] Properly use NewItem method. Reorganize code --- .../lib/include/queryresults_wrapper.h | 14 ++++++- pyreindexer/lib/include/transaction_wrapper.h | 30 +++++++-------- pyreindexer/lib/src/rawpyreindexer.cc | 38 +++++++++---------- pyreindexer/lib/src/rawpyreindexer.h | 10 ++--- pyreindexer/lib/src/reindexerinterface.cc | 22 +++++------ pyreindexer/lib/src/reindexerinterface.h | 17 +++++++-- 6 files changed, 75 insertions(+), 56 deletions(-) diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index 2d1e554..5fd3e08 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -28,21 +28,30 @@ class QueryResultsWrapper { void Wrap(QueryResultsT&& qres) { qres_ = std::move(qres); it_ = qres_.begin(); + wrap_ = true; } Error Select(const std::string& query) { return db_->Select(query, *this); } - size_t Count() const { return qres_.Count(); } + size_t Count() const { + assert(wrap_); + return qres_.Count(); + } - void GetItemJSON(reindexer::WrSerializer& wrser, bool withHdrLen) { it_.GetJSON(wrser, withHdrLen); } + void GetItemJSON(reindexer::WrSerializer& wrser, bool withHdrLen) { + assert(wrap_); + it_.GetJSON(wrser, withHdrLen); + } void Next() { + assert(wrap_); db_->FetchResults(*this); } void FetchResults() { + assert(wrap_); ++it_; if (it_ == qres_.end()) { it_ = qres_.begin(); @@ -56,6 +65,7 @@ class QueryResultsWrapper { DBInterface* db_{nullptr}; QueryResultsT qres_; QueryResultsT::Iterator it_; + bool wrap_{false}; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/include/transaction_wrapper.h b/pyreindexer/lib/include/transaction_wrapper.h index cc19468..9c7ce71 100644 --- a/pyreindexer/lib/include/transaction_wrapper.h +++ b/pyreindexer/lib/include/transaction_wrapper.h @@ -26,39 +26,39 @@ class TransactionWrapper { assert(db_); } - void Wrap(std::string_view ns, TransactionT&& transaction) { + void Wrap(TransactionT&& transaction) { transaction_ = std::move(transaction); - ns_ = ns; + wrap_ = true; } Error Start(std::string_view ns) { return db_->StartTransaction(ns, *this); } - Error Commit() { - assert(!ns_.empty()); - return db_->CommitTransaction(transaction_); - } - - Error Rollback() { - assert(!ns_.empty()); - return db_->RollbackTransaction(transaction_); + ItemT NewItem() { + assert(wrap_); + return db_->NewItem(transaction_); } Error Modify(ItemT&& item, ItemModifyMode mode) { - assert(!ns_.empty()); + assert(wrap_); return db_->Modify(transaction_, std::move(item), mode); } - ItemT NewItem() { - assert(!ns_.empty()); - return db_->NewItem(ns_); + Error Commit() { + assert(wrap_); + return db_->CommitTransaction(transaction_); + } + + Error Rollback() { + assert(wrap_); + return db_->RollbackTransaction(transaction_); } private: DBInterface* db_{nullptr}; TransactionT transaction_; - std::string ns_; + bool wrap_{false}; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 3d80d92..058250b 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -476,25 +476,6 @@ static PyObject* StartTransaction(PyObject* self, PyObject* args) { return Py_BuildValue("isk", err.code(), err.what().c_str(), reinterpret_cast(transaction)); } -enum class StopTransactionMode : bool { Rollback = false, Commit = true }; -static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransactionMode stopMode) { - uintptr_t transactionWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { - return nullptr; - } - - auto transaction = getTransactionWrapper(transactionWrapperAddr); - - assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); - Error err = (StopTransactionMode::Commit == stopMode) ? transaction->Commit() : transaction->Rollback(); - - transactionWrapperDelete(transactionWrapperAddr); - - return pyErr(err); -} -static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Commit); } -static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Rollback); } - static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t transactionWrapperAddr = 0; PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple @@ -573,4 +554,23 @@ static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args) { return static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpsert); } static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeDelete); } +enum class StopTransactionMode : bool { Rollback = false, Commit = true }; +static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransactionMode stopMode) { + uintptr_t transactionWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { + return nullptr; + } + + auto transaction = getTransactionWrapper(transactionWrapperAddr); + + assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); + Error err = (StopTransactionMode::Commit == stopMode) ? transaction->Commit() : transaction->Rollback(); + + transactionWrapperDelete(transactionWrapperAddr); + + return pyErr(err); +} +static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Commit); } +static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Rollback); } + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index bc89aea..6f4eeff 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -55,12 +55,12 @@ static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); static PyObject* GetAggregationResults(PyObject* self, PyObject* args); static PyObject* StartTransaction(PyObject* self, PyObject* args); -static PyObject* CommitTransaction(PyObject* self, PyObject* args); -static PyObject* RollbackTransaction(PyObject* self, PyObject* args); static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args); static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args); static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args); static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args); +static PyObject* CommitTransaction(PyObject* self, PyObject* args); +static PyObject* RollbackTransaction(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { @@ -88,13 +88,13 @@ static PyMethodDef module_methods[] = { {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, {"get_agg_results", GetAggregationResults, METH_VARARGS, "get aggregation results"}, - {"start_transaction", StartTransaction, METH_VARARGS, "start transaction"}, - {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction"}, - {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction"}, + {"start_transaction", StartTransaction, METH_VARARGS, "start transaction. New transaction"}, {"item_insert_transaction", ItemInsertTransaction, METH_VARARGS, "item insert transaction"}, {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, {"item_upsert_transaction", ItemUpsertTransaction, METH_VARARGS, "item upsert transaction"}, {"item_delete_transaction", ItemDeleteTransaction, METH_VARARGS, "item delete transaction"}, + {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction. Free transaction buffer"}, + {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction. Free transaction buffer"}, {nullptr, nullptr, 0, nullptr} }; diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 6b3df8d..0c1fe6d 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -75,21 +75,11 @@ Error ReindexerInterface::StartTransaction(std::string_view ns, Transaction return execute([this, ns, &transactionWrapper] { auto transaction = startTransaction(ns); auto error = transaction.Status(); - transactionWrapper.Wrap(ns, std::move(transaction)); + transactionWrapper.Wrap(std::move(transaction)); return error; }); } -template <> -Error ReindexerInterface::commitTransaction(reindexer::Transaction& transaction) { - reindexer::QueryResults resultDummy; - return db_.CommitTransaction(transaction, resultDummy); -} -template <> -Error ReindexerInterface::commitTransaction(reindexer::client::CoroTransaction& transaction) { - return db_.CommitTransaction(transaction); -} - template <> Error ReindexerInterface::modify(reindexer::Transaction& transaction, reindexer::Item&& item, ItemModifyMode mode) { @@ -102,6 +92,16 @@ Error ReindexerInterface::modify(reindexer::cl return transaction.Modify(std::move(item), mode); } +template <> +Error ReindexerInterface::commitTransaction(reindexer::Transaction& transaction) { + reindexer::QueryResults resultDummy; + return db_.CommitTransaction(transaction, resultDummy); +} +template <> +Error ReindexerInterface::commitTransaction(reindexer::client::CoroTransaction& transaction) { + return db_.CommitTransaction(transaction); +} + template <> Error ReindexerInterface::execute(std::function f) { return f(); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 226baa3..e696544 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -111,15 +111,23 @@ class ReindexerInterface { } Error FetchResults(QueryResultsWrapper& result); Error StartTransaction(std::string_view ns, TransactionWrapper& transactionWrapper); + typename DBT::ItemT NewItem(typename DBT::TransactionT& tr) { + typename DBT::ItemT item; + execute([this, &tr, &item] { + item = newItem(tr); + return item.Status(); + }); + return item; + } + Error Modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode) { + return execute([this, &tr, &item, mode] { return modify(tr, std::move(item), mode); }); + } Error CommitTransaction(typename DBT::TransactionT& tr) { return execute([this, &tr] { return commitTransaction(tr); }); } Error RollbackTransaction(typename DBT::TransactionT& tr) { return execute([this, &tr] { return rollbackTransaction(tr); }); } - Error Modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode) { - return execute([this, &tr, &item, mode] { return modify(tr, std::move(item), mode); }); - } private: Error execute(std::function f); @@ -143,9 +151,10 @@ class ReindexerInterface { Error select(const std::string& query, typename DBT::QueryResultsT& result) { return db_.Select(query, result); } Error enumNamespaces(std::vector& defs, EnumNamespacesOpts opts) { return db_.EnumNamespaces(defs, opts); } typename DBT::TransactionT startTransaction(std::string_view ns) { return db_.NewTransaction({ns.data(), ns.size()}); } + typename DBT::ItemT newItem(typename DBT::TransactionT& tr) { return tr.NewItem(); } + Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); Error commitTransaction(typename DBT::TransactionT& tr); Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } - Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); Error stop(); DBT db_; From 1cbd051b1cbe5581ac2805bf94c2937acde1a164 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 15 Oct 2024 17:01:31 +0300 Subject: [PATCH 008/125] Implement transaction py API --- pyreindexer/example/main.py | 34 +++++- pyreindexer/lib/src/rawpyreindexer.cc | 2 +- pyreindexer/lib/src/rawpyreindexer.h | 4 +- pyreindexer/query_results.py | 4 +- pyreindexer/rx_connector.py | 20 ++++ pyreindexer/transaction.py | 155 ++++++++++++++++++++++++++ readmegen.sh | 2 +- 7 files changed, 214 insertions(+), 7 deletions(-) create mode 100644 pyreindexer/transaction.py diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 99371ac..319ebe3 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -46,7 +46,7 @@ def create_items_example(db, namespace): items_count = 10 for i in range(0, items_count): - item = {'id': i + 1, 'name': 'item_' + str(i % 2)} + item = {'id': i + 1, 'name': 'item_' + str(i % 2), 'value': 'check'} db.item_upsert(namespace, item) @@ -56,6 +56,34 @@ def select_item_query_example(db, namespace): return db.select("SELECT * FROM " + namespace + " WHERE name='" + item_name_for_lookup + "'") +def using_transaction_example(db, namespace, items_in_base): + transaction = db.new_transaction(namespace) + + items_count = len(items_in_base) + + # delete first few items + for i in range(int(items_count/2)): + transaction.delete(items_in_base[i]) + + # update last one item, overwrite field 'value' + item = items_in_base[items_count - 1] + item['value'] = 'the transaction was here' + transaction.update(item) + + # stop transaction and commit changes to namespace + transaction.commit() + + # print records from namespace + selected_items_tr = select_item_query_example(db, namespace) + + res_count = selected_items_tr.count() + print('Transaction results count: ', res_count) + + # disposable QueryResults iterator + for item in selected_items_tr: + print('Item: ', item) + + def rx_example(): db = RxConnector('builtin:///tmp/pyrx') # db = RxConnector('cproto://127.0.0.1:6534/pyrx') @@ -75,13 +103,17 @@ def rx_example(): print('Results count: ', res_count) # disposable QueryResults iterator + items_copy = [] for item in selected_items: + items_copy.append(item) print('Item: ', item) # won't be iterated again for item in selected_items: print('Item: ', item) + using_transaction_example(db, namespace, items_copy) + db.close() diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 058250b..852a75b 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -457,7 +457,7 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { return res; } -static PyObject* StartTransaction(PyObject* self, PyObject* args) { +static PyObject* NewTransaction(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 6f4eeff..a89cc9c 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -54,7 +54,7 @@ static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); static PyObject* GetAggregationResults(PyObject* self, PyObject* args); -static PyObject* StartTransaction(PyObject* self, PyObject* args); +static PyObject* NewTransaction(PyObject* self, PyObject* args); static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args); static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args); static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args); @@ -88,7 +88,7 @@ static PyMethodDef module_methods[] = { {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, {"get_agg_results", GetAggregationResults, METH_VARARGS, "get aggregation results"}, - {"start_transaction", StartTransaction, METH_VARARGS, "start transaction. New transaction"}, + {"new_transaction", NewTransaction, METH_VARARGS, "start new transaction"}, {"item_insert_transaction", ItemInsertTransaction, METH_VARARGS, "item insert transaction"}, {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, {"item_upsert_transaction", ItemUpsertTransaction, METH_VARARGS, "item upsert transaction"}, diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index be81ffc..2141941 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -6,7 +6,7 @@ class QueryResults: api (module): An API module for Reindexer calls err_code (int): the API error code err_msg (string): the API error message - qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object + qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations pos (int): The current result position in iterator """ @@ -16,7 +16,7 @@ def __init__(self, api, qres_wrapper_ptr, qres_iter_count): # Arguments: api (module): An API module for Reindexer calls - qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object + qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations """ diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index ab18024..08f9a56 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -1,5 +1,6 @@ from pyreindexer.raiser_mixin import RaiserMixin from pyreindexer.query_results import QueryResults +from pyreindexer.transaction import Transaction class RxConnector(RaiserMixin): @@ -340,6 +341,25 @@ def select(self, query): self.raise_on_error() return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + def new_transaction(self, namespace): + """Start a new transaction and return the transaction object to processing + + # Arguments: + namespace (string): A name of a namespace + + # Returns: + (:obj:`Transaction`): A new transaction. + + # Raises: + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. + Exception: Raises with an error message of API return on non-zero error code. + + """ + self.raise_on_not_init() + self.err_code, self.err_msg, transaction_wrapper_ptr = self.api.new_transaction(self.rx, namespace) + self.raise_on_error() + return Transaction(self.api, transaction_wrapper_ptr) + def _api_import(self, dsn): """Imports an API dynamically depending on protocol specified in dsn. diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py new file mode 100644 index 0000000..2dd5f11 --- /dev/null +++ b/pyreindexer/transaction.py @@ -0,0 +1,155 @@ +class Transaction: + """ An object representing the context of a Reindexer transaction. + + # Attributes: + api (module): An API module for Reindexer calls + transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object + err_code (int): the API error code + err_msg (string): the API error message + + """ + + def __init__(self, api, transaction_wrapper_ptr): + """Constructs a new Reindexer transaction object. + + # Arguments: + api (module): An API module for Reindexer calls + transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object + + """ + + self.api = api + self.transaction_wrapper_ptr = transaction_wrapper_ptr + self.err_code = 0 + self.err_msg = "" + + def __del__(self): + """Roll back a transaction if it was not previously stopped + + """ + + if self.transaction_wrapper_ptr > 0: + _, _ = self.api.rollback_transaction(self.transaction_wrapper_ptr) + + def _raise_on_error(self): + """Checks if there is an error code and raises with an error message. + + # Raises: + Exception: Raises with an error message of API return on non-zero error code. + + """ + + if self.err_code: + raise Exception(self.err_msg) + + def _raise_on_is_over(self): + """Checks if there is an error code and raises with an error message. + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + + """ + + if self.transaction_wrapper_ptr <= 0: + raise Exception("Transaction is over") + + def insert(self, item_def, precepts=None): + """Inserts an item with its precepts to the transaction. + + # Arguments: + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return on non-zero error code. + + """ + + self._raise_on_is_over() + if precepts is None: + precepts = [] + self.err_code, self.err_msg = self.api.item_insert_transaction(self.transaction_wrapper_ptr, item_def, precepts) + self._raise_on_error() + + def update(self, item_def, precepts=None): + """Update an item with its precepts to the transaction. + + # Arguments: + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return on non-zero error code. + + """ + + self._raise_on_is_over() + if precepts is None: + precepts = [] + self.err_code, self.err_msg = self.api.item_update_transaction(self.transaction_wrapper_ptr, item_def, precepts) + self._raise_on_error() + + def upsert(self, item_def, precepts=None): + """Update an item with its precepts to the transaction. Creates the item if it not exists. + + # Arguments: + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return on non-zero error code. + + """ + + self._raise_on_is_over() + if precepts is None: + precepts = [] + self.err_code, self.err_msg = self.api.item_upsert_transaction(self.transaction_wrapper_ptr, item_def, precepts) + self._raise_on_error() + + def delete(self, item_def): + """Delete an item from the transaction. + + # Arguments: + item_def (dict): A dictionary of item definition + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return on non-zero error code. + + """ + + self._raise_on_is_over() + self.err_code, self.err_msg = self.api.item_delete_transaction(self.transaction_wrapper_ptr, item_def) + self._raise_on_error() + + def commit(self): + """Commit a transaction + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return on non-zero error code. + + """ + + self._raise_on_is_over() + self.err_code, self.err_msg = self.api.commit_transaction(self.transaction_wrapper_ptr) + self.transaction_wrapper_ptr = 0 + self._raise_on_error() + + def rollback(self): + """Roll back a transaction + + # Raises: + Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return on non-zero error code. + + """ + + self._raise_on_is_over() + self.err_code, self.err_msg = self.api.rollback_transaction(self.transaction_wrapper_ptr) + self.transaction_wrapper_ptr = 0 + self._raise_on_error() diff --git a/readmegen.sh b/readmegen.sh index abda2be..46cb058 100755 --- a/readmegen.sh +++ b/readmegen.sh @@ -1,3 +1,3 @@ #!/bin/sh -pydocmd simple pyreindexer++ pyreindexer.rx_connector++ pyreindexer.query_results++ pyreindexer.index_definition++ > README.md +pydocmd simple pyreindexer++ pyreindexer.rx_connector++ pyreindexer.query_results++ pyreindexer.index_definition++ pyreindexer.transaction++ > README.md From 1a77b8e04e3897f7fd79971aff1ce9fda88dd4ae Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 15 Oct 2024 21:27:41 +0300 Subject: [PATCH 009/125] Add transaction py-tests --- pyreindexer/tests/helpers/items.py | 2 +- pyreindexer/tests/helpers/transaction.py | 87 ++++++++++ pyreindexer/tests/tests/test_items.py | 22 +-- pyreindexer/tests/tests/test_transaction.py | 170 ++++++++++++++++++++ setup.py | 2 + 5 files changed, 267 insertions(+), 16 deletions(-) create mode 100644 pyreindexer/tests/helpers/transaction.py create mode 100644 pyreindexer/tests/tests/test_transaction.py diff --git a/pyreindexer/tests/helpers/items.py b/pyreindexer/tests/helpers/items.py index c83dd26..b3bdaf3 100644 --- a/pyreindexer/tests/helpers/items.py +++ b/pyreindexer/tests/helpers/items.py @@ -21,7 +21,7 @@ def upsert_item(namespace, item_def): def update_item(namespace, item_def): """ - Update item to namespace + Update item in namespace """ db, namespace_name = namespace log_operation.info(f"Update item: {item_def} to namespace {namespace_name}") diff --git a/pyreindexer/tests/helpers/transaction.py b/pyreindexer/tests/helpers/transaction.py new file mode 100644 index 0000000..b7c39d7 --- /dev/null +++ b/pyreindexer/tests/helpers/transaction.py @@ -0,0 +1,87 @@ +from tests.helpers.log_helper import log_operation + + +def insert_item_transaction(namespace, item_definition): + """ + Insert an item into namespace using transaction + """ + db, namespace_name = namespace + log_operation.info(f"Insert item: {item_definition} to namespace {namespace_name} by transaction") + transaction = db.new_transaction(namespace_name) + transaction.insert(item_definition) + transaction.commit() + + +def upsert_item_transaction(namespace, item_definition): + """ + Insert or update an item into namespace using transaction + """ + db, namespace_name = namespace + log_operation.info(f"Upsert item: {item_definition} to namespace {namespace_name} by transaction") + transaction = db.new_transaction(namespace_name) + transaction.upsert(item_definition) + transaction.commit() + + +def update_item_transaction(namespace, item_definition): + """ + Update an item in namespace using transaction + """ + db, namespace_name = namespace + log_operation.info(f"Update item: {item_definition} in namespace {namespace_name} by transaction") + transaction = db.new_transaction(namespace_name) + transaction.update(item_definition) + transaction.commit() + + +def delete_item_transaction(namespace, item_definition): + """ + Delete item from namespace using transaction + """ + db, namespace_name = namespace + log_operation.info(f"Delete item: {item_definition} from namespace {namespace_name} by transaction") + transaction = db.new_transaction(namespace_name) + transaction.delete(item_definition) + transaction.commit() + + +def commit_transaction(transaction): + """ + Wrap a method call as a function (Commit) + """ + transaction.commit() + + +def rollback_transaction(transaction): + """ + Wrap a method call as a function (Rollback) + """ + transaction.rollback() + + +def insert_transaction(transaction, item_def): + """ + Wrap a method call as a function (Insert) + """ + transaction.insert(item_def) + + +def update_transaction(transaction, item_def): + """ + Wrap a method call as a function (Update) + """ + transaction.update(item_def) + + +def upsert_transaction(transaction, item_def): + """ + Wrap a method call as a function (Upsert) + """ + transaction.upsert(item_def) + + +def delete_transaction(transaction, item_def): + """ + Wrap a method call as a function (Delete) + """ + transaction.delete(item_def) diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index d732051..6331df8 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -21,11 +21,8 @@ def test_create_item_insert(self, namespace, index): insert_item(namespace, item_definition) # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(1), - "Item wasn't created") - assert_that(select_result, has_item(item_definition), - "Item wasn't created" - ) + assert_that(select_result, has_length(1), "Item wasn't created") + assert_that(select_result, has_item(item_definition), "Item wasn't created") delete_item(namespace, item_definition) def test_create_item_insert_with_precepts(self, namespace, index): @@ -37,8 +34,7 @@ def test_create_item_insert_with_precepts(self, namespace, index): db.item_insert(namespace_name, {"id": 100, "field": "value"}, ["id=serial()"]) # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(number_items), - "Items wasn't created") + assert_that(select_result, has_length(number_items), "Items wasn't created") for i in range(number_items): assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), "Items wasn't created") @@ -52,10 +48,8 @@ def test_create_item_upsert(self, namespace, index): upsert_item(namespace, item_definition) # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(1), - "Item wasn't created") - assert_that(select_result, has_item(item_definition), - "Item wasn't created") + assert_that(select_result, has_length(1), "Item wasn't created") + assert_that(select_result, has_item(item_definition), "Item wasn't created") delete_item(namespace, item_definition) def test_update_item_upsert(self, namespace, index, item): @@ -66,10 +60,8 @@ def test_update_item_upsert(self, namespace, index, item): upsert_item(namespace, item_definition_updated) # Then ("Check that item is updated") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(1), - "Item wasn't updated") - assert_that(select_result, has_item(item_definition_updated), - "Item wasn't updated") + assert_that(select_result, has_length(1), "Item wasn't updated") + assert_that(select_result, has_item(item_definition_updated), "Item wasn't updated") def test_update_item_update(self, namespace, index, item): # Given("Create namespace with item") diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py new file mode 100644 index 0000000..b9b28db --- /dev/null +++ b/pyreindexer/tests/tests/test_transaction.py @@ -0,0 +1,170 @@ +from hamcrest import * + +from tests.helpers.items import * +from tests.helpers.transaction import * +from tests.test_data.constants import item_definition + + +class TestCrudTransaction: + def test_initial_namespace_has_no_items(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Get namespace information") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + # Then ("Check that list of items in namespace is empty") + assert_that(select_result, has_length(0), "Transaction: item list is not empty") + assert_that(select_result, equal_to([]), "Transaction: item list is not empty") + + def test_commit_after_rollback(self, namespace): + # Given("Create namespace") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Rollback transaction") + transaction.rollback() + # Then ("Commit transaction") + assert_that(calling(commit_transaction).with_args(transaction), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_rollback_after_commit(self, namespace): + # Given("Create namespace") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Commit transaction") + transaction.commit() + # Then ("Rollback transaction") + assert_that(calling(rollback_transaction).with_args(transaction), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_insert_after_rollback(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Rollback transaction") + transaction.rollback() + # Then ("Insert transaction") + assert_that(calling(insert_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_update_after_rollback(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Rollback transaction") + transaction.rollback() + # Then ("Update transaction") + assert_that(calling(update_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_upsert_after_rollback(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Rollback transaction") + transaction.rollback() + # Then ("Upsert transaction") + assert_that(calling(upsert_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_delete_after_rollback(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Rollback transaction") + transaction.rollback() + # Then ("Delete transaction") + assert_that(calling(delete_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_create_item_insert(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Insert item into namespace") + insert_item_transaction(namespace, item_definition) + # Then ("Check that item is added") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + assert_that(select_result, has_length(1), "Transaction: item wasn't created") + assert_that(select_result, has_item(item_definition), "Transaction: item wasn't created") + delete_item(namespace, item_definition) + + def test_create_item_insert_with_precepts(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Insert items into namespace") + transaction = db.new_transaction(namespace_name) + number_items = 5 + for _ in range(number_items): + transaction.insert({"id": 100, "field": "value"}, ["id=serial()"]) + transaction.commit() + # Then ("Check that item is added") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + assert_that(select_result, has_length(number_items), "Transaction: items wasn't created") + for i in range(number_items): + assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), + "Transaction: items wasn't created") + for i in range(number_items): + db.item_delete(namespace_name, {'id': i}) + + def test_create_item_upsert(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Upsert item into namespace") + upsert_item_transaction(namespace, item_definition) + # Then ("Check that item is added") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + assert_that(select_result, has_length(1), "Transaction: item wasn't created") + assert_that(select_result, has_item(item_definition), "Transaction: item wasn't created") + delete_item(namespace, item_definition) + + def test_update_item_upsert(self, namespace, index, item): + # Given("Create namespace with item") + db, namespace_name = namespace + # When ("Upsert item") + item_definition_updated = {'id': 100, 'val': "new_value"} + upsert_item_transaction(namespace, item_definition_updated) + # Then ("Check that item is updated") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + assert_that(select_result, has_length(1), "Transaction: item wasn't updated") + assert_that(select_result, has_item(item_definition_updated), "Transaction: item wasn't updated") + + def test_update_item_update(self, namespace, index, item): + # Given("Create namespace with item") + db, namespace_name = namespace + # When ("Update item") + item_definition_updated = {'id': 100, 'val': "new_value"} + update_item_transaction(namespace, item_definition_updated) + # Then ("Check that item is updated") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + assert_that(select_result, has_length(1), "Transaction: item wasn't updated") + assert_that(select_result, has_item(item_definition_updated), "Transaction: item wasn't updated") + + def test_delete_item(self, namespace, index, item): + # Given("Create namespace with item") + db, namespace_name = namespace + # When ("Delete item") + delete_item_transaction(namespace, item) + # Then ("Check that item is deleted") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + assert_that(select_result, has_length(0), "Transaction: item wasn't deleted") + assert_that(select_result, equal_to([]), "Transaction: item wasn't deleted") + + def test_rollback_transaction(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Insert items into namespace") + transaction = db.new_transaction(namespace_name) + number_items = 5 + for _ in range(number_items): + transaction.insert({"id": 100, "field": "value"}, ["id=serial()"]) + # Then ("Rollback transaction") + transaction.rollback() + # When ("Get namespace information") + select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + # Then ("Check that list of items in namespace is empty") + assert_that(select_result, has_length(0), "Transaction: item list is not empty") + assert_that(select_result, equal_to([]), "Transaction: item list is not empty") diff --git a/setup.py b/setup.py index d7be32a..6800711 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ def build_cmake(self, ext): 'tests/tests/test_database.py', 'tests/tests/test_namespace.py', 'tests/tests/test_metadata.py', + 'tests/tests/test_transaction.py', 'tests/tests/__init__.py', 'tests/helpers/namespace.py', 'tests/helpers/sql.py', @@ -99,6 +100,7 @@ def build_cmake(self, ext): 'tests/helpers/index.py', 'tests/helpers/metadata.py', 'tests/helpers/log_helper.py', + 'tests/helpers/transaction.py', 'tests/helpers/__init__.py', 'tests/__init__.py' ]}, From 3ac34853d23fa453d84745ee28d6ff696837473a Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 16 Oct 2024 17:17:41 +0300 Subject: [PATCH 010/125] Update Readme.md --- README.md | 234 ++++++++++++++++++++++++++------ pyreindexer/index_definition.py | 2 +- pyreindexer/query_results.py | 14 +- pyreindexer/raiser_mixin.py | 10 +- pyreindexer/rx_connector.py | 110 +++++++-------- pyreindexer/transaction.py | 44 +++--- 6 files changed, 286 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index de8383e..0005b0c 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

namespace_close

@@ -66,8 +66,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

namespace_drop

@@ -83,8 +83,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

namespaces_enum

@@ -97,16 +97,16 @@ Gets a list of namespaces available __Arguments:__ enum_not_opened (bool, optional): An enumeration mode flag. If it is - set then closed namespaces are in result list too. Defaults to False. + set then closed namespaces are in result list too. Defaults to False __Returns:__ - (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace. + (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

index_add

@@ -123,8 +123,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

index_update

@@ -141,8 +141,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

index_drop

@@ -159,8 +159,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

item_insert

@@ -178,8 +178,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

item_update

@@ -197,8 +197,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

item_upsert

@@ -206,7 +206,7 @@ __Raises:__ ```python RxConnector.item_upsert(self, namespace, item_def, precepts=[]) ``` -Updates an item with its precepts in the namespace specified. Creates the item if it not exist. +Updates an item with its precepts in the namespace specified. Creates the item if it not exists __Arguments:__ @@ -216,8 +216,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

item_delete

@@ -225,7 +225,7 @@ __Raises:__ ```python RxConnector.item_delete(self, namespace, item_def) ``` -Deletes an item from the namespace specified. +Deletes an item from the namespace specified __Arguments:__ @@ -234,8 +234,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

meta_put

@@ -253,8 +253,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

meta_get

@@ -275,8 +275,8 @@ __Returns:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

meta_delete

@@ -293,8 +293,8 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

meta_enum

@@ -310,12 +310,12 @@ __Arguments:__ __Returns:__ - (:obj:`list` of :obj:`str`): A list of all metadata keys. + (:obj:`list` of :obj:`str`): A list of all metadata keys __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

select

@@ -331,12 +331,33 @@ __Arguments:__ __Returns:__ - (:obj:`QueryResults`): A QueryResults iterator. + (:obj:`QueryResults`): A QueryResults iterator __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code + + +

new_transaction

+ +```python +RxConnector.new_transaction(self, namespace) +``` +Start a new transaction and return the transaction object to processing + +__Arguments:__ + + namespace (string): A name of a namespace + +__Returns:__ + + (:obj:`Transaction`): A new transaction + +__Raises:__ + + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code

pyreindexer.query_results

@@ -347,7 +368,7 @@ __Raises:__ ```python QueryResults(self, api, qres_wrapper_ptr, qres_iter_count) ``` -QueryResults is a disposable iterator of Reindexer results for such queries as SELECT and etc. +QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer __Attributes:__ @@ -359,6 +380,7 @@ __Attributes:__ qres_iter_count (int): A count of results for iterations pos (int): The current result position in iterator +

count

```python @@ -368,7 +390,23 @@ Returns a count of results __Returns__ -`int`: A count of results + `int`: A count of results + + +

get_agg_results

+ +```python +QueryResults.get_agg_results(self) +``` +Returns aggregation results for the current query + +__Returns__ + + (:obj:`dict`): Dictionary with all results for the current query + +__Raises__ + + Exception: Raises with an error message of API return on non-zero error code

pyreindexer.index_definition

@@ -377,7 +415,7 @@ __Returns__

IndexDefinition

```python -IndexDefinition(self, /, *args, **kwargs) +IndexDefinition(dict) ``` IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. @@ -398,3 +436,117 @@ __Arguments:__ sort_order_letters (str): Order for a sort sequence for a custom collate mode. config (dict): A config for a fulltext engine. [More](https://github.com/Restream/reindexer/blob/master/fulltext.md) . + +

pyreindexer.transaction

+ + +

Transaction

+ +```python +Transaction(self, api, transaction_wrapper_ptr) +``` +An object representing the context of a Reindexer transaction + +__Attributes:__ + + api (module): An API module for Reindexer calls + transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object + err_code (int): the API error code + err_msg (string): the API error message + + +

commit

+ +```python +Transaction.commit(self) +``` +Commit a transaction + +__Raises:__ + + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + +

delete

+ +```python +Transaction.delete(self, item_def) +``` +Delete an item from the transaction. + +__Arguments:__ + + item_def (dict): A dictionary of item definition + +__Raises:__ + + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + +

insert

+ +```python +Transaction.insert(self, item_def, precepts=None) +``` +Inserts an item with its precepts to the transaction + +__Arguments:__ + + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition + +__Raises:__ + + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + +

rollback

+ +```python +Transaction.rollback(self) +``` +Roll back a transaction + +__Raises:__ + + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + +

update

+ +```python +Transaction.update(self, item_def, precepts=None) +``` +Update an item with its precepts to the transaction + +__Arguments:__ + + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition + +__Raises:__ + + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + +

upsert

+ +```python +Transaction.upsert(self, item_def, precepts=None) +``` +Update an item with its precepts to the transaction. Creates the item if it not exists + +__Arguments:__ + + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition + +__Raises:__ + + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code diff --git a/pyreindexer/index_definition.py b/pyreindexer/index_definition.py index 30e4fad..2898d9c 100644 --- a/pyreindexer/index_definition.py +++ b/pyreindexer/index_definition.py @@ -25,7 +25,7 @@ class IndexDefinition(dict): `none`, `ascii`, `utf8`, `numeric`, `custom`. sort_order_letters (str): Order for a sort sequence for a custom collate mode. config (dict): A config for a fulltext engine. - [More](https://github.com/Restream/reindexer/blob/master/fulltext.md) . + [More](https://github.com/Restream/reindexer/blob/master/fulltext.md). """ def __getitem__(self, attr): diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 2141941..6bd0449 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -12,7 +12,7 @@ class QueryResults: """ def __init__(self, api, qres_wrapper_ptr, qres_iter_count): - """Constructs a new Reindexer query results iterator object. + """Constructs a new Reindexer query results iterator object # Arguments: api (module): An API module for Reindexer calls @@ -29,17 +29,17 @@ def __init__(self, api, qres_wrapper_ptr, qres_iter_count): self.err_msg = "" def __iter__(self): - """Returns the current iteration result. + """Returns the current iteration result """ return self def __next__(self): - """Returns the next iteration result. + """Returns the next iteration result # Raises: - StopIteration: Frees results on end of iterator and raises with iteration stop. + StopIteration: Frees results on end of iterator and raises with iteration stop """ @@ -82,6 +82,12 @@ def _close_iterator(self): def get_agg_results(self): """Returns aggregation results for the current query + # Returns + (:obj:`dict`): Dictionary with all results for the current query + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ self.err_code, self.err_msg, res = self.api.get_agg_results(self.qres_wrapper_ptr) diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index e7c62d8..010c4e0 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -1,5 +1,5 @@ class RaiserMixin(object): - """ RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity. + """ RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity """ err_code: int @@ -7,10 +7,10 @@ class RaiserMixin(object): rx: int def raise_on_error(self): - """Checks if there is an error code and raises with an error message. + """Checks if there is an error code and raises with an error message # Raises: - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return on non-zero error code """ @@ -18,10 +18,10 @@ def raise_on_error(self): raise Exception(self.err_msg) def raise_on_not_init(self): - """Checks if there is an error code and raises with an error message. + """Checks if there is an error code and raises with an error message # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet """ diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 08f9a56..9e690b7 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -19,10 +19,10 @@ class RxConnector(RaiserMixin): def __init__(self, dsn): """Constructs a new connector object. - Initializes an error code and a Reindexer instance descriptor to zero. + Initializes an error code and a Reindexer instance descriptor to zero # Arguments: - dsn (string): The connection string which contains a protocol. + dsn (string): The connection string which contains a protocol Examples: 'builtin:///tmp/pyrx', 'cproto://127.0.0.1:6534/pyrx """ @@ -55,8 +55,8 @@ def namespace_open(self, namespace): namespace (string): A name of a namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -71,8 +71,8 @@ def namespace_close(self, namespace): namespace (string): A name of a namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -87,8 +87,8 @@ def namespace_drop(self, namespace): namespace (string): A name of a namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -101,14 +101,14 @@ def namespaces_enum(self, enum_not_opened=False): # Arguments: enum_not_opened (bool, optional): An enumeration mode flag. If it is - set then closed namespaces are in result list too. Defaults to False. + set then closed namespaces are in result list too. Defaults to False # Returns: - (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace. + (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -125,8 +125,8 @@ def index_add(self, namespace, index_def): index_def (dict): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -142,8 +142,8 @@ def index_update(self, namespace, index_def): index_def (dict): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -159,8 +159,8 @@ def index_drop(self, namespace, index_name): index_name (string): A name of an index # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -169,7 +169,7 @@ def index_drop(self, namespace, index_name): self.raise_on_error() def item_insert(self, namespace, item_def, precepts=None): - """Inserts an item with its precepts to the namespace specified. + """Inserts an item with its precepts to the namespace specified # Arguments: namespace (string): A name of a namespace @@ -177,8 +177,8 @@ def item_insert(self, namespace, item_def, precepts=None): precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -189,7 +189,7 @@ def item_insert(self, namespace, item_def, precepts=None): self.raise_on_error() def item_update(self, namespace, item_def, precepts=None): - """Updates an item with its precepts in the namespace specified. + """Updates an item with its precepts in the namespace specified # Arguments: namespace (string): A name of a namespace @@ -197,8 +197,8 @@ def item_update(self, namespace, item_def, precepts=None): precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -209,7 +209,7 @@ def item_update(self, namespace, item_def, precepts=None): self.raise_on_error() def item_upsert(self, namespace, item_def, precepts=None): - """Updates an item with its precepts in the namespace specified. Creates the item if it not exists. + """Updates an item with its precepts in the namespace specified. Creates the item if it not exists # Arguments: namespace (string): A name of a namespace @@ -217,8 +217,8 @@ def item_upsert(self, namespace, item_def, precepts=None): precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -229,15 +229,15 @@ def item_upsert(self, namespace, item_def, precepts=None): self.raise_on_error() def item_delete(self, namespace, item_def): - """Deletes an item from the namespace specified. + """Deletes an item from the namespace specified # Arguments: namespace (string): A name of a namespace item_def (dict): A dictionary of item definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -254,8 +254,8 @@ def meta_put(self, namespace, key, value): value (string): A metadata for storage # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -274,8 +274,8 @@ def meta_get(self, namespace, key): string: A metadata value # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -292,8 +292,8 @@ def meta_delete(self, namespace, key): key (string): A key in a storage of Reindexer where metadata is kept # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -308,11 +308,11 @@ def meta_enum(self, namespace): namespace (string): A name of a namespace # Returns: - (:obj:`list` of :obj:`str`): A list of all metadata keys. + (:obj:`list` of :obj:`str`): A list of all metadata keys # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -328,11 +328,11 @@ def select(self, query): query (string): An SQL query # Returns: - (:obj:`QueryResults`): A QueryResults iterator. + (:obj:`QueryResults`): A QueryResults iterator # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -348,11 +348,11 @@ def new_transaction(self, namespace): namespace (string): A name of a namespace # Returns: - (:obj:`Transaction`): A new transaction. + (:obj:`Transaction`): A new transaction # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ self.raise_on_not_init() @@ -361,13 +361,13 @@ def new_transaction(self, namespace): return Transaction(self.api, transaction_wrapper_ptr) def _api_import(self, dsn): - """Imports an API dynamically depending on protocol specified in dsn. + """Imports an API dynamically depending on protocol specified in dsn # Arguments: - dsn (string): The connection string which contains a protocol. + dsn (string): The connection string which contains a protocol # Raises: - Exception: Raises an exception if a connection protocol is unrecognized. + Exception: Raises an exception if a connection protocol is unrecognized """ @@ -380,15 +380,15 @@ def _api_import(self, dsn): "Unknown Reindexer connection protocol for dsn: ", dsn) def _api_init(self, dsn): - """Initializes Reindexer instance and connects to a database specified in dsn. - Obtains a pointer to Reindexer instance. + """Initializes Reindexer instance and connects to a database specified in dsn + Obtains a pointer to Reindexer instance # Arguments: - dsn (string): The connection string which contains a protocol. + dsn (string): The connection string which contains a protocol # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code """ @@ -398,10 +398,10 @@ def _api_init(self, dsn): self.raise_on_error() def _api_close(self): - """Destructs Reindexer instance correctly and resets memory pointer. + """Destructs Reindexer instance correctly and resets memory pointer # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet. + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet """ diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 2dd5f11..8336e35 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,5 +1,5 @@ class Transaction: - """ An object representing the context of a Reindexer transaction. + """ An object representing the context of a Reindexer transaction # Attributes: api (module): An API module for Reindexer calls @@ -10,7 +10,7 @@ class Transaction: """ def __init__(self, api, transaction_wrapper_ptr): - """Constructs a new Reindexer transaction object. + """Constructs a new Reindexer transaction object # Arguments: api (module): An API module for Reindexer calls @@ -32,10 +32,10 @@ def __del__(self): _, _ = self.api.rollback_transaction(self.transaction_wrapper_ptr) def _raise_on_error(self): - """Checks if there is an error code and raises with an error message. + """Checks if there is an error code and raises with an error message # Raises: - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return on non-zero error code """ @@ -43,10 +43,10 @@ def _raise_on_error(self): raise Exception(self.err_msg) def _raise_on_is_over(self): - """Checks if there is an error code and raises with an error message. + """Checks if there is an error code and raises with an error message # Raises: - Exception: Raises with an error message of API return if Transaction is over. + Exception: Raises with an error message of API return if Transaction is over """ @@ -54,15 +54,15 @@ def _raise_on_is_over(self): raise Exception("Transaction is over") def insert(self, item_def, precepts=None): - """Inserts an item with its precepts to the transaction. + """Inserts an item with its precepts to the transaction # Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Transaction is over. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code """ @@ -73,15 +73,15 @@ def insert(self, item_def, precepts=None): self._raise_on_error() def update(self, item_def, precepts=None): - """Update an item with its precepts to the transaction. + """Update an item with its precepts to the transaction # Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Transaction is over. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code """ @@ -92,15 +92,15 @@ def update(self, item_def, precepts=None): self._raise_on_error() def upsert(self, item_def, precepts=None): - """Update an item with its precepts to the transaction. Creates the item if it not exists. + """Update an item with its precepts to the transaction. Creates the item if it not exists # Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Transaction is over. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code """ @@ -111,14 +111,14 @@ def upsert(self, item_def, precepts=None): self._raise_on_error() def delete(self, item_def): - """Delete an item from the transaction. + """Delete an item from the transaction # Arguments: item_def (dict): A dictionary of item definition # Raises: - Exception: Raises with an error message of API return if Transaction is over. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code """ @@ -130,8 +130,8 @@ def commit(self): """Commit a transaction # Raises: - Exception: Raises with an error message of API return if Transaction is over. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code """ @@ -144,8 +144,8 @@ def rollback(self): """Roll back a transaction # Raises: - Exception: Raises with an error message of API return if Transaction is over. - Exception: Raises with an error message of API return on non-zero error code. + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code """ From 0d44728b75a938a0760e145aa986879bc8ff3331 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 16 Oct 2024 20:21:57 +0300 Subject: [PATCH 011/125] [fix] Fix build pyreindexer with reindexer v.4.15 --- pyreindexer/lib/include/queryresults_wrapper.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index 5fd3e08..c907ed5 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -58,8 +58,8 @@ class QueryResultsWrapper { } } - const std::vector& GetAggregationResults() const& { return qres_.GetAggregationResults(); } - const std::vector& GetAggregationResults() const&& = delete; + const std::vector& GetAggregationResults() & { return qres_.GetAggregationResults(); } + const std::vector& GetAggregationResults() && = delete; private: DBInterface* db_{nullptr}; From 2f015962a8f55abd27d25f34d460e9f0f40a568d Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 17 Oct 2024 14:45:04 +0300 Subject: [PATCH 012/125] Try fix MacOS tests --- pyreindexer/lib/src/rawpyreindexer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 9f4295a..525cea7 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -320,7 +320,8 @@ static PyObject* EnumMeta(PyObject* self, PyObject* args) { Py_ssize_t pos = 0; for (const auto& key : keys) { - PyList_SetItem(list, pos, Py_BuildValue("s", key.c_str())); // stolen ref + PyObject* pyKey = PyUnicode_FromStringAndSize(key.data(), key.size()); // new ref + PyList_SetItem(list, pos, pyKey); // stolen ref ++pos; } From da03def957f01279614b4c12ab74c438c6f84062 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 17 Oct 2024 15:02:37 +0300 Subject: [PATCH 013/125] Correct style --- pyreindexer/lib/src/rawpyreindexer.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 227ef42..7382be1 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -50,7 +50,7 @@ static PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { return Py_BuildValue("is{}", err.code(), err.what().c_str()); } - PyObject* res = Py_BuildValue("isO", errOK, "", dictFromJson); // new ref + PyObject* res = Py_BuildValue("isO", errOK, "", dictFromJson); // new ref Py_DECREF(dictFromJson); return res; @@ -125,7 +125,7 @@ static PyObject* NamespaceDrop(PyObject* self, PyObject* args) { static PyObject* IndexAdd(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; - PyObject* indexDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* indexDefDict = nullptr; // borrowed ref after ParseTuple if (!PyArg_ParseTuple(args, "ksO!", &rx, &ns, &PyDict_Type, &indexDefDict)) { return nullptr; } @@ -155,7 +155,7 @@ static PyObject* IndexAdd(PyObject* self, PyObject* args) { static PyObject* IndexUpdate(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; - PyObject* indexDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* indexDefDict = nullptr; // borrowed ref after ParseTuple if (!PyArg_ParseTuple(args, "ksO!", &rx, &ns, &PyDict_Type, &indexDefDict)) { return nullptr; } @@ -196,8 +196,8 @@ static PyObject* IndexDrop(PyObject* self, PyObject* args) { static PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t rx = 0; char* ns = nullptr; - PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple - PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "ksO!|O!", &rx, &ns, &PyDict_Type, &itemDefDict, &PyList_Type, &preceptsList)) { return nullptr; } @@ -348,7 +348,7 @@ static PyObject* EnumMeta(PyObject* self, PyObject* args) { Py_ssize_t pos = 0; for (const auto& key : keys) { - PyObject* pyKey = PyUnicode_FromStringAndSize(key.data(), key.size()); // new ref + PyObject* pyKey = PyUnicode_FromStringAndSize(key.data(), key.size()); // new ref PyList_SetItem(list, pos, pyKey); // stolen ref ++pos; } @@ -445,14 +445,14 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { PyObject* dictFromJson = nullptr; try { - dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref + dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref } catch (const Error& err) { Py_XDECREF(dictFromJson); return Py_BuildValue("is{}", err.code(), err.what().c_str()); } - PyObject* res = Py_BuildValue("isO", errOK, "", dictFromJson); // new ref + PyObject* res = Py_BuildValue("isO", errOK, "", dictFromJson); // new ref Py_DECREF(dictFromJson); return res; @@ -479,8 +479,8 @@ static PyObject* NewTransaction(PyObject* self, PyObject* args) { static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t transactionWrapperAddr = 0; - PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple - PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "kO!|O!", &transactionWrapperAddr, &PyDict_Type, &itemDefDict, &PyList_Type, &preceptsList)) { return nullptr; } From e016c336c097363e8fb9b7dbcc6f71a3a56042f5 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 17 Oct 2024 18:52:27 +0300 Subject: [PATCH 014/125] Update tests --- pyreindexer/tests/tests/test_transaction.py | 65 ++++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index b9b28db..649b06e 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -6,16 +6,7 @@ class TestCrudTransaction: - def test_initial_namespace_has_no_items(self, namespace, index): - # Given("Create namespace with index") - db, namespace_name = namespace - # When ("Get namespace information") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - # Then ("Check that list of items in namespace is empty") - assert_that(select_result, has_length(0), "Transaction: item list is not empty") - assert_that(select_result, equal_to([]), "Transaction: item list is not empty") - - def test_commit_after_rollback(self, namespace): + def test_negative_commit_after_rollback(self, namespace): # Given("Create namespace") db, namespace_name = namespace # When ("Start new transaction") @@ -26,7 +17,7 @@ def test_commit_after_rollback(self, namespace): assert_that(calling(commit_transaction).with_args(transaction), raises(Exception, matching=has_string("Transaction is over"))) - def test_rollback_after_commit(self, namespace): + def test_negative_rollback_after_commit(self, namespace): # Given("Create namespace") db, namespace_name = namespace # When ("Start new transaction") @@ -37,7 +28,7 @@ def test_rollback_after_commit(self, namespace): assert_that(calling(rollback_transaction).with_args(transaction), raises(Exception, matching=has_string("Transaction is over"))) - def test_insert_after_rollback(self, namespace, index): + def test_negative_insert_after_rollback(self, namespace, index): # Given("Create namespace with index") db, namespace_name = namespace # When ("Start new transaction") @@ -48,7 +39,7 @@ def test_insert_after_rollback(self, namespace, index): assert_that(calling(insert_transaction).with_args(transaction, item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_update_after_rollback(self, namespace, index): + def test_negative_update_after_rollback(self, namespace, index): # Given("Create namespace with index") db, namespace_name = namespace # When ("Start new transaction") @@ -59,7 +50,7 @@ def test_update_after_rollback(self, namespace, index): assert_that(calling(update_transaction).with_args(transaction, item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_upsert_after_rollback(self, namespace, index): + def test_negative_upsert_after_rollback(self, namespace, index): # Given("Create namespace with index") db, namespace_name = namespace # When ("Start new transaction") @@ -70,7 +61,7 @@ def test_upsert_after_rollback(self, namespace, index): assert_that(calling(upsert_transaction).with_args(transaction, item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_delete_after_rollback(self, namespace, index): + def test_negative_delete_after_rollback(self, namespace, index): # Given("Create namespace with index") db, namespace_name = namespace # When ("Start new transaction") @@ -81,6 +72,50 @@ def test_delete_after_rollback(self, namespace, index): assert_that(calling(delete_transaction).with_args(transaction, item_definition), raises(Exception, matching=has_string("Transaction is over"))) + def test_negative_insert_after_commit(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Commit transaction") + transaction.commit() + # Then ("Insert transaction") + assert_that(calling(insert_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_negative_update_after_commit(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Commit transaction") + transaction.commit() + # Then ("Update transaction") + assert_that(calling(update_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_negative_upsert_after_commit(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Commit transaction") + transaction.commit() + # Then ("Upsert transaction") + assert_that(calling(upsert_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + + def test_negative_delete_after_commit(self, namespace, index): + # Given("Create namespace with index") + db, namespace_name = namespace + # When ("Start new transaction") + transaction = db.new_transaction(namespace_name) + # Then ("Commit transaction") + transaction.commit() + # Then ("Delete transaction") + assert_that(calling(delete_transaction).with_args(transaction, item_definition), + raises(Exception, matching=has_string("Transaction is over"))) + def test_create_item_insert(self, namespace, index): # Given("Create namespace with index") db, namespace_name = namespace From a7b03c2a6b48bd3d237e285eeded5b9dcb1e04c9 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 17 Oct 2024 19:18:47 +0300 Subject: [PATCH 015/125] Reorganize README --- README.md | 56 ++++++++++++++++++++++++++-------------------------- readmegen.sh | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 0005b0c..fd10acf 100644 --- a/README.md +++ b/README.md @@ -409,34 +409,6 @@ __Raises__ Exception: Raises with an error message of API return on non-zero error code -

pyreindexer.index_definition

- - -

IndexDefinition

- -```python -IndexDefinition(dict) -``` -IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. -NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. - -__Arguments:__ - - name (str): An index name. - json_paths (:obj:`list` of :obj:`str`): A name for mapping a value to a json field. - field_type (str): A type of a field. Possible values are: `int`, `int64`, `double`, `string`, `bool`, `composite`. - index_type (str): An index type. Possible values are: `hash`, `tree`, `text`, `-`. - is_pk (bool): True if a field is a primary key. - is_array (bool): True if an index is an array. - is_dense (bool): True if an index is dense. reduce index size. Saves 8 bytes per unique key value for 'hash' and 'tree' index types. - For '-' index type saves 4-8 bytes per each element. Useful for indexes with high selectivity, but for tree and hash indexes with low selectivity could - significantly decrease update performance. - is_sparse (bool): True if a value of an index may be not presented. - collate_mode (str): Sets an order of values by collate mode. Possible values are: `none`, `ascii`, `utf8`, `numeric`, `custom`. - sort_order_letters (str): Order for a sort sequence for a custom collate mode. - config (dict): A config for a fulltext engine. [More](https://github.com/Restream/reindexer/blob/master/fulltext.md) . - -

pyreindexer.transaction

@@ -550,3 +522,31 @@ __Raises:__ Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code + + +

pyreindexer.index_definition

+ + +

IndexDefinition

+ +```python +IndexDefinition(dict) +``` +IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. +NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. + +__Arguments:__ + + name (str): An index name. + json_paths (:obj:`list` of :obj:`str`): A name for mapping a value to a json field. + field_type (str): A type of a field. Possible values are: `int`, `int64`, `double`, `string`, `bool`, `composite`. + index_type (str): An index type. Possible values are: `hash`, `tree`, `text`, `-`. + is_pk (bool): True if a field is a primary key. + is_array (bool): True if an index is an array. + is_dense (bool): True if an index is dense. reduce index size. Saves 8 bytes per unique key value for 'hash' and 'tree' index types. + For '-' index type saves 4-8 bytes per each element. Useful for indexes with high selectivity, but for tree and hash indexes with low selectivity could + significantly decrease update performance. + is_sparse (bool): True if a value of an index may be not presented. + collate_mode (str): Sets an order of values by collate mode. Possible values are: `none`, `ascii`, `utf8`, `numeric`, `custom`. + sort_order_letters (str): Order for a sort sequence for a custom collate mode. + config (dict): A config for a fulltext engine. [More](https://github.com/Restream/reindexer/blob/master/fulltext.md). diff --git a/readmegen.sh b/readmegen.sh index 46cb058..64d852c 100755 --- a/readmegen.sh +++ b/readmegen.sh @@ -1,3 +1,3 @@ #!/bin/sh -pydocmd simple pyreindexer++ pyreindexer.rx_connector++ pyreindexer.query_results++ pyreindexer.index_definition++ pyreindexer.transaction++ > README.md +pydocmd simple pyreindexer++ pyreindexer.rx_connector++ pyreindexer.query_results++ pyreindexer.transaction++ pyreindexer.index_definition++ > README.md From d0d732d091675d7e4cbb6bceb4bf299cc7010e28 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 18 Oct 2024 11:47:23 +0300 Subject: [PATCH 016/125] Correct test_item --- pyreindexer/tests/tests/test_items.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index 6331df8..f554dc4 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -38,8 +38,6 @@ def test_create_item_insert_with_precepts(self, namespace, index): for i in range(number_items): assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), "Items wasn't created") - for i in range(number_items): - db.item_delete(namespace_name, {'id': i}) def test_create_item_upsert(self, namespace, index): # Given("Create namespace with index") From 99099fafe9a03673999457e232e9a596d9087802 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 23 Oct 2024 17:03:28 +0300 Subject: [PATCH 017/125] Update transaction test --- pyreindexer/tests/tests/test_transaction.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index 649b06e..39d2642 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -133,17 +133,16 @@ def test_create_item_insert_with_precepts(self, namespace, index): # When ("Insert items into namespace") transaction = db.new_transaction(namespace_name) number_items = 5 - for _ in range(number_items): - transaction.insert({"id": 100, "field": "value"}, ["id=serial()"]) + for i in range(number_items): + transaction.insert({'id': 100, 'field': 'value' + str(100 + i)}, ['id=serial()']) transaction.commit() # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) assert_that(select_result, has_length(number_items), "Transaction: items wasn't created") for i in range(number_items): - assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), + assert_that(select_result[i], + equal_to({'id': i + 1, 'field': 'value' + str(100 + i)}), "Transaction: items wasn't created") - for i in range(number_items): - db.item_delete(namespace_name, {'id': i}) def test_create_item_upsert(self, namespace, index): # Given("Create namespace with index") @@ -195,7 +194,7 @@ def test_rollback_transaction(self, namespace, index): transaction = db.new_transaction(namespace_name) number_items = 5 for _ in range(number_items): - transaction.insert({"id": 100, "field": "value"}, ["id=serial()"]) + transaction.insert({"id": 100, "field": "value"}) # Then ("Rollback transaction") transaction.rollback() # When ("Get namespace information") From ae9dcd48f2e38cfd8807d5b98acd7fbfd8832883 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 23 Oct 2024 17:04:10 +0300 Subject: [PATCH 018/125] Simplify code --- pyreindexer/example/main.py | 4 +-- pyreindexer/lib/src/rawpyreindexer.cc | 43 ++++++++++++++------------- pyreindexer/lib/src/rawpyreindexer.h | 20 ++----------- 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 319ebe3..691677f 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -56,7 +56,7 @@ def select_item_query_example(db, namespace): return db.select("SELECT * FROM " + namespace + " WHERE name='" + item_name_for_lookup + "'") -def using_transaction_example(db, namespace, items_in_base): +def transaction_example(db, namespace, items_in_base): transaction = db.new_transaction(namespace) items_count = len(items_in_base) @@ -112,7 +112,7 @@ def rx_example(): for item in selected_items: print('Item: ', item) - using_transaction_example(db, namespace, items_copy) + transaction_example(db, namespace, items_copy) db.close() diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 7382be1..c488b04 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1,7 +1,17 @@ #include "rawpyreindexer.h" +#include "pyobjtools.h" +#include "queryresults_wrapper.h" +#include "transaction_wrapper.h" +#include "tools/serializer.h" + namespace pyreindexer { +using reindexer::Error; +using reindexer::IndexDef; +using reindexer::NamespaceDef; +using reindexer::WrSerializer; + static uintptr_t initReindexer() { DBInterface* db = new DBInterface(); return reinterpret_cast(db); @@ -16,26 +26,19 @@ static void destroyReindexer(uintptr_t rx) { static PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } -static QueryResultsWrapper* getQueryResultsWrapper(uintptr_t qresWrapperAddr) { - return reinterpret_cast(qresWrapperAddr); -} - -static void queryResultsWrapperDelete(uintptr_t qresWrapperAddr) { - QueryResultsWrapper* qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); - delete qresWrapperPtr; -} - -static TransactionWrapper* getTransactionWrapper(uintptr_t transactionWrapperAddr) { - return reinterpret_cast(transactionWrapperAddr); +template +static T* getWrapper(uintptr_t wrapperAddr) { + return reinterpret_cast(wrapperAddr); } -static void transactionWrapperDelete(uintptr_t transactionWrapperAddr) { - TransactionWrapper* transactionWrapperPtr = getTransactionWrapper(transactionWrapperAddr); - delete transactionWrapperPtr; +template +static void wrapperDelete(uintptr_t wrapperAddr) { + T* queryWrapperPtr = getWrapper(wrapperAddr); + delete queryWrapperPtr; } static PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { - QueryResultsWrapper* qresWrapperPtr = getQueryResultsWrapper(qresWrapperAddr); + QueryResultsWrapper* qresWrapperPtr = getWrapper(qresWrapperAddr); WrSerializer wrSer; qresWrapperPtr->GetItemJSON(wrSer, false); @@ -418,7 +421,7 @@ static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args) { return nullptr; } - queryResultsWrapperDelete(qresWrapperAddr); + wrapperDelete(qresWrapperAddr); Py_RETURN_NONE; } @@ -429,7 +432,7 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { return nullptr; } - QueryResultsWrapper* qresWrapper = getQueryResultsWrapper(qresWrapperAddr); + QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); const auto& aggResults = qresWrapper->GetAggregationResults(); WrSerializer wrSer; @@ -488,7 +491,7 @@ static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModif Py_INCREF(itemDefDict); Py_XINCREF(preceptsList); - auto transaction = getTransactionWrapper(transactionWrapperAddr); + auto transaction = getWrapper(transactionWrapperAddr); auto item = transaction->NewItem(); Error err = item.Status(); @@ -562,12 +565,12 @@ static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransaction return nullptr; } - auto transaction = getTransactionWrapper(transactionWrapperAddr); + auto transaction = getWrapper(transactionWrapperAddr); assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); Error err = (StopTransactionMode::Commit == stopMode) ? transaction->Commit() : transaction->Rollback(); - transactionWrapperDelete(transactionWrapperAddr); + wrapperDelete(transactionWrapperAddr); return pyErr(err); } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index a89cc9c..d24379c 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -12,24 +12,8 @@ #include -#include "pyobjtools.h" -#include "queryresults_wrapper.h" -#include "transaction_wrapper.h" -#include "tools/serializer.h" - -#ifdef PYREINDEXER_CPROTO -#include "client/cororeindexer.h" -#else -#include "core/reindexer.h" -#endif - namespace pyreindexer { -using reindexer::Error; -using reindexer::IndexDef; -using reindexer::NamespaceDef; -using reindexer::WrSerializer; - static PyObject* Init(PyObject* self, PyObject* args); static PyObject* Destroy(PyObject* self, PyObject* args); static PyObject* Connect(PyObject* self, PyObject* args); @@ -93,8 +77,8 @@ static PyMethodDef module_methods[] = { {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, {"item_upsert_transaction", ItemUpsertTransaction, METH_VARARGS, "item upsert transaction"}, {"item_delete_transaction", ItemDeleteTransaction, METH_VARARGS, "item delete transaction"}, - {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction. Free transaction buffer"}, - {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction. Free transaction buffer"}, + {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction. Free transaction object memory"}, + {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction. Free transaction object memory"}, {nullptr, nullptr, 0, nullptr} }; From 5454cf94aa5ce0a05b0c926c879dcca170328d11 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 23 Oct 2024 17:04:28 +0300 Subject: [PATCH 019/125] Style changes --- pyreindexer/query_results.py | 3 ++- pyreindexer/transaction.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 6bd0449..04f5a94 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,4 +1,4 @@ -class QueryResults: +class QueryResults(object): """ QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer @@ -9,6 +9,7 @@ class QueryResults: qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations pos (int): The current result position in iterator + """ def __init__(self, api, qres_wrapper_ptr, qres_iter_count): diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 8336e35..91489be 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,4 +1,4 @@ -class Transaction: +class Transaction(object): """ An object representing the context of a Reindexer transaction # Attributes: From 247b231eb4e84c23239f49a2899a0d2be9d1845f Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 23 Oct 2024 17:08:09 +0300 Subject: [PATCH 020/125] Start of implementation of query builder --- pyreindexer/example/main.py | 7 ++ pyreindexer/lib/include/query_wrapper.h | 19 ++++ pyreindexer/lib/src/rawpyreindexer.cc | 25 +++++ pyreindexer/lib/src/rawpyreindexer.h | 6 ++ pyreindexer/query.py | 130 ++++++++++++++++++++++++ pyreindexer/rx_connector.py | 57 +++++++---- setup.py | 1 + 7 files changed, 227 insertions(+), 18 deletions(-) create mode 100644 pyreindexer/lib/include/query_wrapper.h create mode 100644 pyreindexer/query.py diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 691677f..379666a 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -83,6 +83,11 @@ def transaction_example(db, namespace, items_in_base): for item in selected_items_tr: print('Item: ', item) +def query_example(db, namespace): + transaction = db.new_transaction(namespace) + query = db.new_query(namespace, transaction) + query.where("name = 'item_0'") + transaction.commit() def rx_example(): db = RxConnector('builtin:///tmp/pyrx') @@ -114,6 +119,8 @@ def rx_example(): transaction_example(db, namespace, items_copy) + query_example(db, namespace) + db.close() diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h new file mode 100644 index 0000000..8cddd91 --- /dev/null +++ b/pyreindexer/lib/include/query_wrapper.h @@ -0,0 +1,19 @@ +#pragma once + +#include "core/query/query.h" + +namespace pyreindexer { + +class QueryWrapper { +public: + QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { + assert(db_); + query_ = std::make_unique(ns); + } + +private: + DBInterface* db_{nullptr}; + std::unique_ptr query_; +}; + +} // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index c488b04..6fceabb 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -2,6 +2,7 @@ #include "pyobjtools.h" #include "queryresults_wrapper.h" +#include "query_wrapper.h" #include "transaction_wrapper.h" #include "tools/serializer.h" @@ -577,4 +578,28 @@ static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransaction static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Commit); } static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Rollback); } + +static PyObject* CreateQuery(PyObject* self, PyObject* args) { + uintptr_t rx = 0; + char* ns = nullptr; + if (!PyArg_ParseTuple(args, "ks", &rx, &ns)) { + return nullptr; + } + + auto db = getDB(rx); + auto query = new QueryWrapper(db, ns); + return Py_BuildValue("isK", errOK, "", reinterpret_cast(query)); +} + +static PyObject* DeleteQuery(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + wrapperDelete(queryWrapperAddr); + + Py_RETURN_NONE; +} + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index d24379c..ab6dc66 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -46,6 +46,9 @@ static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args); static PyObject* CommitTransaction(PyObject* self, PyObject* args); static PyObject* RollbackTransaction(PyObject* self, PyObject* args); +static PyObject* CreateQuery(PyObject* self, PyObject* args); +static PyObject* DeleteQuery(PyObject* self, PyObject* args); + // clang-format off static PyMethodDef module_methods[] = { {"init", Init, METH_NOARGS, "init reindexer instance"}, @@ -80,6 +83,9 @@ static PyMethodDef module_methods[] = { {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction. Free transaction object memory"}, {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction. Free transaction object memory"}, + {"create_query", CreateQuery, METH_VARARGS, "create new query"}, + {"delete_query", DeleteQuery, METH_VARARGS, "delete query. Free query object memory"}, + {nullptr, nullptr, 0, nullptr} }; // clang-format on diff --git a/pyreindexer/query.py b/pyreindexer/query.py new file mode 100644 index 0000000..9957512 --- /dev/null +++ b/pyreindexer/query.py @@ -0,0 +1,130 @@ +from typing import List +from enum import Enum + +class CondType(Enum): + CondAny = 0 + CondEq = 1 + CondLt = 2 + CondLe = 3 + CondGt = 4 + CondGe = 5 + CondRange = 6 + CondSet = 7 + CondAllSet = 8 + CondEmpty = 9 + CondLike = 10 + CondDWithin = 11 + +class Query(object): + """ An object representing the context of a Reindexer query + + # Attributes: + api (module): An API module for Reindexer calls + query_wrapper_ptr (int): A memory pointer to Reindexer query object + + """ + + def __init__(self, api, query_wrapper_ptr): + """Constructs a new Reindexer query object + + # Arguments: + api (module): An API module for Reindexer calls + query_wrapper_ptr (int): A memory pointer to Reindexer query object + + """ + + self.api = api + self.query_wrapper_ptr = query_wrapper_ptr + + def __del__(self): + """Free query memory + + """ + + if self.query_wrapper_ptr > 0: + self.api.delete_query(self.query_wrapper_ptr) + + def Where(self, index: str, condition: CondType, keys: List[str]): + """Where - Add where condition to DB query + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + #Returns: + (:obj:`Query`): Query object ready to be executed + + """ + + return self + +#func (q *Query) WhereQuery(subQuery *Query, condition int, keys interface{}) *Query { +#func (q *Query) WhereBetweenFields(firstField string, condition int, secondField string) *Query { +#func (q *Query) OpenBracket() *Query { +#func (q *Query) CloseBracket() *Query { +#func (q *Query) WhereInt(index string, condition int, keys ...int) *Query { +#func (q *Query) WhereInt32(index string, condition int, keys ...int32) *Query { +#func (q *Query) WhereInt64(index string, condition int, keys ...int64) *Query { +#func (q *Query) WhereString(index string, condition int, keys ...string) *Query { +#func (q *Query) WhereUuid(index string, condition int, keys ...string) *Query { +#func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { +#func (q *Query) Match(index string, keys ...string) *Query { +#func (q *Query) WhereBool(index string, condition int, keys ...bool) *Query { +#func (q *Query) WhereDouble(index string, condition int, keys ...float64) *Query { +#func (q *Query) DWithin(index string, point Point, distance float64) *Query { +#func (q *Query) AggregateSum(field string) *Query { +#func (q *Query) AggregateAvg(field string) *Query { +#func (q *Query) AggregateMin(field string) *Query { +#func (q *Query) AggregateMax(field string) *Query { +#type AggregateFacetRequest struct { +# query *Query +#} +#// fields should not be empty. +#func (q *Query) AggregateFacet(fields ...string) *AggregateFacetRequest { +#func (r *AggregateFacetRequest) Limit(limit int) *AggregateFacetRequest { +#func (r *AggregateFacetRequest) Offset(offset int) *AggregateFacetRequest { +#func (r *AggregateFacetRequest) Sort(field string, desc bool) *AggregateFacetRequest { +#func (q *Query) Sort(sortIndex string, desc bool, values ...interface{}) *Query { +#func (q *Query) SortStPointDistance(field string, p Point, desc bool) *Query { +#func (q *Query) SortStFieldDistance(field1 string, field2 string, desc bool) *Query { +#func (q *Query) And() *Query { +#func (q *Query) Or() *Query { +#func (q *Query) Not() *Query { +#func (q *Query) Distinct(distinctIndex string) *Query { +#func (q *Query) ReqTotal(totalNames ...string) *Query { +#func (q *Query) CachedTotal(totalNames ...string) *Query { +#func (q *Query) Limit(limitItems int) *Query { +#func (q *Query) Offset(startOffset int) *Query { +#func (q *Query) Debug(level int) *Query { +#func (q *Query) Strict(mode QueryStrictMode) *Query { +#func (q *Query) Explain() *Query { +#func (q *Query) SetContext(ctx interface{}) *Query { +#func (q *Query) Exec() *Iterator { +#func (q *Query) ExecCtx(ctx context.Context) *Iterator { +#func (q *Query) ExecToJson(jsonRoots ...string) *JSONIterator { +#func (q *Query) ExecToJsonCtx(ctx context.Context, jsonRoots ...string) *JSONIterator { +#func (q *Query) Delete() (int, error) +#func (q *Query) DeleteCtx(ctx context.Context) (int, error) { +#func (q *Query) SetObject(field string, values interface{}) *Query { +#func (q *Query) Set(field string, values interface{}) *Query { +#func (q *Query) Drop(field string) *Query { +#func (q *Query) SetExpression(field string, value string) *Query { +#func (q *Query) Update() *Iterator { +#func (q *Query) UpdateCtx(ctx context.Context) *Iterator { +#func (q *Query) MustExec() *Iterator { +#func (q *Query) MustExecCtx(ctx context.Context) *Iterator { +#func (q *Query) Get() (item interface{}, found bool) { +#func (q *Query) GetCtx(ctx context.Context) (item interface{}, found bool) { +#func (q *Query) GetJson() (json []byte, found bool) { +#func (q *Query) GetJsonCtx(ctx context.Context) (json []byte, found bool) { +#func (q *Query) InnerJoin(q2 *Query, field string) *Query { +#func (q *Query) Join(q2 *Query, field string) *Query { +#func (q *Query) LeftJoin(q2 *Query, field string) *Query { +#func (q *Query) JoinHandler(field string, handler JoinHandler) *Query { +#func (q *Query) Merge(q2 *Query) *Query { +#func (q *Query) On(index string, condition int, joinIndex string) *Query { +#func (q *Query) Select(fields ...string) *Query { +#func (q *Query) FetchCount(n int) *Query { +#func (q *Query) Functions(fields ...string) *Query { +#func (q *Query) EqualPosition(fields ...string) *Query { diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 9e690b7..01e5b28 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -1,6 +1,8 @@ +from typing import List, Dict from pyreindexer.raiser_mixin import RaiserMixin from pyreindexer.query_results import QueryResults from pyreindexer.transaction import Transaction +from pyreindexer.query import Query class RxConnector(RaiserMixin): @@ -41,14 +43,14 @@ def __del__(self): if self.rx > 0: self._api_close() - def close(self): + def close(self) -> None: """Closes an API instance with Reindexer resources freeing """ self._api_close() - def namespace_open(self, namespace): + def namespace_open(self, namespace) -> None: """Opens a namespace specified or creates a namespace if it does not exist # Arguments: @@ -64,7 +66,7 @@ def namespace_open(self, namespace): self.err_code, self.err_msg = self.api.namespace_open(self.rx, namespace) self.raise_on_error() - def namespace_close(self, namespace): + def namespace_close(self, namespace) -> None: """Closes a namespace specified # Arguments: @@ -80,7 +82,7 @@ def namespace_close(self, namespace): self.err_code, self.err_msg = self.api.namespace_close(self.rx, namespace) self.raise_on_error() - def namespace_drop(self, namespace): + def namespace_drop(self, namespace) -> None: """Drops a namespace specified # Arguments: @@ -96,7 +98,7 @@ def namespace_drop(self, namespace): self.err_code, self.err_msg = self.api.namespace_drop(self.rx, namespace) self.raise_on_error() - def namespaces_enum(self, enum_not_opened=False): + def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: """Gets a list of namespaces available # Arguments: @@ -117,7 +119,7 @@ def namespaces_enum(self, enum_not_opened=False): self.raise_on_error() return res - def index_add(self, namespace, index_def): + def index_add(self, namespace, index_def) -> None: """Adds an index to the namespace specified # Arguments: @@ -134,7 +136,7 @@ def index_add(self, namespace, index_def): self.err_code, self.err_msg = self.api.index_add(self.rx, namespace, index_def) self.raise_on_error() - def index_update(self, namespace, index_def): + def index_update(self, namespace, index_def) -> None: """Updates an index in the namespace specified # Arguments: @@ -151,7 +153,7 @@ def index_update(self, namespace, index_def): self.err_code, self.err_msg = self.api.index_update(self.rx, namespace, index_def) self.raise_on_error() - def index_drop(self, namespace, index_name): + def index_drop(self, namespace, index_name) -> None: """Drops an index from the namespace specified # Arguments: @@ -168,7 +170,7 @@ def index_drop(self, namespace, index_name): self.err_code, self.err_msg = self.api.index_drop(self.rx, namespace, index_name) self.raise_on_error() - def item_insert(self, namespace, item_def, precepts=None): + def item_insert(self, namespace, item_def, precepts=None) -> None: """Inserts an item with its precepts to the namespace specified # Arguments: @@ -188,7 +190,7 @@ def item_insert(self, namespace, item_def, precepts=None): self.err_code, self.err_msg = self.api.item_insert(self.rx, namespace, item_def, precepts) self.raise_on_error() - def item_update(self, namespace, item_def, precepts=None): + def item_update(self, namespace, item_def, precepts=None) -> None: """Updates an item with its precepts in the namespace specified # Arguments: @@ -208,7 +210,7 @@ def item_update(self, namespace, item_def, precepts=None): self.err_code, self.err_msg = self.api.item_update(self.rx, namespace, item_def, precepts) self.raise_on_error() - def item_upsert(self, namespace, item_def, precepts=None): + def item_upsert(self, namespace, item_def, precepts=None) -> None: """Updates an item with its precepts in the namespace specified. Creates the item if it not exists # Arguments: @@ -228,7 +230,7 @@ def item_upsert(self, namespace, item_def, precepts=None): self.err_code, self.err_msg = self.api.item_upsert(self.rx, namespace, item_def, precepts) self.raise_on_error() - def item_delete(self, namespace, item_def): + def item_delete(self, namespace, item_def) -> None: """Deletes an item from the namespace specified # Arguments: @@ -245,7 +247,7 @@ def item_delete(self, namespace, item_def): self.err_code, self.err_msg = self.api.item_delete(self.rx, namespace, item_def) self.raise_on_error() - def meta_put(self, namespace, key, value): + def meta_put(self, namespace, key, value) -> None: """Puts metadata to a storage of Reindexer by key # Arguments: @@ -263,7 +265,7 @@ def meta_put(self, namespace, key, value): self.err_code, self.err_msg = self.api.meta_put(self.rx, namespace, key, value) self.raise_on_error() - def meta_get(self, namespace, key): + def meta_get(self, namespace, key) -> str : """Gets metadata from a storage of Reindexer by key specified # Arguments: @@ -284,7 +286,7 @@ def meta_get(self, namespace, key): self.raise_on_error() return res - def meta_delete(self, namespace, key): + def meta_delete(self, namespace, key) -> None: """Deletes metadata from a storage of Reindexer by key specified # Arguments: @@ -301,7 +303,7 @@ def meta_delete(self, namespace, key): self.err_code, self.err_msg = self.api.meta_delete(self.rx, namespace, key) self.raise_on_error() - def meta_enum(self, namespace): + def meta_enum(self, namespace) -> List[str] : """Gets a list of metadata keys from a storage of Reindexer # Arguments: @@ -321,7 +323,7 @@ def meta_enum(self, namespace): self.raise_on_error() return res - def select(self, query): + def select(self, query) -> QueryResults: """Executes an SQL query and returns query results # Arguments: @@ -341,7 +343,7 @@ def select(self, query): self.raise_on_error() return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) - def new_transaction(self, namespace): + def new_transaction(self, namespace) -> Transaction: """Start a new transaction and return the transaction object to processing # Arguments: @@ -355,11 +357,30 @@ def new_transaction(self, namespace): Exception: Raises with an error message of API return on non-zero error code """ + self.raise_on_not_init() self.err_code, self.err_msg, transaction_wrapper_ptr = self.api.new_transaction(self.rx, namespace) self.raise_on_error() return Transaction(self.api, transaction_wrapper_ptr) + def new_query(self, namespace: str) -> Query: + """Create a new query and return the query object to processing + + # Arguments: + namespace (string): A name of a namespace + + # Returns: + (:obj:`Query`): A new query + + # Raises: + Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + + """ + + self.raise_on_not_init() + query_wrapper_ptr = self.api.create_query(self.rx, namespace) + return Query(self.api, query_wrapper_ptr) + def _api_import(self, dsn): """Imports an API dynamically depending on protocol specified in dsn diff --git a/setup.py b/setup.py index 6800711..a0d7888 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ def build_cmake(self, ext): 'lib/include/pyobjtools.h', 'lib/include/pyobjtools.cc', 'lib/include/queryresults_wrapper.h', + 'lib/include/query_wrapper.h', 'lib/include/transaction_wrapper.h', 'lib/src/rawpyreindexer.h', 'lib/src/rawpyreindexer.cc', From b3fad40a28639dcefc2a8d233f13ddbff66343d3 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 24 Oct 2024 20:36:41 +0300 Subject: [PATCH 021/125] Part I: start of implementation of query builder --- pyreindexer/example/main.py | 13 +- pyreindexer/lib/include/pyobjtools.h | 21 ++- pyreindexer/lib/include/query_wrapper.h | 42 ++++- pyreindexer/lib/src/rawpyreindexer.cc | 198 ++++++++++++++++++++++++ pyreindexer/lib/src/rawpyreindexer.h | 25 ++- pyreindexer/query.py | 163 +++++++++++++++++-- pyreindexer/rx_connector.py | 3 +- pyreindexer/transaction.py | 2 +- 8 files changed, 442 insertions(+), 25 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 379666a..d480a96 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -1,4 +1,5 @@ from pyreindexer import RxConnector +from pyreindexer.query import CondType def create_index_example(db, namespace): @@ -84,10 +85,14 @@ def transaction_example(db, namespace, items_in_base): print('Item: ', item) def query_example(db, namespace): - transaction = db.new_transaction(namespace) - query = db.new_query(namespace, transaction) - query.where("name = 'item_0'") - transaction.commit() + (db.new_query(namespace) + .open_bracket() + .where_between_fields('fld1', CondType.CondEq, 'fld2') + .close_bracket() + .where_int64('fld2', CondType.CondLe, [42])) + + query = db.new_query(namespace) + query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) def rx_example(): db = RxConnector('builtin:///tmp/pyrx') diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index 89e98e2..4d46cee 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -8,7 +8,26 @@ namespace pyreindexer { -std::vector ParseListToStrVec(PyObject** dict); +std::vector ParseListToStrVec(PyObject** list); + +template +std::vector ParseListToIntVec(PyObject** list) { + std::vector vec; + + Py_ssize_t sz = PyList_Size(*list); + for (Py_ssize_t i = 0; i < sz; i++) { + PyObject* item = PyList_GetItem(*list, i); + + if (!PyLong_Check(item)) { + throw reindexer::Error(errParseJson, std::string("Integer expected, got ") + Py_TYPE(item)->tp_name); + } + + long v = PyLong_AsLong(item); + vec.push_back(T(v)); + } + + return vec; +} void PyObjectToJson(PyObject** dict, reindexer::WrSerializer& wrSer); PyObject* PyObjectFromJson(reindexer::span json); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 8cddd91..f565c25 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -1,19 +1,55 @@ #pragma once +#include "core/type_consts.h" #include "core/query/query.h" +#include "core/keyvalue/uuid.h" namespace pyreindexer { class QueryWrapper { public: - QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { + QueryWrapper(DBInterface* db, std::string_view ns) : db_{db}, query_(ns) { assert(db_); - query_ = std::make_unique(ns); + } + + void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { + query_ = std::move(query_.WhereBetweenFields(firstField, condition, secondField)); + } + + void OpenBracket() { + query_ = std::move(query_.OpenBracket()); + } + + void CloseBracket() { + query_ = std::move(query_.CloseBracket()); + } + +// template +// void Where(std::string&& index, CondType condition, const std::vector& keys) { +// query_ = std::move(query_.Where(std::move(index), condition, keys)); +// } + template + void Where(std::string_view index, CondType condition, const std::vector& keys) { + query_ = std::move(query_.Where(index, condition, keys)); + } + + Error WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { + try { + for (const auto& key : keys) { + auto uuid = reindexer::Uuid(key); + (void)uuid; // check format only + } + } catch (const Error& err) { + return err; + } + + query_ = std::move(query_.Where(index, condition, keys)); + return errOK; } private: DBInterface* db_{nullptr}; - std::unique_ptr query_; + reindexer::Query query_; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 6fceabb..955ace2 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -602,4 +602,202 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { Py_RETURN_NONE; } +static PyObject* WhereBetweenFields(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* first_field = nullptr; + unsigned condition = 0; + char* second_field = nullptr; + if (!PyArg_ParseTuple(args, "ksIs", &queryWrapperAddr, &first_field, &condition, &second_field)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->WhereBetweenFields(first_field, CondType(condition), second_field); + + Py_RETURN_NONE; +} + +static PyObject* OpenBracket(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->OpenBracket(); + + Py_RETURN_NONE; +} + +static PyObject* CloseBracket(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->CloseBracket(); + + Py_RETURN_NONE; +} + +static PyObject* WhereInt(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToIntVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + +static PyObject* WhereInt32(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToIntVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + +static PyObject* WhereInt64(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToIntVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + +static PyObject* WhereString(PyObject* self, PyObject* args) +{ + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToStrVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + +static PyObject* WhereUUID(PyObject* self, PyObject* args) +{ + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToStrVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + auto err = query->WhereUUID(index, CondType(condition), keys); + return pyErr(err); +} + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index ab6dc66..9f27149 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -49,6 +49,15 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args); static PyObject* CreateQuery(PyObject* self, PyObject* args); static PyObject* DeleteQuery(PyObject* self, PyObject* args); +static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); +static PyObject* OpenBracket(PyObject* self, PyObject* args); +static PyObject* CloseBracket(PyObject* self, PyObject* args); +static PyObject* WhereInt(PyObject* self, PyObject* args); +static PyObject* WhereInt32(PyObject* self, PyObject* args); +static PyObject* WhereInt64(PyObject* self, PyObject* args); +static PyObject* WhereString(PyObject* self, PyObject* args); +static PyObject* WhereUUID(PyObject* self, PyObject* args); + // clang-format off static PyMethodDef module_methods[] = { {"init", Init, METH_NOARGS, "init reindexer instance"}, @@ -70,11 +79,11 @@ static PyMethodDef module_methods[] = { {"meta_delete", DeleteMeta, METH_VARARGS, "delete meta"}, {"meta_enum", EnumMeta, METH_VARARGS, "enum meta"}, {"select", Select, METH_VARARGS, "select query"}, - + // query results {"query_results_iterate", QueryResultsWrapperIterate, METH_VARARGS, "get query result"}, {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, {"get_agg_results", GetAggregationResults, METH_VARARGS, "get aggregation results"}, - + // transaction (sync) {"new_transaction", NewTransaction, METH_VARARGS, "start new transaction"}, {"item_insert_transaction", ItemInsertTransaction, METH_VARARGS, "item insert transaction"}, {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, @@ -82,10 +91,20 @@ static PyMethodDef module_methods[] = { {"item_delete_transaction", ItemDeleteTransaction, METH_VARARGS, "item delete transaction"}, {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction. Free transaction object memory"}, {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction. Free transaction object memory"}, - + // query {"create_query", CreateQuery, METH_VARARGS, "create new query"}, {"delete_query", DeleteQuery, METH_VARARGS, "delete query. Free query object memory"}, + //{"where", Where, METH_VARARGS, "add where condition"}, + {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, + {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, + {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, + {"where_int", WhereInt, METH_VARARGS, "add where condition with int args"}, + {"where_int32", WhereInt32, METH_VARARGS, "add where condition with int32 args"}, + {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, + {"where_string", WhereString, METH_VARARGS, "add where condition with string args"}, + {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUID as string args"}, + {nullptr, nullptr, 0, nullptr} }; // clang-format on diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 9957512..037c015 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,3 +1,4 @@ +from __future__ import annotations from typing import List from enum import Enum @@ -21,10 +22,12 @@ class Query(object): # Attributes: api (module): An API module for Reindexer calls query_wrapper_ptr (int): A memory pointer to Reindexer query object + err_code (int): the API error code + err_msg (string): the API error message """ - def __init__(self, api, query_wrapper_ptr): + def __init__(self, api, query_wrapper_ptr: int): """Constructs a new Reindexer query object # Arguments: @@ -35,6 +38,8 @@ def __init__(self, api, query_wrapper_ptr): self.api = api self.query_wrapper_ptr = query_wrapper_ptr + self.err_code = 0 + self.err_msg = "" def __del__(self): """Free query memory @@ -44,29 +49,163 @@ def __del__(self): if self.query_wrapper_ptr > 0: self.api.delete_query(self.query_wrapper_ptr) - def Where(self, index: str, condition: CondType, keys: List[str]): - """Where - Add where condition to DB query + def _raise_on_error(self): + """Checks if there is an error code and raises with an error message + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + if self.err_code: + raise Exception(self.err_msg) + +################################################################### +#func (q *Query) Where(index string, condition int, keys interface{}) *Query { +#func (q *Query) WhereQuery(subQuery *Query, condition int, keys interface{}) *Query { +################################################################# + + def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: # ToDo + """Where - Add comparing two fields where condition to DB query + + # Arguments: + first_field (string): First field name used in condition clause + condition (:enum:`CondType`): Type of condition + second_field (string): Second field name used in condition clause + + # Returns: + (:obj:`Query`): Query object ready to be executed + + """ + + self.api.where_between_fields(self.query_wrapper_ptr, first_field, condition.value, second_field) + return self + + def open_bracket(self) -> Query: + """OpenBracket - Open bracket for where condition to DB query + + # Returns: + (:obj:`Query`): Query object ready to be executed + + """ + + self.api.open_bracket(self.query_wrapper_ptr) + return self + + def close_bracket(self) -> Query: + """CloseBracket - Close bracket for where condition to DB query + + # Returns: + (:obj:`Query`): Query object ready to be executed + + """ + + self.api.close_bracket(self.query_wrapper_ptr) + return self + + def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: + """Where - Add where condition to DB query with int args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[int]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_int(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + + def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query: + """Where - Add where condition to DB query with Int32 args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[Int32]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_int32(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + + def where_int64(self, index: str, condition: CondType, keys: List[int]) -> Query: + """Where - Add where condition to DB query with Int64 args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[Int64]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_int64(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + + def where_string(self, index: str, condition: CondType, keys: List[str]) -> Query: + """Where - Add where condition to DB query with string args # Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - #Returns: + # Returns: (:obj:`Query`): Query object ready to be executed + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ + self.err_code, self.err_msg = self.api.where_string(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() return self -#func (q *Query) WhereQuery(subQuery *Query, condition int, keys interface{}) *Query { -#func (q *Query) WhereBetweenFields(firstField string, condition int, secondField string) *Query { -#func (q *Query) OpenBracket() *Query { -#func (q *Query) CloseBracket() *Query { -#func (q *Query) WhereInt(index string, condition int, keys ...int) *Query { -#func (q *Query) WhereInt32(index string, condition int, keys ...int32) *Query { -#func (q *Query) WhereInt64(index string, condition int, keys ...int64) *Query { -#func (q *Query) WhereString(index string, condition int, keys ...string) *Query { + def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: + """Where - Add where condition to DB query with UUID as string args. + This function applies binary encoding to the UUID value. + 'index' MUST be declared as uuid index in this case + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + +################################################################ #func (q *Query) WhereUuid(index string, condition int, keys ...string) *Query { #func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { #func (q *Query) Match(index string, keys ...string) *Query { diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 01e5b28..6c83dd5 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -378,7 +378,8 @@ def new_query(self, namespace: str) -> Query: """ self.raise_on_not_init() - query_wrapper_ptr = self.api.create_query(self.rx, namespace) + self.err_code, self.err_msg, query_wrapper_ptr = self.api.create_query(self.rx, namespace) + self.raise_on_error() return Query(self.api, query_wrapper_ptr) def _api_import(self, dsn): diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 91489be..cc33e08 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -9,7 +9,7 @@ class Transaction(object): """ - def __init__(self, api, transaction_wrapper_ptr): + def __init__(self, api, transaction_wrapper_ptr: int): """Constructs a new Reindexer transaction object # Arguments: From 1f5324113b0229ed2a7a9187252b2e741fff9a24 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 25 Oct 2024 09:26:51 +0300 Subject: [PATCH 022/125] Part II: start of implementation of query builder --- pyreindexer/lib/include/pyobjtools.cc | 40 ++++++++++++++++ pyreindexer/lib/include/pyobjtools.h | 2 + pyreindexer/lib/include/query_wrapper.h | 4 -- pyreindexer/lib/src/rawpyreindexer.cc | 63 +++++++++++++++++++++++++ pyreindexer/lib/src/rawpyreindexer.h | 4 ++ pyreindexer/query.py | 46 ++++++++++++++++-- 6 files changed, 151 insertions(+), 8 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 975d017..e2f6239 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -112,6 +112,46 @@ std::vector ParseListToStrVec(PyObject** list) { return vec; } +std::vector ParseListToBoolVec(PyObject** list) { + std::vector vec; + + Py_ssize_t sz = PyList_Size(*list); + for (Py_ssize_t i = 0; i < sz; i++) { + PyObject* item = PyList_GetItem(*list, i); + + if (!PyBool_Check(item)) { + throw reindexer::Error(errParseJson, std::string("Bool expected, got ") + Py_TYPE(item)->tp_name); + } + + vec.push_back(PyLong_AsLong(item) != 0); + } + + return vec; +} + +std::vector ParseListToDoubleVec(PyObject** list) { + std::vector vec; + + Py_ssize_t sz = PyList_Size(*list); + for (Py_ssize_t i = 0; i < sz; i++) { + PyObject* item = PyList_GetItem(*list, i); + + if (!PyFloat_Check(item)) { + throw reindexer::Error(errParseJson, std::string("Double expected, got ") + Py_TYPE(item)->tp_name); + } + + double v = PyFloat_AsDouble(item); + double intpart = 0.0; + if (std::modf(v, &intpart) == 0.0) { + vec.push_back(int64_t(v)); + } else { + vec.push_back(v); + } + } + + return vec; +} + PyObject* pyValueFromJsonValue(const gason::JsonValue& value) { PyObject* pyValue = nullptr; diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index 4d46cee..f72e9c1 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -9,6 +9,8 @@ namespace pyreindexer { std::vector ParseListToStrVec(PyObject** list); +std::vector ParseListToBoolVec(PyObject** list); +std::vector ParseListToDoubleVec(PyObject** list); template std::vector ParseListToIntVec(PyObject** list) { diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index f565c25..6b7de9e 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -24,10 +24,6 @@ class QueryWrapper { query_ = std::move(query_.CloseBracket()); } -// template -// void Where(std::string&& index, CondType condition, const std::vector& keys) { -// query_ = std::move(query_.Where(std::move(index), condition, keys)); -// } template void Where(std::string_view index, CondType condition, const std::vector& keys) { query_ = std::move(query_.Where(index, condition, keys)); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 955ace2..d72cdfb 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -800,4 +800,67 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) return pyErr(err); } +static PyObject* WhereBool(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToBoolVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + +static PyObject* WhereDouble(PyObject* self, PyObject* args) + { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToDoubleVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 9f27149..2d767eb 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -57,6 +57,8 @@ static PyObject* WhereInt32(PyObject* self, PyObject* args); static PyObject* WhereInt64(PyObject* self, PyObject* args); static PyObject* WhereString(PyObject* self, PyObject* args); static PyObject* WhereUUID(PyObject* self, PyObject* args); +static PyObject* WhereBool(PyObject* self, PyObject* args); +static PyObject* WhereDouble(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { @@ -104,6 +106,8 @@ static PyMethodDef module_methods[] = { {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, {"where_string", WhereString, METH_VARARGS, "add where condition with string args"}, {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUID as string args"}, + {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, + {"where_float64", WhereDouble, METH_VARARGS, "add where condition with double args"}, {nullptr, nullptr, 0, nullptr} }; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 037c015..50c1426 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -65,7 +65,7 @@ def _raise_on_error(self): #func (q *Query) WhereQuery(subQuery *Query, condition int, keys interface{}) *Query { ################################################################# - def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: # ToDo + def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: """Where - Add comparing two fields where condition to DB query # Arguments: @@ -205,12 +205,50 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: self._raise_on_error() return self + def where_bool(self, index: str, condition: CondType, keys: List[bool]) -> Query: + """Where - Add where condition to DB query with bool args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[bool]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_bool(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + + def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Query: + """Where - Add where condition to DB query with float args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[float]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_float64(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + ################################################################ -#func (q *Query) WhereUuid(index string, condition int, keys ...string) *Query { +#func (q *Query) WhereDouble(index string, condition int, keys ...float64) *Query { #func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { #func (q *Query) Match(index string, keys ...string) *Query { -#func (q *Query) WhereBool(index string, condition int, keys ...bool) *Query { -#func (q *Query) WhereDouble(index string, condition int, keys ...float64) *Query { #func (q *Query) DWithin(index string, point Point, distance float64) *Query { #func (q *Query) AggregateSum(field string) *Query { #func (q *Query) AggregateAvg(field string) *Query { From 122c00c58aac040b41b4b3c529f1adea2e805286 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 25 Oct 2024 14:47:51 +0300 Subject: [PATCH 023/125] Part III: start of implementation of query builder --- pyreindexer/example/main.py | 3 +- pyreindexer/lib/include/query_wrapper.h | 12 +++ pyreindexer/lib/src/rawpyreindexer.cc | 50 +++++++++-- pyreindexer/lib/src/rawpyreindexer.h | 16 +++- pyreindexer/query.py | 108 ++++++++++++++++++++---- 5 files changed, 162 insertions(+), 27 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index d480a96..473d825 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -92,7 +92,8 @@ def query_example(db, namespace): .where_int64('fld2', CondType.CondLe, [42])) query = db.new_query(namespace) - query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) + (query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) + .NOT().where_float64('fld2', CondType.CondSet, [3.14])) def rx_example(): db = RxConnector('builtin:///tmp/pyrx') diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 6b7de9e..9af875c 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -43,6 +43,18 @@ class QueryWrapper { return errOK; } + void AND() { + query_ = std::move(query_.And()); + } + + void OR() { + query_ = std::move(query_.Or()); + } + + void NOT() { + query_ = std::move(query_.Not()); + } + private: DBInterface* db_{nullptr}; reindexer::Query query_; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index d72cdfb..d6de1cc 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -737,8 +737,7 @@ static PyObject* WhereInt64(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* WhereString(PyObject* self, PyObject* args) -{ +static PyObject* WhereString(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned condition = 0; @@ -769,8 +768,7 @@ static PyObject* WhereString(PyObject* self, PyObject* args) return pyErr(errOK); } -static PyObject* WhereUUID(PyObject* self, PyObject* args) -{ +static PyObject* WhereUUID(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned condition = 0; @@ -831,8 +829,7 @@ static PyObject* WhereBool(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* WhereDouble(PyObject* self, PyObject* args) - { +static PyObject* WhereDouble(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned condition = 0; @@ -863,4 +860,45 @@ static PyObject* WhereDouble(PyObject* self, PyObject* args) return pyErr(errOK); } +static PyObject* AND(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->AND(); + + Py_RETURN_NONE; +} + + +static PyObject* OR(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->OR(); + + Py_RETURN_NONE; +} + + +static PyObject* NOT(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->NOT(); + + Py_RETURN_NONE; +} + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 2d767eb..1806167 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -60,6 +60,10 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* WhereBool(PyObject* self, PyObject* args); static PyObject* WhereDouble(PyObject* self, PyObject* args); +static PyObject* AND(PyObject* self, PyObject* args); +static PyObject* OR(PyObject* self, PyObject* args); +static PyObject* NOT(PyObject* self, PyObject* args); + // clang-format off static PyMethodDef module_methods[] = { {"init", Init, METH_NOARGS, "init reindexer instance"}, @@ -98,17 +102,23 @@ static PyMethodDef module_methods[] = { {"delete_query", DeleteQuery, METH_VARARGS, "delete query. Free query object memory"}, //{"where", Where, METH_VARARGS, "add where condition"}, + //{"where_query", WhereQuery, METH_VARARGS, "add where condition"}, + {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, - {"where_int", WhereInt, METH_VARARGS, "add where condition with int args"}, - {"where_int32", WhereInt32, METH_VARARGS, "add where condition with int32 args"}, - {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, + {"where_int", WhereInt, METH_VARARGS, "add where condition with int args"}, + {"where_int32", WhereInt32, METH_VARARGS, "add where condition with int32 args"}, + {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, {"where_string", WhereString, METH_VARARGS, "add where condition with string args"}, {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUID as string args"}, {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, {"where_float64", WhereDouble, METH_VARARGS, "add where condition with double args"}, + {"AND", AND, METH_VARARGS, "next condition will be added with AND"}, + {"OR", OR, METH_VARARGS, "next condition will be added with OR"}, + {"NOT", NOT, METH_VARARGS, "next condition will be added with NOT AND"}, + {nullptr, nullptr, 0, nullptr} }; // clang-format on diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 50c1426..08edbc6 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -60,13 +60,13 @@ def _raise_on_error(self): if self.err_code: raise Exception(self.err_msg) -################################################################### +################################################################### ToDo #func (q *Query) Where(index string, condition int, keys interface{}) *Query { #func (q *Query) WhereQuery(subQuery *Query, condition int, keys interface{}) *Query { ################################################################# def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: - """Where - Add comparing two fields where condition to DB query + """Add comparing two fields where condition to DB query # Arguments: first_field (string): First field name used in condition clause @@ -82,7 +82,7 @@ def where_between_fields(self, first_field: str, condition: CondType, second_fie return self def open_bracket(self) -> Query: - """OpenBracket - Open bracket for where condition to DB query + """Open bracket for where condition to DB query # Returns: (:obj:`Query`): Query object ready to be executed @@ -104,7 +104,7 @@ def close_bracket(self) -> Query: return self def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Where - Add where condition to DB query with int args + """Add where condition to DB query with int args # Arguments: index (string): Field name used in condition clause @@ -124,7 +124,7 @@ def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: return self def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Where - Add where condition to DB query with Int32 args + """Add where condition to DB query with Int32 args # Arguments: index (string): Field name used in condition clause @@ -144,7 +144,7 @@ def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query return self def where_int64(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Where - Add where condition to DB query with Int64 args + """Add where condition to DB query with Int64 args # Arguments: index (string): Field name used in condition clause @@ -164,7 +164,7 @@ def where_int64(self, index: str, condition: CondType, keys: List[int]) -> Query return self def where_string(self, index: str, condition: CondType, keys: List[str]) -> Query: - """Where - Add where condition to DB query with string args + """Add where condition to DB query with string args # Arguments: index (string): Field name used in condition clause @@ -184,7 +184,7 @@ def where_string(self, index: str, condition: CondType, keys: List[str]) -> Quer return self def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: - """Where - Add where condition to DB query with UUID as string args. + """Add where condition to DB query with UUID as string args. This function applies binary encoding to the UUID value. 'index' MUST be declared as uuid index in this case @@ -206,7 +206,7 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: return self def where_bool(self, index: str, condition: CondType, keys: List[bool]) -> Query: - """Where - Add where condition to DB query with bool args + """Add where condition to DB query with bool args # Arguments: index (string): Field name used in condition clause @@ -226,7 +226,7 @@ def where_bool(self, index: str, condition: CondType, keys: List[bool]) -> Query return self def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Query: - """Where - Add where condition to DB query with float args + """Add where condition to DB query with float args # Arguments: index (string): Field name used in condition clause @@ -245,10 +245,30 @@ def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Q self._raise_on_error() return self +################################################################ ToDo +#func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { // ToDo ################################################################ -#func (q *Query) WhereDouble(index string, condition int, keys ...float64) *Query { -#func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { -#func (q *Query) Match(index string, keys ...string) *Query { + + def match(self, index: str, keys: List[str]) -> Query: + """Add string EQ-condition to DB query with string args + + # Arguments: + index (string): Field name used in condition clause + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object ready to be executed + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_string(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) + self._raise_on_error() + return self + +################################################################ ToDo #func (q *Query) DWithin(index string, point Point, distance float64) *Query { #func (q *Query) AggregateSum(field string) *Query { #func (q *Query) AggregateAvg(field string) *Query { @@ -264,10 +284,63 @@ def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Q #func (r *AggregateFacetRequest) Sort(field string, desc bool) *AggregateFacetRequest { #func (q *Query) Sort(sortIndex string, desc bool, values ...interface{}) *Query { #func (q *Query) SortStPointDistance(field string, p Point, desc bool) *Query { -#func (q *Query) SortStFieldDistance(field1 string, field2 string, desc bool) *Query { -#func (q *Query) And() *Query { -#func (q *Query) Or() *Query { -#func (q *Query) Not() *Query { +################################################################ + +#func (q *Query) SortStFieldDistance(field1 string, field2 string, desc bool) *Query { // ToDo +# def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) -> Query: +# """Wrapper for geometry sorting by shortest distance between 2 geometry fields (ST_Distance) + +# # Arguments: +# first_field (string): First field name used in condition +# second_field (string): Second field name used in condition +# desc (bool): Descending flag + +# # Returns: +# (:obj:`Query`): Query object ready to be executed + +# # Raises: +# Exception: Raises with an error message of API return on non-zero error code + +# """ + +# request : str = "ST_Distance(" +# request += first_field +# request += ',' +# request += second_field +# request += ')' + +# return self.sort(request, desc) +################################################################ + + def AND(self) -> Query: + """Next condition will be added with AND. + This is the default operation for WHERE statement. Do not have to be called explicitly in user's code. + Used in DSL conversion + + """ + self.api.AND(self.query_wrapper_ptr) + return self + + def OR(self) -> Query: + """Next condition will be added with OR. + Implements short-circuiting: + if the previous condition is successful the next will not be evaluated, but except Join conditions + + + """ + self.api.OR(self.query_wrapper_ptr) + return self + + def NOT(self) -> Query: + """Next condition will be added with NOT AND. + Implements short-circuiting: if the previous condition is failed the next will not be evaluated + + """ + + self.api.NOT(self.query_wrapper_ptr) + return self + +################################################################ ToDo #func (q *Query) Distinct(distinctIndex string) *Query { #func (q *Query) ReqTotal(totalNames ...string) *Query { #func (q *Query) CachedTotal(totalNames ...string) *Query { @@ -305,3 +378,4 @@ def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Q #func (q *Query) FetchCount(n int) *Query { #func (q *Query) Functions(fields ...string) *Query { #func (q *Query) EqualPosition(fields ...string) *Query { +# 66 / 10 + 1 + 3 \ No newline at end of file From 25c868a14b6d3e0a63f0d74e022a56bb6ca0a04f Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 25 Oct 2024 18:29:22 +0300 Subject: [PATCH 024/125] Part IV: start of implementation of query builder --- pyreindexer/example/main.py | 2 +- pyreindexer/lib/include/query_wrapper.h | 50 +++++++- pyreindexer/lib/src/rawpyreindexer.cc | 85 +++++++++++-- pyreindexer/lib/src/rawpyreindexer.h | 27 ++-- pyreindexer/query.py | 160 ++++++++++++++++++++---- 5 files changed, 275 insertions(+), 49 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 473d825..8af0703 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -93,7 +93,7 @@ def query_example(db, namespace): query = db.new_query(namespace) (query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) - .NOT().where_float64('fld2', CondType.CondSet, [3.14])) + .not_op().where_float64('fld2', CondType.CondSet, [3.14])) def rx_example(): db = RxConnector('builtin:///tmp/pyrx') diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 9af875c..3d7e0ec 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -6,6 +6,12 @@ namespace pyreindexer { +enum class LogOpType { + And = 1, + Or = 2, + Not = 3 +}; + class QueryWrapper { public: QueryWrapper(DBInterface* db, std::string_view ns) : db_{db}, query_(ns) { @@ -43,16 +49,48 @@ class QueryWrapper { return errOK; } - void AND() { - query_ = std::move(query_.And()); + void LogOp(LogOpType op) { + switch (op) { + case LogOpType::And: + query_ = std::move(query_.And()); + break; + case LogOpType::Or: + query_ = std::move(query_.Or()); + break; + case LogOpType::Not: + query_ = std::move(query_.Not()); + break; + default: + assert(false); + } + } + + void Distinct(const std::string& index) { + query_ = std::move(query_.Distinct(index)); + } + + void ReqTotal() { + query_ = std::move(query_.ReqTotal()); + } + + void CachedTotal() { + query_ = std::move(query_.CachedTotal()); + } + + void Limit(unsigned limitItems) { + query_ = std::move(query_.Limit(limitItems)); + } + + void Offset(int startOffset) { + query_ = std::move(query_.Offset(startOffset)); } - void OR() { - query_ = std::move(query_.Or()); + void Debug(int level) { + query_ = std::move(query_.Debug(level)); } - void NOT() { - query_ = std::move(query_.Not()); + void Strict(StrictMode mode) { + query_ = std::move(query_.Strict(mode)); } private: diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index d6de1cc..cd9c3da 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -860,21 +860,35 @@ static PyObject* WhereDouble(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* AND(PyObject* self, PyObject* args) { +static PyObject* LogOp(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + unsigned opID = 0; + if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &opID)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->AND(); + query->LogOp(LogOpType(opID)); Py_RETURN_NONE; } +static PyObject* Distinct(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &index)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); -static PyObject* OR(PyObject* self, PyObject* args) { + query->Distinct(index); + + Py_RETURN_NONE; +} + +static PyObject* ReqTotal(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -882,13 +896,12 @@ static PyObject* OR(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->OR(); + query->ReqTotal(); Py_RETURN_NONE; } - -static PyObject* NOT(PyObject* self, PyObject* args) { +static PyObject* CachedTotal(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -896,7 +909,63 @@ static PyObject* NOT(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->NOT(); + query->CachedTotal(); + + Py_RETURN_NONE; +} + +static PyObject* Limit(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + unsigned limitItems = 0; + if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &limitItems)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->Limit(limitItems); + + Py_RETURN_NONE; +} + +static PyObject* Offset(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + int startOffset = 0; + if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &startOffset)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->Offset(startOffset); + + Py_RETURN_NONE; +} + +static PyObject* Debug(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + unsigned level = 0; + if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &level)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->Debug(level); + + Py_RETURN_NONE; +} + +static PyObject* Strict(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + unsigned mode = 0; + if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &mode)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->Strict(StrictMode(mode)); Py_RETURN_NONE; } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 1806167..9ec21be 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -59,10 +59,15 @@ static PyObject* WhereString(PyObject* self, PyObject* args); static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* WhereBool(PyObject* self, PyObject* args); static PyObject* WhereDouble(PyObject* self, PyObject* args); +static PyObject* LogOp(PyObject* self, PyObject* args); +static PyObject* Distinct(PyObject* self, PyObject* args); +static PyObject* ReqTotal(PyObject* self, PyObject* args); +static PyObject* CachedTotal(PyObject* self, PyObject* args); -static PyObject* AND(PyObject* self, PyObject* args); -static PyObject* OR(PyObject* self, PyObject* args); -static PyObject* NOT(PyObject* self, PyObject* args); +static PyObject* Limit(PyObject* self, PyObject* args); +static PyObject* Offset(PyObject* self, PyObject* args); +static PyObject* Debug(PyObject* self, PyObject* args); +static PyObject* Strict(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { @@ -110,14 +115,20 @@ static PyMethodDef module_methods[] = { {"where_int", WhereInt, METH_VARARGS, "add where condition with int args"}, {"where_int32", WhereInt32, METH_VARARGS, "add where condition with int32 args"}, {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, - {"where_string", WhereString, METH_VARARGS, "add where condition with string args"}, - {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUID as string args"}, + {"where_string", WhereString, METH_VARARGS, "add where condition with strings"}, + {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, {"where_float64", WhereDouble, METH_VARARGS, "add where condition with double args"}, + {"log_op", LogOp, METH_VARARGS, "next condition will be added with AND|OR|NOT AND"}, - {"AND", AND, METH_VARARGS, "next condition will be added with AND"}, - {"OR", OR, METH_VARARGS, "next condition will be added with OR"}, - {"NOT", NOT, METH_VARARGS, "next condition will be added with NOT AND"}, + {"distinct", Distinct, METH_VARARGS, "perform distinct for index"}, + {"request_total", ReqTotal, METH_VARARGS, "request total items calculation"}, + {"cached_total", CachedTotal, METH_VARARGS, "request cached total items calculation"}, + + {"limit", Limit, METH_VARARGS, "request cached total items calculation"}, + {"offset", Offset, METH_VARARGS, "request cached total items calculation"}, + {"debug", Debug, METH_VARARGS, "request cached total items calculation"}, + {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, {nullptr, nullptr, 0, nullptr} }; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 08edbc6..9cc1d02 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -16,6 +16,12 @@ class CondType(Enum): CondLike = 10 CondDWithin = 11 +class StrictMode(Enum): + NotSet = 0 + Empty = 1 + Names = 2 + Indexes = 3 + class Query(object): """ An object representing the context of a Reindexer query @@ -26,6 +32,10 @@ class Query(object): err_msg (string): the API error message """ + class _LogOp(Enum): + And = 1 + Or = 2 + Not = 3 def __init__(self, api, query_wrapper_ptr: int): """Constructs a new Reindexer query object @@ -74,7 +84,7 @@ def where_between_fields(self, first_field: str, condition: CondType, second_fie second_field (string): Second field name used in condition clause # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations """ @@ -85,7 +95,7 @@ def open_bracket(self) -> Query: """Open bracket for where condition to DB query # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations """ @@ -96,7 +106,7 @@ def close_bracket(self) -> Query: """CloseBracket - Close bracket for where condition to DB query # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations """ @@ -112,7 +122,7 @@ def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: keys (list[int]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -132,7 +142,7 @@ def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query keys (list[Int32]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -152,7 +162,7 @@ def where_int64(self, index: str, condition: CondType, keys: List[int]) -> Query keys (list[Int64]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -172,7 +182,7 @@ def where_string(self, index: str, condition: CondType, keys: List[str]) -> Quer keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -194,7 +204,7 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -214,7 +224,7 @@ def where_bool(self, index: str, condition: CondType, keys: List[bool]) -> Query keys (list[bool]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -234,7 +244,7 @@ def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Q keys (list[float]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -257,7 +267,7 @@ def match(self, index: str, keys: List[str]) -> Query: keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: - (:obj:`Query`): Query object ready to be executed + (:obj:`Query`): Query object for further customizations # Raises: Exception: Raises with an error message of API return on non-zero error code @@ -296,7 +306,7 @@ def match(self, index: str, keys: List[str]) -> Query: # desc (bool): Descending flag # # Returns: -# (:obj:`Query`): Query object ready to be executed +# (:obj:`Query`): Query object for further customizations # # Raises: # Exception: Raises with an error message of API return on non-zero error code @@ -312,42 +322,140 @@ def match(self, index: str, keys: List[str]) -> Query: # return self.sort(request, desc) ################################################################ - def AND(self) -> Query: + def and_op(self) -> Query: """Next condition will be added with AND. This is the default operation for WHERE statement. Do not have to be called explicitly in user's code. Used in DSL conversion + # Returns: + (:obj:`Query`): Query object for further customizations """ - self.api.AND(self.query_wrapper_ptr) + + self.api.log_op(self.query_wrapper_ptr, self._LogOp.And.value) return self - def OR(self) -> Query: + def or_op(self) -> Query: """Next condition will be added with OR. Implements short-circuiting: if the previous condition is successful the next will not be evaluated, but except Join conditions + # Returns: + (:obj:`Query`): Query object for further customizations """ - self.api.OR(self.query_wrapper_ptr) + + self.api.log_op(self.query_wrapper_ptr, self._LogOp.Or.value) return self - def NOT(self) -> Query: + def not_op(self) -> Query: """Next condition will be added with NOT AND. Implements short-circuiting: if the previous condition is failed the next will not be evaluated + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.log_op(self.query_wrapper_ptr, self._LogOp.Not.value) + return self + + def distinct(self, index: str) -> Query: + """Performs distinct for a certain index. + Return only items with uniq value of field + + # Arguments: + index (string): Field name for distinct operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.distinct(self.query_wrapper_ptr, index) + return self + + def request_total(self) -> Query: + """Request total items calculation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + # q.totalName = totalNames[0] // ToDo + self.api.request_total(self.query_wrapper_ptr) + return self + + def cached_total(self) -> Query: + """Request cached total items calculation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + # q.totalName = totalNames[0] // ToDo + self.api.cached_total(self.query_wrapper_ptr) + return self + + def limit(self, limit_items: int) -> Query: + """Set a limit (count) of returned items. + Analog to sql LIMIT rowsNumber + + # Arguments: + limit_items (int): Number of rows to get from result set + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.limit(self.query_wrapper_ptr, limit_items) + return self + + def offset(self, start_offset: int) -> Query: + """Sets the number of the first selected row from result query + + # Arguments: + limit_items (int): Index of the first row to get from result set + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.offset(self.query_wrapper_ptr, start_offset) + return self + + def debug(self, level: int) -> Query: + """Changes debug level + + # Arguments: + level (int): Debug level + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.debug(self.query_wrapper_ptr, level) + return self + + def strict(self, mode: StrictMode) -> Query: + """Changes strict mode + + # Arguments: + level (:enum:`StrictMode`): Strict mode + + # Returns: + (:obj:`Query`): Query object for further customizations + """ - self.api.NOT(self.query_wrapper_ptr) + self.api.strict(self.query_wrapper_ptr, mode.value) return self ################################################################ ToDo -#func (q *Query) Distinct(distinctIndex string) *Query { -#func (q *Query) ReqTotal(totalNames ...string) *Query { -#func (q *Query) CachedTotal(totalNames ...string) *Query { -#func (q *Query) Limit(limitItems int) *Query { -#func (q *Query) Offset(startOffset int) *Query { -#func (q *Query) Debug(level int) *Query { -#func (q *Query) Strict(mode QueryStrictMode) *Query { #func (q *Query) Explain() *Query { #func (q *Query) SetContext(ctx interface{}) *Query { #func (q *Query) Exec() *Iterator { @@ -378,4 +486,4 @@ def NOT(self) -> Query: #func (q *Query) FetchCount(n int) *Query { #func (q *Query) Functions(fields ...string) *Query { #func (q *Query) EqualPosition(fields ...string) *Query { -# 66 / 10 + 1 + 3 \ No newline at end of file +# 66 / 21 \ No newline at end of file From 5b10cebe61b9836490adbfdff6f2503b829c7ecd Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 25 Oct 2024 20:17:17 +0300 Subject: [PATCH 025/125] Part V: start of implementation of query builder --- pyreindexer/example/main.py | 13 ++++++++++--- pyreindexer/lib/include/query_wrapper.h | 5 +++++ pyreindexer/lib/src/rawpyreindexer.cc | 13 +++++++++++++ pyreindexer/lib/src/rawpyreindexer.h | 4 ++-- pyreindexer/query.py | 14 ++++++++++++-- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 8af0703..326ffeb 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -84,16 +84,23 @@ def transaction_example(db, namespace, items_in_base): for item in selected_items_tr: print('Item: ', item) + def query_example(db, namespace): (db.new_query(namespace) .open_bracket() .where_between_fields('fld1', CondType.CondEq, 'fld2') .close_bracket() - .where_int64('fld2', CondType.CondLe, [42])) + .where_int64('fld2', CondType.CondLe, [42]) + .limit(10) + .debug(1) + .request_total()) - query = db.new_query(namespace) + query = db.new_query(namespace).offset(1).cached_total() (query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) - .not_op().where_float64('fld2', CondType.CondSet, [3.14])) + .not_op() + .where_float64('fld2', CondType.CondSet, [3.14]) + .explain()) + def rx_example(): db = RxConnector('builtin:///tmp/pyrx') diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 3d7e0ec..4956241 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -93,6 +93,11 @@ class QueryWrapper { query_ = std::move(query_.Strict(mode)); } + void Explain() { + constexpr static bool on = true; + query_ = std::move(query_.Explain(on)); + } + private: DBInterface* db_{nullptr}; reindexer::Query query_; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index cd9c3da..ed3ec5b 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -970,4 +970,17 @@ static PyObject* Strict(PyObject* self, PyObject* args) { Py_RETURN_NONE; } +static PyObject* Explain(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->Explain(); + + Py_RETURN_NONE; +} + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 9ec21be..83329fe 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -68,6 +68,7 @@ static PyObject* Limit(PyObject* self, PyObject* args); static PyObject* Offset(PyObject* self, PyObject* args); static PyObject* Debug(PyObject* self, PyObject* args); static PyObject* Strict(PyObject* self, PyObject* args); +static PyObject* Explain(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { @@ -120,15 +121,14 @@ static PyMethodDef module_methods[] = { {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, {"where_float64", WhereDouble, METH_VARARGS, "add where condition with double args"}, {"log_op", LogOp, METH_VARARGS, "next condition will be added with AND|OR|NOT AND"}, - {"distinct", Distinct, METH_VARARGS, "perform distinct for index"}, {"request_total", ReqTotal, METH_VARARGS, "request total items calculation"}, {"cached_total", CachedTotal, METH_VARARGS, "request cached total items calculation"}, - {"limit", Limit, METH_VARARGS, "request cached total items calculation"}, {"offset", Offset, METH_VARARGS, "request cached total items calculation"}, {"debug", Debug, METH_VARARGS, "request cached total items calculation"}, {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, + {"explain", Explain, METH_VARARGS, "enable explain query"}, {nullptr, nullptr, 0, nullptr} }; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 9cc1d02..f230375 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -455,8 +455,18 @@ def strict(self, mode: StrictMode) -> Query: self.api.strict(self.query_wrapper_ptr, mode.value) return self + def explain(self) -> Query: + """Enable explain query + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.explain(self.query_wrapper_ptr) + return self + ################################################################ ToDo -#func (q *Query) Explain() *Query { #func (q *Query) SetContext(ctx interface{}) *Query { #func (q *Query) Exec() *Iterator { #func (q *Query) ExecCtx(ctx context.Context) *Iterator { @@ -486,4 +496,4 @@ def strict(self, mode: StrictMode) -> Query: #func (q *Query) FetchCount(n int) *Query { #func (q *Query) Functions(fields ...string) *Query { #func (q *Query) EqualPosition(fields ...string) *Query { -# 66 / 21 \ No newline at end of file +# 66 / 22 \ No newline at end of file From cbc34aabbfec553b4805935a9bf185818a8e6a55 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Mon, 28 Oct 2024 15:33:24 +0300 Subject: [PATCH 026/125] Part VI: start of implementation of query builder --- pyreindexer/example/main.py | 1 + pyreindexer/lib/include/query_wrapper.h | 30 ++++++ pyreindexer/lib/src/rawpyreindexer.cc | 132 ++++++++++++++++++++++++ pyreindexer/lib/src/rawpyreindexer.h | 13 ++- pyreindexer/query.py | 117 +++++++++++++++++++-- 5 files changed, 284 insertions(+), 9 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 326ffeb..8ce7364 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -100,6 +100,7 @@ def query_example(db, namespace): .not_op() .where_float64('fld2', CondType.CondSet, [3.14]) .explain()) + query.expression("fld1", "array_remove(integer_array, [5,6,7,8]) || [1,2,3]").drop("fld2") def rx_example(): diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 4956241..fdcf79f 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -98,6 +98,36 @@ class QueryWrapper { query_ = std::move(query_.Explain(on)); } + void Drop(std::string_view index) { + query_ = std::move(query_.Drop(index)); + } + + void SetExpression(std::string_view field, std::string_view value) { + constexpr static bool hasExpressions = true; + query_ = std::move(query_.Set(field, value, hasExpressions)); + } + + void On(std::string_view index, CondType condition, std::string_view joinIndex) { + // ToDo + (void)index; + (void)condition; + (void)joinIndex; + } + + void Select(const std::vector& fields) { + query_ = std::move(query_.Select(fields)); + } + + void AddFunctions(const std::vector& functions) { + for (const auto& function : functions) { + query_.AddFunction(function); + } + } + + void AddEqualPosition(const std::vector& equalPositions) { + query_ = std::move(query_.AddEqualPosition(equalPositions)); + } + private: DBInterface* db_{nullptr}; reindexer::Query query_; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index ed3ec5b..f95a4e7 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -983,4 +983,136 @@ static PyObject* Explain(PyObject* self, PyObject* args) { Py_RETURN_NONE; } +static PyObject* Drop(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &index)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->Drop(index); + + Py_RETURN_NONE; +} + +static PyObject* SetExpression(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* field = nullptr; + char* value = nullptr; + if (!PyArg_ParseTuple(args, "kss", &queryWrapperAddr, &field, &value)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->SetExpression(field, value); + + Py_RETURN_NONE; +} + +static PyObject* On(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + char* joinIndex = nullptr; + if (!PyArg_ParseTuple(args, "ksIs", &queryWrapperAddr, &index, &condition, &joinIndex)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->On(index, CondType(condition), joinIndex); + + Py_RETURN_NONE; +} + +static PyObject* SelectQuery(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &fieldsList)) { + return nullptr; + } + + Py_XINCREF(fieldsList); + + std::vector fields; + if (fieldsList != nullptr) { + try { + fields = ParseListToStrVec(&fieldsList); + } catch (const Error& err) { + Py_DECREF(fieldsList); + + return pyErr(err); + } + } + + Py_XDECREF(fieldsList); + + auto query = getWrapper(queryWrapperAddr); + + query->Select(fields); + + return pyErr(errOK); +} + +static PyObject* AddFunctions(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + PyObject* functionsList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &functionsList)) { + return nullptr; + } + + Py_XINCREF(functionsList); + + std::vector functions; + if (functionsList != nullptr) { + try { + functions = ParseListToStrVec(&functionsList); + } catch (const Error& err) { + Py_DECREF(functionsList); + + return pyErr(err); + } + } + + Py_XDECREF(functionsList); + + auto query = getWrapper(queryWrapperAddr); + + query->AddFunctions(functions); + + return pyErr(errOK); +} + +static PyObject* AddEqualPosition(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + PyObject* equalPosesList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &equalPosesList)) { + return nullptr; + } + + Py_XINCREF(equalPosesList); + + std::vector equalPoses; + if (equalPosesList != nullptr) { + try { + equalPoses = ParseListToStrVec(&equalPosesList); + } catch (const Error& err) { + Py_DECREF(equalPosesList); + + return pyErr(err); + } + } + + Py_XDECREF(equalPosesList); + + auto query = getWrapper(queryWrapperAddr); + + query->AddEqualPosition(equalPoses); + + return pyErr(errOK); +} + } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 83329fe..08cf996 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -63,12 +63,17 @@ static PyObject* LogOp(PyObject* self, PyObject* args); static PyObject* Distinct(PyObject* self, PyObject* args); static PyObject* ReqTotal(PyObject* self, PyObject* args); static PyObject* CachedTotal(PyObject* self, PyObject* args); - static PyObject* Limit(PyObject* self, PyObject* args); static PyObject* Offset(PyObject* self, PyObject* args); static PyObject* Debug(PyObject* self, PyObject* args); static PyObject* Strict(PyObject* self, PyObject* args); static PyObject* Explain(PyObject* self, PyObject* args); +static PyObject* Drop(PyObject* self, PyObject* args); +static PyObject* SetExpression(PyObject* self, PyObject* args); +static PyObject* On(PyObject* self, PyObject* args); +static PyObject* SelectQuery(PyObject* self, PyObject* args); +static PyObject* AddFunctions(PyObject* self, PyObject* args); +static PyObject* AddEqualPosition(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { @@ -129,6 +134,12 @@ static PyMethodDef module_methods[] = { {"debug", Debug, METH_VARARGS, "request cached total items calculation"}, {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, {"explain", Explain, METH_VARARGS, "enable explain query"}, + {"drop", Drop, METH_VARARGS, "drop values"}, + {"expression", SetExpression, METH_VARARGS, "set expression"}, + {"on", On, METH_VARARGS, "on specifies join condition"}, + {"select_query", SelectQuery, METH_VARARGS, "select add filter to fields of result's objects"}, + {"functions", AddFunctions, METH_VARARGS, "add sql-functions to query"}, + {"equal_position", AddEqualPosition, METH_VARARGS, "add equal position fields"}, {nullptr, nullptr, 0, nullptr} }; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index f230375..42279dc 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -445,7 +445,7 @@ def strict(self, mode: StrictMode) -> Query: """Changes strict mode # Arguments: - level (:enum:`StrictMode`): Strict mode + mode (:enum:`StrictMode`): Strict mode # Returns: (:obj:`Query`): Query object for further customizations @@ -476,8 +476,38 @@ def explain(self) -> Query: #func (q *Query) DeleteCtx(ctx context.Context) (int, error) { #func (q *Query) SetObject(field string, values interface{}) *Query { #func (q *Query) Set(field string, values interface{}) *Query { -#func (q *Query) Drop(field string) *Query { -#func (q *Query) SetExpression(field string, value string) *Query { +################################################################ + + def drop(self, index: str) -> Query: + """Drops a value for a field + + # Arguments: + index (string): Field name for drop operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.drop(self.query_wrapper_ptr, index) + return self + + def expression(self, field: str, value: str) -> Query: + """Updates indexed field by arithmetical expression + + # Arguments: + field (string): Field name + value (string): New value expression for field + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.expression(self.query_wrapper_ptr, field, value) + return self + +################################################################ // ToDo #func (q *Query) Update() *Iterator { #func (q *Query) UpdateCtx(ctx context.Context) *Iterator { #func (q *Query) MustExec() *Iterator { @@ -491,9 +521,80 @@ def explain(self) -> Query: #func (q *Query) LeftJoin(q2 *Query, field string) *Query { #func (q *Query) JoinHandler(field string, handler JoinHandler) *Query { #func (q *Query) Merge(q2 *Query) *Query { -#func (q *Query) On(index string, condition int, joinIndex string) *Query { -#func (q *Query) Select(fields ...string) *Query { +################################################################ + + def on(self, index: str, condition: CondType, join_index: str) -> Query: + """On specifies join condition. + + # Arguments: + index (string): Field name from `Query` namespace should be used during join + condition (:enum:`CondType`): Type of condition, specifies how `Query` will be joined with the latest join query issued on `Query` (e.g. `EQ`/`GT`/`SET`/...) + join_index (string): Index-field name from namespace for the latest join query issued on `Query` should be used during join + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.on(self.query_wrapper_ptr, index, condition, join_index) + return self + + def select(self, fields: List[str]) -> Query: + """Sets list of columns in this namespace to be finally selected. + The columns should be specified in the same case as the jsonpaths corresponding to them. + Non-existent fields and fields in the wrong case are ignored. + If there are no fields in this list that meet these conditions, then the filter works as "*". + + # Arguments: + fields (list[string]): List of columns to be selected + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ + + self.err_code, self.err_msg = self.api.select_query(self.query_wrapper_ptr, fields) + self._raise_on_error() + return self + +################################################################ // ToDo #func (q *Query) FetchCount(n int) *Query { -#func (q *Query) Functions(fields ...string) *Query { -#func (q *Query) EqualPosition(fields ...string) *Query { -# 66 / 22 \ No newline at end of file +################################################################ + + def functions(self, functions: List[str]) -> Query: + """Adds sql-functions to query + + # Arguments: + functions (list[string]): Functions declaration + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ + + self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, functions) + self._raise_on_error() + return self + + def equal_position(self, equal_position: List[str]) -> Query: + """Adds equal position fields to arrays queries + + # Arguments: + equal_poses (list[string]): Equal position fields to arrays queries + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ + + self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) + self._raise_on_error() + return self + +# 66 / 28 \ No newline at end of file From cd35c307ccff565eaf1eeaca42a4c1f03fe18e04 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Mon, 28 Oct 2024 22:10:27 +0300 Subject: [PATCH 027/125] Add commit_with_count() --- pyreindexer/lib/include/transaction_wrapper.h | 6 +++-- pyreindexer/lib/src/rawpyreindexer.cc | 24 +++++++++++++++---- pyreindexer/lib/src/rawpyreindexer.h | 4 ++-- pyreindexer/lib/src/reindexerinterface.cc | 14 +++++------ pyreindexer/lib/src/reindexerinterface.h | 6 ++--- pyreindexer/transaction.py | 21 +++++++++++++--- 6 files changed, 52 insertions(+), 23 deletions(-) diff --git a/pyreindexer/lib/include/transaction_wrapper.h b/pyreindexer/lib/include/transaction_wrapper.h index 9c7ce71..cc0a9c3 100644 --- a/pyreindexer/lib/include/transaction_wrapper.h +++ b/pyreindexer/lib/include/transaction_wrapper.h @@ -13,10 +13,12 @@ namespace pyreindexer { #ifdef PYREINDEXER_CPROTO using DBInterface = ReindexerInterface; using TransactionT = reindexer::client::CoroTransaction; +using QueryResultsT = reindexer::client::CoroQueryResults; using ItemT = reindexer::client::Item; #else using DBInterface = ReindexerInterface; using TransactionT = reindexer::Transaction; +using QueryResultsT = reindexer::QueryResults; using ItemT = reindexer::Item; #endif @@ -45,9 +47,9 @@ class TransactionWrapper { return db_->Modify(transaction_, std::move(item), mode); } - Error Commit() { + Error Commit(size_t& count) { assert(wrap_); - return db_->CommitTransaction(transaction_); + return db_->CommitTransaction(transaction_, count); } Error Rollback() { diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index c488b04..cbf0020 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -558,8 +558,7 @@ static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args) { return static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpsert); } static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeDelete); } -enum class StopTransactionMode : bool { Rollback = false, Commit = true }; -static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransactionMode stopMode) { +static PyObject* CommitTransaction(PyObject* self, PyObject* args) { uintptr_t transactionWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { return nullptr; @@ -568,13 +567,28 @@ static PyObject* stopTransaction(PyObject* self, PyObject* args, StopTransaction auto transaction = getWrapper(transactionWrapperAddr); assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); - Error err = (StopTransactionMode::Commit == stopMode) ? transaction->Commit() : transaction->Rollback(); + size_t count = 0; + Error err = transaction->CommitTransaction(count); + + wrapperDelete(transactionWrapperAddr); + + return Py_BuildValue("isI", err.code(), err.what().c_str(), count); +} + +static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { + uintptr_t transactionWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { + return nullptr; + } + + auto transaction = getWrapper(transactionWrapperAddr); + + assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); + Error err = transaction->Rollback(); wrapperDelete(transactionWrapperAddr); return pyErr(err); } -static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Commit); } -static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return stopTransaction(self, args, StopTransactionMode::Rollback); } } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index d24379c..d5b988f 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -77,8 +77,8 @@ static PyMethodDef module_methods[] = { {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, {"item_upsert_transaction", ItemUpsertTransaction, METH_VARARGS, "item upsert transaction"}, {"item_delete_transaction", ItemDeleteTransaction, METH_VARARGS, "item delete transaction"}, - {"commit_transaction", CommitTransaction, METH_VARARGS, "commit transaction. Free transaction object memory"}, - {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback transaction. Free transaction object memory"}, + {"commit_transaction", CommitTransaction, METH_VARARGS, "apply changes. Free transaction object memory"}, + {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback changes. Free transaction object memory"}, {nullptr, nullptr, 0, nullptr} }; diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 0c1fe6d..e905245 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -92,14 +92,12 @@ Error ReindexerInterface::modify(reindexer::cl return transaction.Modify(std::move(item), mode); } -template <> -Error ReindexerInterface::commitTransaction(reindexer::Transaction& transaction) { - reindexer::QueryResults resultDummy; - return db_.CommitTransaction(transaction, resultDummy); -} -template <> -Error ReindexerInterface::commitTransaction(reindexer::client::CoroTransaction& transaction) { - return db_.CommitTransaction(transaction); +template +Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& transaction, size_t& count) { + typename DBT::QueryResultsT qr; + auto err = db_.CommitTransaction(transaction, qr); + count = qr.Count(); + return err; } template <> diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index e696544..aded2ab 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -122,8 +122,8 @@ class ReindexerInterface { Error Modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode) { return execute([this, &tr, &item, mode] { return modify(tr, std::move(item), mode); }); } - Error CommitTransaction(typename DBT::TransactionT& tr) { - return execute([this, &tr] { return commitTransaction(tr); }); + Error CommitTransaction(typename DBT::TransactionT& tr, size_t& count) { + return execute([this, &tr, &count] { return commitTransaction(tr, count); }); } Error RollbackTransaction(typename DBT::TransactionT& tr) { return execute([this, &tr] { return rollbackTransaction(tr); }); @@ -153,7 +153,7 @@ class ReindexerInterface { typename DBT::TransactionT startTransaction(std::string_view ns) { return db_.NewTransaction({ns.data(), ns.size()}); } typename DBT::ItemT newItem(typename DBT::TransactionT& tr) { return tr.NewItem(); } Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); - Error commitTransaction(typename DBT::TransactionT& tr); + Error commitTransaction(typename DBT::TransactionT& transaction, size_t& count); Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } Error stop(); diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 91489be..202e99b 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -127,7 +127,7 @@ def delete(self, item_def): self._raise_on_error() def commit(self): - """Commit a transaction + """Apply changes # Raises: Exception: Raises with an error message of API return if Transaction is over @@ -136,12 +136,27 @@ def commit(self): """ self._raise_on_is_over() - self.err_code, self.err_msg = self.api.commit_transaction(self.transaction_wrapper_ptr) + self.err_code, self.err_msg, _ = self.api.commit_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 self._raise_on_error() + def commit_with_count(self) -> int: + """Apply changes and return the number of count of changed items + + # Raises: + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + """ + + self._raise_on_is_over() + self.err_code, self.err_msg, count = self.api.commit_transaction(self.transaction_wrapper_ptr) + self.transaction_wrapper_ptr = 0 + self._raise_on_error() + return count + def rollback(self): - """Roll back a transaction + """Rollback changes # Raises: Exception: Raises with an error message of API return if Transaction is over From 8f7542d63e3b027dda84c7d898b4baa70b327618 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 29 Oct 2024 11:22:01 +0300 Subject: [PATCH 028/125] Add commit_with_count(). Fix code and check in test --- pyreindexer/lib/src/rawpyreindexer.cc | 2 +- pyreindexer/tests/tests/test_transaction.py | 3 ++- pyreindexer/transaction.py | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index cbf0020..ab89279 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -568,7 +568,7 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args) { assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); size_t count = 0; - Error err = transaction->CommitTransaction(count); + Error err = transaction->Commit(count); wrapperDelete(transactionWrapperAddr); diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index 39d2642..db7181e 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -135,7 +135,8 @@ def test_create_item_insert_with_precepts(self, namespace, index): number_items = 5 for i in range(number_items): transaction.insert({'id': 100, 'field': 'value' + str(100 + i)}, ['id=serial()']) - transaction.commit() + count = transaction.commit_with_count() + assert_that(count, equal_to(number_items), "Transaction: items wasn't created") # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) assert_that(select_result, has_length(number_items), "Transaction: items wasn't created") diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 202e99b..4ef4e36 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -149,11 +149,11 @@ def commit_with_count(self) -> int: """ - self._raise_on_is_over() - self.err_code, self.err_msg, count = self.api.commit_transaction(self.transaction_wrapper_ptr) - self.transaction_wrapper_ptr = 0 - self._raise_on_error() - return count + self._raise_on_is_over() + self.err_code, self.err_msg, count = self.api.commit_transaction(self.transaction_wrapper_ptr) + self.transaction_wrapper_ptr = 0 + self._raise_on_error() + return count def rollback(self): """Rollback changes From c9070b4e61138fb9124489cbaad558ca590f3196 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 29 Oct 2024 22:25:42 +0300 Subject: [PATCH 029/125] Rewrite query_wrapper --- pyreindexer/example/main.py | 5 +- pyreindexer/lib/include/query_wrapper.h | 208 +++++++++++++++++------- pyreindexer/lib/src/rawpyreindexer.cc | 130 ++++++++------- pyreindexer/lib/src/rawpyreindexer.h | 14 +- pyreindexer/query.py | 74 ++++++--- 5 files changed, 280 insertions(+), 151 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 8ce7364..dfb2d8c 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -97,9 +97,10 @@ def query_example(db, namespace): query = db.new_query(namespace).offset(1).cached_total() (query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) - .not_op() + .op_not() .where_float64('fld2', CondType.CondSet, [3.14]) - .explain()) + .explain() + .fetch_count(10)) query.expression("fld1", "array_remove(integer_array, [5,6,7,8]) || [1,2,3]").drop("fld2") diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index fdcf79f..65acdec 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -1,136 +1,224 @@ #pragma once +#include +#include #include "core/type_consts.h" -#include "core/query/query.h" #include "core/keyvalue/uuid.h" +#include "core/keyvalue/variant.h" +#include "tools/serializer.h" +#include "tools/errors.h" namespace pyreindexer { -enum class LogOpType { - And = 1, - Or = 2, - Not = 3 -}; +namespace { + const int VALUE_INT_64 = 0; + const int VALUE_DOUBLE = 1; + const int VALUE_STRING = 2; + const int VALUE_BOOL = 3; + const int VALUE_NULL = 4; + const int VALUE_INT = 8; + const int VALUE_UNDEFINED = 9; + const int VALUE_COMPOSITE = 10; + const int VALUE_TUPLE = 11; + const int VALUE_UUID = 12; +} class QueryWrapper { public: - QueryWrapper(DBInterface* db, std::string_view ns) : db_{db}, query_(ns) { + QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { assert(db_); + ser_.PutVString(ns); } void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { - query_ = std::move(query_.WhereBetweenFields(firstField, condition, secondField)); + ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(firstField); + ser_.PutVarUint(condition); + ser_.PutVString(secondField); + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + } + + Error OpenBracket() { + ser_.PutVarUint(QueryItemType::QueryOpenBracket); + ser_.PutVarUint(nextOperation_); + nextOperation_ = OpType::OpAnd; + openedBrackets_.push_back(queriesCount_); + ++queriesCount_; + return errOK; } - void OpenBracket() { - query_ = std::move(query_.OpenBracket()); - } + Error CloseBracket() { + if (nextOperation_ != OpType::OpAnd) { + return Error(errLogic, "Operation before close bracket"); + } - void CloseBracket() { - query_ = std::move(query_.CloseBracket()); - } + if (openedBrackets_.empty()) { + return Error(errLogic, "Close bracket before open it"); + } - template - void Where(std::string_view index, CondType condition, const std::vector& keys) { - query_ = std::move(query_.Where(index, condition, keys)); + ser_.PutVarUint(QueryItemType::QueryCloseBracket); + openedBrackets_.pop_back(); + return errOK; } - Error WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { - try { - for (const auto& key : keys) { + template // ToDo -------------------------------------------------------------- + void Where(std::string_view index, CondType condition, const std::vector& keys) { + (void)index; + (void)condition; + (void)keys; + } //--------------------------------------------------------- + + void WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QueryCondition); + ser_.PutVString(index); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + try { auto uuid = reindexer::Uuid(key); - (void)uuid; // check format only + ser_.PutVarUint(VALUE_UUID); + ser_.PutUuid(uuid); + } catch (const Error& err) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(key); } - } catch (const Error& err) { - return err; } - - query_ = std::move(query_.Where(index, condition, keys)); - return errOK; } - void LogOp(LogOpType op) { + void LogOp(OpType op) { switch (op) { - case LogOpType::And: - query_ = std::move(query_.And()); - break; - case LogOpType::Or: - query_ = std::move(query_.Or()); - break; - case LogOpType::Not: - query_ = std::move(query_.Not()); + case OpType::OpAnd: + case OpType::OpOr: + case OpType::OpNot: break; default: assert(false); } + nextOperation_ = op; } - void Distinct(const std::string& index) { - query_ = std::move(query_.Distinct(index)); + void Distinct(std::string_view index) { + ser_.PutVarUint(QueryItemType::QueryAggregation); + ser_.PutVarUint(AggType::AggDistinct); + ser_.PutVarUint(1); + ser_.PutVString(index); } - void ReqTotal() { - query_ = std::move(query_.ReqTotal()); + void ReqTotal(std::string_view totalName) { + ser_.PutVarUint(QueryItemType::QueryReqTotal); + ser_.PutVarUint(CalcTotalMode::ModeAccurateTotal); + if (!totalName.empty()) { + totalName_ = totalName; + } } - void CachedTotal() { - query_ = std::move(query_.CachedTotal()); + void CachedTotal(std::string_view totalName) { + ser_.PutVarUint(QueryItemType::QueryReqTotal); + ser_.PutVarUint(CalcTotalMode::ModeCachedTotal); + if (!totalName.empty()) { + totalName_ = totalName; + } } void Limit(unsigned limitItems) { - query_ = std::move(query_.Limit(limitItems)); + int limit = std::min(std::numeric_limits::max(), limitItems); + ser_.PutVarUint(QueryItemType::QueryLimit); + ser_.PutVarUint(limit); } void Offset(int startOffset) { - query_ = std::move(query_.Offset(startOffset)); + int offset = std::min(std::numeric_limits::max(), startOffset); + ser_.PutVarUint(QueryItemType::QueryOffset); + ser_.PutVarUint(offset); } void Debug(int level) { - query_ = std::move(query_.Debug(level)); + ser_.PutVarUint(QueryItemType::QueryDebugLevel); + ser_.PutVarUint(level); } void Strict(StrictMode mode) { - query_ = std::move(query_.Strict(mode)); + ser_.PutVarUint(QueryItemType::QueryStrictMode); + ser_.PutVarUint(mode); } - void Explain() { - constexpr static bool on = true; - query_ = std::move(query_.Explain(on)); + void Modifier(QueryItemType type) { + ser_.PutVarUint(type); } - void Drop(std::string_view index) { - query_ = std::move(query_.Drop(index)); + void Drop(std::string_view field) { + ser_.PutVarUint(QueryItemType::QueryDropField); + ser_.PutVString(field); } void SetExpression(std::string_view field, std::string_view value) { - constexpr static bool hasExpressions = true; - query_ = std::move(query_.Set(field, value, hasExpressions)); + ser_.PutVarUint(QueryItemType::QueryUpdateField); + ser_.PutVString(field); + + ser_.PutVarUint(1); // size + ser_.PutVarUint(1); // is expression + ser_.PutVString(value); // ToDo q.putValue(value); } - void On(std::string_view index, CondType condition, std::string_view joinIndex) { + Error On(std::string_view joinField, CondType condition, std::string_view joinIndex) { // ToDo - (void)index; - (void)condition; - (void)joinIndex; + /*if q.closed { + q.panicTrace("query.On call on already closed query. You should create new Query") + } + if q.root == nil { + panic(fmt.Errorf("Can't join on root query")) + }*/ + ser_.PutVarUint(QueryItemType::QueryJoinOn); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + ser_.PutVString(joinField); + ser_.PutVString(joinIndex); + nextOperation_ = OpType::OpAnd; + return errOK; } void Select(const std::vector& fields) { - query_ = std::move(query_.Select(fields)); + for (const auto& field : fields) { + ser_.PutVarUint(QueryItemType::QuerySelectFilter); + ser_.PutVString(field); + } + } + + void FetchCount(int count) { + fetchCount_ = count; } void AddFunctions(const std::vector& functions) { for (const auto& function : functions) { - query_.AddFunction(function); + ser_.PutVarUint(QueryItemType::QuerySelectFunction); + ser_.PutVString(function); } } void AddEqualPosition(const std::vector& equalPositions) { - query_ = std::move(query_.AddEqualPosition(equalPositions)); + ser_.PutVarUint(QueryItemType::QueryEqualPosition); + ser_.PutVarUint(openedBrackets_.empty()? 0 : int(openedBrackets_.back() + 1)); + ser_.PutVarUint(equalPositions.size()); + for (const auto& position : equalPositions) { + ser_.PutVString(position); + } } private: DBInterface* db_{nullptr}; - reindexer::Query query_; + reindexer::WrSerializer ser_; + + OpType nextOperation_{OpType::OpAnd}; + unsigned queriesCount_{0}; + std::deque openedBrackets_; + std::string totalName_; + int fetchCount_{0}; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 650486e..b7ac37f 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -89,7 +89,7 @@ static PyObject* Connect(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->Connect(dsn); + auto err = getDB(rx)->Connect(dsn); return pyErr(err); } @@ -100,7 +100,7 @@ static PyObject* NamespaceOpen(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->OpenNamespace(ns); + auto err = getDB(rx)->OpenNamespace(ns); return pyErr(err); } @@ -111,7 +111,7 @@ static PyObject* NamespaceClose(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->CloseNamespace(ns); + auto err = getDB(rx)->CloseNamespace(ns); return pyErr(err); } @@ -122,7 +122,7 @@ static PyObject* NamespaceDrop(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->DropNamespace(ns); + auto err = getDB(rx)->DropNamespace(ns); return pyErr(err); } @@ -149,7 +149,7 @@ static PyObject* IndexAdd(PyObject* self, PyObject* args) { Py_DECREF(indexDefDict); IndexDef indexDef; - Error err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); + auto err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); if (err.ok()) { err = getDB(rx)->AddIndex(ns, indexDef); } @@ -179,7 +179,7 @@ static PyObject* IndexUpdate(PyObject* self, PyObject* args) { Py_DECREF(indexDefDict); IndexDef indexDef; - Error err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); + auto err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); if (err.ok()) { err = getDB(rx)->UpdateIndex(ns, indexDef); } @@ -193,7 +193,7 @@ static PyObject* IndexDrop(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->DropIndex(ns, IndexDef(indexName)); + auto err = getDB(rx)->DropIndex(ns, IndexDef(indexName)); return pyErr(err); } @@ -210,7 +210,7 @@ static PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) Py_XINCREF(preceptsList); auto item = getDB(rx)->NewItem(ns); - Error err = item.Status(); + auto err = item.Status(); if (!err.ok()) { return pyErr(err); } @@ -285,7 +285,7 @@ static PyObject* PutMeta(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->PutMeta(ns, key, value); + auto err = getDB(rx)->PutMeta(ns, key, value); return pyErr(err); } @@ -297,7 +297,7 @@ static PyObject* GetMeta(PyObject* self, PyObject* args) { } std::string value; - Error err = getDB(rx)->GetMeta(ns, key, value); + auto err = getDB(rx)->GetMeta(ns, key, value); return Py_BuildValue("iss", err.code(), err.what().c_str(), value.c_str()); } @@ -308,7 +308,7 @@ static PyObject* DeleteMeta(PyObject* self, PyObject* args) { return nullptr; } - Error err = getDB(rx)->DeleteMeta(ns, key); + auto err = getDB(rx)->DeleteMeta(ns, key); return pyErr(err); } @@ -321,7 +321,7 @@ static PyObject* Select(PyObject* self, PyObject* args) { auto db = getDB(rx); auto qresWrapper = new QueryResultsWrapper(db); - Error err = qresWrapper->Select(query); + auto err = qresWrapper->Select(query); if (!err.ok()) { delete qresWrapper; @@ -340,7 +340,7 @@ static PyObject* EnumMeta(PyObject* self, PyObject* args) { } std::vector keys; - Error err = getDB(rx)->EnumMeta(ns, keys); + auto err = getDB(rx)->EnumMeta(ns, keys); if (!err.ok()) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); } @@ -371,7 +371,7 @@ static PyObject* EnumNamespaces(PyObject* self, PyObject* args) { } std::vector nsDefs; - Error err = getDB(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); + auto err = getDB(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); if (!err.ok()) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); } @@ -471,7 +471,7 @@ static PyObject* NewTransaction(PyObject* self, PyObject* args) { auto db = getDB(rx); auto transaction = new TransactionWrapper(db); - Error err = transaction->Start(ns); + auto err = transaction->Start(ns); if (!err.ok()) { delete transaction; @@ -495,7 +495,7 @@ static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModif auto transaction = getWrapper(transactionWrapperAddr); auto item = transaction->NewItem(); - Error err = item.Status(); + auto err = item.Status(); if (!err.ok()) { Py_DECREF(itemDefDict); Py_XDECREF(preceptsList); @@ -569,7 +569,7 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args) { assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); size_t count = 0; - Error err = transaction->Commit(count); + auto err = transaction->Commit(count); wrapperDelete(transactionWrapperAddr); @@ -585,7 +585,7 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { auto transaction = getWrapper(transactionWrapperAddr); assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); - Error err = transaction->Rollback(); + auto err = transaction->Rollback(); wrapperDelete(transactionWrapperAddr); @@ -632,7 +632,8 @@ static PyObject* WhereBetweenFields(PyObject* self, PyObject* args) { Py_RETURN_NONE; } -static PyObject* OpenBracket(PyObject* self, PyObject* args) { +enum class BracketType { Open, Closed }; +static PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -640,23 +641,11 @@ static PyObject* OpenBracket(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->OpenBracket(); - - Py_RETURN_NONE; -} - -static PyObject* CloseBracket(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->CloseBracket(); - - Py_RETURN_NONE; + auto err = (type == BracketType::Open)? query->OpenBracket() : query->CloseBracket(); + return pyErr(err); } +static PyObject* OpenBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Open); } +static PyObject* CloseBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Closed); } static PyObject* WhereInt(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -684,7 +673,7 @@ static PyObject* WhereInt(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); + query->Where(index, CondType(condition), keys); // ToDo return pyErr(errOK); } @@ -715,7 +704,7 @@ static PyObject* WhereInt32(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); + query->Where(index, CondType(condition), keys); // ToDo return pyErr(errOK); } @@ -746,7 +735,7 @@ static PyObject* WhereInt64(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); + query->Where(index, CondType(condition), keys); // ToDo return pyErr(errOK); } @@ -777,7 +766,7 @@ static PyObject* WhereString(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); + query->Where(index, CondType(condition), keys); // ToDo return pyErr(errOK); } @@ -808,8 +797,9 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - auto err = query->WhereUUID(index, CondType(condition), keys); - return pyErr(err); + query->WhereUUID(index, CondType(condition), keys); + + return pyErr(errOK); } static PyObject* WhereBool(PyObject* self, PyObject* args) { @@ -874,19 +864,21 @@ static PyObject* WhereDouble(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* LogOp(PyObject* self, PyObject* args) { +static PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { uintptr_t queryWrapperAddr = 0; - unsigned opID = 0; - if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &opID)) { + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->LogOp(LogOpType(opID)); + query->LogOp(opID); Py_RETURN_NONE; } +static PyObject* And(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpAnd); } +static PyObject* Or(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpOr); } +static PyObject* Not(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpNot); } static PyObject* Distinct(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -904,26 +896,28 @@ static PyObject* Distinct(PyObject* self, PyObject* args) { static PyObject* ReqTotal(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + char* totalName = nullptr; + if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &totalName)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->ReqTotal(); + query->ReqTotal(totalName); Py_RETURN_NONE; } static PyObject* CachedTotal(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + char* totalName = nullptr; + if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &totalName)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->CachedTotal(); + query->CachedTotal(totalName); Py_RETURN_NONE; } @@ -958,8 +952,8 @@ static PyObject* Offset(PyObject* self, PyObject* args) { static PyObject* Debug(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - unsigned level = 0; - if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &level)) { + int level = 0; + if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &level)) { return nullptr; } @@ -984,18 +978,15 @@ static PyObject* Strict(PyObject* self, PyObject* args) { Py_RETURN_NONE; } -static PyObject* Explain(PyObject* self, PyObject* args) { +static void modifier(PyObject* self, PyObject* args, QueryItemType type) { uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { - return nullptr; + if (PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + auto query = getWrapper(queryWrapperAddr); + query->Modifier(type); } - - auto query = getWrapper(queryWrapperAddr); - - query->Explain(); - - Py_RETURN_NONE; } +static PyObject* Explain(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryExplain); Py_RETURN_NONE; } +static PyObject* WithRank(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryWithRank); Py_RETURN_NONE; } static PyObject* Drop(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -1037,9 +1028,8 @@ static PyObject* On(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->On(index, CondType(condition), joinIndex); - - Py_RETURN_NONE; + auto err = query->On(index, CondType(condition), joinIndex); + return pyErr(err); } static PyObject* SelectQuery(PyObject* self, PyObject* args) { @@ -1071,6 +1061,20 @@ static PyObject* SelectQuery(PyObject* self, PyObject* args) { return pyErr(errOK); } +static PyObject* FetchCount(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + int count = 0; + if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &count)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->FetchCount(count); + + Py_RETURN_NONE; +} + static PyObject* AddFunctions(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; PyObject* functionsList = nullptr; // borrowed ref after ParseTuple if passed diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 7c259d1..813dba7 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -59,7 +59,9 @@ static PyObject* WhereString(PyObject* self, PyObject* args); static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* WhereBool(PyObject* self, PyObject* args); static PyObject* WhereDouble(PyObject* self, PyObject* args); -static PyObject* LogOp(PyObject* self, PyObject* args); +static PyObject* And(PyObject* self, PyObject* args); +static PyObject* Or(PyObject* self, PyObject* args); +static PyObject* Not(PyObject* self, PyObject* args); static PyObject* Distinct(PyObject* self, PyObject* args); static PyObject* ReqTotal(PyObject* self, PyObject* args); static PyObject* CachedTotal(PyObject* self, PyObject* args); @@ -68,10 +70,12 @@ static PyObject* Offset(PyObject* self, PyObject* args); static PyObject* Debug(PyObject* self, PyObject* args); static PyObject* Strict(PyObject* self, PyObject* args); static PyObject* Explain(PyObject* self, PyObject* args); +static PyObject* WithRank(PyObject* self, PyObject* args); static PyObject* Drop(PyObject* self, PyObject* args); static PyObject* SetExpression(PyObject* self, PyObject* args); static PyObject* On(PyObject* self, PyObject* args); static PyObject* SelectQuery(PyObject* self, PyObject* args); +static PyObject* FetchCount(PyObject* self, PyObject* args); static PyObject* AddFunctions(PyObject* self, PyObject* args); static PyObject* AddEqualPosition(PyObject* self, PyObject* args); @@ -111,10 +115,8 @@ static PyMethodDef module_methods[] = { // query {"create_query", CreateQuery, METH_VARARGS, "create new query"}, {"delete_query", DeleteQuery, METH_VARARGS, "delete query. Free query object memory"}, - //{"where", Where, METH_VARARGS, "add where condition"}, //{"where_query", WhereQuery, METH_VARARGS, "add where condition"}, - {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, @@ -125,7 +127,9 @@ static PyMethodDef module_methods[] = { {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, {"where_float64", WhereDouble, METH_VARARGS, "add where condition with double args"}, - {"log_op", LogOp, METH_VARARGS, "next condition will be added with AND|OR|NOT AND"}, + {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, + {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, + {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, {"distinct", Distinct, METH_VARARGS, "perform distinct for index"}, {"request_total", ReqTotal, METH_VARARGS, "request total items calculation"}, {"cached_total", CachedTotal, METH_VARARGS, "request cached total items calculation"}, @@ -134,10 +138,12 @@ static PyMethodDef module_methods[] = { {"debug", Debug, METH_VARARGS, "request cached total items calculation"}, {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, {"explain", Explain, METH_VARARGS, "enable explain query"}, + {"with_rank", WithRank, METH_VARARGS, "enable fulltext rank"}, {"drop", Drop, METH_VARARGS, "drop values"}, {"expression", SetExpression, METH_VARARGS, "set expression"}, {"on", On, METH_VARARGS, "on specifies join condition"}, {"select_query", SelectQuery, METH_VARARGS, "select add filter to fields of result's objects"}, + {"fetch_count", FetchCount, METH_VARARGS, "limit number of items"}, {"functions", AddFunctions, METH_VARARGS, "add sql-functions to query"}, {"equal_position", AddEqualPosition, METH_VARARGS, "add equal position fields"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 42279dc..94c5fa6 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -32,10 +32,6 @@ class Query(object): err_msg (string): the API error message """ - class _LogOp(Enum): - And = 1 - Or = 2 - Not = 3 def __init__(self, api, query_wrapper_ptr: int): """Constructs a new Reindexer query object @@ -97,9 +93,13 @@ def open_bracket(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ - self.api.open_bracket(self.query_wrapper_ptr) + self.err_code, self.err_msg = self.api.open_bracket(self.query_wrapper_ptr) + self._raise_on_error() return self def close_bracket(self) -> Query: @@ -108,9 +108,13 @@ def close_bracket(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ - self.api.close_bracket(self.query_wrapper_ptr) + self.err_code, self.err_msg = self.api.close_bracket(self.query_wrapper_ptr) + self._raise_on_error() return self def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: @@ -322,7 +326,7 @@ def match(self, index: str, keys: List[str]) -> Query: # return self.sort(request, desc) ################################################################ - def and_op(self) -> Query: + def op_and(self) -> Query: """Next condition will be added with AND. This is the default operation for WHERE statement. Do not have to be called explicitly in user's code. Used in DSL conversion @@ -331,10 +335,10 @@ def and_op(self) -> Query: (:obj:`Query`): Query object for further customizations """ - self.api.log_op(self.query_wrapper_ptr, self._LogOp.And.value) + self.api.op_and(self.query_wrapper_ptr) return self - def or_op(self) -> Query: + def op_or(self) -> Query: """Next condition will be added with OR. Implements short-circuiting: if the previous condition is successful the next will not be evaluated, but except Join conditions @@ -344,10 +348,10 @@ def or_op(self) -> Query: """ - self.api.log_op(self.query_wrapper_ptr, self._LogOp.Or.value) + self.api.op_or(self.query_wrapper_ptr) return self - def not_op(self) -> Query: + def op_not(self) -> Query: """Next condition will be added with NOT AND. Implements short-circuiting: if the previous condition is failed the next will not be evaluated @@ -356,7 +360,7 @@ def not_op(self) -> Query: """ - self.api.log_op(self.query_wrapper_ptr, self._LogOp.Not.value) + self.api.op_not(self.query_wrapper_ptr) return self def distinct(self, index: str) -> Query: @@ -374,28 +378,32 @@ def distinct(self, index: str) -> Query: self.api.distinct(self.query_wrapper_ptr, index) return self - def request_total(self) -> Query: + def request_total(self, total_name: str= '') -> Query: """Request total items calculation + # Arguments: + total_name (string): Name to be requested + # Returns: (:obj:`Query`): Query object for further customizations """ - # q.totalName = totalNames[0] // ToDo - self.api.request_total(self.query_wrapper_ptr) + self.api.request_total(self.query_wrapper_ptr, total_name) return self - def cached_total(self) -> Query: + def cached_total(self, total_name: str= '') -> Query: """Request cached total items calculation + # Arguments: + total_name (string): Name to be requested + # Returns: (:obj:`Query`): Query object for further customizations """ - # q.totalName = totalNames[0] // ToDo - self.api.cached_total(self.query_wrapper_ptr) + self.api.cached_total(self.query_wrapper_ptr, total_name) return self def limit(self, limit_items: int) -> Query: @@ -466,6 +474,17 @@ def explain(self) -> Query: self.api.explain(self.query_wrapper_ptr) return self + def with_rank(self) -> Query: + """Output fulltext rank. Allowed only with fulltext query + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.with_rank(self.query_wrapper_ptr) + return self + ################################################################ ToDo #func (q *Query) SetContext(ctx interface{}) *Query { #func (q *Query) Exec() *Iterator { @@ -559,9 +578,20 @@ def select(self, fields: List[str]) -> Query: self._raise_on_error() return self -################################################################ // ToDo -#func (q *Query) FetchCount(n int) *Query { -################################################################ + def fetch_count(self, n: int) -> Query: + """Sets the number of items that will be fetched by one operation. + When n <= 0 query will fetch all results in one operation + + # Arguments: + n (int): Number of items + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.fetch_count(self.query_wrapper_ptr, n) + return self def functions(self, functions: List[str]) -> Query: """Adds sql-functions to query @@ -597,4 +627,4 @@ def equal_position(self, equal_position: List[str]) -> Query: self._raise_on_error() return self -# 66 / 28 \ No newline at end of file +# 66 / 30 \ No newline at end of file From 30b880611d6d07fc6fd15903c25b5be4e0ef3429 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 30 Oct 2024 14:27:38 +0300 Subject: [PATCH 030/125] Rewrite query_wrapper. Part II --- pyreindexer/CMakeLists.txt | 2 +- pyreindexer/example/main.py | 2 +- pyreindexer/lib/include/query_wrapper.cc | 255 +++++++++++++++++++++++ pyreindexer/lib/include/query_wrapper.h | 212 ++++--------------- pyreindexer/lib/src/rawpyreindexer.cc | 4 +- pyreindexer/lib/src/rawpyreindexer.h | 2 +- pyreindexer/query.py | 34 ++- 7 files changed, 328 insertions(+), 183 deletions(-) create mode 100644 pyreindexer/lib/include/query_wrapper.cc diff --git a/pyreindexer/CMakeLists.txt b/pyreindexer/CMakeLists.txt index c1939b5..7ac89e4 100644 --- a/pyreindexer/CMakeLists.txt +++ b/pyreindexer/CMakeLists.txt @@ -5,7 +5,7 @@ project(pyreindexer) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE "RelWithDebInfo") + set(CMAKE_BUILD_TYPE "RelWithDebInfo") endif() enable_testing() diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index dfb2d8c..d5f12e4 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -98,7 +98,7 @@ def query_example(db, namespace): query = db.new_query(namespace).offset(1).cached_total() (query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) .op_not() - .where_float64('fld2', CondType.CondSet, [3.14]) + .where_float32('fld2', CondType.CondSet, [3.14]) .explain() .fetch_count(10)) query.expression("fld1", "array_remove(integer_array, [5,6,7,8]) || [1,2,3]").drop("fld2") diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc new file mode 100644 index 0000000..095800d --- /dev/null +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -0,0 +1,255 @@ +#include "query_wrapper.h" + +#include + +#include "core/keyvalue/uuid.h" + +namespace pyreindexer { + +namespace { + const int VALUE_INT_64 = 0; + const int VALUE_DOUBLE = 1; + const int VALUE_STRING = 2; + const int VALUE_BOOL = 3; + const int VALUE_NULL = 4; + const int VALUE_INT = 8; + const int VALUE_UNDEFINED = 9; + const int VALUE_COMPOSITE = 10; + const int VALUE_TUPLE = 11; + const int VALUE_UUID = 12; +} + + QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { + assert(db_); + ser_.PutVString(ns); + } + + void QueryWrapper::WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { + ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(firstField); + ser_.PutVarUint(condition); + ser_.PutVString(secondField); + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + } + + Error QueryWrapper::OpenBracket() { + ser_.PutVarUint(QueryItemType::QueryOpenBracket); + ser_.PutVarUint(nextOperation_); + nextOperation_ = OpType::OpAnd; + openedBrackets_.push_back(queriesCount_); + ++queriesCount_; + return errOK; + } + + reindexer::Error QueryWrapper::CloseBracket() { + if (nextOperation_ != OpType::OpAnd) { + return reindexer::Error(errLogic, "Operation before close bracket"); + } + + if (openedBrackets_.empty()) { + return reindexer::Error(errLogic, "Close bracket before open it"); + } + + ser_.PutVarUint(QueryItemType::QueryCloseBracket); + openedBrackets_.pop_back(); + return errOK; + } + + void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QueryCondition); + ser_.PutVString(index); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + try { + auto uuid = reindexer::Uuid(key); + ser_.PutVarUint(VALUE_UUID); + ser_.PutUuid(uuid); + } catch (const Error& err) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(key); + } + } + } + + void QueryWrapper::LogOp(OpType op) { + switch (op) { + case OpType::OpAnd: + case OpType::OpOr: + case OpType::OpNot: + break; + default: + assert(false); + } + nextOperation_ = op; + } + + void QueryWrapper::Distinct(std::string_view index) { + ser_.PutVarUint(QueryItemType::QueryAggregation); + ser_.PutVarUint(AggType::AggDistinct); + ser_.PutVarUint(1); + ser_.PutVString(index); + } + + void QueryWrapper::ReqTotal(std::string_view totalName) { + ser_.PutVarUint(QueryItemType::QueryReqTotal); + ser_.PutVarUint(CalcTotalMode::ModeAccurateTotal); + if (!totalName.empty()) { + totalName_ = totalName; + } + } + + void QueryWrapper::CachedTotal(std::string_view totalName) { + ser_.PutVarUint(QueryItemType::QueryReqTotal); + ser_.PutVarUint(CalcTotalMode::ModeCachedTotal); + if (!totalName.empty()) { + totalName_ = totalName; + } + } + + void QueryWrapper::Limit(unsigned limitItems) { + ser_.PutVarUint(QueryItemType::QueryLimit); + ser_.PutVarUint(limitItems); + } + + void QueryWrapper::Offset(unsigned startOffset) { + int offset = std::min(std::numeric_limits::max(), startOffset); + ser_.PutVarUint(QueryItemType::QueryOffset); + ser_.PutVarUint(offset); + } + + void QueryWrapper::Debug(unsigned level) { + ser_.PutVarUint(QueryItemType::QueryDebugLevel); + ser_.PutVarUint(level); + } + + void QueryWrapper::Strict(StrictMode mode) { + ser_.PutVarUint(QueryItemType::QueryStrictMode); + ser_.PutVarUint(mode); + } + + void QueryWrapper::Modifier(QueryItemType type) { + ser_.PutVarUint(type); + } + + void QueryWrapper::Drop(std::string_view field) { + ser_.PutVarUint(QueryItemType::QueryDropField); + ser_.PutVString(field); + } + + void QueryWrapper::SetExpression(std::string_view field, std::string_view value) { + ser_.PutVarUint(QueryItemType::QueryUpdateField); + ser_.PutVString(field); + + ser_.PutVarUint(1); // size + ser_.PutVarUint(1); // is expression + ser_.PutVString(value); // ToDo q.putValue(value); + } + + reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { + // ToDo + /*if q.closed { + q.panicTrace("query.On call on already closed query. You should create new Query") + } + if q.root == nil { + panic(fmt.Errorf("Can't join on root query")) + }*/ + ser_.PutVarUint(QueryItemType::QueryJoinOn); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + ser_.PutVString(joinField); + ser_.PutVString(joinIndex); + nextOperation_ = OpType::OpAnd; + return errOK; + } + + void QueryWrapper::Select(const std::vector& fields) { + for (const auto& field : fields) { + ser_.PutVarUint(QueryItemType::QuerySelectFilter); + ser_.PutVString(field); + } + } + + void QueryWrapper::FetchCount(int count) { + fetchCount_ = count; + } + + void QueryWrapper::AddFunctions(const std::vector& functions) { + for (const auto& function : functions) { + ser_.PutVarUint(QueryItemType::QuerySelectFunction); + ser_.PutVString(function); + } + } + + void QueryWrapper::AddEqualPosition(const std::vector& equalPositions) { + ser_.PutVarUint(QueryItemType::QueryEqualPosition); + ser_.PutVarUint(openedBrackets_.empty()? 0 : int(openedBrackets_.back() + 1)); + ser_.PutVarUint(equalPositions.size()); + for (const auto& position : equalPositions) { + ser_.PutVString(position); + } + } + + template <> + void QueryWrapper::putValue(int8_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(value); + } + template <> + void QueryWrapper::putValue(uint8_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(int64_t(value)); + } + template <> + void QueryWrapper::putValue(int16_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(value); + } + template <> + void QueryWrapper::putValue(uint16_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(int64_t(value)); + } + template <> + void QueryWrapper::putValue(int32_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(value); + } + template <> + void QueryWrapper::putValue(uint32_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(int64_t(value)); + } + template <> + void QueryWrapper::putValue(int64_t value) { + ser_.PutVarUint(VALUE_INT_64); + ser_.PutVarint(value); + } + template <> + void QueryWrapper::putValue(uint64_t value) { + ser_.PutVarUint(VALUE_INT_64); + ser_.PutVarint(value); + } + template <> + void QueryWrapper::putValue(std::string_view value) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(value); + } + template <> + void QueryWrapper::putValue(bool value) { + ser_.PutVarUint(VALUE_BOOL); + ser_.PutBool(value); + } + template <> + void QueryWrapper::putValue(double value) { + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(value); + } + +} // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 65acdec..1ba3dc2 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -1,214 +1,84 @@ #pragma once #include -#include + #include "core/type_consts.h" -#include "core/keyvalue/uuid.h" -#include "core/keyvalue/variant.h" #include "tools/serializer.h" #include "tools/errors.h" +#ifdef PYREINDEXER_CPROTO +#include "client/cororeindexer.h" +#else +#include "core/reindexer.h" +#endif + +#include "reindexerinterface.h" namespace pyreindexer { -namespace { - const int VALUE_INT_64 = 0; - const int VALUE_DOUBLE = 1; - const int VALUE_STRING = 2; - const int VALUE_BOOL = 3; - const int VALUE_NULL = 4; - const int VALUE_INT = 8; - const int VALUE_UNDEFINED = 9; - const int VALUE_COMPOSITE = 10; - const int VALUE_TUPLE = 11; - const int VALUE_UUID = 12; -} +#ifdef PYREINDEXER_CPROTO +using DBInterface = ReindexerInterface; +#else +using DBInterface = ReindexerInterface; +#endif class QueryWrapper { public: - QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { - assert(db_); - ser_.PutVString(ns); - } - - void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { - ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); - ser_.PutVarUint(nextOperation_); - ser_.PutVString(firstField); - ser_.PutVarUint(condition); - ser_.PutVString(secondField); - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - } + QueryWrapper(DBInterface* db, std::string_view ns); - Error OpenBracket() { - ser_.PutVarUint(QueryItemType::QueryOpenBracket); - ser_.PutVarUint(nextOperation_); - nextOperation_ = OpType::OpAnd; - openedBrackets_.push_back(queriesCount_); - ++queriesCount_; - return errOK; - } + void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField); - Error CloseBracket() { - if (nextOperation_ != OpType::OpAnd) { - return Error(errLogic, "Operation before close bracket"); - } + reindexer::Error OpenBracket(); + reindexer::Error CloseBracket(); - if (openedBrackets_.empty()) { - return Error(errLogic, "Close bracket before open it"); - } - - ser_.PutVarUint(QueryItemType::QueryCloseBracket); - openedBrackets_.pop_back(); - return errOK; - } - - template // ToDo -------------------------------------------------------------- + template void Where(std::string_view index, CondType condition, const std::vector& keys) { - (void)index; - (void)condition; - (void)keys; - } //--------------------------------------------------------- - - void WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { ser_.PutVarUint(QueryItemType::QueryCondition); ser_.PutVString(index); ser_.PutVarUint(nextOperation_); ser_.PutVarUint(condition); - nextOperation_ = OpType::OpAnd; - ++queriesCount_; ser_.PutVarUint(keys.size()); for (const auto& key : keys) { - try { - auto uuid = reindexer::Uuid(key); - ser_.PutVarUint(VALUE_UUID); - ser_.PutUuid(uuid); - } catch (const Error& err) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(key); - } + putValue(key); } - } - - void LogOp(OpType op) { - switch (op) { - case OpType::OpAnd: - case OpType::OpOr: - case OpType::OpNot: - break; - default: - assert(false); - } - nextOperation_ = op; - } - void Distinct(std::string_view index) { - ser_.PutVarUint(QueryItemType::QueryAggregation); - ser_.PutVarUint(AggType::AggDistinct); - ser_.PutVarUint(1); - ser_.PutVString(index); - } - - void ReqTotal(std::string_view totalName) { - ser_.PutVarUint(QueryItemType::QueryReqTotal); - ser_.PutVarUint(CalcTotalMode::ModeAccurateTotal); - if (!totalName.empty()) { - totalName_ = totalName; - } + nextOperation_ = OpType::OpAnd; + ++queriesCount_; } - void CachedTotal(std::string_view totalName) { - ser_.PutVarUint(QueryItemType::QueryReqTotal); - ser_.PutVarUint(CalcTotalMode::ModeCachedTotal); - if (!totalName.empty()) { - totalName_ = totalName; - } - } + void WhereUUID(std::string_view index, CondType condition, const std::vector& keys); - void Limit(unsigned limitItems) { - int limit = std::min(std::numeric_limits::max(), limitItems); - ser_.PutVarUint(QueryItemType::QueryLimit); - ser_.PutVarUint(limit); - } + void LogOp(OpType op); - void Offset(int startOffset) { - int offset = std::min(std::numeric_limits::max(), startOffset); - ser_.PutVarUint(QueryItemType::QueryOffset); - ser_.PutVarUint(offset); - } + void Distinct(std::string_view index); - void Debug(int level) { - ser_.PutVarUint(QueryItemType::QueryDebugLevel); - ser_.PutVarUint(level); - } + void ReqTotal(std::string_view totalName); + void CachedTotal(std::string_view totalName); - void Strict(StrictMode mode) { - ser_.PutVarUint(QueryItemType::QueryStrictMode); - ser_.PutVarUint(mode); - } + void Limit(unsigned limitItems); + void Offset(unsigned startOffset); + void Debug(unsigned level); + void Strict(StrictMode mode); - void Modifier(QueryItemType type) { - ser_.PutVarUint(type); - } + void Modifier(QueryItemType type); - void Drop(std::string_view field) { - ser_.PutVarUint(QueryItemType::QueryDropField); - ser_.PutVString(field); - } + void Drop(std::string_view field); - void SetExpression(std::string_view field, std::string_view value) { - ser_.PutVarUint(QueryItemType::QueryUpdateField); - ser_.PutVString(field); + void SetExpression(std::string_view field, std::string_view value); - ser_.PutVarUint(1); // size - ser_.PutVarUint(1); // is expression - ser_.PutVString(value); // ToDo q.putValue(value); - } + reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); - Error On(std::string_view joinField, CondType condition, std::string_view joinIndex) { - // ToDo - /*if q.closed { - q.panicTrace("query.On call on already closed query. You should create new Query") - } - if q.root == nil { - panic(fmt.Errorf("Can't join on root query")) - }*/ - ser_.PutVarUint(QueryItemType::QueryJoinOn); - ser_.PutVarUint(nextOperation_); - ser_.PutVarUint(condition); - ser_.PutVString(joinField); - ser_.PutVString(joinIndex); - nextOperation_ = OpType::OpAnd; - return errOK; - } + void Select(const std::vector& fields); - void Select(const std::vector& fields) { - for (const auto& field : fields) { - ser_.PutVarUint(QueryItemType::QuerySelectFilter); - ser_.PutVString(field); - } - } + void FetchCount(int count); - void FetchCount(int count) { - fetchCount_ = count; - } + void AddFunctions(const std::vector& functions); - void AddFunctions(const std::vector& functions) { - for (const auto& function : functions) { - ser_.PutVarUint(QueryItemType::QuerySelectFunction); - ser_.PutVString(function); - } - } + void AddEqualPosition(const std::vector& equalPositions); - void AddEqualPosition(const std::vector& equalPositions) { - ser_.PutVarUint(QueryItemType::QueryEqualPosition); - ser_.PutVarUint(openedBrackets_.empty()? 0 : int(openedBrackets_.back() + 1)); - ser_.PutVarUint(equalPositions.size()); - for (const auto& position : equalPositions) { - ser_.PutVString(position); - } - } +private: + template + void putValue(T) {} private: DBInterface* db_{nullptr}; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index b7ac37f..cf8f0c3 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -924,8 +924,8 @@ static PyObject* CachedTotal(PyObject* self, PyObject* args) { static PyObject* Limit(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - unsigned limitItems = 0; - if (!PyArg_ParseTuple(args, "kI", &queryWrapperAddr, &limitItems)) { + int limitItems = 0; + if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &limitItems)) { return nullptr; } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 813dba7..76913cd 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -126,7 +126,7 @@ static PyMethodDef module_methods[] = { {"where_string", WhereString, METH_VARARGS, "add where condition with strings"}, {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, - {"where_float64", WhereDouble, METH_VARARGS, "add where condition with double args"}, + {"where_float32", WhereDouble, METH_VARARGS, "add where condition with double args"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 94c5fa6..8541db3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -117,7 +117,7 @@ def close_bracket(self) -> Query: self._raise_on_error() return self - def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: + def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query: """Add where condition to DB query with int args # Arguments: @@ -137,7 +137,7 @@ def where_int(self, index: str, condition: CondType, keys: List[int]) -> Query: self._raise_on_error() return self - def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query: + def where_uint32(self, index: str, condition: CondType, keys: List[int]) -> Query: """Add where condition to DB query with Int32 args # Arguments: @@ -153,7 +153,7 @@ def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query """ - self.err_code, self.err_msg = self.api.where_int32(self.query_wrapper_ptr, index, condition.value, keys) + self.err_code, self.err_msg = self.api.where_uint32(self.query_wrapper_ptr, index, condition.value, keys) self._raise_on_error() return self @@ -177,6 +177,26 @@ def where_int64(self, index: str, condition: CondType, keys: List[int]) -> Query self._raise_on_error() return self + def where_uint64(self, index: str, condition: CondType, keys: List[int]) -> Query: + """Add where condition to DB query with Int64 args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[Int64]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where_uint64(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + def where_string(self, index: str, condition: CondType, keys: List[str]) -> Query: """Add where condition to DB query with string args @@ -239,7 +259,7 @@ def where_bool(self, index: str, condition: CondType, keys: List[bool]) -> Query self._raise_on_error() return self - def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Query: + def where_float32(self, index: str, condition: CondType, keys: List[float]) -> Query: """Add where condition to DB query with float args # Arguments: @@ -255,7 +275,7 @@ def where_float64(self, index: str, condition: CondType, keys: List[float]) -> Q """ - self.err_code, self.err_msg = self.api.where_float64(self.query_wrapper_ptr, index, condition.value, keys) + self.err_code, self.err_msg = self.api.where_float32(self.query_wrapper_ptr, index, condition.value, keys) self._raise_on_error() return self @@ -382,7 +402,7 @@ def request_total(self, total_name: str= '') -> Query: """Request total items calculation # Arguments: - total_name (string): Name to be requested + total_name (string, optional): Name to be requested # Returns: (:obj:`Query`): Query object for further customizations @@ -396,7 +416,7 @@ def cached_total(self, total_name: str= '') -> Query: """Request cached total items calculation # Arguments: - total_name (string): Name to be requested + total_name (string, optional): Name to be requested # Returns: (:obj:`Query`): Query object for further customizations From bd55939a2ed2d9f4e6f57555b087b718ec585bc5 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 30 Oct 2024 15:27:39 +0300 Subject: [PATCH 031/125] Rewrite query_wrapper. Part III: add where for Variant --- pyreindexer/example/main.py | 6 +-- pyreindexer/lib/include/pyobjtools.cc | 61 +++++++++++++++++++----- pyreindexer/lib/include/pyobjtools.h | 14 +++--- pyreindexer/lib/include/query_wrapper.cc | 5 ++ pyreindexer/lib/src/rawpyreindexer.cc | 39 +++++++++++++-- pyreindexer/lib/src/rawpyreindexer.h | 2 + pyreindexer/query.py | 22 ++++++++- 7 files changed, 124 insertions(+), 25 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index d5f12e4..734b4f4 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -90,15 +90,15 @@ def query_example(db, namespace): .open_bracket() .where_between_fields('fld1', CondType.CondEq, 'fld2') .close_bracket() - .where_int64('fld2', CondType.CondLe, [42]) + .where('fld2', CondType.CondLe, [42]) .limit(10) .debug(1) .request_total()) query = db.new_query(namespace).offset(1).cached_total() - (query.where_string('fld1', CondType.CondSet, ['s','t','o','p']) + (query.where('fld1', CondType.CondSet, ['s','t','o','p']) .op_not() - .where_float32('fld2', CondType.CondSet, [3.14]) + .where('fld2', CondType.CondSet, [3.14]) .explain() .fetch_count(10)) query.expression("fld1", "array_remove(integer_array, [5,6,7,8]) || [1,2,3]").drop("fld2") diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index e2f6239..769bbcd 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -1,6 +1,8 @@ #include "pyobjtools.h" -#include +//#include +#include "tools/serializer.h" +#include "vendor/gason/gason.h" namespace pyreindexer { @@ -96,9 +98,10 @@ void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer) { } std::vector ParseListToStrVec(PyObject** list) { - std::vector vec; + std::vector result; Py_ssize_t sz = PyList_Size(*list); + result.reserve(sz); for (Py_ssize_t i = 0; i < sz; i++) { PyObject* item = PyList_GetItem(*list, i); @@ -106,16 +109,17 @@ std::vector ParseListToStrVec(PyObject** list) { throw reindexer::Error(errParseJson, std::string("String expected, got ") + Py_TYPE(item)->tp_name); } - vec.push_back(PyUnicode_AsUTF8(item)); + result.push_back(PyUnicode_AsUTF8(item)); } - return vec; + return result; } std::vector ParseListToBoolVec(PyObject** list) { - std::vector vec; + std::vector result; Py_ssize_t sz = PyList_Size(*list); + result.reserve(sz); for (Py_ssize_t i = 0; i < sz; i++) { PyObject* item = PyList_GetItem(*list, i); @@ -123,16 +127,17 @@ std::vector ParseListToBoolVec(PyObject** list) { throw reindexer::Error(errParseJson, std::string("Bool expected, got ") + Py_TYPE(item)->tp_name); } - vec.push_back(PyLong_AsLong(item) != 0); + result.push_back(PyLong_AsLong(item) != 0); } - return vec; + return result; } std::vector ParseListToDoubleVec(PyObject** list) { - std::vector vec; + std::vector result; Py_ssize_t sz = PyList_Size(*list); + result.reserve(sz); for (Py_ssize_t i = 0; i < sz; i++) { PyObject* item = PyList_GetItem(*list, i); @@ -143,13 +148,47 @@ std::vector ParseListToDoubleVec(PyObject** list) { double v = PyFloat_AsDouble(item); double intpart = 0.0; if (std::modf(v, &intpart) == 0.0) { - vec.push_back(int64_t(v)); + result.push_back(int64_t(v)); } else { - vec.push_back(v); + result.push_back(v); } } - return vec; + return result; +} + +reindexer::Variant convert(PyObject** value) { + if (PyFloat_Check(*value)) { + double v = PyFloat_AsDouble(*value); + double intpart = 0.0; + if (std::modf(v, &intpart) == 0.0) { + return reindexer::Variant(int64_t(v)); + } else { + return reindexer::Variant(v); + } + } else if (PyBool_Check(*value)) { + return reindexer::Variant(PyLong_AsLong(*value) != 0); + } else if (PyLong_Check(*value)) { + return reindexer::Variant(int64_t(PyLong_AsLong(*value))); + } else if (PyUnicode_Check(*value)) { + return reindexer::Variant(std::string_view(PyUnicode_AsUTF8(*value))); + } else { + throw reindexer::Error(errParseJson, std::string("Unexpected type, got ") + Py_TYPE(*value)->tp_name); + } + return {}; +} + +std::vector ParseListToVec(PyObject** list) { + std::vector result; + + Py_ssize_t sz = PyList_Size(*list); + result.reserve(sz); + for (Py_ssize_t i = 0; i < sz; i++) { + PyObject* item = PyList_GetItem(*list, i); + result.push_back(convert(&item)); + } + + return result; } PyObject* pyValueFromJsonValue(const gason::JsonValue& value) { diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index f72e9c1..f62b3eb 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -2,21 +2,23 @@ #include #include -#include "estl/span.h" -#include "vendor/gason/gason.h" -#include "tools/serializer.h" +//#include "estl/span.h" +#include "core/keyvalue/variant.h" +//#include "tools/serializer.h" namespace pyreindexer { std::vector ParseListToStrVec(PyObject** list); std::vector ParseListToBoolVec(PyObject** list); std::vector ParseListToDoubleVec(PyObject** list); +std::vector ParseListToVec(PyObject** list); template std::vector ParseListToIntVec(PyObject** list) { - std::vector vec; + std::vector result; Py_ssize_t sz = PyList_Size(*list); + result.reserve(sz); for (Py_ssize_t i = 0; i < sz; i++) { PyObject* item = PyList_GetItem(*list, i); @@ -25,10 +27,10 @@ std::vector ParseListToIntVec(PyObject** list) { } long v = PyLong_AsLong(item); - vec.push_back(T(v)); + result.push_back(T(v)); } - return vec; + return result; } void PyObjectToJson(PyObject** dict, reindexer::WrSerializer& wrSer); diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 095800d..fc68fb7 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -251,5 +251,10 @@ namespace { ser_.PutVarUint(VALUE_DOUBLE); ser_.PutDouble(value); } + template <> + void QueryWrapper::putValue(const reindexer::Variant& value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVariant(value); + } } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index cf8f0c3..5db36d8 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -647,6 +647,37 @@ static PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { static PyObject* OpenBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Open); } static PyObject* CloseBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Closed); } +static PyObject* Where(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Where(index, CondType(condition), keys); + + return pyErr(errOK); +} + static PyObject* WhereInt(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; @@ -673,7 +704,7 @@ static PyObject* WhereInt(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); // ToDo + query->Where(index, CondType(condition), keys); return pyErr(errOK); } @@ -704,7 +735,7 @@ static PyObject* WhereInt32(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); // ToDo + query->Where(index, CondType(condition), keys); return pyErr(errOK); } @@ -735,7 +766,7 @@ static PyObject* WhereInt64(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); // ToDo + query->Where(index, CondType(condition), keys); return pyErr(errOK); } @@ -766,7 +797,7 @@ static PyObject* WhereString(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Where(index, CondType(condition), keys); // ToDo + query->Where(index, CondType(condition), keys); return pyErr(errOK); } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 76913cd..dee539d 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -52,6 +52,7 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args); static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); static PyObject* OpenBracket(PyObject* self, PyObject* args); static PyObject* CloseBracket(PyObject* self, PyObject* args); +static PyObject* Where(PyObject* self, PyObject* args); static PyObject* WhereInt(PyObject* self, PyObject* args); static PyObject* WhereInt32(PyObject* self, PyObject* args); static PyObject* WhereInt64(PyObject* self, PyObject* args); @@ -120,6 +121,7 @@ static PyMethodDef module_methods[] = { {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, + {"where", Where, METH_VARARGS, "add where condition with args"}, {"where_int", WhereInt, METH_VARARGS, "add where condition with int args"}, {"where_int32", WhereInt32, METH_VARARGS, "add where condition with int32 args"}, {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 8541db3..133b795 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List +from typing import List, Union from enum import Enum class CondType(Enum): @@ -117,6 +117,26 @@ def close_bracket(self) -> Query: self._raise_on_error() return self + def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float,str]] = []) -> Query: + """Add where condition to DB query with int args + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[Union[int,bool,float,str]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() + return self + def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query: """Add where condition to DB query with int args From 2d647570bd8764639ccf189a11ff154f433e50ea Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 30 Oct 2024 15:41:23 +0300 Subject: [PATCH 032/125] Rewrite query_wrapper. Part IV: add where for Variant. Clean code --- pyreindexer/lib/include/pyobjtools.cc | 42 ----- pyreindexer/lib/include/pyobjtools.h | 22 --- pyreindexer/lib/include/query_wrapper.cc | 10 ++ pyreindexer/lib/src/rawpyreindexer.cc | 186 ----------------------- pyreindexer/lib/src/rawpyreindexer.h | 12 -- pyreindexer/query.py | 140 ----------------- 6 files changed, 10 insertions(+), 402 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 769bbcd..e777794 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -115,48 +115,6 @@ std::vector ParseListToStrVec(PyObject** list) { return result; } -std::vector ParseListToBoolVec(PyObject** list) { - std::vector result; - - Py_ssize_t sz = PyList_Size(*list); - result.reserve(sz); - for (Py_ssize_t i = 0; i < sz; i++) { - PyObject* item = PyList_GetItem(*list, i); - - if (!PyBool_Check(item)) { - throw reindexer::Error(errParseJson, std::string("Bool expected, got ") + Py_TYPE(item)->tp_name); - } - - result.push_back(PyLong_AsLong(item) != 0); - } - - return result; -} - -std::vector ParseListToDoubleVec(PyObject** list) { - std::vector result; - - Py_ssize_t sz = PyList_Size(*list); - result.reserve(sz); - for (Py_ssize_t i = 0; i < sz; i++) { - PyObject* item = PyList_GetItem(*list, i); - - if (!PyFloat_Check(item)) { - throw reindexer::Error(errParseJson, std::string("Double expected, got ") + Py_TYPE(item)->tp_name); - } - - double v = PyFloat_AsDouble(item); - double intpart = 0.0; - if (std::modf(v, &intpart) == 0.0) { - result.push_back(int64_t(v)); - } else { - result.push_back(v); - } - } - - return result; -} - reindexer::Variant convert(PyObject** value) { if (PyFloat_Check(*value)) { double v = PyFloat_AsDouble(*value); diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index f62b3eb..c91e453 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -9,30 +9,8 @@ namespace pyreindexer { std::vector ParseListToStrVec(PyObject** list); -std::vector ParseListToBoolVec(PyObject** list); -std::vector ParseListToDoubleVec(PyObject** list); std::vector ParseListToVec(PyObject** list); -template -std::vector ParseListToIntVec(PyObject** list) { - std::vector result; - - Py_ssize_t sz = PyList_Size(*list); - result.reserve(sz); - for (Py_ssize_t i = 0; i < sz; i++) { - PyObject* item = PyList_GetItem(*list, i); - - if (!PyLong_Check(item)) { - throw reindexer::Error(errParseJson, std::string("Integer expected, got ") + Py_TYPE(item)->tp_name); - } - - long v = PyLong_AsLong(item); - result.push_back(T(v)); - } - - return result; -} - void PyObjectToJson(PyObject** dict, reindexer::WrSerializer& wrSer); PyObject* PyObjectFromJson(reindexer::span json); diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index fc68fb7..4e05d39 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -242,11 +242,21 @@ namespace { ser_.PutVString(value); } template <> + void QueryWrapper::putValue(const std::string& value) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(value); + } + template <> void QueryWrapper::putValue(bool value) { ser_.PutVarUint(VALUE_BOOL); ser_.PutBool(value); } template <> + void QueryWrapper::putValue(float value) { + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(value); + } + template <> void QueryWrapper::putValue(double value) { ser_.PutVarUint(VALUE_DOUBLE); ser_.PutDouble(value); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 5db36d8..ad8df3b 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -678,130 +678,6 @@ static PyObject* Where(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* WhereInt(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { - return nullptr; - } - - Py_XINCREF(keysList); - - std::vector keys; - if (keysList != nullptr) { - try { - keys = ParseListToIntVec(&keysList); - } catch (const Error& err) { - Py_DECREF(keysList); - - return pyErr(err); - } - } - - Py_XDECREF(keysList); - - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); - - return pyErr(errOK); -} - -static PyObject* WhereInt32(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { - return nullptr; - } - - Py_XINCREF(keysList); - - std::vector keys; - if (keysList != nullptr) { - try { - keys = ParseListToIntVec(&keysList); - } catch (const Error& err) { - Py_DECREF(keysList); - - return pyErr(err); - } - } - - Py_XDECREF(keysList); - - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); - - return pyErr(errOK); -} - -static PyObject* WhereInt64(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { - return nullptr; - } - - Py_XINCREF(keysList); - - std::vector keys; - if (keysList != nullptr) { - try { - keys = ParseListToIntVec(&keysList); - } catch (const Error& err) { - Py_DECREF(keysList); - - return pyErr(err); - } - } - - Py_XDECREF(keysList); - - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); - - return pyErr(errOK); -} - -static PyObject* WhereString(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { - return nullptr; - } - - Py_XINCREF(keysList); - - std::vector keys; - if (keysList != nullptr) { - try { - keys = ParseListToStrVec(&keysList); - } catch (const Error& err) { - Py_DECREF(keysList); - - return pyErr(err); - } - } - - Py_XDECREF(keysList); - - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); - - return pyErr(errOK); -} - static PyObject* WhereUUID(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; @@ -833,68 +709,6 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* WhereBool(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { - return nullptr; - } - - Py_XINCREF(keysList); - - std::vector keys; - if (keysList != nullptr) { - try { - keys = ParseListToBoolVec(&keysList); - } catch (const Error& err) { - Py_DECREF(keysList); - - return pyErr(err); - } - } - - Py_XDECREF(keysList); - - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); - - return pyErr(errOK); -} - -static PyObject* WhereDouble(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { - return nullptr; - } - - Py_XINCREF(keysList); - - std::vector keys; - if (keysList != nullptr) { - try { - keys = ParseListToDoubleVec(&keysList); - } catch (const Error& err) { - Py_DECREF(keysList); - - return pyErr(err); - } - } - - Py_XDECREF(keysList); - - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); - - return pyErr(errOK); -} - static PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index dee539d..877bf6a 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -53,13 +53,7 @@ static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); static PyObject* OpenBracket(PyObject* self, PyObject* args); static PyObject* CloseBracket(PyObject* self, PyObject* args); static PyObject* Where(PyObject* self, PyObject* args); -static PyObject* WhereInt(PyObject* self, PyObject* args); -static PyObject* WhereInt32(PyObject* self, PyObject* args); -static PyObject* WhereInt64(PyObject* self, PyObject* args); -static PyObject* WhereString(PyObject* self, PyObject* args); static PyObject* WhereUUID(PyObject* self, PyObject* args); -static PyObject* WhereBool(PyObject* self, PyObject* args); -static PyObject* WhereDouble(PyObject* self, PyObject* args); static PyObject* And(PyObject* self, PyObject* args); static PyObject* Or(PyObject* self, PyObject* args); static PyObject* Not(PyObject* self, PyObject* args); @@ -122,13 +116,7 @@ static PyMethodDef module_methods[] = { {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, {"where", Where, METH_VARARGS, "add where condition with args"}, - {"where_int", WhereInt, METH_VARARGS, "add where condition with int args"}, - {"where_int32", WhereInt32, METH_VARARGS, "add where condition with int32 args"}, - {"where_int64", WhereInt64, METH_VARARGS, "add where condition with int64 args"}, - {"where_string", WhereString, METH_VARARGS, "add where condition with strings"}, {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, - {"where_bool", WhereBool, METH_VARARGS, "add where condition with bool args"}, - {"where_float32", WhereDouble, METH_VARARGS, "add where condition with double args"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 133b795..48dc63c 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -137,106 +137,6 @@ def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float self._raise_on_error() return self - def where_int32(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Add where condition to DB query with int args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[int]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_int(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - - def where_uint32(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Add where condition to DB query with Int32 args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[Int32]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_uint32(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - - def where_int64(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Add where condition to DB query with Int64 args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[Int64]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_int64(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - - def where_uint64(self, index: str, condition: CondType, keys: List[int]) -> Query: - """Add where condition to DB query with Int64 args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[Int64]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_uint64(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - - def where_string(self, index: str, condition: CondType, keys: List[str]) -> Query: - """Add where condition to DB query with string args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_string(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """Add where condition to DB query with UUID as string args. This function applies binary encoding to the UUID value. @@ -259,46 +159,6 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: self._raise_on_error() return self - def where_bool(self, index: str, condition: CondType, keys: List[bool]) -> Query: - """Add where condition to DB query with bool args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[bool]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_bool(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - - def where_float32(self, index: str, condition: CondType, keys: List[float]) -> Query: - """Add where condition to DB query with float args - - # Arguments: - index (string): Field name used in condition clause - condition (:enum:`CondType`): Type of condition - keys (list[float]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - - # Returns: - (:obj:`Query`): Query object for further customizations - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.err_code, self.err_msg = self.api.where_float32(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() - return self - ################################################################ ToDo #func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { // ToDo ################################################################ From 2d6423f05d1fb1f67b18c4f58777c873c08448e6 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 30 Oct 2024 18:09:08 +0300 Subject: [PATCH 033/125] Part VII: start of implementation of query builder. Add where --- pyreindexer/example/main.py | 1 + pyreindexer/lib/include/pyobjtools.cc | 1 - pyreindexer/lib/include/pyobjtools.h | 2 - pyreindexer/lib/include/query_wrapper.cc | 11 ++ pyreindexer/lib/include/query_wrapper.h | 16 +- pyreindexer/lib/src/rawpyreindexer.cc | 227 +++++++++++++++-------- pyreindexer/lib/src/rawpyreindexer.h | 36 ++-- pyreindexer/query.py | 97 ++++++---- 8 files changed, 263 insertions(+), 128 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 734b4f4..bb3986b 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -99,6 +99,7 @@ def query_example(db, namespace): (query.where('fld1', CondType.CondSet, ['s','t','o','p']) .op_not() .where('fld2', CondType.CondSet, [3.14]) + .where_query(db.new_query(namespace), CondType.CondSet, ['to','check']) .explain() .fetch_count(10)) query.expression("fld1", "array_remove(integer_array, [5,6,7,8]) || [1,2,3]").drop("fld2") diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index e777794..7301923 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -1,6 +1,5 @@ #include "pyobjtools.h" -//#include #include "tools/serializer.h" #include "vendor/gason/gason.h" diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index c91e453..8997d14 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -2,9 +2,7 @@ #include #include -//#include "estl/span.h" #include "core/keyvalue/variant.h" -//#include "tools/serializer.h" namespace pyreindexer { diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 4e05d39..ac35086 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -57,6 +57,17 @@ namespace { return errOK; } + void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { + ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(index); + ser_.PutVarUint(condition); + ser_.PutVString(query.ser_.Slice()); + + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + } + void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { ser_.PutVarUint(QueryItemType::QueryCondition); ser_.PutVString(index); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 1ba3dc2..3ce61ed 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -45,7 +45,22 @@ class QueryWrapper { nextOperation_ = OpType::OpAnd; ++queriesCount_; } + template + void WhereQuery(QueryWrapper& query, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QuerySubQueryCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(query.ser_.Slice()); + ser_.PutVarUint(condition); + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + putValue(key); + } + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + } + void WhereComposite(std::string_view index, CondType condition, QueryWrapper& query); void WhereUUID(std::string_view index, CondType condition, const std::vector& keys); void LogOp(OpType op); @@ -73,7 +88,6 @@ class QueryWrapper { void FetchCount(int count); void AddFunctions(const std::vector& functions); - void AddEqualPosition(const std::vector& equalPositions); private: diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index ad8df3b..af0452c 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -13,32 +13,33 @@ using reindexer::IndexDef; using reindexer::NamespaceDef; using reindexer::WrSerializer; -static uintptr_t initReindexer() { +namespace { +uintptr_t initReindexer() { DBInterface* db = new DBInterface(); return reinterpret_cast(db); } -static DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } +DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } -static void destroyReindexer(uintptr_t rx) { +void destroyReindexer(uintptr_t rx) { DBInterface* db = getDB(rx); delete db; } -static PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } +PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } template -static T* getWrapper(uintptr_t wrapperAddr) { +T* getWrapper(uintptr_t wrapperAddr) { return reinterpret_cast(wrapperAddr); } template -static void wrapperDelete(uintptr_t wrapperAddr) { +void wrapperDelete(uintptr_t wrapperAddr) { T* queryWrapperPtr = getWrapper(wrapperAddr); delete queryWrapperPtr; } -static PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { +PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { QueryResultsWrapper* qresWrapperPtr = getWrapper(qresWrapperAddr); WrSerializer wrSer; @@ -59,6 +60,9 @@ static PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { return res; } +} // namespace + +// common -------------------------------------------------------------------------------------------------------------- static PyObject* Init(PyObject* self, PyObject* args) { uintptr_t rx = initReindexer(); @@ -93,6 +97,28 @@ static PyObject* Connect(PyObject* self, PyObject* args) { return pyErr(err); } +static PyObject* Select(PyObject* self, PyObject* args) { + uintptr_t rx = 0; + char* query = nullptr; + if (!PyArg_ParseTuple(args, "ks", &rx, &query)) { + return nullptr; + } + + auto db = getDB(rx); + auto qresWrapper = new QueryResultsWrapper(db); + auto err = qresWrapper->Select(query); + + if (!err.ok()) { + delete qresWrapper; + + return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); + } + + return Py_BuildValue("iskI", err.code(), err.what().c_str(), reinterpret_cast(qresWrapper), qresWrapper->Count()); +} + +// namespace ---------------------------------------------------------------------------------------------------------- + static PyObject* NamespaceOpen(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; @@ -126,6 +152,52 @@ static PyObject* NamespaceDrop(PyObject* self, PyObject* args) { return pyErr(err); } +static PyObject* EnumNamespaces(PyObject* self, PyObject* args) { + uintptr_t rx = 0; + unsigned enumAll = 0; + if (!PyArg_ParseTuple(args, "kI", &rx, &enumAll)) { + return nullptr; + } + + std::vector nsDefs; + auto err = getDB(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); + if (!err.ok()) { + return Py_BuildValue("is[]", err.code(), err.what().c_str()); + } + + PyObject* list = PyList_New(nsDefs.size()); // new ref + if (!list) { + return nullptr; + } + + WrSerializer wrSer; + Py_ssize_t pos = 0; + for (const auto& ns : nsDefs) { + wrSer.Reset(); + ns.GetJSON(wrSer, false); + + PyObject* dictFromJson = nullptr; + try { + dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref + } catch (const Error& err) { + Py_XDECREF(dictFromJson); + Py_DECREF(list); + + return Py_BuildValue("is{}", err.code(), err.what().c_str()); + } + + PyList_SetItem(list, pos, dictFromJson); // stolen ref + ++pos; + } + + PyObject* res = Py_BuildValue("isO", err.code(), err.what().c_str(), list); + Py_DECREF(list); + + return res; +} + +// index --------------------------------------------------------------------------------------------------------------- + static PyObject* IndexAdd(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; @@ -197,7 +269,10 @@ static PyObject* IndexDrop(PyObject* self, PyObject* args) { return pyErr(err); } -static PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { +// item ---------------------------------------------------------------------------------------------------------------- + +namespace { +PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t rx = 0; char* ns = nullptr; PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple @@ -272,12 +347,14 @@ static PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) return pyErr(err); } - +} // namespace static PyObject* ItemInsert(PyObject* self, PyObject* args) { return itemModify(self, args, ModeInsert); } static PyObject* ItemUpdate(PyObject* self, PyObject* args) { return itemModify(self, args, ModeUpdate); } static PyObject* ItemUpsert(PyObject* self, PyObject* args) { return itemModify(self, args, ModeUpsert); } static PyObject* ItemDelete(PyObject* self, PyObject* args) { return itemModify(self, args, ModeDelete); } +// meta ---------------------------------------------------------------------------------------------------------------- + static PyObject* PutMeta(PyObject* self, PyObject* args) { uintptr_t rx = 0; char *ns = nullptr, *key = nullptr, *value = nullptr; @@ -312,26 +389,6 @@ static PyObject* DeleteMeta(PyObject* self, PyObject* args) { return pyErr(err); } -static PyObject* Select(PyObject* self, PyObject* args) { - uintptr_t rx = 0; - char* query = nullptr; - if (!PyArg_ParseTuple(args, "ks", &rx, &query)) { - return nullptr; - } - - auto db = getDB(rx); - auto qresWrapper = new QueryResultsWrapper(db); - auto err = qresWrapper->Select(query); - - if (!err.ok()) { - delete qresWrapper; - - return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); - } - - return Py_BuildValue("iskI", err.code(), err.what().c_str(), reinterpret_cast(qresWrapper), qresWrapper->Count()); -} - static PyObject* EnumMeta(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; @@ -363,49 +420,7 @@ static PyObject* EnumMeta(PyObject* self, PyObject* args) { return res; } -static PyObject* EnumNamespaces(PyObject* self, PyObject* args) { - uintptr_t rx = 0; - unsigned enumAll = 0; - if (!PyArg_ParseTuple(args, "kI", &rx, &enumAll)) { - return nullptr; - } - - std::vector nsDefs; - auto err = getDB(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); - if (!err.ok()) { - return Py_BuildValue("is[]", err.code(), err.what().c_str()); - } - - PyObject* list = PyList_New(nsDefs.size()); // new ref - if (!list) { - return nullptr; - } - - WrSerializer wrSer; - Py_ssize_t pos = 0; - for (const auto& ns : nsDefs) { - wrSer.Reset(); - ns.GetJSON(wrSer, false); - - PyObject* dictFromJson = nullptr; - try { - dictFromJson = PyObjectFromJson(reindexer::giftStr(wrSer.Slice())); // stolen ref - } catch (const Error& err) { - Py_XDECREF(dictFromJson); - Py_DECREF(list); - - return Py_BuildValue("is{}", err.code(), err.what().c_str()); - } - - PyList_SetItem(list, pos, dictFromJson); // stolen ref - ++pos; - } - - PyObject* res = Py_BuildValue("isO", err.code(), err.what().c_str(), list); - Py_DECREF(list); - - return res; -} +// query results ------------------------------------------------------------------------------------------------------- static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args) { uintptr_t qresWrapperAddr = 0; @@ -462,6 +477,8 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { return res; } +// transaction --------------------------------------------------------------------------------------------------------- + static PyObject* NewTransaction(PyObject* self, PyObject* args) { uintptr_t rx = 0; char* ns = nullptr; @@ -481,7 +498,8 @@ static PyObject* NewTransaction(PyObject* self, PyObject* args) { return Py_BuildValue("isk", err.code(), err.what().c_str(), reinterpret_cast(transaction)); } -static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { +namespace { +PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t transactionWrapperAddr = 0; PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed @@ -554,6 +572,7 @@ static PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModif return nullptr; } +} // namespace static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeInsert); } static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpdate); } static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpsert); } @@ -592,6 +611,7 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return pyErr(err); } +// query --------------------------------------------------------------------------------------------------------------- static PyObject* CreateQuery(PyObject* self, PyObject* args) { uintptr_t rx = 0; @@ -632,8 +652,9 @@ static PyObject* WhereBetweenFields(PyObject* self, PyObject* args) { Py_RETURN_NONE; } +namespace { enum class BracketType { Open, Closed }; -static PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { +PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -644,6 +665,7 @@ static PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { auto err = (type == BracketType::Open)? query->OpenBracket() : query->CloseBracket(); return pyErr(err); } +} // namespace static PyObject* OpenBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Open); } static PyObject* CloseBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Closed); } @@ -678,6 +700,55 @@ static PyObject* Where(PyObject* self, PyObject* args) { return pyErr(errOK); } +static PyObject* WhereQuery(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + uintptr_t subQueryWrapperAddr = 0; + unsigned condition = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kkIO!", &queryWrapperAddr, &subQueryWrapperAddr, &condition, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + auto subQuery = getWrapper(subQueryWrapperAddr); + + query->WhereQuery(*subQuery, CondType(condition), keys); + + return pyErr(errOK); +} + +static PyObject* WhereComposite(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned condition = 0; + uintptr_t subQueryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "ksIk", &queryWrapperAddr, &index, &condition, &subQueryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + auto subQuery = getWrapper(subQueryWrapperAddr); + + query->WhereComposite(index, CondType(condition), *subQuery); + + Py_RETURN_NONE; +} + static PyObject* WhereUUID(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; @@ -709,7 +780,8 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { +namespace { +PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -721,6 +793,7 @@ static PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { Py_RETURN_NONE; } +} // namespace static PyObject* And(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpAnd); } static PyObject* Or(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpOr); } static PyObject* Not(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpNot); } @@ -823,13 +896,15 @@ static PyObject* Strict(PyObject* self, PyObject* args) { Py_RETURN_NONE; } -static void modifier(PyObject* self, PyObject* args, QueryItemType type) { +namespace { +void modifier(PyObject* self, PyObject* args, QueryItemType type) { uintptr_t queryWrapperAddr = 0; if (PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { auto query = getWrapper(queryWrapperAddr); query->Modifier(type); } } +} // namespace static PyObject* Explain(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryExplain); Py_RETURN_NONE; } static PyObject* WithRank(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryWithRank); Py_RETURN_NONE; } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 877bf6a..3b6f196 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -14,30 +14,35 @@ namespace pyreindexer { +// common static PyObject* Init(PyObject* self, PyObject* args); static PyObject* Destroy(PyObject* self, PyObject* args); static PyObject* Connect(PyObject* self, PyObject* args); +static PyObject* Select(PyObject* self, PyObject* args); +// namespace static PyObject* NamespaceOpen(PyObject* self, PyObject* args); static PyObject* NamespaceClose(PyObject* self, PyObject* args); static PyObject* NamespaceDrop(PyObject* self, PyObject* args); +static PyObject* EnumNamespaces(PyObject* self, PyObject* args); +// index static PyObject* IndexAdd(PyObject* self, PyObject* args); static PyObject* IndexUpdate(PyObject* self, PyObject* args); static PyObject* IndexDrop(PyObject* self, PyObject* args); +// item static PyObject* ItemInsert(PyObject* self, PyObject* args); static PyObject* ItemUpdate(PyObject* self, PyObject* args); static PyObject* ItemUpsert(PyObject* self, PyObject* args); static PyObject* ItemDelete(PyObject* self, PyObject* args); +// meta static PyObject* PutMeta(PyObject* self, PyObject* args); static PyObject* GetMeta(PyObject* self, PyObject* args); static PyObject* DeleteMeta(PyObject* self, PyObject* args); -static PyObject* Select(PyObject* self, PyObject* args); static PyObject* EnumMeta(PyObject* self, PyObject* args); -static PyObject* EnumNamespaces(PyObject* self, PyObject* args); - +// query results static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); static PyObject* GetAggregationResults(PyObject* self, PyObject* args); - +// transaction (sync) static PyObject* NewTransaction(PyObject* self, PyObject* args); static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args); static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args); @@ -45,15 +50,16 @@ static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args); static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args); static PyObject* CommitTransaction(PyObject* self, PyObject* args); static PyObject* RollbackTransaction(PyObject* self, PyObject* args); - +// query static PyObject* CreateQuery(PyObject* self, PyObject* args); static PyObject* DeleteQuery(PyObject* self, PyObject* args); - +static PyObject* Where(PyObject* self, PyObject* args); +static PyObject* WhereQuery(PyObject* self, PyObject* args); +static PyObject* WhereComposite(PyObject* self, PyObject* args); +static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); static PyObject* OpenBracket(PyObject* self, PyObject* args); static PyObject* CloseBracket(PyObject* self, PyObject* args); -static PyObject* Where(PyObject* self, PyObject* args); -static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* And(PyObject* self, PyObject* args); static PyObject* Or(PyObject* self, PyObject* args); static PyObject* Not(PyObject* self, PyObject* args); @@ -79,22 +85,26 @@ static PyMethodDef module_methods[] = { {"init", Init, METH_NOARGS, "init reindexer instance"}, {"destroy", Destroy, METH_VARARGS, "destroy reindexer instance"}, {"connect", Connect, METH_VARARGS, "connect to reindexer database"}, + {"select", Select, METH_VARARGS, "select query"}, + // namespace {"namespace_open", NamespaceOpen, METH_VARARGS, "open namespace"}, {"namespace_close", NamespaceClose, METH_VARARGS, "close namespace"}, {"namespace_drop", NamespaceDrop, METH_VARARGS, "drop namespace"}, {"namespaces_enum", EnumNamespaces, METH_VARARGS, "enum namespaces"}, + // index {"index_add", IndexAdd, METH_VARARGS, "add index"}, {"index_update", IndexUpdate, METH_VARARGS, "update index"}, {"index_drop", IndexDrop, METH_VARARGS, "drop index"}, + // item {"item_insert", ItemInsert, METH_VARARGS, "insert item"}, {"item_update", ItemUpdate, METH_VARARGS, "update item"}, {"item_upsert", ItemUpsert, METH_VARARGS, "upsert item"}, {"item_delete", ItemDelete, METH_VARARGS, "delete item"}, + // meta {"meta_put", PutMeta, METH_VARARGS, "put meta"}, {"meta_get", GetMeta, METH_VARARGS, "get meta"}, {"meta_delete", DeleteMeta, METH_VARARGS, "delete meta"}, {"meta_enum", EnumMeta, METH_VARARGS, "enum meta"}, - {"select", Select, METH_VARARGS, "select query"}, // query results {"query_results_iterate", QueryResultsWrapperIterate, METH_VARARGS, "get query result"}, {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, @@ -110,13 +120,13 @@ static PyMethodDef module_methods[] = { // query {"create_query", CreateQuery, METH_VARARGS, "create new query"}, {"delete_query", DeleteQuery, METH_VARARGS, "delete query. Free query object memory"}, - //{"where", Where, METH_VARARGS, "add where condition"}, - //{"where_query", WhereQuery, METH_VARARGS, "add where condition"}, + {"where", Where, METH_VARARGS, "add where condition with args"}, + {"where_query", WhereQuery, METH_VARARGS, "add sub-query where condition"}, + {"where_composite", WhereComposite, METH_VARARGS, "add where condition for composite indexes"}, + {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, - {"where", Where, METH_VARARGS, "add where condition with args"}, - {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 48dc63c..417e053 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -66,29 +66,33 @@ def _raise_on_error(self): if self.err_code: raise Exception(self.err_msg) -################################################################### ToDo -#func (q *Query) Where(index string, condition int, keys interface{}) *Query { -#func (q *Query) WhereQuery(subQuery *Query, condition int, keys interface{}) *Query { -################################################################# - - def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: - """Add comparing two fields where condition to DB query + def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float,str]] = ()) -> Query: + """Add where condition to DB query with int args # Arguments: - first_field (string): First field name used in condition clause + index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - second_field (string): Second field name used in condition clause + keys (list[Union[int,bool,float,str]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return on non-zero error code + """ - self.api.where_between_fields(self.query_wrapper_ptr, first_field, condition.value, second_field) + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, keys) + self._raise_on_error() return self - def open_bracket(self) -> Query: - """Open bracket for where condition to DB query + def where_query(self, sub_query: Query, condition: CondType, keys: List[Union[int,bool,float,str]] = ()) -> Query: + """Add sub query where condition to DB query with int args + + # Arguments: + sub_query (:obj:`Query`): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[Union[int,bool,float,str]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: (:obj:`Query`): Query object for further customizations @@ -98,32 +102,35 @@ def open_bracket(self) -> Query: """ - self.err_code, self.err_msg = self.api.open_bracket(self.query_wrapper_ptr) + self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, keys) self._raise_on_error() return self - def close_bracket(self) -> Query: - """CloseBracket - Close bracket for where condition to DB query + def where_composite(self, index: str, condition: CondType, sub_query: Query) -> Query: + """Add where condition to DB query with interface args for composite indexes + + # Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + sub_query (:obj:`Query`): Field name used in condition clause # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return on non-zero error code - """ - self.err_code, self.err_msg = self.api.close_bracket(self.query_wrapper_ptr) - self._raise_on_error() + self.api.where_composite(self.query_wrapper_ptr, index, condition.value, sub_query.query_wrapper_ptr) return self - def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float,str]] = []) -> Query: - """Add where condition to DB query with int args + def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: + """Add where condition to DB query with UUID as string args. + This function applies binary encoding to the UUID value. + 'index' MUST be declared as uuid index in this case # Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[Union[int,bool,float,str]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: (:obj:`Query`): Query object for further customizations @@ -133,19 +140,28 @@ def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float """ - self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, keys) + self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) self._raise_on_error() return self - def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: - """Add where condition to DB query with UUID as string args. - This function applies binary encoding to the UUID value. - 'index' MUST be declared as uuid index in this case + def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: + """Add comparing two fields where condition to DB query # Arguments: - index (string): Field name used in condition clause + first_field (string): First field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + second_field (string): Second field name used in condition clause + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.where_between_fields(self.query_wrapper_ptr, first_field, condition.value, second_field) + return self + + def open_bracket(self) -> Query: + """Open bracket for where condition to DB query # Returns: (:obj:`Query`): Query object for further customizations @@ -155,13 +171,24 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """ - self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) + self.err_code, self.err_msg = self.api.open_bracket(self.query_wrapper_ptr) self._raise_on_error() return self -################################################################ ToDo -#func (q *Query) WhereComposite(index string, condition int, keys ...interface{}) *Query { // ToDo -################################################################ + def close_bracket(self) -> Query: + """CloseBracket - Close bracket for where condition to DB query + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.close_bracket(self.query_wrapper_ptr) + self._raise_on_error() + return self def match(self, index: str, keys: List[str]) -> Query: """Add string EQ-condition to DB query with string args @@ -178,7 +205,7 @@ def match(self, index: str, keys: List[str]) -> Query: """ - self.err_code, self.err_msg = self.api.where_string(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) self._raise_on_error() return self From 5280a2b305fff5b2d3fd4bbde936d9d43cc5e658 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 30 Oct 2024 19:30:15 +0300 Subject: [PATCH 034/125] Part VIII: start of implementation of query builder. Add Point and dwithin --- pyreindexer/index_definition.py | 2 +- pyreindexer/lib/include/query_wrapper.cc | 85 +++++++++++++++--------- pyreindexer/lib/include/query_wrapper.h | 12 ++-- pyreindexer/lib/src/rawpyreindexer.cc | 80 +++++++++++++--------- pyreindexer/lib/src/rawpyreindexer.h | 2 + pyreindexer/point.py | 19 ++++++ pyreindexer/query.py | 21 +++++- pyreindexer/query_results.py | 4 +- pyreindexer/raiser_mixin.py | 2 +- pyreindexer/rx_connector.py | 8 +-- pyreindexer/transaction.py | 2 +- 11 files changed, 156 insertions(+), 81 deletions(-) create mode 100644 pyreindexer/point.py diff --git a/pyreindexer/index_definition.py b/pyreindexer/index_definition.py index 2898d9c..ce031e3 100644 --- a/pyreindexer/index_definition.py +++ b/pyreindexer/index_definition.py @@ -5,7 +5,7 @@ class IndexDefinition(dict): - """ IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. + """IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. # Arguments: diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index ac35086..3335335 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -7,16 +7,16 @@ namespace pyreindexer { namespace { - const int VALUE_INT_64 = 0; - const int VALUE_DOUBLE = 1; - const int VALUE_STRING = 2; - const int VALUE_BOOL = 3; - const int VALUE_NULL = 4; - const int VALUE_INT = 8; + const int VALUE_INT_64 = 0; + const int VALUE_DOUBLE = 1; + const int VALUE_STRING = 2; + const int VALUE_BOOL = 3; + const int VALUE_NULL = 4; + const int VALUE_INT = 8; const int VALUE_UNDEFINED = 9; const int VALUE_COMPOSITE = 10; - const int VALUE_TUPLE = 11; - const int VALUE_UUID = 12; + const int VALUE_TUPLE = 11; + const int VALUE_UUID = 12; } QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { @@ -24,6 +24,39 @@ namespace { ser_.PutVString(ns); } + void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { + ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(index); + ser_.PutVarUint(condition); + ser_.PutVString(query.ser_.Slice()); + + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + } + + void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QueryCondition); + ser_.PutVString(index); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + nextOperation_ = OpType::OpAnd; + ++queriesCount_; + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + try { + auto uuid = reindexer::Uuid(key); + ser_.PutVarUint(VALUE_UUID); + ser_.PutUuid(uuid); + } catch (const Error& err) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(key); + } + } + } + + void QueryWrapper::WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); ser_.PutVarUint(nextOperation_); @@ -37,8 +70,8 @@ namespace { Error QueryWrapper::OpenBracket() { ser_.PutVarUint(QueryItemType::QueryOpenBracket); ser_.PutVarUint(nextOperation_); - nextOperation_ = OpType::OpAnd; openedBrackets_.push_back(queriesCount_); + nextOperation_ = OpType::OpAnd; ++queriesCount_; return errOK; } @@ -57,36 +90,22 @@ namespace { return errOK; } - void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { - ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); - ser_.PutVarUint(nextOperation_); - ser_.PutVString(index); - ser_.PutVarUint(condition); - ser_.PutVString(query.ser_.Slice()); - - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - } - - void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { + void QueryWrapper::DWithin(std::string_view index, double x, double y, double distance) { ser_.PutVarUint(QueryItemType::QueryCondition); ser_.PutVString(index); ser_.PutVarUint(nextOperation_); - ser_.PutVarUint(condition); + ser_.PutVarUint(CondType::CondDWithin); + nextOperation_ = OpType::OpAnd; ++queriesCount_; - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - try { - auto uuid = reindexer::Uuid(key); - ser_.PutVarUint(VALUE_UUID); - ser_.PutUuid(uuid); - } catch (const Error& err) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(key); - } - } + ser_.PutVarUint(3); + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(x); + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(y); + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(distance); } void QueryWrapper::LogOp(OpType op) { diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 3ce61ed..34620f4 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -25,11 +25,6 @@ class QueryWrapper { public: QueryWrapper(DBInterface* db, std::string_view ns); - void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField); - - reindexer::Error OpenBracket(); - reindexer::Error CloseBracket(); - template void Where(std::string_view index, CondType condition, const std::vector& keys) { ser_.PutVarUint(QueryItemType::QueryCondition); @@ -63,6 +58,13 @@ class QueryWrapper { void WhereComposite(std::string_view index, CondType condition, QueryWrapper& query); void WhereUUID(std::string_view index, CondType condition, const std::vector& keys); + void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField); + + reindexer::Error OpenBracket(); + reindexer::Error CloseBracket(); + + void DWithin(std::string_view index, double x, double y, double distance); + void LogOp(OpType op); void Distinct(std::string_view index); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index af0452c..4eca961 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -636,38 +636,6 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { Py_RETURN_NONE; } -static PyObject* WhereBetweenFields(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* first_field = nullptr; - unsigned condition = 0; - char* second_field = nullptr; - if (!PyArg_ParseTuple(args, "ksIs", &queryWrapperAddr, &first_field, &condition, &second_field)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->WhereBetweenFields(first_field, CondType(condition), second_field); - - Py_RETURN_NONE; -} - -namespace { -enum class BracketType { Open, Closed }; -PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { - uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - auto err = (type == BracketType::Open)? query->OpenBracket() : query->CloseBracket(); - return pyErr(err); -} -} // namespace -static PyObject* OpenBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Open); } -static PyObject* CloseBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Closed); } static PyObject* Where(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -780,6 +748,54 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { return pyErr(errOK); } +static PyObject* WhereBetweenFields(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* first_field = nullptr; + unsigned condition = 0; + char* second_field = nullptr; + if (!PyArg_ParseTuple(args, "ksIs", &queryWrapperAddr, &first_field, &condition, &second_field)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->WhereBetweenFields(first_field, CondType(condition), second_field); + + Py_RETURN_NONE; +} + +namespace { +enum class BracketType { Open, Closed }; +PyObject* addBracket(PyObject* self, PyObject* args, BracketType type) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + auto err = (type == BracketType::Open)? query->OpenBracket() : query->CloseBracket(); + return pyErr(err); +} +} // namespace +static PyObject* OpenBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Open); } +static PyObject* CloseBracket(PyObject* self, PyObject* args) { return addBracket(self, args, BracketType::Closed); } + +static PyObject* DWithin(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + double x = 0, y = 0, distance = 0; + if (!PyArg_ParseTuple(args, "ksddd", &queryWrapperAddr, &x, &y, &distance)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->DWithin(index, x, y, distance); + + Py_RETURN_NONE; +} + namespace { PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { uintptr_t queryWrapperAddr = 0; diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 3b6f196..48b18e9 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -60,6 +60,7 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); static PyObject* OpenBracket(PyObject* self, PyObject* args); static PyObject* CloseBracket(PyObject* self, PyObject* args); +static PyObject* DWithin(PyObject* self, PyObject* args); static PyObject* And(PyObject* self, PyObject* args); static PyObject* Or(PyObject* self, PyObject* args); static PyObject* Not(PyObject* self, PyObject* args); @@ -127,6 +128,7 @@ static PyMethodDef module_methods[] = { {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, + {"dwithin", DWithin, METH_VARARGS, "add dwithin condition"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, diff --git a/pyreindexer/point.py b/pyreindexer/point.py new file mode 100644 index 0000000..ee3affc --- /dev/null +++ b/pyreindexer/point.py @@ -0,0 +1,19 @@ +class Point(object): + """An object representing the context of a Reindexer 2D point + + # Attributes: + x (float): x coordinate of the point + y (float): y coordinate of the point + """ + + def __init__(self, x, y): + """Constructs a new Reindexer query object + + # Arguments: + x (float): x coordinate of the point + y (float): y coordinate of the point + + """ + + self.x = x + self.y = y diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 417e053..eee5583 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import List, Union from enum import Enum +from pyreindexer.point import Point class CondType(Enum): CondAny = 0 @@ -23,7 +24,7 @@ class StrictMode(Enum): Indexes = 3 class Query(object): - """ An object representing the context of a Reindexer query + """An object representing the context of a Reindexer query # Attributes: api (module): An API module for Reindexer calls @@ -209,8 +210,24 @@ def match(self, index: str, keys: List[str]) -> Query: self._raise_on_error() return self + #func (q *Query) DWithin(index string, point Point, distance float64) *Query { + def dwithin(self, index: str, point: Point, distance: float) -> Query: + """Add DWithin condition to DB query + + # Arguments: + index (string): Field name used in condition clause + point (:obj:`Point`): Point object used in condition clause + distance (float): Distance in meters between point + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.dwithin(self.query_wrapper_ptr, index, point.x, point.y, distance) + return self + ################################################################ ToDo -#func (q *Query) DWithin(index string, point Point, distance float64) *Query { #func (q *Query) AggregateSum(field string) *Query { #func (q *Query) AggregateAvg(field string) *Query { #func (q *Query) AggregateMin(field string) *Query { diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 04f5a94..2b5696a 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,6 +1,6 @@ class QueryResults(object): - """ QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. - When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer + """QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. + When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer # Attributes: api (module): An API module for Reindexer calls diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index 010c4e0..d45cc23 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -1,5 +1,5 @@ class RaiserMixin(object): - """ RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity + """RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity """ err_code: int diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 6c83dd5..25c7fa7 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -6,10 +6,10 @@ class RxConnector(RaiserMixin): - """ RxConnector provides a binding to Reindexer upon two shared libraries (hereinafter - APIs): 'rawpyreindexerb.so' - and 'rawpyreindexerc.so'. The first one is aimed to a builtin way usage. That API embeds Reindexer, so it could be - used right in-place as is. The second one acts as a lightweight client which establishes a connection to Reindexer - server via RPC. The APIs interfaces are completely the same. + """RxConnector provides a binding to Reindexer upon two shared libraries (hereinafter - APIs): 'rawpyreindexerb.so' + and 'rawpyreindexerc.so'. The first one is aimed to a builtin way usage. That API embeds Reindexer, so it could + be used right in-place as is. The second one acts as a lightweight client which establishes a connection to + Reindexer server via RPC. The APIs interfaces are completely the same. # Attributes: api (module): An API module loaded dynamically for Reindexer calls diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 0b42e96..b26e9f1 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,5 +1,5 @@ class Transaction(object): - """ An object representing the context of a Reindexer transaction + """An object representing the context of a Reindexer transaction # Attributes: api (module): An API module for Reindexer calls From dd7acebe6a67353ef720b3fe8d79b2d753ad6334 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 30 Oct 2024 20:50:00 +0300 Subject: [PATCH 035/125] Part IX: start of implementation of query builder. Add simple aggregate --- pyreindexer/lib/include/query_wrapper.cc | 479 +++++++++++------------ pyreindexer/lib/include/query_wrapper.h | 9 +- pyreindexer/lib/src/rawpyreindexer.cc | 49 ++- pyreindexer/lib/src/rawpyreindexer.h | 12 +- pyreindexer/query.py | 93 ++++- 5 files changed, 348 insertions(+), 294 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 3335335..9bf5d41 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -17,284 +17,283 @@ namespace { const int VALUE_COMPOSITE = 10; const int VALUE_TUPLE = 11; const int VALUE_UUID = 12; -} +} // namespace - QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { - assert(db_); - ser_.PutVString(ns); - } +QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { + assert(db_); + ser_.PutVString(ns); +} - void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { - ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); - ser_.PutVarUint(nextOperation_); - ser_.PutVString(index); - ser_.PutVarUint(condition); - ser_.PutVString(query.ser_.Slice()); +void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { + ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(index); + ser_.PutVarUint(condition); + ser_.PutVString(query.ser_.Slice()); - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - } + nextOperation_ = OpType::OpAnd; + ++queriesCount_; +} - void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { - ser_.PutVarUint(QueryItemType::QueryCondition); - ser_.PutVString(index); - ser_.PutVarUint(nextOperation_); - ser_.PutVarUint(condition); - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - try { - auto uuid = reindexer::Uuid(key); - ser_.PutVarUint(VALUE_UUID); - ser_.PutUuid(uuid); - } catch (const Error& err) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(key); - } +void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QueryCondition); + ser_.PutVString(index); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + try { + auto uuid = reindexer::Uuid(key); + ser_.PutVarUint(VALUE_UUID); + ser_.PutUuid(uuid); + } catch (const Error& err) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(key); } } + nextOperation_ = OpType::OpAnd; + ++queriesCount_; +} - void QueryWrapper::WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { - ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); - ser_.PutVarUint(nextOperation_); - ser_.PutVString(firstField); - ser_.PutVarUint(condition); - ser_.PutVString(secondField); - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - } - Error QueryWrapper::OpenBracket() { - ser_.PutVarUint(QueryItemType::QueryOpenBracket); - ser_.PutVarUint(nextOperation_); - openedBrackets_.push_back(queriesCount_); - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - return errOK; - } +void QueryWrapper::WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { + ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(firstField); + ser_.PutVarUint(condition); + ser_.PutVString(secondField); - reindexer::Error QueryWrapper::CloseBracket() { - if (nextOperation_ != OpType::OpAnd) { - return reindexer::Error(errLogic, "Operation before close bracket"); - } + nextOperation_ = OpType::OpAnd; + ++queriesCount_; +} - if (openedBrackets_.empty()) { - return reindexer::Error(errLogic, "Close bracket before open it"); - } +Error QueryWrapper::OpenBracket() { + ser_.PutVarUint(QueryItemType::QueryOpenBracket); + ser_.PutVarUint(nextOperation_); + openedBrackets_.push_back(queriesCount_); - ser_.PutVarUint(QueryItemType::QueryCloseBracket); - openedBrackets_.pop_back(); - return errOK; - } + nextOperation_ = OpType::OpAnd; + ++queriesCount_; - void QueryWrapper::DWithin(std::string_view index, double x, double y, double distance) { - ser_.PutVarUint(QueryItemType::QueryCondition); - ser_.PutVString(index); - ser_.PutVarUint(nextOperation_); - ser_.PutVarUint(CondType::CondDWithin); - - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - - ser_.PutVarUint(3); - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(x); - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(y); - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(distance); - } + return errOK; +} - void QueryWrapper::LogOp(OpType op) { - switch (op) { - case OpType::OpAnd: - case OpType::OpOr: - case OpType::OpNot: - break; - default: - assert(false); - } - nextOperation_ = op; +reindexer::Error QueryWrapper::CloseBracket() { + if (nextOperation_ != OpType::OpAnd) { + return reindexer::Error(errLogic, "Operation before close bracket"); } - void QueryWrapper::Distinct(std::string_view index) { - ser_.PutVarUint(QueryItemType::QueryAggregation); - ser_.PutVarUint(AggType::AggDistinct); - ser_.PutVarUint(1); - ser_.PutVString(index); + if (openedBrackets_.empty()) { + return reindexer::Error(errLogic, "Close bracket before open it"); } - void QueryWrapper::ReqTotal(std::string_view totalName) { - ser_.PutVarUint(QueryItemType::QueryReqTotal); - ser_.PutVarUint(CalcTotalMode::ModeAccurateTotal); - if (!totalName.empty()) { - totalName_ = totalName; - } - } + ser_.PutVarUint(QueryItemType::QueryCloseBracket); + openedBrackets_.pop_back(); - void QueryWrapper::CachedTotal(std::string_view totalName) { - ser_.PutVarUint(QueryItemType::QueryReqTotal); - ser_.PutVarUint(CalcTotalMode::ModeCachedTotal); - if (!totalName.empty()) { - totalName_ = totalName; - } - } + return errOK; +} - void QueryWrapper::Limit(unsigned limitItems) { - ser_.PutVarUint(QueryItemType::QueryLimit); - ser_.PutVarUint(limitItems); - } +void QueryWrapper::DWithin(std::string_view index, double x, double y, double distance) { + ser_.PutVarUint(QueryItemType::QueryCondition); + ser_.PutVString(index); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(CondType::CondDWithin); + + ser_.PutVarUint(3); + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(x); + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(y); + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(distance); + + nextOperation_ = OpType::OpAnd; + ++queriesCount_; +} - void QueryWrapper::Offset(unsigned startOffset) { - int offset = std::min(std::numeric_limits::max(), startOffset); - ser_.PutVarUint(QueryItemType::QueryOffset); - ser_.PutVarUint(offset); - } +void QueryWrapper::Aggregate(std::string_view index, AggType type) { + ser_.PutVarUint(QueryItemType::QueryAggregation); + ser_.PutVarUint(type); + ser_.PutVarUint(1); + ser_.PutVString(index); +} - void QueryWrapper::Debug(unsigned level) { - ser_.PutVarUint(QueryItemType::QueryDebugLevel); - ser_.PutVarUint(level); +void QueryWrapper::LogOp(OpType op) { + switch (op) { + case OpType::OpAnd: + case OpType::OpOr: + case OpType::OpNot: + break; + default: + assert(false); } + nextOperation_ = op; +} - void QueryWrapper::Strict(StrictMode mode) { - ser_.PutVarUint(QueryItemType::QueryStrictMode); - ser_.PutVarUint(mode); +void QueryWrapper::Total(std::string_view totalName, CalcTotalMode mode) { + ser_.PutVarUint(QueryItemType::QueryReqTotal); + ser_.PutVarUint(mode); + if (!totalName.empty()) { + totalName_ = totalName; } +} - void QueryWrapper::Modifier(QueryItemType type) { - ser_.PutVarUint(type); - } +void QueryWrapper::Limit(unsigned limitItems) { + ser_.PutVarUint(QueryItemType::QueryLimit); + ser_.PutVarUint(limitItems); +} - void QueryWrapper::Drop(std::string_view field) { - ser_.PutVarUint(QueryItemType::QueryDropField); - ser_.PutVString(field); - } +void QueryWrapper::Offset(unsigned startOffset) { + ser_.PutVarUint(QueryItemType::QueryOffset); + ser_.PutVarUint(startOffset); +} - void QueryWrapper::SetExpression(std::string_view field, std::string_view value) { - ser_.PutVarUint(QueryItemType::QueryUpdateField); - ser_.PutVString(field); +void QueryWrapper::Debug(unsigned level) { + ser_.PutVarUint(QueryItemType::QueryDebugLevel); + ser_.PutVarUint(level); +} - ser_.PutVarUint(1); // size - ser_.PutVarUint(1); // is expression - ser_.PutVString(value); // ToDo q.putValue(value); - } +void QueryWrapper::Strict(StrictMode mode) { + ser_.PutVarUint(QueryItemType::QueryStrictMode); + ser_.PutVarUint(mode); +} - reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { - // ToDo - /*if q.closed { - q.panicTrace("query.On call on already closed query. You should create new Query") - } - if q.root == nil { - panic(fmt.Errorf("Can't join on root query")) - }*/ - ser_.PutVarUint(QueryItemType::QueryJoinOn); - ser_.PutVarUint(nextOperation_); - ser_.PutVarUint(condition); - ser_.PutVString(joinField); - ser_.PutVString(joinIndex); - nextOperation_ = OpType::OpAnd; - return errOK; - } +void QueryWrapper::Modifier(QueryItemType type) { + ser_.PutVarUint(type); +} - void QueryWrapper::Select(const std::vector& fields) { - for (const auto& field : fields) { - ser_.PutVarUint(QueryItemType::QuerySelectFilter); - ser_.PutVString(field); - } - } +void QueryWrapper::Drop(std::string_view field) { + ser_.PutVarUint(QueryItemType::QueryDropField); + ser_.PutVString(field); +} - void QueryWrapper::FetchCount(int count) { - fetchCount_ = count; - } +void QueryWrapper::SetExpression(std::string_view field, std::string_view value) { + ser_.PutVarUint(QueryItemType::QueryUpdateField); + ser_.PutVString(field); - void QueryWrapper::AddFunctions(const std::vector& functions) { - for (const auto& function : functions) { - ser_.PutVarUint(QueryItemType::QuerySelectFunction); - ser_.PutVString(function); - } - } + ser_.PutVarUint(1); // size + ser_.PutVarUint(1); // is expression + ser_.PutVString(value); // ToDo q.putValue(value); +} - void QueryWrapper::AddEqualPosition(const std::vector& equalPositions) { - ser_.PutVarUint(QueryItemType::QueryEqualPosition); - ser_.PutVarUint(openedBrackets_.empty()? 0 : int(openedBrackets_.back() + 1)); - ser_.PutVarUint(equalPositions.size()); - for (const auto& position : equalPositions) { - ser_.PutVString(position); - } +reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { + // ToDo + /*if q.closed { + q.panicTrace("query.On call on already closed query. You should create new Query") } + if q.root == nil { + panic(fmt.Errorf("Can't join on root query")) + }*/ + ser_.PutVarUint(QueryItemType::QueryJoinOn); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + ser_.PutVString(joinField); + ser_.PutVString(joinIndex); - template <> - void QueryWrapper::putValue(int8_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(value); - } - template <> - void QueryWrapper::putValue(uint8_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(int64_t(value)); - } - template <> - void QueryWrapper::putValue(int16_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(value); - } - template <> - void QueryWrapper::putValue(uint16_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(int64_t(value)); - } - template <> - void QueryWrapper::putValue(int32_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(value); - } - template <> - void QueryWrapper::putValue(uint32_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(int64_t(value)); - } - template <> - void QueryWrapper::putValue(int64_t value) { - ser_.PutVarUint(VALUE_INT_64); - ser_.PutVarint(value); - } - template <> - void QueryWrapper::putValue(uint64_t value) { - ser_.PutVarUint(VALUE_INT_64); - ser_.PutVarint(value); - } - template <> - void QueryWrapper::putValue(std::string_view value) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(value); - } - template <> - void QueryWrapper::putValue(const std::string& value) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(value); - } - template <> - void QueryWrapper::putValue(bool value) { - ser_.PutVarUint(VALUE_BOOL); - ser_.PutBool(value); - } - template <> - void QueryWrapper::putValue(float value) { - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(value); + nextOperation_ = OpType::OpAnd; + + return errOK; +} + +void QueryWrapper::Select(const std::vector& fields) { + for (const auto& field : fields) { + ser_.PutVarUint(QueryItemType::QuerySelectFilter); + ser_.PutVString(field); } - template <> - void QueryWrapper::putValue(double value) { - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(value); +} + +void QueryWrapper::FetchCount(int count) { + fetchCount_ = count; +} + +void QueryWrapper::AddFunctions(const std::vector& functions) { + for (const auto& function : functions) { + ser_.PutVarUint(QueryItemType::QuerySelectFunction); + ser_.PutVString(function); } - template <> - void QueryWrapper::putValue(const reindexer::Variant& value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVariant(value); +} + +void QueryWrapper::AddEqualPosition(const std::vector& equalPositions) { + ser_.PutVarUint(QueryItemType::QueryEqualPosition); + ser_.PutVarUint(openedBrackets_.empty()? 0 : int(openedBrackets_.back() + 1)); + ser_.PutVarUint(equalPositions.size()); + for (const auto& position : equalPositions) { + ser_.PutVString(position); } +} + + +template <> +void QueryWrapper::putValue(int8_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(value); +} +template <> +void QueryWrapper::putValue(uint8_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(int64_t(value)); +} +template <> +void QueryWrapper::putValue(int16_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(value); +} +template <> +void QueryWrapper::putValue(uint16_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(int64_t(value)); +} +template <> +void QueryWrapper::putValue(int32_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(value); +} +template <> +void QueryWrapper::putValue(uint32_t value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVarint(int64_t(value)); +} +template <> +void QueryWrapper::putValue(int64_t value) { + ser_.PutVarUint(VALUE_INT_64); + ser_.PutVarint(value); +} +template <> +void QueryWrapper::putValue(uint64_t value) { + ser_.PutVarUint(VALUE_INT_64); + ser_.PutVarint(value); +} +template <> +void QueryWrapper::putValue(std::string_view value) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(value); +} +template <> +void QueryWrapper::putValue(const std::string& value) { + ser_.PutVarUint(VALUE_STRING); + ser_.PutVString(value); +} +template <> +void QueryWrapper::putValue(bool value) { + ser_.PutVarUint(VALUE_BOOL); + ser_.PutBool(value); +} +template <> +void QueryWrapper::putValue(float value) { + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(value); +} +template <> +void QueryWrapper::putValue(double value) { + ser_.PutVarUint(VALUE_DOUBLE); + ser_.PutDouble(value); +} +template <> +void QueryWrapper::putValue(const reindexer::Variant& value) { + ser_.PutVarUint(VALUE_INT); + ser_.PutVariant(value); +} } // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 34620f4..e2ad36a 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -65,12 +65,11 @@ class QueryWrapper { void DWithin(std::string_view index, double x, double y, double distance); - void LogOp(OpType op); + void Aggregate(std::string_view index, AggType type); - void Distinct(std::string_view index); + void LogOp(OpType op); - void ReqTotal(std::string_view totalName); - void CachedTotal(std::string_view totalName); + void Total(std::string_view totalName, CalcTotalMode mode); void Limit(unsigned limitItems); void Offset(unsigned startOffset); @@ -97,7 +96,7 @@ class QueryWrapper { void putValue(T) {} private: - DBInterface* db_{nullptr}; + DBInterface* db_{nullptr}; // ToDo reindexer::WrSerializer ser_; OpType nextOperation_{OpType::OpAnd}; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 4eca961..8dfb01b 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -797,52 +797,46 @@ static PyObject* DWithin(PyObject* self, PyObject* args) { } namespace { -PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { +PyObject* aggregate(PyObject* self, PyObject* args, AggType type) { uintptr_t queryWrapperAddr = 0; - if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + char* field = nullptr; + if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &field)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->LogOp(opID); + query->Aggregate(field, type); Py_RETURN_NONE; } } // namespace -static PyObject* And(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpAnd); } -static PyObject* Or(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpOr); } -static PyObject* Not(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpNot); } - -static PyObject* Distinct(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - char* index = nullptr; - if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &index)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->Distinct(index); +static PyObject* AggregateDistinct(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggDistinct); } +static PyObject* AggregateSum(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggSum); } +static PyObject* AggregateAvg(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggAvg); } +static PyObject* AggregateMin(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMin); } +static PyObject* AggregateMax(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMax); } - Py_RETURN_NONE; -} - -static PyObject* ReqTotal(PyObject* self, PyObject* args) { +namespace { +PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { uintptr_t queryWrapperAddr = 0; - char* totalName = nullptr; - if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &totalName)) { + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->ReqTotal(totalName); + query->LogOp(opID); Py_RETURN_NONE; } +} // namespace +static PyObject* And(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpAnd); } +static PyObject* Or(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpOr); } +static PyObject* Not(PyObject* self, PyObject* args) { return logOp(self, args, OpType::OpNot); } -static PyObject* CachedTotal(PyObject* self, PyObject* args) { +namespace { +static PyObject* total(PyObject* self, PyObject* args, CalcTotalMode mode) { uintptr_t queryWrapperAddr = 0; char* totalName = nullptr; if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &totalName)) { @@ -851,10 +845,13 @@ static PyObject* CachedTotal(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->CachedTotal(totalName); + query->Total(totalName, mode); Py_RETURN_NONE; } +} // namespace +static PyObject* ReqTotal(PyObject* self, PyObject* args) { return total(self, args, CalcTotalMode::ModeAccurateTotal); } +static PyObject* CachedTotal(PyObject* self, PyObject* args) { return total(self, args, CalcTotalMode::ModeCachedTotal); } static PyObject* Limit(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 48b18e9..76b9171 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -61,10 +61,14 @@ static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); static PyObject* OpenBracket(PyObject* self, PyObject* args); static PyObject* CloseBracket(PyObject* self, PyObject* args); static PyObject* DWithin(PyObject* self, PyObject* args); +static PyObject* AggregateDistinct(PyObject* self, PyObject* args); +static PyObject* AggregateSum(PyObject* self, PyObject* args); +static PyObject* AggregateAvg(PyObject* self, PyObject* args); +static PyObject* AggregateMin(PyObject* self, PyObject* args); +static PyObject* AggregateMax(PyObject* self, PyObject* args); static PyObject* And(PyObject* self, PyObject* args); static PyObject* Or(PyObject* self, PyObject* args); static PyObject* Not(PyObject* self, PyObject* args); -static PyObject* Distinct(PyObject* self, PyObject* args); static PyObject* ReqTotal(PyObject* self, PyObject* args); static PyObject* CachedTotal(PyObject* self, PyObject* args); static PyObject* Limit(PyObject* self, PyObject* args); @@ -129,10 +133,14 @@ static PyMethodDef module_methods[] = { {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, {"close_bracket", CloseBracket, METH_VARARGS, "close bracket for where condition"}, {"dwithin", DWithin, METH_VARARGS, "add dwithin condition"}, + {"aggregate_distinct", AggregateDistinct, METH_VARARGS, "list of unique values of field"}, + {"aggregate_sum", AggregateSum, METH_VARARGS, "sum field value"}, + {"aggregate_avg", AggregateAvg, METH_VARARGS, "average field value"}, + {"aggregate_min", AggregateMin, METH_VARARGS, "minimum field value"}, + {"aggregate_max", AggregateMax, METH_VARARGS, "maximum field value"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, - {"distinct", Distinct, METH_VARARGS, "perform distinct for index"}, {"request_total", ReqTotal, METH_VARARGS, "request total items calculation"}, {"cached_total", CachedTotal, METH_VARARGS, "request cached total items calculation"}, {"limit", Limit, METH_VARARGS, "request cached total items calculation"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index eee5583..ef632e2 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -210,7 +210,6 @@ def match(self, index: str, keys: List[str]) -> Query: self._raise_on_error() return self - #func (q *Query) DWithin(index string, point Point, distance float64) *Query { def dwithin(self, index: str, point: Point, distance: float) -> Query: """Add DWithin condition to DB query @@ -227,11 +226,78 @@ def dwithin(self, index: str, point: Point, distance: float) -> Query: self.api.dwithin(self.query_wrapper_ptr, index, point.x, point.y, distance) return self + def distinct(self, index: str) -> Query: + """Performs distinct for a certain index. + Return only items with uniq value of field + + # Arguments: + index (string): Field name for distinct operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.aggregate_distinct(self.query_wrapper_ptr, index) + return self + + def aggregate_sum(self, index: str) -> Query: + """Performs a summation of values for a specified index + + # Arguments: + index (string): Field name for sum operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.aggregate_sum(self.query_wrapper_ptr, index) + return self + + def aggregate_avg(self, index: str) -> Query: + """Finds for the average at the specified index + + # Arguments: + index (string): Field name for sum operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.aggregate_avg(self.query_wrapper_ptr, index) + return self + + def aggregate_min(self, index: str) -> Query: + """Finds for the minimum at the specified index + + # Arguments: + index (string): Field name for sum operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.aggregate_min(self.query_wrapper_ptr, index) + return self + + def aggregate_max(self, index: str) -> Query: + """Finds for the maximum at the specified index + + # Arguments: + index (string): Field name for sum operation + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.aggregate_max(self.query_wrapper_ptr, index) + return self + ################################################################ ToDo -#func (q *Query) AggregateSum(field string) *Query { -#func (q *Query) AggregateAvg(field string) *Query { -#func (q *Query) AggregateMin(field string) *Query { -#func (q *Query) AggregateMax(field string) *Query { #type AggregateFacetRequest struct { # query *Query #} @@ -307,21 +373,6 @@ def op_not(self) -> Query: self.api.op_not(self.query_wrapper_ptr) return self - def distinct(self, index: str) -> Query: - """Performs distinct for a certain index. - Return only items with uniq value of field - - # Arguments: - index (string): Field name for distinct operation - - # Returns: - (:obj:`Query`): Query object for further customizations - - """ - - self.api.distinct(self.query_wrapper_ptr, index) - return self - def request_total(self, total_name: str= '') -> Query: """Request total items calculation @@ -571,4 +622,4 @@ def equal_position(self, equal_position: List[str]) -> Query: self._raise_on_error() return self -# 66 / 30 \ No newline at end of file +# ToDo 66/33 \ No newline at end of file From 35dd3ede463b54aea6b2ee31c0f39f2dc4a985ba Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 31 Oct 2024 14:35:42 +0300 Subject: [PATCH 036/125] Part X: start of implementation of query builder. Add sort --- pyreindexer/example/main.py | 4 +- pyreindexer/lib/include/query_wrapper.h | 12 ++++ pyreindexer/lib/src/rawpyreindexer.cc | 31 +++++++++ pyreindexer/lib/src/rawpyreindexer.h | 2 + pyreindexer/query.py | 87 ++++++++++++++++++------- 5 files changed, 111 insertions(+), 25 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index bb3986b..93e645c 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -102,7 +102,9 @@ def query_example(db, namespace): .where_query(db.new_query(namespace), CondType.CondSet, ['to','check']) .explain() .fetch_count(10)) - query.expression("fld1", "array_remove(integer_array, [5,6,7,8]) || [1,2,3]").drop("fld2") + query.expression('fld1', 'array_remove(integer_array, [5,6,7,8]) || [1,2,3]').drop('fld2') + + db.new_query(namespace).sort_stfield_distance('fldGeom1', 'fldGeom2', False) def rx_example(): diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index e2ad36a..562f67b 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -67,6 +67,18 @@ class QueryWrapper { void Aggregate(std::string_view index, AggType type); + template + void Sort(std::string_view index, bool desc, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QuerySortIndex); + ser_.PutVString(index); + ser_.PutVarUint(desc? 1 : 0); + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + putValue(key); + } + } + void LogOp(OpType op); void Total(std::string_view totalName, CalcTotalMode mode); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 8dfb01b..997e600 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -817,6 +817,37 @@ static PyObject* AggregateAvg(PyObject* self, PyObject* args) { return aggregate static PyObject* AggregateMin(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMin); } static PyObject* AggregateMax(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMax); } +static PyObject* Sort(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* index = nullptr; + unsigned desc = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &desc, &PyList_Type, &keysList)) { + return nullptr; + } + + Py_XINCREF(keysList); + + std::vector keys; + if (keysList != nullptr) { + try { + keys = ParseListToVec(&keysList); + } catch (const Error& err) { + Py_DECREF(keysList); + + return pyErr(err); + } + } + + Py_XDECREF(keysList); + + auto query = getWrapper(queryWrapperAddr); + + query->Sort(index, (desc != 0), keys); + + return pyErr(errOK); +} + namespace { PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { uintptr_t queryWrapperAddr = 0; diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 76b9171..ff54dc2 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -66,6 +66,7 @@ static PyObject* AggregateSum(PyObject* self, PyObject* args); static PyObject* AggregateAvg(PyObject* self, PyObject* args); static PyObject* AggregateMin(PyObject* self, PyObject* args); static PyObject* AggregateMax(PyObject* self, PyObject* args); +static PyObject* Sort(PyObject* self, PyObject* args); static PyObject* And(PyObject* self, PyObject* args); static PyObject* Or(PyObject* self, PyObject* args); static PyObject* Not(PyObject* self, PyObject* args); @@ -138,6 +139,7 @@ static PyMethodDef module_methods[] = { {"aggregate_avg", AggregateAvg, METH_VARARGS, "average field value"}, {"aggregate_min", AggregateMin, METH_VARARGS, "minimum field value"}, {"aggregate_max", AggregateMax, METH_VARARGS, "maximum field value"}, + {"sort", Sort, METH_VARARGS, "apply sort order"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, {"op_not", Not, METH_VARARGS, "next condition will be added with NOT AND"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index ef632e2..69539e3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -67,7 +67,7 @@ def _raise_on_error(self): if self.err_code: raise Exception(self.err_msg) - def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float,str]] = ()) -> Query: + def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float,str]] = None) -> Query: """Add where condition to DB query with int args # Arguments: @@ -83,11 +83,12 @@ def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float """ + keys = keys or [] self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, keys) self._raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, keys: List[Union[int,bool,float,str]] = ()) -> Query: + def where_query(self, sub_query: Query, condition: CondType, keys: List[Union[int,bool,float,str]] = None) -> Query: """Add sub query where condition to DB query with int args # Arguments: @@ -103,6 +104,7 @@ def where_query(self, sub_query: Query, condition: CondType, keys: List[Union[in """ + keys = keys or [] self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, keys) self._raise_on_error() return self @@ -306,35 +308,72 @@ def aggregate_max(self, index: str) -> Query: #func (r *AggregateFacetRequest) Limit(limit int) *AggregateFacetRequest { #func (r *AggregateFacetRequest) Offset(offset int) *AggregateFacetRequest { #func (r *AggregateFacetRequest) Sort(field string, desc bool) *AggregateFacetRequest { -#func (q *Query) Sort(sortIndex string, desc bool, values ...interface{}) *Query { -#func (q *Query) SortStPointDistance(field string, p Point, desc bool) *Query { ################################################################ -#func (q *Query) SortStFieldDistance(field1 string, field2 string, desc bool) *Query { // ToDo -# def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) -> Query: -# """Wrapper for geometry sorting by shortest distance between 2 geometry fields (ST_Distance) + def sort(self, index: str, desc: bool, keys: List[Union[int,bool,float,str]] = None) -> Query: + """Apply sort order to return from query items. If values argument specified, then items equal to values, + if found will be placed in the top positions. Forced sort is support for the first sorting field only -# # Arguments: -# first_field (string): First field name used in condition -# second_field (string): Second field name used in condition -# desc (bool): Descending flag + # Arguments: + index (string): The index name + desc (bool): True if sorting in descending order + keys (list[Union[int,bool,float,str]]): Value of index to match. For composite indexes keys must be list, with value of each subindex + # ToDo List[List[Union[int,bool,float,str]]] + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ -# # Returns: -# (:obj:`Query`): Query object for further customizations + keys = keys or [] + self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, keys) + self._raise_on_error() + return self -# # Raises: -# Exception: Raises with an error message of API return on non-zero error code + def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: + """Apply geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance + between geometry field and point (ST_Distance) -# """ + # Arguments: + index (string): The index name + point (:obj:`Point`): Point object used in sorting operation + desc (bool): True if sorting in descending order -# request : str = "ST_Distance(" -# request += first_field -# request += ',' -# request += second_field -# request += ')' + # Returns: + (:obj:`Query`): Query object for further customizations -# return self.sort(request, desc) -################################################################ + """ + + request : str = "ST_Distance(" + index + request += ",ST_GeomFromText('point(" + request += "{:.10f}".format(point.x) + " " + "{:.10f}".format(point.y) + request += ")'))" + + return self.sort(request, desc) + + def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) -> Query: + """Apply geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance + between 2 geometry fields (ST_Distance) + + # Arguments: + first_field (string): First field name used in condition + second_field (string): Second field name used in condition + desc (bool): Descending flag + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + request : str = 'ST_Distance(' + first_field + ',' + second_field + ')' + + return self.sort(request, desc) def op_and(self) -> Query: """Next condition will be added with AND. @@ -622,4 +661,4 @@ def equal_position(self, equal_position: List[str]) -> Query: self._raise_on_error() return self -# ToDo 66/33 \ No newline at end of file +# ToDo 66/35 \ No newline at end of file From eee1335d797504fcab74fd8463cd8dada8ad7d0b Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 31 Oct 2024 22:05:28 +0300 Subject: [PATCH 037/125] Part XI: start of implementation of query builder. Add aggregation --- pyreindexer/example/main.py | 6 +- pyreindexer/lib/include/query_wrapper.cc | 32 ++--- pyreindexer/lib/include/query_wrapper.h | 7 +- pyreindexer/lib/src/rawpyreindexer.cc | 108 +++++++++------- pyreindexer/lib/src/rawpyreindexer.h | 8 ++ pyreindexer/query.py | 151 ++++++++++++++++++----- 6 files changed, 221 insertions(+), 91 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 93e645c..c588d45 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -1,5 +1,5 @@ from pyreindexer import RxConnector -from pyreindexer.query import CondType +from pyreindexer.query import CondType, StrictMode def create_index_example(db, namespace): @@ -102,10 +102,12 @@ def query_example(db, namespace): .where_query(db.new_query(namespace), CondType.CondSet, ['to','check']) .explain() .fetch_count(10)) - query.expression('fld1', 'array_remove(integer_array, [5,6,7,8]) || [1,2,3]').drop('fld2') + query.expression('fld1', 'array_remove(integer_array, [5,6,7,8]) || [1,2,3]').drop('fld2').strict(StrictMode.Names) db.new_query(namespace).sort_stfield_distance('fldGeom1', 'fldGeom2', False) + db.new_query(namespace).aggregate_facet(['fld1', 'fld2']).limit(100) + def rx_example(): db = RxConnector('builtin:///tmp/pyrx') diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 9bf5d41..1fa0c6c 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -113,6 +113,12 @@ void QueryWrapper::DWithin(std::string_view index, double x, double y, double di ++queriesCount_; } +void QueryWrapper::AggregationSort(std::string_view field, bool desc) { + ser_.PutVarUint(QueryItemType::QueryAggregationSort); + ser_.PutVString(field); + ser_.PutVarUint(desc? 1 : 0); +} + void QueryWrapper::Aggregate(std::string_view index, AggType type) { ser_.PutVarUint(QueryItemType::QueryAggregation); ser_.PutVarUint(type); @@ -120,6 +126,15 @@ void QueryWrapper::Aggregate(std::string_view index, AggType type) { ser_.PutVString(index); } +void QueryWrapper::Aggregation(const std::vector& fields) { + ser_.PutVarUint(QueryItemType::QueryAggregation); + ser_.PutVarUint(AggType::AggFacet); + ser_.PutVarUint(fields.size()); + for (const auto& field : fields) { + ser_.PutVString(field); + } +} + void QueryWrapper::LogOp(OpType op) { switch (op) { case OpType::OpAnd: @@ -140,19 +155,9 @@ void QueryWrapper::Total(std::string_view totalName, CalcTotalMode mode) { } } -void QueryWrapper::Limit(unsigned limitItems) { - ser_.PutVarUint(QueryItemType::QueryLimit); - ser_.PutVarUint(limitItems); -} - -void QueryWrapper::Offset(unsigned startOffset) { - ser_.PutVarUint(QueryItemType::QueryOffset); - ser_.PutVarUint(startOffset); -} - -void QueryWrapper::Debug(unsigned level) { - ser_.PutVarUint(QueryItemType::QueryDebugLevel); - ser_.PutVarUint(level); +void QueryWrapper::AddValue(QueryItemType type, unsigned value) { + ser_.PutVarUint(type); + ser_.PutVarUint(value); } void QueryWrapper::Strict(StrictMode mode) { @@ -292,7 +297,6 @@ void QueryWrapper::putValue(double value) { } template <> void QueryWrapper::putValue(const reindexer::Variant& value) { - ser_.PutVarUint(VALUE_INT); ser_.PutVariant(value); } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 562f67b..fa1fbdd 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -66,6 +66,8 @@ class QueryWrapper { void DWithin(std::string_view index, double x, double y, double distance); void Aggregate(std::string_view index, AggType type); + void Aggregation(const std::vector& fields); + void AggregationSort(std::string_view field, bool desc); template void Sort(std::string_view index, bool desc, const std::vector& keys) { @@ -83,9 +85,8 @@ class QueryWrapper { void Total(std::string_view totalName, CalcTotalMode mode); - void Limit(unsigned limitItems); - void Offset(unsigned startOffset); - void Debug(unsigned level); + void AddValue(QueryItemType type, unsigned value); + void Strict(StrictMode mode); void Modifier(QueryItemType type); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 997e600..c649edd 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -669,7 +669,7 @@ static PyObject* Where(PyObject* self, PyObject* args) { } static PyObject* WhereQuery(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; + uintptr_t queryWrapperAddr = 0; uintptr_t subQueryWrapperAddr = 0; unsigned condition = 0; PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed @@ -817,6 +817,68 @@ static PyObject* AggregateAvg(PyObject* self, PyObject* args) { return aggregate static PyObject* AggregateMin(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMin); } static PyObject* AggregateMax(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMax); } +namespace { +static PyObject* addValue(PyObject* self, PyObject* args, QueryItemType type) { + uintptr_t queryWrapperAddr = 0; + int value = 0; + if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &value)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->AddValue(type, value); + + Py_RETURN_NONE; +} +} // namespace +static PyObject* AggregationLimit(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryAggregationLimit); } +static PyObject* AggregationOffset(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryAggregationOffset); } + +static PyObject* AggregationSort(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* field = nullptr; + unsigned desc = 0; + if (!PyArg_ParseTuple(args, "ksI", &queryWrapperAddr, &field, &desc)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + query->AggregationSort(field, (desc != 0)); + + return pyErr(errOK); +} + +static PyObject* Aggregation(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &fieldsList)) { + return nullptr; + } + + Py_XINCREF(fieldsList); + + std::vector fields; + if (fieldsList != nullptr) { + try { + fields = ParseListToStrVec(&fieldsList); + } catch (const Error& err) { + Py_DECREF(fieldsList); + + return pyErr(err); + } + } + + Py_XDECREF(fieldsList); + + auto query = getWrapper(queryWrapperAddr); + + query->Aggregation(fields); + + return pyErr(errOK); +} + static PyObject* Sort(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; @@ -884,47 +946,9 @@ static PyObject* total(PyObject* self, PyObject* args, CalcTotalMode mode) { static PyObject* ReqTotal(PyObject* self, PyObject* args) { return total(self, args, CalcTotalMode::ModeAccurateTotal); } static PyObject* CachedTotal(PyObject* self, PyObject* args) { return total(self, args, CalcTotalMode::ModeCachedTotal); } -static PyObject* Limit(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - int limitItems = 0; - if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &limitItems)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->Limit(limitItems); - - Py_RETURN_NONE; -} - -static PyObject* Offset(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - int startOffset = 0; - if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &startOffset)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->Offset(startOffset); - - Py_RETURN_NONE; -} - -static PyObject* Debug(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - int level = 0; - if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &level)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->Debug(level); - - Py_RETURN_NONE; -} +static PyObject* Limit(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryLimit); } +static PyObject* Offset(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryOffset); } +static PyObject* Debug(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryDebugLevel); } static PyObject* Strict(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index ff54dc2..c980ac5 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -66,6 +66,10 @@ static PyObject* AggregateSum(PyObject* self, PyObject* args); static PyObject* AggregateAvg(PyObject* self, PyObject* args); static PyObject* AggregateMin(PyObject* self, PyObject* args); static PyObject* AggregateMax(PyObject* self, PyObject* args); +static PyObject* AggregationLimit(PyObject* self, PyObject* args); +static PyObject* AggregationOffset(PyObject* self, PyObject* args); +static PyObject* AggregationSort(PyObject* self, PyObject* args); +static PyObject* Aggregation(PyObject* self, PyObject* args); static PyObject* Sort(PyObject* self, PyObject* args); static PyObject* And(PyObject* self, PyObject* args); static PyObject* Or(PyObject* self, PyObject* args); @@ -139,6 +143,10 @@ static PyMethodDef module_methods[] = { {"aggregate_avg", AggregateAvg, METH_VARARGS, "average field value"}, {"aggregate_min", AggregateMin, METH_VARARGS, "minimum field value"}, {"aggregate_max", AggregateMax, METH_VARARGS, "maximum field value"}, + {"aggregation_limit", AggregationLimit, METH_VARARGS, "limit facet results"}, + {"aggregation_offset", AggregationOffset, METH_VARARGS, "set offset for facet results"}, + {"aggregation_sort", AggregationSort, METH_VARARGS, "sort facets"}, + {"aggregation", Aggregation, METH_VARARGS, "get fields facet value"}, {"sort", Sort, METH_VARARGS, "apply sort order"}, {"op_and", And, METH_VARARGS, "next condition will be added with AND AND"}, {"op_or", Or, METH_VARARGS, "next condition will be added with OR AND"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 69539e3..f5ad0bf 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -44,9 +44,9 @@ def __init__(self, api, query_wrapper_ptr: int): """ self.api = api - self.query_wrapper_ptr = query_wrapper_ptr - self.err_code = 0 - self.err_msg = "" + self.query_wrapper_ptr: int = query_wrapper_ptr + self.err_code: int = 0 + self.err_msg: str = '' def __del__(self): """Free query memory @@ -67,13 +67,30 @@ def _raise_on_error(self): if self.err_code: raise Exception(self.err_msg) - def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float,str]] = None) -> Query: + @staticmethod + def _convert_to_list(param: Union[int, bool, float, str, List[Union[int, bool, float, str]]]) -> List[Union[int,bool,float,str]]: + """Convert an input parameter to a list + + # Arguments: + param (Union[None, int, bool, float, str, List[Union[int, bool, float, str]]]): The input parameter + + # Returns: + List[Union[int, bool, float, str]]: Always converted to a list + + """ + + param_copy = param.copy() if param is not None else [] + param_copy = param_copy if not isinstance(param_copy, list) else [param_copy] + return param_copy + + def where(self, index: str, condition: CondType, keys: Union[int, bool, float, str, List[Union[int, bool, float, str]]] = None) -> Query: """Add where condition to DB query with int args # Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[Union[int,bool,float,str]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (Union[None, int, bool, float, str, list[Union[int, bool, float, str]]]): + Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: (:obj:`Query`): Query object for further customizations @@ -83,18 +100,20 @@ def where(self, index: str, condition: CondType, keys: List[Union[int,bool,float """ - keys = keys or [] - self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, keys) + params: list = self._convert_to_list(keys) + + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) self._raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, keys: List[Union[int,bool,float,str]] = None) -> Query: + def where_query(self, sub_query: Query, condition: CondType, keys: Union[int, bool, float, str, List[Union[int, bool, float, str]]] = None) -> Query: """Add sub query where condition to DB query with int args # Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[Union[int,bool,float,str]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (Union[None, int, bool, float, str, List[Union[int, bool, float, str]]]): + Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: (:obj:`Query`): Query object for further customizations @@ -104,8 +123,9 @@ def where_query(self, sub_query: Query, condition: CondType, keys: List[Union[in """ - keys = keys or [] - self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, keys) + params: list = self._convert_to_list(keys) + + self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) self._raise_on_error() return self @@ -299,26 +319,96 @@ def aggregate_max(self, index: str) -> Query: self.api.aggregate_max(self.query_wrapper_ptr, index) return self -################################################################ ToDo -#type AggregateFacetRequest struct { -# query *Query -#} -#// fields should not be empty. -#func (q *Query) AggregateFacet(fields ...string) *AggregateFacetRequest { -#func (r *AggregateFacetRequest) Limit(limit int) *AggregateFacetRequest { -#func (r *AggregateFacetRequest) Offset(offset int) *AggregateFacetRequest { -#func (r *AggregateFacetRequest) Sort(field string, desc bool) *AggregateFacetRequest { -################################################################ + class _AggregateFacet(object) : + """An object representing the context of a Reindexer aggregate facet + + # Attributes: + api (module): An API module for Reindexer calls + query_wrapper_ptr (int): A memory pointer to Reindexer query object + + """ + + def __init__(self, query: Query): + """Constructs a new Reindexer AggregateFacetRequest object + + # Arguments: + api (module): An API module for Reindexer calls + query_wrapper_ptr (int): A memory pointer to Reindexer query object + + """ + + self.api = query.api + self.query_wrapper_ptr = query.query_wrapper_ptr + + def limit(self, limit: int) -> Query._AggregateFacet: + """Limit facet aggregation results + + # Arguments: + limit (int): Limit of aggregation of facet - def sort(self, index: str, desc: bool, keys: List[Union[int,bool,float,str]] = None) -> Query: + # Returns: + (:obj:`_AggregateFacet`): Facet object for further customizations + + """ + + self.api.aggregation_limit(self.query_wrapper_ptr, limit) + return self + + def offset(self, offset: int) -> Query._AggregateFacet: + """Set offset of the facet aggregation results + + # Arguments: + limit (int): Offset in facet aggregation results + + # Returns: + (:obj:`_AggregateFacet`): Facet object for further customizations + + """ + + self.api.aggregation_offset(self.query_wrapper_ptr, offset) + return self + + def sort(self, field: str, desc: bool) -> Query._AggregateFacet: + """Sort facets by field value + + # Arguments: + field (str): Item field. Use field 'count' to sort by facet's count value + desc (bool): Sort in descending order + + # Returns: + (:obj:`_AggregateFacet`): Facet object for further customizations + + """ + + self.api.aggregation_sort(self.query_wrapper_ptr, field, desc) + return self + + def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: + """Get fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data + column or 'count' and cut off by offset and limit. In order to support this functionality this method + returns AggregationFacetRequest which has methods sort, limit and offset + + # Arguments: + fields (list[string]): Fields any data column name or 'count', fields should not be empty + + # Returns: + (:obj:`_AggregateFacet`): Request object for further customizations + + """ + + self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, fields) + self._raise_on_error() + return self._AggregateFacet(self) + + def sort(self, index: str, desc: bool, keys: Union[int, bool, float, str, List[Union[int, bool, float, str]]] = None) -> Query: """Apply sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only # Arguments: index (string): The index name - desc (bool): True if sorting in descending order - keys (list[Union[int,bool,float,str]]): Value of index to match. For composite indexes keys must be list, with value of each subindex - # ToDo List[List[Union[int,bool,float,str]]] + desc (bool): Sort in descending order + keys (Union[None, int, bool, float, str, List[Union[int, bool, float, str]]]): + Value of index to match. For composite indexes keys must be list, with value of each subindex # Returns: (:obj:`Query`): Query object for further customizations @@ -328,8 +418,9 @@ def sort(self, index: str, desc: bool, keys: List[Union[int,bool,float,str]] = N """ - keys = keys or [] - self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, keys) + params: list = self._convert_to_list(keys) + + self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, params) self._raise_on_error() return self @@ -340,7 +431,7 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: # Arguments: index (string): The index name point (:obj:`Point`): Point object used in sorting operation - desc (bool): True if sorting in descending order + desc (bool): Sort in descending order # Returns: (:obj:`Query`): Query object for further customizations @@ -361,7 +452,7 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) # Arguments: first_field (string): First field name used in condition second_field (string): Second field name used in condition - desc (bool): Descending flag + desc (bool): Sort in descending order # Returns: (:obj:`Query`): Query object for further customizations @@ -661,4 +752,4 @@ def equal_position(self, equal_position: List[str]) -> Query: self._raise_on_error() return self -# ToDo 66/35 \ No newline at end of file +# ToDo 66/39 \ No newline at end of file From 87cf4a69d2ad7c7e99e9f8c89ea8a5f979645584 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 1 Nov 2024 11:43:45 +0300 Subject: [PATCH 038/125] Part XI: start of implementation of query builder. Add aggregation. Fix --- pyreindexer/query.py | 50 ++++++++++++++++++++------------------ pyreindexer/transaction.py | 38 ++++++++++++++--------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index f5ad0bf..6c912d9 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -23,6 +23,8 @@ class StrictMode(Enum): Names = 2 Indexes = 3 +simple_types = Union[int, str, bool, float] + class Query(object): """An object representing the context of a Reindexer query @@ -56,7 +58,7 @@ def __del__(self): if self.query_wrapper_ptr > 0: self.api.delete_query(self.query_wrapper_ptr) - def _raise_on_error(self): + def __raise_on_error(self): """Checks if there is an error code and raises with an error message # Raises: @@ -68,7 +70,7 @@ def _raise_on_error(self): raise Exception(self.err_msg) @staticmethod - def _convert_to_list(param: Union[int, bool, float, str, List[Union[int, bool, float, str]]]) -> List[Union[int,bool,float,str]]: + def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[simple_types]: """Convert an input parameter to a list # Arguments: @@ -79,11 +81,11 @@ def _convert_to_list(param: Union[int, bool, float, str, List[Union[int, bool, f """ - param_copy = param.copy() if param is not None else [] - param_copy = param_copy if not isinstance(param_copy, list) else [param_copy] - return param_copy + param = [] if param is None else param + param = param if isinstance(param, list) else [param] + return param - def where(self, index: str, condition: CondType, keys: Union[int, bool, float, str, List[Union[int, bool, float, str]]] = None) -> Query: + def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: """Add where condition to DB query with int args # Arguments: @@ -100,13 +102,13 @@ def where(self, index: str, condition: CondType, keys: Union[int, bool, float, s """ - params: list = self._convert_to_list(keys) + params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) - self._raise_on_error() + self.__raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, keys: Union[int, bool, float, str, List[Union[int, bool, float, str]]] = None) -> Query: + def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: """Add sub query where condition to DB query with int args # Arguments: @@ -123,10 +125,10 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[int, bo """ - params: list = self._convert_to_list(keys) + params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) - self._raise_on_error() + self.__raise_on_error() return self def where_composite(self, index: str, condition: CondType, sub_query: Query) -> Query: @@ -164,7 +166,7 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """ self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) - self._raise_on_error() + self.__raise_on_error() return self def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: @@ -195,7 +197,7 @@ def open_bracket(self) -> Query: """ self.err_code, self.err_msg = self.api.open_bracket(self.query_wrapper_ptr) - self._raise_on_error() + self.__raise_on_error() return self def close_bracket(self) -> Query: @@ -210,7 +212,7 @@ def close_bracket(self) -> Query: """ self.err_code, self.err_msg = self.api.close_bracket(self.query_wrapper_ptr) - self._raise_on_error() + self.__raise_on_error() return self def match(self, index: str, keys: List[str]) -> Query: @@ -229,7 +231,7 @@ def match(self, index: str, keys: List[str]) -> Query: """ self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) - self._raise_on_error() + self.__raise_on_error() return self def dwithin(self, index: str, point: Point, distance: float) -> Query: @@ -397,10 +399,10 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: """ self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, fields) - self._raise_on_error() + self.__raise_on_error() return self._AggregateFacet(self) - def sort(self, index: str, desc: bool, keys: Union[int, bool, float, str, List[Union[int, bool, float, str]]] = None) -> Query: + def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_types]]=None) -> Query: """Apply sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only @@ -418,10 +420,10 @@ def sort(self, index: str, desc: bool, keys: Union[int, bool, float, str, List[U """ - params: list = self._convert_to_list(keys) + params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, params) - self._raise_on_error() + self.__raise_on_error() return self def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: @@ -438,7 +440,7 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: """ - request : str = "ST_Distance(" + index + request: str = "ST_Distance(" + index request += ",ST_GeomFromText('point(" request += "{:.10f}".format(point.x) + " " + "{:.10f}".format(point.y) request += ")'))" @@ -462,7 +464,7 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) """ - request : str = 'ST_Distance(' + first_field + ',' + second_field + ')' + request: str = 'ST_Distance(' + first_field + ',' + second_field + ')' return self.sort(request, desc) @@ -700,7 +702,7 @@ def select(self, fields: List[str]) -> Query: """ self.err_code, self.err_msg = self.api.select_query(self.query_wrapper_ptr, fields) - self._raise_on_error() + self.__raise_on_error() return self def fetch_count(self, n: int) -> Query: @@ -732,7 +734,7 @@ def functions(self, functions: List[str]) -> Query: """ self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, functions) - self._raise_on_error() + self.__raise_on_error() return self def equal_position(self, equal_position: List[str]) -> Query: @@ -749,7 +751,7 @@ def equal_position(self, equal_position: List[str]) -> Query: """ self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) - self._raise_on_error() + self.__raise_on_error() return self # ToDo 66/39 \ No newline at end of file diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index b26e9f1..c1ed65c 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -31,7 +31,7 @@ def __del__(self): if self.transaction_wrapper_ptr > 0: _, _ = self.api.rollback_transaction(self.transaction_wrapper_ptr) - def _raise_on_error(self): + def __raise_on_error(self): """Checks if there is an error code and raises with an error message # Raises: @@ -42,7 +42,7 @@ def _raise_on_error(self): if self.err_code: raise Exception(self.err_msg) - def _raise_on_is_over(self): + def __raise_on_is_over(self): """Checks if there is an error code and raises with an error message # Raises: @@ -66,11 +66,11 @@ def insert(self, item_def, precepts=None): """ - self._raise_on_is_over() + self.__raise_on_is_over() if precepts is None: precepts = [] self.err_code, self.err_msg = self.api.item_insert_transaction(self.transaction_wrapper_ptr, item_def, precepts) - self._raise_on_error() + self.__raise_on_error() def update(self, item_def, precepts=None): """Update an item with its precepts to the transaction @@ -85,11 +85,10 @@ def update(self, item_def, precepts=None): """ - self._raise_on_is_over() - if precepts is None: - precepts = [] + self.__raise_on_is_over() + precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_update_transaction(self.transaction_wrapper_ptr, item_def, precepts) - self._raise_on_error() + self.__raise_on_error() def upsert(self, item_def, precepts=None): """Update an item with its precepts to the transaction. Creates the item if it not exists @@ -104,11 +103,10 @@ def upsert(self, item_def, precepts=None): """ - self._raise_on_is_over() - if precepts is None: - precepts = [] + self.__raise_on_is_over() + precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_upsert_transaction(self.transaction_wrapper_ptr, item_def, precepts) - self._raise_on_error() + self.__raise_on_error() def delete(self, item_def): """Delete an item from the transaction @@ -122,9 +120,9 @@ def delete(self, item_def): """ - self._raise_on_is_over() + self.__raise_on_is_over() self.err_code, self.err_msg = self.api.item_delete_transaction(self.transaction_wrapper_ptr, item_def) - self._raise_on_error() + self.__raise_on_error() def commit(self): """Apply changes @@ -135,10 +133,10 @@ def commit(self): """ - self._raise_on_is_over() + self.__raise_on_is_over() self.err_code, self.err_msg, _ = self.api.commit_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 - self._raise_on_error() + self.__raise_on_error() def commit_with_count(self) -> int: """Apply changes and return the number of count of changed items @@ -149,10 +147,10 @@ def commit_with_count(self) -> int: """ - self._raise_on_is_over() + self.__raise_on_is_over() self.err_code, self.err_msg, count = self.api.commit_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 - self._raise_on_error() + self.__raise_on_error() return count def rollback(self): @@ -164,7 +162,7 @@ def rollback(self): """ - self._raise_on_is_over() + self.__raise_on_is_over() self.err_code, self.err_msg = self.api.rollback_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 - self._raise_on_error() + self.__raise_on_error() From 3e04ec2b438fa8c9bc3c14e4a68682ecf4b3a772 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 1 Nov 2024 14:40:43 +0300 Subject: [PATCH 039/125] Part XII: start of implementation of query builder. Add SetObject --- pyreindexer/lib/include/pyobjtools.cc | 35 ++++++++++++++++- pyreindexer/lib/include/pyobjtools.h | 5 ++- pyreindexer/lib/include/query_wrapper.h | 18 +++++++++ pyreindexer/lib/src/rawpyreindexer.cc | 51 +++++++++++++++++++++---- pyreindexer/lib/src/rawpyreindexer.h | 4 ++ pyreindexer/query.py | 48 ++++++++++++++++++++++- 6 files changed, 150 insertions(+), 11 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 7301923..6123f45 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -96,7 +96,40 @@ void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer) { } } -std::vector ParseListToStrVec(PyObject** list) { +std::vector PyObjectToJson(PyObject** obj) { + std::vector values; + + reindexer::WrSerializer wrSer; + if (PyDict_Check(*obj)) { + Py_ssize_t sz = PyDict_Size(*obj); + if (sz) { + PyObject *key = nullptr, *value = nullptr; + Py_ssize_t pos = 0; + while (PyDict_Next(*obj, &pos, &key, &value)) { + const char* k = PyUnicode_AsUTF8(key); + wrSer.PrintJsonString(k); + wrSer << ':'; + pyValueSerialize(&value, wrSer); + values.emplace_back(wrSer.Slice()); + wrSer.Reset(); + } + } + } else if (PyList_Check(*obj) ) { + Py_ssize_t sz = PyList_Size(*obj); + for (Py_ssize_t i = 0; i < sz; ++i) { + PyObject* value = PyList_GetItem(*obj, i); + pyValueSerialize(&value, wrSer); + values.emplace_back(wrSer.Slice()); + wrSer.Reset(); + } + } else { + pyValueSerialize(obj, wrSer); + values.emplace_back(wrSer.Slice()); + } + return values; +} + +std::vector ParseStrListToStrVec(PyObject** list) { std::vector result; Py_ssize_t sz = PyList_Size(*list); diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index 8997d14..5b691e2 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -6,10 +6,11 @@ namespace pyreindexer { -std::vector ParseListToStrVec(PyObject** list); +std::vector ParseStrListToStrVec(PyObject** list); std::vector ParseListToVec(PyObject** list); -void PyObjectToJson(PyObject** dict, reindexer::WrSerializer& wrSer); +void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer); +std::vector PyObjectToJson(PyObject** obj); PyObject* PyObjectFromJson(reindexer::span json); } // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index fa1fbdd..653153b 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -91,6 +91,24 @@ class QueryWrapper { void Modifier(QueryItemType type); + void SetObject(std::string_view field, const std::vector& values, QueryItemType type) { + ser_.PutVarUint(type); + ser_.PutVString(field); + if (type == QueryItemType::QueryUpdateObject) { + ser_.PutVarUint(values.size()); // values count + } + if (type != QueryItemType::QueryUpdateField) { + ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag + } + if (type != QueryItemType::QueryUpdateObject) { + ser_.PutVarUint(values.size()); // values count + } + for (const auto& value : values) { + ser_.PutVarUint(0); // function/value flag + putValue(value); + } + } + void Drop(std::string_view field); void SetExpression(std::string_view field, std::string_view value); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index c649edd..992a602 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -315,7 +315,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { std::vector itemPrecepts; try { - itemPrecepts = ParseListToStrVec(&preceptsList); + itemPrecepts = ParseStrListToStrVec(&preceptsList); } catch (const Error& err) { Py_DECREF(preceptsList); @@ -546,7 +546,7 @@ PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode m std::vector itemPrecepts; try { - itemPrecepts = ParseListToStrVec(&preceptsList); + itemPrecepts = ParseStrListToStrVec(&preceptsList); } catch (const Error& err) { Py_DECREF(preceptsList); @@ -731,7 +731,7 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { std::vector keys; if (keysList != nullptr) { try { - keys = ParseListToStrVec(&keysList); + keys = ParseStrListToStrVec(&keysList); } catch (const Error& err) { Py_DECREF(keysList); @@ -862,7 +862,7 @@ static PyObject* Aggregation(PyObject* self, PyObject* args) { std::vector fields; if (fieldsList != nullptr) { try { - fields = ParseListToStrVec(&fieldsList); + fields = ParseStrListToStrVec(&fieldsList); } catch (const Error& err) { Py_DECREF(fieldsList); @@ -976,6 +976,43 @@ void modifier(PyObject* self, PyObject* args, QueryItemType type) { static PyObject* Explain(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryExplain); Py_RETURN_NONE; } static PyObject* WithRank(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryWithRank); Py_RETURN_NONE; } +namespace { +static PyObject* setObject(PyObject* self, PyObject* args, QueryItemType type) { + uintptr_t queryWrapperAddr = 0; + char* field = nullptr; + PyObject* valuesList = nullptr; // borrowed ref after ParseTuple + if (!PyArg_ParseTuple(args, "ksO!", &queryWrapperAddr, &field, &PyList_Type, &valuesList)) { + return nullptr; + } + + Py_INCREF(valuesList); + + std::vector values; + if (valuesList != nullptr) { + try { + values = PyObjectToJson(&valuesList); + } catch (const Error& err) { + Py_DECREF(valuesList); + + return pyErr(err); + } + } + + Py_DECREF(valuesList); + + if ((type == QueryItemType::QueryUpdateField) && (values.size() > 1)) { + type = QueryItemType::QueryUpdateFieldV2; + } + auto query = getWrapper(queryWrapperAddr); + + query->SetObject(field, values, type); + + return pyErr(errOK); +} +} // namespace +static PyObject* SetObject(PyObject* self, PyObject* args) { return setObject(self, args, QueryItemType::QueryUpdateObject); } +static PyObject* Set(PyObject* self, PyObject* args) { return setObject(self, args, QueryItemType::QueryUpdateField); } + static PyObject* Drop(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; @@ -1032,7 +1069,7 @@ static PyObject* SelectQuery(PyObject* self, PyObject* args) { std::vector fields; if (fieldsList != nullptr) { try { - fields = ParseListToStrVec(&fieldsList); + fields = ParseStrListToStrVec(&fieldsList); } catch (const Error& err) { Py_DECREF(fieldsList); @@ -1075,7 +1112,7 @@ static PyObject* AddFunctions(PyObject* self, PyObject* args) { std::vector functions; if (functionsList != nullptr) { try { - functions = ParseListToStrVec(&functionsList); + functions = ParseStrListToStrVec(&functionsList); } catch (const Error& err) { Py_DECREF(functionsList); @@ -1104,7 +1141,7 @@ static PyObject* AddEqualPosition(PyObject* self, PyObject* args) { std::vector equalPoses; if (equalPosesList != nullptr) { try { - equalPoses = ParseListToStrVec(&equalPosesList); + equalPoses = ParseStrListToStrVec(&equalPosesList); } catch (const Error& err) { Py_DECREF(equalPosesList); diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index c980ac5..001b0b8 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -82,6 +82,8 @@ static PyObject* Debug(PyObject* self, PyObject* args); static PyObject* Strict(PyObject* self, PyObject* args); static PyObject* Explain(PyObject* self, PyObject* args); static PyObject* WithRank(PyObject* self, PyObject* args); +static PyObject* SetObject(PyObject* self, PyObject* args); +static PyObject* Set(PyObject* self, PyObject* args); static PyObject* Drop(PyObject* self, PyObject* args); static PyObject* SetExpression(PyObject* self, PyObject* args); static PyObject* On(PyObject* self, PyObject* args); @@ -159,6 +161,8 @@ static PyMethodDef module_methods[] = { {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, {"explain", Explain, METH_VARARGS, "enable explain query"}, {"with_rank", WithRank, METH_VARARGS, "enable fulltext rank"}, + {"set_object", SetObject, METH_VARARGS, "add update query"}, + {"set", Set, METH_VARARGS, "add field update"}, {"drop", Drop, METH_VARARGS, "drop values"}, {"expression", SetExpression, METH_VARARGS, "set expression"}, {"on", On, METH_VARARGS, "on specifies join condition"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 6c912d9..06fb067 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -165,6 +165,7 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """ + keys = [] if keys is None else keys self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) self.__raise_on_error() return self @@ -230,6 +231,7 @@ def match(self, index: str, keys: List[str]) -> Query: """ + keys = [] if keys is None else keys self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) self.__raise_on_error() return self @@ -398,6 +400,7 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: """ + fields = [] if fields is None else fields self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, fields) self.__raise_on_error() return self._AggregateFacet(self) @@ -624,6 +627,46 @@ def with_rank(self) -> Query: #func (q *Query) Set(field string, values interface{}) *Query { ################################################################ + def set_object(self, field: str, values: List[simple_types]) -> Query: + """Adds an update query to an object field for an update query + + # Arguments: + field (string): Field name + values (list[Union[int, str, bool, float]]): List of values to add + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + values = [] if values is None else values + self.err_code, self.err_msg = self.api.set_object(self.query_wrapper_ptr, field, values) + self.__raise_on_error() + return self + + def set(self, field: str, values: List[simple_types]) -> Query: + """Adds a field update request to the update request + + # Arguments: + field (string): Field name + values (list[Union[int, str, bool, float]]): List of values to add + + # Returns: + (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + values = [] if values is None else values + self.err_code, self.err_msg = self.api.set(self.query_wrapper_ptr, field, values) + self.__raise_on_error() + return self + def drop(self, index: str) -> Query: """Drops a value for a field @@ -701,6 +744,7 @@ def select(self, fields: List[str]) -> Query: Exception: Raises with an error message of API return on non-zero error code """ + fields = [] if fields is None else fields self.err_code, self.err_msg = self.api.select_query(self.query_wrapper_ptr, fields) self.__raise_on_error() return self @@ -733,6 +777,7 @@ def functions(self, functions: List[str]) -> Query: Exception: Raises with an error message of API return on non-zero error code """ + functions = [] if functions is None else functions self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, functions) self.__raise_on_error() return self @@ -750,8 +795,9 @@ def equal_position(self, equal_position: List[str]) -> Query: Exception: Raises with an error message of API return on non-zero error code """ + equal_position = [] if equal_position is None else equal_position self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) self.__raise_on_error() return self -# ToDo 66/39 \ No newline at end of file +# ToDo 66/41 \ No newline at end of file From ea30af21bcaebedec1621c7a129cfbb64af861f9 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 1 Nov 2024 18:06:00 +0300 Subject: [PATCH 040/125] Part XIII: start of implementation of query builder. Add Join --- README.md | 12 +- pyreindexer/lib/include/query_wrapper.cc | 17 +-- pyreindexer/lib/include/query_wrapper.h | 2 + pyreindexer/lib/src/rawpyreindexer.cc | 14 +++ pyreindexer/lib/src/rawpyreindexer.h | 2 + pyreindexer/query.py | 135 ++++++++++++++++++++--- pyreindexer/query_results.py | 4 +- pyreindexer/rx_connector.py | 4 +- pyreindexer/transaction.py | 4 +- 9 files changed, 157 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index fd10acf..5b4752a 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ __Attributes:__ api (module): An API module loaded dynamically for Reindexer calls rx (int): A memory pointer to Reindexer instance - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message

close

@@ -374,8 +374,8 @@ When the results are fetched the iterator closes and frees a memory of results b __Attributes:__ api (module): An API module for Reindexer calls - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations pos (int): The current result position in iterator @@ -423,8 +423,8 @@ __Attributes:__ api (module): An API module for Reindexer calls transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message

commit

diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 1fa0c6c..230fbbd 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -183,14 +183,17 @@ void QueryWrapper::SetExpression(std::string_view field, std::string_view value) ser_.PutVString(value); // ToDo q.putValue(value); } -reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { - // ToDo - /*if q.closed { - q.panicTrace("query.On call on already closed query. You should create new Query") +void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex) { + if ((type == JoinType::InnerJoin) && (nextOperation_ == OpType::OpOr)) { + nextOperation_ = OpType::OpAnd; + type = JoinType::OrInnerJoin; } - if q.root == nil { - panic(fmt.Errorf("Can't join on root query")) - }*/ + ser_.PutVarUint(QueryJoinCondition); + ser_.PutVarUint(type); + ser_.PutVarUint(joinQueryIndex); +} + +reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { ser_.PutVarUint(QueryItemType::QueryJoinOn); ser_.PutVarUint(nextOperation_); ser_.PutVarUint(condition); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 653153b..b5f1d60 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -113,6 +113,8 @@ class QueryWrapper { void SetExpression(std::string_view field, std::string_view value); + void Join(JoinType type, unsigned joinQueryIndex); + reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); void Select(const std::vector& fields); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 992a602..3503c94 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1042,6 +1042,20 @@ static PyObject* SetExpression(PyObject* self, PyObject* args) { Py_RETURN_NONE; } +static PyObject* Join(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + unsigned type = 0; + unsigned index = 0; + if (!PyArg_ParseTuple(args, "kII", &queryWrapperAddr, &type, &index)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + query->Join(JoinType(type), index); + + Py_RETURN_NONE; +} + static PyObject* On(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 001b0b8..0b4717d 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -86,6 +86,7 @@ static PyObject* SetObject(PyObject* self, PyObject* args); static PyObject* Set(PyObject* self, PyObject* args); static PyObject* Drop(PyObject* self, PyObject* args); static PyObject* SetExpression(PyObject* self, PyObject* args); +static PyObject* Join(PyObject* self, PyObject* args); static PyObject* On(PyObject* self, PyObject* args); static PyObject* SelectQuery(PyObject* self, PyObject* args); static PyObject* FetchCount(PyObject* self, PyObject* args); @@ -165,6 +166,7 @@ static PyMethodDef module_methods[] = { {"set", Set, METH_VARARGS, "add field update"}, {"drop", Drop, METH_VARARGS, "drop values"}, {"expression", SetExpression, METH_VARARGS, "set expression"}, + {"join", Join, METH_VARARGS, "join 2 query"}, {"on", On, METH_VARARGS, "on specifies join condition"}, {"select_query", SelectQuery, METH_VARARGS, "select add filter to fields of result's objects"}, {"fetch_count", FetchCount, METH_VARARGS, "limit number of items"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 06fb067..0113491 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -23,6 +23,12 @@ class StrictMode(Enum): Names = 2 Indexes = 3 +class JoinType(Enum): + LeftJoin = 0 + InnerJoin = 1 + OrInnerJoin = 2 + Merge = 3 + simple_types = Union[int, str, bool, float] class Query(object): @@ -31,8 +37,11 @@ class Query(object): # Attributes: api (module): An API module for Reindexer calls query_wrapper_ptr (int): A memory pointer to Reindexer query object - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message + root (:object:`Query`): The root of the Reindexer query + # ToDo + merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects """ @@ -49,6 +58,12 @@ def __init__(self, api, query_wrapper_ptr: int): self.query_wrapper_ptr: int = query_wrapper_ptr self.err_code: int = 0 self.err_msg: str = '' + # ToDo + self.root: Query = None + self.join_type: JoinType = JoinType.LeftJoin + self.join_fields: List[str] = [] + self.join_queries: List[Query] = [] + self.merged_queries: List[Query] = [] def __del__(self): """Free query memory @@ -74,7 +89,7 @@ def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[si """Convert an input parameter to a list # Arguments: - param (Union[None, int, bool, float, str, List[Union[int, bool, float, str]]]): The input parameter + param (Union[None, simple_types, list[simple_types]]): The input parameter # Returns: List[Union[int, bool, float, str]]: Always converted to a list @@ -91,7 +106,7 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ # Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (Union[None, int, bool, float, str, list[Union[int, bool, float, str]]]): + keys (Union[None, simple_types, list[simple_types]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: @@ -114,7 +129,7 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_ # Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (Union[None, int, bool, float, str, List[Union[int, bool, float, str]]]): + keys (Union[None, simple_types, list[simple_types]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex # Returns: @@ -267,7 +282,7 @@ def distinct(self, index: str) -> Query: self.api.aggregate_distinct(self.query_wrapper_ptr, index) return self - def aggregate_sum(self, index: str) -> Query: + def aggregate_sum(self, index: str) -> Query: """Performs a summation of values for a specified index # Arguments: @@ -412,7 +427,7 @@ def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_typ # Arguments: index (string): The index name desc (bool): Sort in descending order - keys (Union[None, int, bool, float, str, List[Union[int, bool, float, str]]]): + keys (Union[None, simple_types, List[simple_types]]): Value of index to match. For composite indexes keys must be list, with value of each subindex # Returns: @@ -623,8 +638,6 @@ def with_rank(self) -> Query: #func (q *Query) ExecToJsonCtx(ctx context.Context, jsonRoots ...string) *JSONIterator { #func (q *Query) Delete() (int, error) #func (q *Query) DeleteCtx(ctx context.Context) (int, error) { -#func (q *Query) SetObject(field string, values interface{}) *Query { -#func (q *Query) Set(field string, values interface{}) *Query { ################################################################ def set_object(self, field: str, values: List[simple_types]) -> Query: @@ -632,7 +645,7 @@ def set_object(self, field: str, values: List[simple_types]) -> Query: # Arguments: field (string): Field name - values (list[Union[int, str, bool, float]]): List of values to add + values (list[simple_types]): List of values to add # Returns: (:obj:`Query`): Query object for further customizations @@ -652,7 +665,7 @@ def set(self, field: str, values: List[simple_types]) -> Query: # Arguments: field (string): Field name - values (list[Union[int, str, bool, float]]): List of values to add + values (list[simple_types]): List of values to add # Returns: (:obj:`Query`): Query object for further customizations @@ -705,15 +718,93 @@ def expression(self, field: str, value: str) -> Query: #func (q *Query) GetCtx(ctx context.Context) (item interface{}, found bool) { #func (q *Query) GetJson() (json []byte, found bool) { #func (q *Query) GetJsonCtx(ctx context.Context) (json []byte, found bool) { -#func (q *Query) InnerJoin(q2 *Query, field string) *Query { -#func (q *Query) Join(q2 *Query, field string) *Query { -#func (q *Query) LeftJoin(q2 *Query, field string) *Query { +################################################################ + + def __join(self, query: Query, field: str, join_type: JoinType) -> Query: + """Joins queries + + # Arguments: + query (:obj:`Query`): Query object to join + field (string): Joined field name + type (:enum:`JoinType`): Join type + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + if self.root is not None: + return self.root.__join(query, field, join_type) + + if query.root is not None: + raise Exception("Query.Join call on already joined query. You should create new Query") + + if join_type is not JoinType.LeftJoin: + # index of join query + self.api.join(self.query_wrapper_ptr, join_type.value, len(self.join_queries)) + + query.join_type = join_type + query.root = self + self.join_queries.append(query) + self.join_fields.append(field) + # ToDo self.join_handlers.append(None) + return query + + def inner_join(self, query: Query, field: str) -> Query: + return self.__join(query, field, JoinType.InnerJoin) + + def join(self, query: Query, field: str) -> Query: + """Join is an alias for LeftJoin + + """ + + return self.__join(query, field, JoinType.LeftJoin) + + def left_join(self, join_query: Query, field: str) -> Query: + """LeftJoin joins 2 queries. + Items from this query are expanded with the data from the join_query. + One of the conditions below must hold for `field` parameter in order for LeftJoin to work: + namespace of `join_query` contains `field` as one of its fields marked as `joined` + # ToDo `this query` has a join handler (registered via `q.JoinHandler(...)` call) with the same `field` value + + # Arguments: + query (:obj:`Query`): Query object to left join + field (string): Joined field name. As unique identifier for the join between this query and `join_query` + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + return self.__join(join_query, field, JoinType.LeftJoin) + +################################################################ ToDo #func (q *Query) JoinHandler(field string, handler JoinHandler) *Query { -#func (q *Query) Merge(q2 *Query) *Query { ################################################################ + def merge(self, query: Query) -> Query: + """Merge queries of the same type + + # Arguments: + query (:obj:`Query`): Query object to merge + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + + if self.root is not None: + return self.root.merge(query) + + if query.root is not None: + query = query.root + + query.root = self + self.merged_queries.append(query) + return self + def on(self, index: str, condition: CondType, join_index: str) -> Query: - """On specifies join condition. + """On specifies join condition # Arguments: index (string): Field name from `Query` namespace should be used during join @@ -725,6 +816,14 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: """ + # ToDo + #if q.closed { + # q.panicTrace("query.On call on already closed query. You should create new Query") + #} + + if self.root is None: + raise Exception("Can't join on root query") + self.api.on(self.query_wrapper_ptr, index, condition, join_index) return self @@ -732,7 +831,7 @@ def select(self, fields: List[str]) -> Query: """Sets list of columns in this namespace to be finally selected. The columns should be specified in the same case as the jsonpaths corresponding to them. Non-existent fields and fields in the wrong case are ignored. - If there are no fields in this list that meet these conditions, then the filter works as "*". + If there are no fields in this list that meet these conditions, then the filter works as "*" # Arguments: fields (list[string]): List of columns to be selected @@ -800,4 +899,4 @@ def equal_position(self, equal_position: List[str]) -> Query: self.__raise_on_error() return self -# ToDo 66/41 \ No newline at end of file +# ToDo 66/45 \ No newline at end of file diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 2b5696a..33c832a 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -4,8 +4,8 @@ class QueryResults(object): # Attributes: api (module): An API module for Reindexer calls - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations pos (int): The current result position in iterator diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 25c7fa7..b5e4a35 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -14,8 +14,8 @@ class RxConnector(RaiserMixin): # Attributes: api (module): An API module loaded dynamically for Reindexer calls rx (int): A memory pointer to Reindexer instance - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message """ diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index c1ed65c..37a7481 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -4,8 +4,8 @@ class Transaction(object): # Attributes: api (module): An API module for Reindexer calls transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object - err_code (int): the API error code - err_msg (string): the API error message + err_code (int): The API error code + err_msg (string): The API error message """ From 066efca55a646d8d53385e5c9f8e9bb7ef9e7b46 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Fri, 1 Nov 2024 21:19:51 +0300 Subject: [PATCH 041/125] Part XIV: start of implementation of query builder. Add Delete. Part I --- README.md | 34 ++++++------ pyreindexer/lib/include/query_wrapper.cc | 6 +++ pyreindexer/lib/include/query_wrapper.h | 4 +- pyreindexer/lib/src/rawpyreindexer.cc | 32 ++++++++--- pyreindexer/lib/src/rawpyreindexer.h | 6 ++- pyreindexer/lib/src/reindexerinterface.cc | 8 +++ pyreindexer/lib/src/reindexerinterface.h | 5 ++ pyreindexer/query.py | 65 +++++++++++++++++++---- pyreindexer/raiser_mixin.py | 2 +- pyreindexer/rx_connector.py | 40 +++++++------- 10 files changed, 143 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 5b4752a..f9e3ecb 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -66,7 +66,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -83,7 +83,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -105,7 +105,7 @@ __Returns:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -123,7 +123,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -141,7 +141,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -159,7 +159,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -178,7 +178,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -197,7 +197,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -216,7 +216,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -234,7 +234,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -253,7 +253,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -275,7 +275,7 @@ __Returns:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -293,7 +293,7 @@ __Arguments:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -314,7 +314,7 @@ __Returns:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -335,7 +335,7 @@ __Returns:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -356,7 +356,7 @@ __Returns:__ __Raises:__ - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 230fbbd..4ffd47a 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -2,6 +2,7 @@ #include +#include "core/query/query.h" #include "core/keyvalue/uuid.h" namespace pyreindexer { @@ -169,6 +170,11 @@ void QueryWrapper::Modifier(QueryItemType type) { ser_.PutVarUint(type); } +reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { + reindexer::Query query; // ToDo 1234 FromJSON or FromSQL. Need implement GetJSON and GetSQL + return db_->DeleteQuery(query, count); +} + void QueryWrapper::Drop(std::string_view field) { ser_.PutVarUint(QueryItemType::QueryDropField); ser_.PutVString(field); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index b5f1d60..931f643 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -91,6 +91,8 @@ class QueryWrapper { void Modifier(QueryItemType type); + reindexer::Error DeleteQuery(size_t& count); + void SetObject(std::string_view field, const std::vector& values, QueryItemType type) { ser_.PutVarUint(type); ser_.PutVString(field); @@ -129,7 +131,7 @@ class QueryWrapper { void putValue(T) {} private: - DBInterface* db_{nullptr}; // ToDo + DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; OpType nextOperation_{OpType::OpAnd}; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 3503c94..9225429 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -625,7 +625,7 @@ static PyObject* CreateQuery(PyObject* self, PyObject* args) { return Py_BuildValue("isK", errOK, "", reinterpret_cast(query)); } -static PyObject* DeleteQuery(PyObject* self, PyObject* args) { +static PyObject* DestroyQuery(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -636,7 +636,6 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { Py_RETURN_NONE; } - static PyObject* Where(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; @@ -965,16 +964,33 @@ static PyObject* Strict(PyObject* self, PyObject* args) { } namespace { -void modifier(PyObject* self, PyObject* args, QueryItemType type) { +PyObject* modifier(PyObject* self, PyObject* args, QueryItemType type) { uintptr_t queryWrapperAddr = 0; - if (PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { - auto query = getWrapper(queryWrapperAddr); - query->Modifier(type); + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; } + + auto query = getWrapper(queryWrapperAddr); + query->Modifier(type); + + Py_RETURN_NONE; } } // namespace -static PyObject* Explain(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryExplain); Py_RETURN_NONE; } -static PyObject* WithRank(PyObject* self, PyObject* args) { modifier(self, args, QueryItemType::QueryWithRank); Py_RETURN_NONE; } +static PyObject* Explain(PyObject* self, PyObject* args) { return modifier(self, args, QueryItemType::QueryExplain); } +static PyObject* WithRank(PyObject* self, PyObject* args) { return modifier(self, args, QueryItemType::QueryWithRank); } + +static PyObject* DeleteQuery(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + size_t count = 0; + auto err = query->DeleteQuery(count); + + return Py_BuildValue("isI", err.code(), err.what().c_str(), count); +} namespace { static PyObject* setObject(PyObject* self, PyObject* args, QueryItemType type) { diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 0b4717d..835de6c 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -52,7 +52,7 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args); static PyObject* RollbackTransaction(PyObject* self, PyObject* args); // query static PyObject* CreateQuery(PyObject* self, PyObject* args); -static PyObject* DeleteQuery(PyObject* self, PyObject* args); +static PyObject* DestroyQuery(PyObject* self, PyObject* args); static PyObject* Where(PyObject* self, PyObject* args); static PyObject* WhereQuery(PyObject* self, PyObject* args); static PyObject* WhereComposite(PyObject* self, PyObject* args); @@ -82,6 +82,7 @@ static PyObject* Debug(PyObject* self, PyObject* args); static PyObject* Strict(PyObject* self, PyObject* args); static PyObject* Explain(PyObject* self, PyObject* args); static PyObject* WithRank(PyObject* self, PyObject* args); +static PyObject* DeleteQuery(PyObject* self, PyObject* args); static PyObject* SetObject(PyObject* self, PyObject* args); static PyObject* Set(PyObject* self, PyObject* args); static PyObject* Drop(PyObject* self, PyObject* args); @@ -132,7 +133,7 @@ static PyMethodDef module_methods[] = { {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback changes. Free transaction object memory"}, // query {"create_query", CreateQuery, METH_VARARGS, "create new query"}, - {"delete_query", DeleteQuery, METH_VARARGS, "delete query. Free query object memory"}, + {"destroy_query", DestroyQuery, METH_VARARGS, "delete query object. Free query object memory"}, {"where", Where, METH_VARARGS, "add where condition with args"}, {"where_query", WhereQuery, METH_VARARGS, "add sub-query where condition"}, {"where_composite", WhereComposite, METH_VARARGS, "add where condition for composite indexes"}, @@ -162,6 +163,7 @@ static PyMethodDef module_methods[] = { {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, {"explain", Explain, METH_VARARGS, "enable explain query"}, {"with_rank", WithRank, METH_VARARGS, "enable fulltext rank"}, + {"delete_query", DeleteQuery, METH_VARARGS, "execute delete query"}, {"set_object", SetObject, METH_VARARGS, "add update query"}, {"set", Set, METH_VARARGS, "add field update"}, {"drop", Drop, METH_VARARGS, "drop values"}, diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index e905245..83f404d 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -100,6 +100,14 @@ Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& tra return err; } +template +Error ReindexerInterface::deleteQuery(const reindexer::Query& query, size_t& count) { + typename DBT::QueryResultsT qr; + auto err = db_.Delete(query, qr); + count = qr.Count(); + return err; +} + template <> Error ReindexerInterface::execute(std::function f) { return f(); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index aded2ab..e8b4dea 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -2,6 +2,7 @@ #include #include +#include "core/query/query.h" #include "core/indexdef.h" #include "core/namespacedef.h" #include "coroutine/channel.h" @@ -128,6 +129,9 @@ class ReindexerInterface { Error RollbackTransaction(typename DBT::TransactionT& tr) { return execute([this, &tr] { return rollbackTransaction(tr); }); } + Error DeleteQuery(const reindexer::Query& query, size_t& count) { + return execute([this, &query, &count] { return deleteQuery(query, count); }); + } private: Error execute(std::function f); @@ -155,6 +159,7 @@ class ReindexerInterface { Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); Error commitTransaction(typename DBT::TransactionT& transaction, size_t& count); Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } + Error deleteQuery(const reindexer::Query& query, size_t& count); Error stop(); DBT db_; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 0113491..b6e16a8 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -39,9 +39,12 @@ class Query(object): query_wrapper_ptr (int): A memory pointer to Reindexer query object err_code (int): The API error code err_msg (string): The API error message - root (:object:`Query`): The root of the Reindexer query - # ToDo + root (:object:`Query`): The root query of the Reindexer query + join_type (:enum:`JoinType`): Join type + join_fields (list[string]): A list of fields (name as unique identification) to join + join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects + closed (bool): Whether the query is closed """ @@ -58,12 +61,12 @@ def __init__(self, api, query_wrapper_ptr: int): self.query_wrapper_ptr: int = query_wrapper_ptr self.err_code: int = 0 self.err_msg: str = '' - # ToDo self.root: Query = None self.join_type: JoinType = JoinType.LeftJoin self.join_fields: List[str] = [] self.join_queries: List[Query] = [] self.merged_queries: List[Query] = [] + self.closed: bool = False def __del__(self): """Free query memory @@ -71,7 +74,28 @@ def __del__(self): """ if self.query_wrapper_ptr > 0: - self.api.delete_query(self.query_wrapper_ptr) + self.api.destroy_query(self.query_wrapper_ptr) + + def __close(self) -> None: + if self.root is not None: + self.root.__close() + + if self.closed : + raise Exception("Close call on already closed query") + + for i in range(len(self.join_queries)): + self.join_queries[i].__close() + self.join_queries[i] = None + + for i in range(len(self.merged_queries)): + self.merged_queries[i].__close() + self.merged_queries[i] = None + + # ToDo + #for i in self.join_handlers: + # self.join_handlers[i] = None + + self.closed = True def __raise_on_error(self): """Checks if there is an error code and raises with an error message @@ -636,7 +660,30 @@ def with_rank(self) -> Query: #func (q *Query) ExecCtx(ctx context.Context) *Iterator { #func (q *Query) ExecToJson(jsonRoots ...string) *JSONIterator { #func (q *Query) ExecToJsonCtx(ctx context.Context, jsonRoots ...string) *JSONIterator { -#func (q *Query) Delete() (int, error) +################################################################ + + def delete(self) -> int: + """Delete will execute query, and delete items, matches query + + # Returns: + (int): Number of deleted elements + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + if (self.root is not None) or (len(self.join_queries) > 0) : + raise Exception("Delete does not support joined queries") + + if self.closed : + raise Exception("Delete call on already closed query. You should create new Query") + + self.err_code, self.err_msg, number = self.api.delete_query(self.query_wrapper_ptr) + self.__close() + return number + +################################################################ ToDo #func (q *Query) DeleteCtx(ctx context.Context) (int, error) { ################################################################ @@ -737,7 +784,7 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: return self.root.__join(query, field, join_type) if query.root is not None: - raise Exception("Query.Join call on already joined query. You should create new Query") + raise Exception("Query.join call on already joined query. You should create new Query") if join_type is not JoinType.LeftJoin: # index of join query @@ -816,10 +863,8 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: """ - # ToDo - #if q.closed { - # q.panicTrace("query.On call on already closed query. You should create new Query") - #} + if self.closed : + raise Exception("Query.on call on already closed query. You should create new Query") if self.root is None: raise Exception("Can't join on root query") diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index d45cc23..daf7bb3 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -21,7 +21,7 @@ def raise_on_not_init(self): """Checks if there is an error code and raises with an error message # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet """ diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index b5e4a35..17ac70e 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -57,7 +57,7 @@ def namespace_open(self, namespace) -> None: namespace (string): A name of a namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -73,7 +73,7 @@ def namespace_close(self, namespace) -> None: namespace (string): A name of a namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -89,7 +89,7 @@ def namespace_drop(self, namespace) -> None: namespace (string): A name of a namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -109,7 +109,7 @@ def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -127,7 +127,7 @@ def index_add(self, namespace, index_def) -> None: index_def (dict): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -144,7 +144,7 @@ def index_update(self, namespace, index_def) -> None: index_def (dict): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -161,7 +161,7 @@ def index_drop(self, namespace, index_name) -> None: index_name (string): A name of an index # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -179,7 +179,7 @@ def item_insert(self, namespace, item_def, precepts=None) -> None: precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -199,7 +199,7 @@ def item_update(self, namespace, item_def, precepts=None) -> None: precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -219,7 +219,7 @@ def item_upsert(self, namespace, item_def, precepts=None) -> None: precepts (:obj:`list` of :obj:`str`): A dictionary of index definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -238,7 +238,7 @@ def item_delete(self, namespace, item_def) -> None: item_def (dict): A dictionary of item definition # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -256,7 +256,7 @@ def meta_put(self, namespace, key, value) -> None: value (string): A metadata for storage # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -276,7 +276,7 @@ def meta_get(self, namespace, key) -> str : string: A metadata value # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -294,7 +294,7 @@ def meta_delete(self, namespace, key) -> None: key (string): A key in a storage of Reindexer where metadata is kept # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -313,7 +313,7 @@ def meta_enum(self, namespace) -> List[str] : (:obj:`list` of :obj:`str`): A list of all metadata keys # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -333,7 +333,7 @@ def select(self, query) -> QueryResults: (:obj:`QueryResults`): A QueryResults iterator # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -353,7 +353,7 @@ def new_transaction(self, namespace) -> Transaction: (:obj:`Transaction`): A new transaction # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -373,7 +373,7 @@ def new_query(self, namespace: str) -> Query: (:obj:`Query`): A new query # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet """ @@ -409,7 +409,7 @@ def _api_init(self, dsn): dsn (string): The connection string which contains a protocol # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code """ @@ -423,7 +423,7 @@ def _api_close(self): """Destructs Reindexer instance correctly and resets memory pointer # Raises: - Exception: Raises with an error message of API return if Reindexer instance is not initialized yet + Exception: Raises with an error message when Reindexer instance is not initialized yet """ From c14a9aea42ce1e071979d7cabe17e79bea7ed158 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Sat, 2 Nov 2024 18:15:57 +0300 Subject: [PATCH 042/125] Part XV: start of implementation of query builder. Add Insert\Delete\Update. Part II --- pyreindexer/lib/include/query_wrapper.cc | 30 ++- pyreindexer/lib/include/query_wrapper.h | 10 +- pyreindexer/lib/src/rawpyreindexer.cc | 39 +++- pyreindexer/lib/src/rawpyreindexer.h | 8 +- pyreindexer/lib/src/reindexerinterface.cc | 22 +- pyreindexer/lib/src/reindexerinterface.h | 8 + pyreindexer/query.py | 264 +++++++++++++++++++--- pyreindexer/rx_connector.py | 58 ++++- pyreindexer/transaction.py | 2 +- 9 files changed, 399 insertions(+), 42 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 4ffd47a..4ad753a 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -170,9 +170,33 @@ void QueryWrapper::Modifier(QueryItemType type) { ser_.PutVarUint(type); } +reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { + reindexer::Serializer ser(ser_.Buf(), ser_.Len()); + reindexer::Query query = reindexer::Query::Deserialize(ser); + + while (!query.Eof()) { // ToDo + const auto joinType = JoinType(query.GetVarUint()); + reindexer::JoinedQuery q1{joinType, reindexer::Query::Deserialize(query)}; + if (q1.joinType == JoinType::Merge) { + q.Merge(std::move(q1)); + } else { + q.AddJoinQuery(std::move(q1)); + } + } + + return db_->SelectQuery(query, qr); +} + reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { - reindexer::Query query; // ToDo 1234 FromJSON or FromSQL. Need implement GetJSON and GetSQL - return db_->DeleteQuery(query, count); + reindexer::Serializer ser(ser_.Buf(), ser_.Len()); + reindexer::Query query = reindexer::Query::Deserialize(ser); + return db_->DeleteQuery(deserializedQuery, count); +} + +reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { + reindexer::Serializer ser(ser_.Buf(), ser_.Len()); + reindexer::Query query = reindexer::Query::Deserialize(ser); + return db_->UpdateQuery(query, qr); } void QueryWrapper::Drop(std::string_view field) { @@ -211,7 +235,7 @@ reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition return errOK; } -void QueryWrapper::Select(const std::vector& fields) { +void QueryWrapper::SelectFilter(const std::vector& fields) { for (const auto& field : fields) { ser_.PutVarUint(QueryItemType::QuerySelectFilter); ser_.PutVString(field); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 931f643..9c2725f 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -12,6 +12,7 @@ #endif #include "reindexerinterface.h" +#include "queryresults_wrapper.h" namespace pyreindexer { @@ -91,7 +92,9 @@ class QueryWrapper { void Modifier(QueryItemType type); + reindexer::Error SelectQuery(QueryResultsWrapper& qr); reindexer::Error DeleteQuery(size_t& count); + reindexer::Error UpdateQuery(QueryResultsWrapper& qr); void SetObject(std::string_view field, const std::vector& values, QueryItemType type) { ser_.PutVarUint(type); @@ -119,18 +122,19 @@ class QueryWrapper { reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); - void Select(const std::vector& fields); + void SelectFilter(const std::vector& fields); void FetchCount(int count); void AddFunctions(const std::vector& functions); void AddEqualPosition(const std::vector& equalPositions); + DBInterface* GetDB() const { return db_; } + private: template void putValue(T) {} -private: DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; @@ -138,7 +142,7 @@ class QueryWrapper { unsigned queriesCount_{0}; std::deque openedBrackets_; std::string totalName_; - int fetchCount_{0}; + int fetchCount_{1000}; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 9225429..65941e0 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -992,6 +992,41 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { return Py_BuildValue("isI", err.code(), err.what().c_str(), count); } +namespace { +enum class ExecuteType { Select, Update }; +static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) { + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + auto qresWrapper = new QueryResultsWrapper(query->GetDB()); + Error err = errOK; + switch (type) { + case ExecuteType::Select: + query->SelectQuery(*qresWrapper); + break; + case ExecuteType::Update: + query->UpdateQuery(*qresWrapper); + break; + default: + assert(false); + } + + if (!err.ok()) { + delete qresWrapper; + + return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); + } + + return Py_BuildValue("iskI", err.code(), err.what().c_str(), reinterpret_cast(qresWrapper), qresWrapper->Count()); +} +} // namespace +static PyObject* SelectQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Select); } +static PyObject* UpdateQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Update); } + namespace { static PyObject* setObject(PyObject* self, PyObject* args, QueryItemType type) { uintptr_t queryWrapperAddr = 0; @@ -1087,7 +1122,7 @@ static PyObject* On(PyObject* self, PyObject* args) { return pyErr(err); } -static PyObject* SelectQuery(PyObject* self, PyObject* args) { +static PyObject* SelectFilter(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &fieldsList)) { @@ -1111,7 +1146,7 @@ static PyObject* SelectQuery(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Select(fields); + query->SelectFilter(fields); return pyErr(errOK); } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 835de6c..fc428c8 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -82,14 +82,16 @@ static PyObject* Debug(PyObject* self, PyObject* args); static PyObject* Strict(PyObject* self, PyObject* args); static PyObject* Explain(PyObject* self, PyObject* args); static PyObject* WithRank(PyObject* self, PyObject* args); +static PyObject* SelectQuery(PyObject* self, PyObject* args); static PyObject* DeleteQuery(PyObject* self, PyObject* args); +static PyObject* UpdateQuery(PyObject* self, PyObject* args); static PyObject* SetObject(PyObject* self, PyObject* args); static PyObject* Set(PyObject* self, PyObject* args); static PyObject* Drop(PyObject* self, PyObject* args); static PyObject* SetExpression(PyObject* self, PyObject* args); static PyObject* Join(PyObject* self, PyObject* args); static PyObject* On(PyObject* self, PyObject* args); -static PyObject* SelectQuery(PyObject* self, PyObject* args); +static PyObject* SelectFilter(PyObject* self, PyObject* args); static PyObject* FetchCount(PyObject* self, PyObject* args); static PyObject* AddFunctions(PyObject* self, PyObject* args); static PyObject* AddEqualPosition(PyObject* self, PyObject* args); @@ -163,14 +165,16 @@ static PyMethodDef module_methods[] = { {"strict", Strict, METH_VARARGS, "request cached total items calculation"}, {"explain", Explain, METH_VARARGS, "enable explain query"}, {"with_rank", WithRank, METH_VARARGS, "enable fulltext rank"}, + {"select_query", SelectQuery, METH_VARARGS, "execute select query"}, {"delete_query", DeleteQuery, METH_VARARGS, "execute delete query"}, + {"update_query", UpdateQuery, METH_VARARGS, "execute update query"}, {"set_object", SetObject, METH_VARARGS, "add update query"}, {"set", Set, METH_VARARGS, "add field update"}, {"drop", Drop, METH_VARARGS, "drop values"}, {"expression", SetExpression, METH_VARARGS, "set expression"}, {"join", Join, METH_VARARGS, "join 2 query"}, {"on", On, METH_VARARGS, "on specifies join condition"}, - {"select_query", SelectQuery, METH_VARARGS, "select add filter to fields of result's objects"}, + {"select_filter", SelectFilter, METH_VARARGS, "select add filter to fields of result's objects"}, {"fetch_count", FetchCount, METH_VARARGS, "limit number of items"}, {"functions", AddFunctions, METH_VARARGS, "add sql-functions to query"}, {"equal_position", AddEqualPosition, METH_VARARGS, "add equal position fields"}, diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 83f404d..4dda981 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -100,11 +100,27 @@ Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& tra return err; } +template +Error ReindexerInterface::selectQuery(const reindexer::Query& query, QueryResultsWrapper& result) { + typename DBT::QueryResultsT qres; + auto err = db_.Select(query, qres); + result.Wrap(std::move(qres)); + return err; +} + template Error ReindexerInterface::deleteQuery(const reindexer::Query& query, size_t& count) { - typename DBT::QueryResultsT qr; - auto err = db_.Delete(query, qr); - count = qr.Count(); + typename DBT::QueryResultsT qres; + auto err = db_.Delete(query, qres); + count = qres.Count(); + return err; +} + +template +Error ReindexerInterface::updateQuery(const reindexer::Query& query, QueryResultsWrapper& result) { + typename DBT::QueryResultsT qres; + auto err = db_.Update(query, qres); + result.Wrap(std::move(qres)); return err; } diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index e8b4dea..96d7d4d 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -129,9 +129,15 @@ class ReindexerInterface { Error RollbackTransaction(typename DBT::TransactionT& tr) { return execute([this, &tr] { return rollbackTransaction(tr); }); } + Error SelectQuery(const reindexer::Query& query, QueryResultsWrapper& result) { + return execute([this, &query, &result] { return selectQuery(query, result); }); + } Error DeleteQuery(const reindexer::Query& query, size_t& count) { return execute([this, &query, &count] { return deleteQuery(query, count); }); } + Error UpdateQuery(const reindexer::Query& query, QueryResultsWrapper& result) { + return execute([this, &query, &result] { return updateQuery(query, result); }); + } private: Error execute(std::function f); @@ -159,7 +165,9 @@ class ReindexerInterface { Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); Error commitTransaction(typename DBT::TransactionT& transaction, size_t& count); Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } + Error selectQuery(const reindexer::Query& query, QueryResultsWrapper& result); Error deleteQuery(const reindexer::Query& query, size_t& count); + Error updateQuery(const reindexer::Query& query, QueryResultsWrapper& result); Error stop(); DBT db_; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index b6e16a8..d444de6 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import List, Union from enum import Enum +from pyreindexer.query_results import QueryResults from pyreindexer.point import Point class CondType(Enum): @@ -41,7 +42,7 @@ class Query(object): err_msg (string): The API error message root (:object:`Query`): The root query of the Reindexer query join_type (:enum:`JoinType`): Join type - join_fields (list[string]): A list of fields (name as unique identification) to join + # ToDo join_fields (list[string]): A list of fields (name as unique identification) to join join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects closed (bool): Whether the query is closed @@ -63,7 +64,7 @@ def __init__(self, api, query_wrapper_ptr: int): self.err_msg: str = '' self.root: Query = None self.join_type: JoinType = JoinType.LeftJoin - self.join_fields: List[str] = [] + # ToDo self.join_fields: List[str] = [] self.join_queries: List[Query] = [] self.merged_queries: List[Query] = [] self.closed: bool = False @@ -76,6 +77,28 @@ def __del__(self): if self.query_wrapper_ptr > 0: self.api.destroy_query(self.query_wrapper_ptr) + def __raise_on_error(self): + """Checks if there is an error code and raises with an error message + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + if self.err_code: + raise Exception(self.err_msg) + + def __raise_on_is_closed(self): + """Checks the state of a query and returns an error message when necessary + + # Raises: + Exception: Raises with an error message of API return if Query is closed + + """ + + if self.closed: + raise Exception("Query is closed. You should create new Query") + def __close(self) -> None: if self.root is not None: self.root.__close() @@ -85,29 +108,16 @@ def __close(self) -> None: for i in range(len(self.join_queries)): self.join_queries[i].__close() - self.join_queries[i] = None + self.join_queries.clear() for i in range(len(self.merged_queries)): self.merged_queries[i].__close() - self.merged_queries[i] = None + self.merged_queries.clear() - # ToDo - #for i in self.join_handlers: - # self.join_handlers[i] = None + # ToDo self.join_handlers.clear() self.closed = True - def __raise_on_error(self): - """Checks if there is an error code and raises with an error message - - # Raises: - Exception: Raises with an error message of API return on non-zero error code - - """ - - if self.err_code: - raise Exception(self.err_msg) - @staticmethod def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[simple_types]: """Convert an input parameter to a list @@ -137,10 +147,12 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) @@ -160,10 +172,12 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_ (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) @@ -181,8 +195,12 @@ def where_composite(self, index: str, condition: CondType, sub_query: Query) -> # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() self.api.where_composite(self.query_wrapper_ptr, index, condition.value, sub_query.query_wrapper_ptr) return self @@ -200,11 +218,15 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + keys = [] if keys is None else keys + self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) self.__raise_on_error() return self @@ -220,8 +242,13 @@ def where_between_fields(self, first_field: str, condition: CondType, second_fie # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.where_between_fields(self.query_wrapper_ptr, first_field, condition.value, second_field) return self @@ -232,10 +259,13 @@ def open_bracket(self) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + self.err_code, self.err_msg = self.api.open_bracket(self.query_wrapper_ptr) self.__raise_on_error() return self @@ -248,9 +278,12 @@ def close_bracket(self) -> Query: # Raises: Exception: Raises with an error message of API return on non-zero error code + Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + self.err_code, self.err_msg = self.api.close_bracket(self.query_wrapper_ptr) self.__raise_on_error() return self @@ -266,11 +299,15 @@ def match(self, index: str, keys: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + keys = [] if keys is None else keys + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) self.__raise_on_error() return self @@ -286,8 +323,13 @@ def dwithin(self, index: str, point: Point, distance: float) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.dwithin(self.query_wrapper_ptr, index, point.x, point.y, distance) return self @@ -301,8 +343,13 @@ def distinct(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.aggregate_distinct(self.query_wrapper_ptr, index) return self @@ -315,8 +362,13 @@ def aggregate_sum(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.aggregate_sum(self.query_wrapper_ptr, index) return self @@ -329,8 +381,13 @@ def aggregate_avg(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.aggregate_avg(self.query_wrapper_ptr, index) return self @@ -343,8 +400,13 @@ def aggregate_min(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.aggregate_min(self.query_wrapper_ptr, index) return self @@ -357,8 +419,13 @@ def aggregate_max(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.aggregate_max(self.query_wrapper_ptr, index) return self @@ -437,9 +504,15 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: # Returns: (:obj:`_AggregateFacet`): Request object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + fields = [] if fields is None else fields + self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, fields) self.__raise_on_error() return self._AggregateFacet(self) @@ -458,10 +531,13 @@ def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_typ (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, params) @@ -480,8 +556,13 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + request: str = "ST_Distance(" + index request += ",ST_GeomFromText('point(" request += "{:.10f}".format(point.x) + " " + "{:.10f}".format(point.y) @@ -502,10 +583,13 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + request: str = 'ST_Distance(' + first_field + ',' + second_field + ')' return self.sort(request, desc) @@ -517,8 +601,14 @@ def op_and(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.op_and(self.query_wrapper_ptr) return self @@ -530,8 +620,13 @@ def op_or(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.op_or(self.query_wrapper_ptr) return self @@ -542,8 +637,13 @@ def op_not(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.op_not(self.query_wrapper_ptr) return self @@ -556,8 +656,13 @@ def request_total(self, total_name: str= '') -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.request_total(self.query_wrapper_ptr, total_name) return self @@ -570,8 +675,13 @@ def cached_total(self, total_name: str= '') -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.cached_total(self.query_wrapper_ptr, total_name) return self @@ -585,8 +695,13 @@ def limit(self, limit_items: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.limit(self.query_wrapper_ptr, limit_items) return self @@ -599,8 +714,13 @@ def offset(self, start_offset: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.offset(self.query_wrapper_ptr, start_offset) return self @@ -613,8 +733,13 @@ def debug(self, level: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.debug(self.query_wrapper_ptr, level) return self @@ -627,8 +752,13 @@ def strict(self, mode: StrictMode) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.strict(self.query_wrapper_ptr, mode.value) return self @@ -638,8 +768,13 @@ def explain(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.explain(self.query_wrapper_ptr) return self @@ -649,8 +784,13 @@ def with_rank(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.with_rank(self.query_wrapper_ptr) return self @@ -669,17 +809,17 @@ def delete(self) -> int: (int): Number of deleted elements # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ if (self.root is not None) or (len(self.join_queries) > 0) : raise Exception("Delete does not support joined queries") - - if self.closed : - raise Exception("Delete call on already closed query. You should create new Query") + self.__raise_on_is_closed() self.err_code, self.err_msg, number = self.api.delete_query(self.query_wrapper_ptr) + self.__raise_on_error() self.__close() return number @@ -698,11 +838,15 @@ def set_object(self, field: str, values: List[simple_types]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + values = [] if values is None else values + self.err_code, self.err_msg = self.api.set_object(self.query_wrapper_ptr, field, values) self.__raise_on_error() return self @@ -718,11 +862,15 @@ def set(self, field: str, values: List[simple_types]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + values = [] if values is None else values + self.err_code, self.err_msg = self.api.set(self.query_wrapper_ptr, field, values) self.__raise_on_error() return self @@ -736,8 +884,13 @@ def drop(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.drop(self.query_wrapper_ptr, index) return self @@ -751,14 +904,42 @@ def expression(self, field: str, value: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.expression(self.query_wrapper_ptr, field, value) return self + def update(self) -> QueryResults: + """Update will execute query, and update fields in items, which matches query + + # Returns: + (:obj:`QueryResults`): A QueryResults iterator + + # Raises: + Exception: Raises with an error message of API return if Query is closed + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.__raise_on_is_closed() + + if (self.root is not None) or (len(self.join_queries) > 0) : + raise Exception("Update does not support joined queries") + + self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.update_query(self.query_wrapper_ptr) + self.__raise_on_error() + return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + ################################################################ // ToDo -#func (q *Query) Update() *Iterator { #func (q *Query) UpdateCtx(ctx context.Context) *Iterator { +################################################################ + +################################################################ // ToDo #func (q *Query) MustExec() *Iterator { #func (q *Query) MustExecCtx(ctx context.Context) *Iterator { #func (q *Query) Get() (item interface{}, found bool) { @@ -778,6 +959,9 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ if self.root is not None: @@ -793,7 +977,7 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: query.join_type = join_type query.root = self self.join_queries.append(query) - self.join_fields.append(field) + # ToDo self.join_fields.append(field) # ToDo self.join_handlers.append(None) return query @@ -838,8 +1022,13 @@ def merge(self, query: Query) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + if self.root is not None: return self.root.merge(query) @@ -861,10 +1050,12 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ - if self.closed : - raise Exception("Query.on call on already closed query. You should create new Query") + self.__raise_on_is_closed() if self.root is None: raise Exception("Can't join on root query") @@ -885,11 +1076,16 @@ def select(self, fields: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code + """ + self.__raise_on_is_closed() + fields = [] if fields is None else fields - self.err_code, self.err_msg = self.api.select_query(self.query_wrapper_ptr, fields) + + self.err_code, self.err_msg = self.api.select_filter(self.query_wrapper_ptr, fields) self.__raise_on_error() return self @@ -903,8 +1099,13 @@ def fetch_count(self, n: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations + # Raises: + Exception: Raises with an error message of API return if Query is closed + """ + self.__raise_on_is_closed() + self.api.fetch_count(self.query_wrapper_ptr, n) return self @@ -918,10 +1119,14 @@ def functions(self, functions: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ + self.__raise_on_is_closed() + functions = [] if functions is None else functions + self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, functions) self.__raise_on_error() return self @@ -936,12 +1141,17 @@ def equal_position(self, equal_position: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: + Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code + """ + self.__raise_on_is_closed() + equal_position = [] if equal_position is None else equal_position + self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) self.__raise_on_error() return self -# ToDo 66/45 \ No newline at end of file +# ToDo 66(58+8ctx)/48 \ No newline at end of file diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 17ac70e..b8ec972 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -323,7 +323,7 @@ def meta_enum(self, namespace) -> List[str] : self.raise_on_error() return res - def select(self, query) -> QueryResults: + def select(self, query: str) -> QueryResults: """Executes an SQL query and returns query results # Arguments: @@ -382,6 +382,62 @@ def new_query(self, namespace: str) -> Query: self.raise_on_error() return Query(self.api, query_wrapper_ptr) + def select_query(self, query: Query) -> QueryResults: + """Executes an query and returns query results + + # Arguments: + query (:obj:`Query`): An query object + + # Returns: + (:obj:`QueryResults`): A QueryResults iterator + + # Raises: + Exception: Raises with an error message when Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.raise_on_not_init() # ToDo return query.select() + self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select_query(query.query_wrapper_ptr) + self.raise_on_error() + return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + + def delete_query(self, query: Query) -> int: + """Delete will execute query, and delete items, matches query + + # Arguments: + query (:obj:`Query`): An query object + + # Returns: + (int): Number of deleted elements + + # Raises: + Exception: Raises with an error message when Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.raise_on_not_init() + return query.delete() + + def update_query(self, query: Query) -> QueryResults: + """Update will execute query, and update fields in items, which matches query + + # Arguments: + query (:obj:`Query`): An query object + + # Returns: + (:obj:`QueryResults`): A QueryResults iterator + + # Raises: + Exception: Raises with an error message when Reindexer instance is not initialized yet + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.raise_on_not_init() + return query.update() + def _api_import(self, dsn): """Imports an API dynamically depending on protocol specified in dsn diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 37a7481..d401ac5 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -43,7 +43,7 @@ def __raise_on_error(self): raise Exception(self.err_msg) def __raise_on_is_over(self): - """Checks if there is an error code and raises with an error message + """Checks the state of a transaction and returns an error message when necessary # Raises: Exception: Raises with an error message of API return if Transaction is over From 5cd9e40549686df3ae4558613c3c2c3c42f88932 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Sat, 2 Nov 2024 20:18:02 +0300 Subject: [PATCH 043/125] Part XV: start of implementation of query builder. Add Insert\Delete\Update. Part III: End --- pyreindexer/lib/include/query_wrapper.cc | 68 +++++--- pyreindexer/lib/include/query_wrapper.h | 10 +- pyreindexer/lib/src/rawpyreindexer.cc | 6 +- pyreindexer/query.py | 211 +---------------------- 4 files changed, 65 insertions(+), 230 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 4ad753a..15e94e5 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -170,32 +170,53 @@ void QueryWrapper::Modifier(QueryItemType type) { ser_.PutVarUint(type); } -reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { - reindexer::Serializer ser(ser_.Buf(), ser_.Len()); - reindexer::Query query = reindexer::Query::Deserialize(ser); - - while (!query.Eof()) { // ToDo - const auto joinType = JoinType(query.GetVarUint()); - reindexer::JoinedQuery q1{joinType, reindexer::Query::Deserialize(query)}; - if (q1.joinType == JoinType::Merge) { - q.Merge(std::move(q1)); - } else { - q.AddJoinQuery(std::move(q1)); - } +reindexer::Serializer QueryWrapper::prepareQueryData(reindexer::WrSerializer& data) { + reindexer::WrSerializer buffer; + buffer.Write(data.Slice()); // do full copy of query data + buffer.PutVarUint(QueryItemType::QueryEnd); // close query data + return {buffer.Buf(), buffer.Len()}; +} + +reindexer::JoinedQuery QueryWrapper::createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data) { + reindexer::Serializer jser = prepareQueryData(data); + return {joinType, reindexer::Query::Deserialize(jser)}; +} + +void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) { + for (auto joinQuery : joinQueries) { + auto jq = createJoinedQuery(joinQuery->joinType_, joinQuery->ser_); + query.AddJoinQuery(std::move(jq)); + } +} + +reindexer::Query QueryWrapper::prepareQuery() { + reindexer::Serializer ser = prepareQueryData(ser_); + auto query = reindexer::Query::Deserialize(ser); + + addJoinQueries(joinQueries_, query); + + for (auto mergedQuery : mergedQueries_) { + auto mq = createJoinedQuery(JoinType::Merge, mergedQuery->ser_); + query.Merge(std::move(mq)); + + addJoinQueries(mergedQuery->joinQueries_, query); } + return query; +} + +reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { + auto query = prepareQuery(); return db_->SelectQuery(query, qr); } reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { - reindexer::Serializer ser(ser_.Buf(), ser_.Len()); - reindexer::Query query = reindexer::Query::Deserialize(ser); - return db_->DeleteQuery(deserializedQuery, count); + auto query = prepareQuery(); + return db_->DeleteQuery(query, count); } reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { - reindexer::Serializer ser(ser_.Buf(), ser_.Len()); - reindexer::Query query = reindexer::Query::Deserialize(ser); + auto query = prepareQuery(); return db_->UpdateQuery(query, qr); } @@ -213,14 +234,19 @@ void QueryWrapper::SetExpression(std::string_view field, std::string_view value) ser_.PutVString(value); // ToDo q.putValue(value); } -void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex) { - if ((type == JoinType::InnerJoin) && (nextOperation_ == OpType::OpOr)) { +void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery) { + assert(joinQuery); + + joinType_ = type; + if ((joinType_ == JoinType::InnerJoin) && (nextOperation_ == OpType::OpOr)) { nextOperation_ = OpType::OpAnd; - type = JoinType::OrInnerJoin; + joinType_ = JoinType::OrInnerJoin; } ser_.PutVarUint(QueryJoinCondition); - ser_.PutVarUint(type); + ser_.PutVarUint(joinType_); ser_.PutVarUint(joinQueryIndex); + + joinQueries_.push_back(joinQuery); } reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 9c2725f..1d65aaa 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -118,7 +118,7 @@ class QueryWrapper { void SetExpression(std::string_view field, std::string_view value); - void Join(JoinType type, unsigned joinQueryIndex); + void Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery); reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); @@ -135,6 +135,11 @@ class QueryWrapper { template void putValue(T) {} + reindexer::Serializer prepareQueryData(reindexer::WrSerializer& data); + reindexer::JoinedQuery createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data); + void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query); + reindexer::Query prepareQuery(); + DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; @@ -143,6 +148,9 @@ class QueryWrapper { std::deque openedBrackets_; std::string totalName_; int fetchCount_{1000}; + JoinType joinType_{JoinType::LeftJoin}; + std::vector joinQueries_; + std::vector mergedQueries_; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 65941e0..9d512b3 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1097,12 +1097,14 @@ static PyObject* Join(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; unsigned type = 0; unsigned index = 0; - if (!PyArg_ParseTuple(args, "kII", &queryWrapperAddr, &type, &index)) { + uintptr_t queryWrapperAddrJoin = 0; + if (!PyArg_ParseTuple(args, "kIIk", &queryWrapperAddr, &type, &index, &queryWrapperAddrJoin)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->Join(JoinType(type), index); + auto joinQuery = getWrapper(queryWrapperAddrJoin); + query->Join(JoinType(type), index, joinQuery); Py_RETURN_NONE; } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index d444de6..77f1118 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -67,7 +67,6 @@ def __init__(self, api, query_wrapper_ptr: int): # ToDo self.join_fields: List[str] = [] self.join_queries: List[Query] = [] self.merged_queries: List[Query] = [] - self.closed: bool = False def __del__(self): """Free query memory @@ -88,36 +87,6 @@ def __raise_on_error(self): if self.err_code: raise Exception(self.err_msg) - def __raise_on_is_closed(self): - """Checks the state of a query and returns an error message when necessary - - # Raises: - Exception: Raises with an error message of API return if Query is closed - - """ - - if self.closed: - raise Exception("Query is closed. You should create new Query") - - def __close(self) -> None: - if self.root is not None: - self.root.__close() - - if self.closed : - raise Exception("Close call on already closed query") - - for i in range(len(self.join_queries)): - self.join_queries[i].__close() - self.join_queries.clear() - - for i in range(len(self.merged_queries)): - self.merged_queries[i].__close() - self.merged_queries.clear() - - # ToDo self.join_handlers.clear() - - self.closed = True - @staticmethod def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[simple_types]: """Convert an input parameter to a list @@ -147,12 +116,10 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) @@ -172,12 +139,10 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_ (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) @@ -195,12 +160,8 @@ def where_composite(self, index: str, condition: CondType, sub_query: Query) -> # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() self.api.where_composite(self.query_wrapper_ptr, index, condition.value, sub_query.query_wrapper_ptr) return self @@ -218,13 +179,10 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - keys = [] if keys is None else keys self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) @@ -242,13 +200,8 @@ def where_between_fields(self, first_field: str, condition: CondType, second_fie # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.where_between_fields(self.query_wrapper_ptr, first_field, condition.value, second_field) return self @@ -259,13 +212,10 @@ def open_bracket(self) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - self.err_code, self.err_msg = self.api.open_bracket(self.query_wrapper_ptr) self.__raise_on_error() return self @@ -278,12 +228,9 @@ def close_bracket(self) -> Query: # Raises: Exception: Raises with an error message of API return on non-zero error code - Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - self.err_code, self.err_msg = self.api.close_bracket(self.query_wrapper_ptr) self.__raise_on_error() return self @@ -299,13 +246,10 @@ def match(self, index: str, keys: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - keys = [] if keys is None else keys self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) @@ -323,13 +267,8 @@ def dwithin(self, index: str, point: Point, distance: float) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.dwithin(self.query_wrapper_ptr, index, point.x, point.y, distance) return self @@ -343,13 +282,8 @@ def distinct(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.aggregate_distinct(self.query_wrapper_ptr, index) return self @@ -362,13 +296,8 @@ def aggregate_sum(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.aggregate_sum(self.query_wrapper_ptr, index) return self @@ -381,13 +310,8 @@ def aggregate_avg(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.aggregate_avg(self.query_wrapper_ptr, index) return self @@ -400,13 +324,8 @@ def aggregate_min(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.aggregate_min(self.query_wrapper_ptr, index) return self @@ -419,13 +338,8 @@ def aggregate_max(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.aggregate_max(self.query_wrapper_ptr, index) return self @@ -504,13 +418,8 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: # Returns: (:obj:`_AggregateFacet`): Request object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - fields = [] if fields is None else fields self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, fields) @@ -531,13 +440,10 @@ def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_typ (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - params: list = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, params) @@ -556,13 +462,8 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - request: str = "ST_Distance(" + index request += ",ST_GeomFromText('point(" request += "{:.10f}".format(point.x) + " " + "{:.10f}".format(point.y) @@ -583,13 +484,10 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - request: str = 'ST_Distance(' + first_field + ',' + second_field + ')' return self.sort(request, desc) @@ -602,13 +500,8 @@ def op_and(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.op_and(self.query_wrapper_ptr) return self @@ -620,13 +513,8 @@ def op_or(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.op_or(self.query_wrapper_ptr) return self @@ -637,13 +525,8 @@ def op_not(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.op_not(self.query_wrapper_ptr) return self @@ -656,13 +539,8 @@ def request_total(self, total_name: str= '') -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.request_total(self.query_wrapper_ptr, total_name) return self @@ -675,13 +553,8 @@ def cached_total(self, total_name: str= '') -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.cached_total(self.query_wrapper_ptr, total_name) return self @@ -695,13 +568,8 @@ def limit(self, limit_items: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.limit(self.query_wrapper_ptr, limit_items) return self @@ -714,13 +582,8 @@ def offset(self, start_offset: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.offset(self.query_wrapper_ptr, start_offset) return self @@ -733,13 +596,8 @@ def debug(self, level: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.debug(self.query_wrapper_ptr, level) return self @@ -752,13 +610,8 @@ def strict(self, mode: StrictMode) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.strict(self.query_wrapper_ptr, mode.value) return self @@ -768,13 +621,8 @@ def explain(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.explain(self.query_wrapper_ptr) return self @@ -784,13 +632,8 @@ def with_rank(self) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.with_rank(self.query_wrapper_ptr) return self @@ -809,18 +652,16 @@ def delete(self) -> int: (int): Number of deleted elements # Raises: - Exception: Raises with an error message of API return if Query is closed + Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code """ if (self.root is not None) or (len(self.join_queries) > 0) : raise Exception("Delete does not support joined queries") - self.__raise_on_is_closed() self.err_code, self.err_msg, number = self.api.delete_query(self.query_wrapper_ptr) self.__raise_on_error() - self.__close() return number ################################################################ ToDo @@ -838,13 +679,10 @@ def set_object(self, field: str, values: List[simple_types]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - values = [] if values is None else values self.err_code, self.err_msg = self.api.set_object(self.query_wrapper_ptr, field, values) @@ -862,13 +700,10 @@ def set(self, field: str, values: List[simple_types]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - values = [] if values is None else values self.err_code, self.err_msg = self.api.set(self.query_wrapper_ptr, field, values) @@ -884,13 +719,8 @@ def drop(self, index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.drop(self.query_wrapper_ptr, index) return self @@ -904,13 +734,8 @@ def expression(self, field: str, value: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.expression(self.query_wrapper_ptr, field, value) return self @@ -921,13 +746,11 @@ def update(self) -> QueryResults: (:obj:`QueryResults`): A QueryResults iterator # Raises: - Exception: Raises with an error message of API return if Query is closed + Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - if (self.root is not None) or (len(self.join_queries) > 0) : raise Exception("Update does not support joined queries") @@ -960,7 +783,7 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed + Exception: Raises with an error message when query is in an invalid state """ @@ -972,7 +795,7 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: if join_type is not JoinType.LeftJoin: # index of join query - self.api.join(self.query_wrapper_ptr, join_type.value, len(self.join_queries)) + self.api.join(self.query_wrapper_ptr, join_type.value, len(self.join_queries), query.query_wrapper_ptr) query.join_type = join_type query.root = self @@ -1022,13 +845,8 @@ def merge(self, query: Query) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - if self.root is not None: return self.root.merge(query) @@ -1036,7 +854,7 @@ def merge(self, query: Query) -> Query: query = query.root query.root = self - self.merged_queries.append(query) + self.merged_queries.append(query) # ToDo return self def on(self, index: str, condition: CondType, join_index: str) -> Query: @@ -1050,13 +868,8 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - if self.root is None: raise Exception("Can't join on root query") @@ -1076,13 +889,10 @@ def select(self, fields: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - fields = [] if fields is None else fields self.err_code, self.err_msg = self.api.select_filter(self.query_wrapper_ptr, fields) @@ -1099,13 +909,8 @@ def fetch_count(self, n: int) -> Query: # Returns: (:obj:`Query`): Query object for further customizations - # Raises: - Exception: Raises with an error message of API return if Query is closed - """ - self.__raise_on_is_closed() - self.api.fetch_count(self.query_wrapper_ptr, n) return self @@ -1119,12 +924,9 @@ def functions(self, functions: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - functions = [] if functions is None else functions self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, functions) @@ -1141,13 +943,10 @@ def equal_position(self, equal_position: List[str]) -> Query: (:obj:`Query`): Query object for further customizations # Raises: - Exception: Raises with an error message of API return if Query is closed Exception: Raises with an error message of API return on non-zero error code """ - self.__raise_on_is_closed() - equal_position = [] if equal_position is None else equal_position self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) From d051dba05fd05a96af64da94d58f4fbeadc8b846 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 5 Nov 2024 14:56:16 +0300 Subject: [PATCH 044/125] Part XV: start of implementation of query builder. Add Insert\Delete\Update. Part IV: Real end and fixes --- pyreindexer/CMakeLists.txt | 12 +++-- pyreindexer/lib/include/pyobjtools.cc | 3 ++ pyreindexer/lib/include/query_wrapper.cc | 5 ++ pyreindexer/lib/include/query_wrapper.h | 7 +-- .../lib/include/queryresults_wrapper.h | 5 ++ pyreindexer/lib/src/rawpyreindexer.cc | 29 ++++++++++- pyreindexer/lib/src/rawpyreindexer.h | 4 ++ pyreindexer/query.py | 50 +++++++++++++++++-- pyreindexer/query_results.py | 12 +++++ pyreindexer/rx_connector.py | 6 +-- 10 files changed, 114 insertions(+), 19 deletions(-) diff --git a/pyreindexer/CMakeLists.txt b/pyreindexer/CMakeLists.txt index 7ac89e4..1bde9ee 100644 --- a/pyreindexer/CMakeLists.txt +++ b/pyreindexer/CMakeLists.txt @@ -11,11 +11,13 @@ endif() enable_testing() set(PY_MIN_VERSION 3.6) -set(RX_MIN_VERSION 3.24.0) +set(RX_MIN_VERSION 3.24.0) # ToDo find_package(PythonInterp ${PY_MIN_VERSION}) find_package(PythonLibs ${PY_MIN_VERSION}) find_package(reindexer CONFIG ${RX_MIN_VERSION}) +option(WITH_GCOV "Enable instrumented code coverage build" OFF) + set(LIB_BUILTIN_NAME "rawpyreindexerb") set(LIB_CPROTO_NAME "rawpyreindexerc") set(LIBS_EXT ".so") @@ -23,8 +25,8 @@ set(LIBS_EXT ".so") set(LIBSRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/src) set(RESOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/include) -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Werror -Wno-unused-parameter -fexceptions") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wswitch-enum") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Werror -Wswitch-enum -Wno-unused-parameter -Wold-style-cast -fexceptions") string (REPLACE "-O2" "-O3" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") file(GLOB_RECURSE SRCS ${RESOURCES_DIR}/*.cc ${LIBSRC_DIR}/*.cc) @@ -54,6 +56,6 @@ set_target_properties(${LIB_BUILTIN_NAME} PROPERTIES PREFIX "") set_target_properties(${LIB_CPROTO_NAME} PROPERTIES PREFIX "") if (WITH_GCOV) - target_link_libraries(${LIB_BUILTIN_NAME} -fprofile-arcs) - target_link_libraries(${LIB_CPROTO_NAME} -fprofile-arcs) + target_link_libraries(${LIB_BUILTIN_NAME} -fprofile-arcs -ftest-coverage) + target_link_libraries(${LIB_CPROTO_NAME} -fprofile-arcs -ftest-coverage) endif (WITH_GCOV) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 6123f45..4db5f9d 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -226,6 +226,9 @@ PyObject* pyValueFromJsonValue(const gason::JsonValue& value) { Py_XDECREF(dictFromJson); } break; + case gason::JSON_EMPTY: + throw gason::Exception("Unexpected 'JSON_EMPTY' tag"); + break; } return pyValue; diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 15e94e5..7aa90ee 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -249,6 +249,11 @@ void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* jo joinQueries_.push_back(joinQuery); } +void QueryWrapper::Merge(QueryWrapper* mergeQuery) { + assert(mergeQuery); + mergedQueries_.push_back(mergeQuery); +} + reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition, std::string_view joinIndex) { ser_.PutVarUint(QueryItemType::QueryJoinOn); ser_.PutVarUint(nextOperation_); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 1d65aaa..17bd066 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -119,6 +119,7 @@ class QueryWrapper { void SetExpression(std::string_view field, std::string_view value); void Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery); + void Merge(QueryWrapper* mergeQuery); reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); @@ -136,9 +137,9 @@ class QueryWrapper { void putValue(T) {} reindexer::Serializer prepareQueryData(reindexer::WrSerializer& data); - reindexer::JoinedQuery createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data); - void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query); - reindexer::Query prepareQuery(); + reindexer::JoinedQuery createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data); + void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query); + reindexer::Query prepareQuery(); DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index c907ed5..3d57600 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -35,6 +35,11 @@ class QueryResultsWrapper { return db_->Select(query, *this); } + Error Status() { + assert(wrap_); + return it_.Status(); + } + size_t Count() const { assert(wrap_); return qres_.Count(); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 9d512b3..716d0ca 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -422,6 +422,17 @@ static PyObject* EnumMeta(PyObject* self, PyObject* args) { // query results ------------------------------------------------------------------------------------------------------- +static PyObject* QueryResultsWrapperStatus(PyObject* self, PyObject* args) { + uintptr_t qresWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &qresWrapperAddr)) { + return nullptr; + } + + QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); + auto err = qresWrapper->Status(); + return pyErr(err); +} + static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args) { uintptr_t qresWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &qresWrapperAddr)) { @@ -1103,8 +1114,22 @@ static PyObject* Join(PyObject* self, PyObject* args) { } auto query = getWrapper(queryWrapperAddr); - auto joinQuery = getWrapper(queryWrapperAddrJoin); - query->Join(JoinType(type), index, joinQuery); + auto queryJoin = getWrapper(queryWrapperAddrJoin); + query->Join(JoinType(type), index, queryJoin); + + Py_RETURN_NONE; +} + +static PyObject* Merge(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + uintptr_t queryWrapperAddrMerge = 0; + if (!PyArg_ParseTuple(args, "kIIk", &queryWrapperAddr, &queryWrapperAddrMerge)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + auto queryMerge = getWrapper(queryWrapperAddrMerge); + query->Merge(queryMerge); Py_RETURN_NONE; } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index fc428c8..bc17dda 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -39,6 +39,7 @@ static PyObject* GetMeta(PyObject* self, PyObject* args); static PyObject* DeleteMeta(PyObject* self, PyObject* args); static PyObject* EnumMeta(PyObject* self, PyObject* args); // query results +static PyObject* QueryResultsWrapperStatus(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); static PyObject* GetAggregationResults(PyObject* self, PyObject* args); @@ -90,6 +91,7 @@ static PyObject* Set(PyObject* self, PyObject* args); static PyObject* Drop(PyObject* self, PyObject* args); static PyObject* SetExpression(PyObject* self, PyObject* args); static PyObject* Join(PyObject* self, PyObject* args); +static PyObject* Merge(PyObject* self, PyObject* args); static PyObject* On(PyObject* self, PyObject* args); static PyObject* SelectFilter(PyObject* self, PyObject* args); static PyObject* FetchCount(PyObject* self, PyObject* args); @@ -122,6 +124,7 @@ static PyMethodDef module_methods[] = { {"meta_delete", DeleteMeta, METH_VARARGS, "delete meta"}, {"meta_enum", EnumMeta, METH_VARARGS, "enum meta"}, // query results + {"query_results_status", QueryResultsWrapperStatus, METH_VARARGS, "get query result status"}, {"query_results_iterate", QueryResultsWrapperIterate, METH_VARARGS, "get query result"}, {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, {"get_agg_results", GetAggregationResults, METH_VARARGS, "get aggregation results"}, @@ -173,6 +176,7 @@ static PyMethodDef module_methods[] = { {"drop", Drop, METH_VARARGS, "drop values"}, {"expression", SetExpression, METH_VARARGS, "set expression"}, {"join", Join, METH_VARARGS, "join 2 query"}, + {"merge", Merge, METH_VARARGS, "merge 2 query"}, {"on", On, METH_VARARGS, "on specifies join condition"}, {"select_filter", SelectFilter, METH_VARARGS, "select add filter to fields of result's objects"}, {"fetch_count", FetchCount, METH_VARARGS, "limit number of items"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 77f1118..3d782ee 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -639,8 +639,30 @@ def with_rank(self) -> Query: ################################################################ ToDo #func (q *Query) SetContext(ctx interface{}) *Query { -#func (q *Query) Exec() *Iterator { +################################################################ + + def execute(self) -> QueryResults: + """Exec will execute query, and return slice of items + + # Returns: + (:obj:`QueryResults`): A QueryResults iterator + + # Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + """ + + if self.root is not None : + return self.root.execute() + + self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select_query(self.query_wrapper_ptr) + self.__raise_on_error() + return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + +################################################################ ToDo #func (q *Query) ExecCtx(ctx context.Context) *Iterator { +################################################################ ToDo - not actual #func (q *Query) ExecToJson(jsonRoots ...string) *JSONIterator { #func (q *Query) ExecToJsonCtx(ctx context.Context, jsonRoots ...string) *JSONIterator { ################################################################ @@ -762,12 +784,31 @@ def update(self) -> QueryResults: #func (q *Query) UpdateCtx(ctx context.Context) *Iterator { ################################################################ + def must_execute(self) -> QueryResults: + """Exec will execute query, and return slice of items, with status check + + # Returns: + (:obj:`QueryResults`): A QueryResults iterator + + # Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + """ + + result = self.execute() + result.status() + return result + ################################################################ // ToDo -#func (q *Query) MustExec() *Iterator { #func (q *Query) MustExecCtx(ctx context.Context) *Iterator { +################################################################ #func (q *Query) Get() (item interface{}, found bool) { +################################################################ // ToDo #func (q *Query) GetCtx(ctx context.Context) (item interface{}, found bool) { +################################################################ #func (q *Query) GetJson() (json []byte, found bool) { +################################################################ // ToDo #func (q *Query) GetJsonCtx(ctx context.Context) (json []byte, found bool) { ################################################################ @@ -854,7 +895,8 @@ def merge(self, query: Query) -> Query: query = query.root query.root = self - self.merged_queries.append(query) # ToDo + self.merged_queries.append(query) + self.api.merge(query) return self def on(self, index: str, condition: CondType, join_index: str) -> Query: @@ -952,5 +994,3 @@ def equal_position(self, equal_position: List[str]) -> Query: self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) self.__raise_on_error() return self - -# ToDo 66(58+8ctx)/48 \ No newline at end of file diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 33c832a..3106422 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -62,6 +62,18 @@ def __del__(self): self._close_iterator() + def status(self): + """Check status + + # Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.query_results_status(self.qres_wrapper_ptr) + if self.err_code: + raise Exception(self.err_msg) + def count(self): """Returns a count of results diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index b8ec972..c050a9a 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -397,10 +397,8 @@ def select_query(self, query: Query) -> QueryResults: """ - self.raise_on_not_init() # ToDo return query.select() - self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select_query(query.query_wrapper_ptr) - self.raise_on_error() - return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + self.raise_on_not_init() + return query.execute() def delete_query(self, query: Query) -> int: """Delete will execute query, and delete items, matches query From cc74f676b9d7ea030978d839dba479776fd0eb7b Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Tue, 5 Nov 2024 15:05:47 +0300 Subject: [PATCH 045/125] [ref] tests --- pyreindexer/tests/conftest.py | 26 +++++++------- pyreindexer/tests/helpers/index.py | 2 +- pyreindexer/tests/helpers/items.py | 4 +-- pyreindexer/tests/helpers/log_helper.py | 2 +- pyreindexer/tests/helpers/metadata.py | 2 +- pyreindexer/tests/helpers/namespace.py | 12 +++---- pyreindexer/tests/helpers/sql.py | 8 ++--- pyreindexer/tests/tests/test_database.py | 24 ++++++------- pyreindexer/tests/tests/test_index.py | 25 ++++++-------- pyreindexer/tests/tests/test_items.py | 41 +++++++---------------- pyreindexer/tests/tests/test_metadata.py | 8 ++--- pyreindexer/tests/tests/test_namespace.py | 7 ++-- pyreindexer/tests/tests/test_sql.py | 41 ++++++++++++----------- 13 files changed, 93 insertions(+), 109 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 6afbc16..67545dc 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -1,13 +1,14 @@ import shutil + import pytest from pyreindexer import RxConnector -from tests.helpers.index import * -from tests.helpers.items import * -from tests.helpers.log_helper import log_fixture -from tests.helpers.metadata import put_metadata -from tests.helpers.namespace import * -from tests.test_data.constants import index_definition, item_definition +from .helpers.index import * +from .helpers.items import * +from .helpers.log_helper import log_fixture +from .helpers.metadata import put_metadata +from .helpers.namespace import * +from .test_data.constants import index_definition, item_definition def pytest_addoption(parser): @@ -16,8 +17,6 @@ def pytest_addoption(parser): @pytest.fixture(scope="session", autouse=True) def log_setup(request): - """ Execute once before test run - """ log_fixture.info("Work with pyreindexer connector using {} mode".format(request.config.getoption("--mode"))) @@ -77,11 +76,12 @@ def items(namespace): """ Create items to namespace """ - for i in range(10): - insert_item(namespace, {"id": i+1, "val": "testval" + str(i+1)}) - yield - for i in range(10): - delete_item(namespace, {"id": i+1, "val": "testval" + str(i+1)}) + items = [{"id": i, "val": f"testval{i}"} for i in range(10)] + for item in items: + insert_item(namespace, item) + yield items + for item in items: + delete_item(namespace, item) @pytest.fixture(scope="function") diff --git a/pyreindexer/tests/helpers/index.py b/pyreindexer/tests/helpers/index.py index dea0fe3..f4f1a4e 100644 --- a/pyreindexer/tests/helpers/index.py +++ b/pyreindexer/tests/helpers/index.py @@ -1,4 +1,4 @@ -from tests.helpers.log_helper import log_operation +from .log_helper import log_operation def create_index(namespace, index_def): diff --git a/pyreindexer/tests/helpers/items.py b/pyreindexer/tests/helpers/items.py index c83dd26..53101b7 100644 --- a/pyreindexer/tests/helpers/items.py +++ b/pyreindexer/tests/helpers/items.py @@ -1,4 +1,4 @@ -from tests.helpers.log_helper import log_operation +from .log_helper import log_operation def insert_item(namespace, item_def): @@ -25,7 +25,7 @@ def update_item(namespace, item_def): """ db, namespace_name = namespace log_operation.info(f"Update item: {item_def} to namespace {namespace_name}") - db.item_upsert(namespace_name, item_def) + db.item_update(namespace_name, item_def) def delete_item(namespace, item_def): diff --git a/pyreindexer/tests/helpers/log_helper.py b/pyreindexer/tests/helpers/log_helper.py index cc5dd4b..fed533c 100644 --- a/pyreindexer/tests/helpers/log_helper.py +++ b/pyreindexer/tests/helpers/log_helper.py @@ -11,7 +11,7 @@ class OneLineExceptionFormatter(logging.Formatter): def formatException(self, exc_info): result = super(OneLineExceptionFormatter, self).formatException(exc_info) - return repr(result) # or format into one line however you want to + return repr(result) def format(self, record): result = super(OneLineExceptionFormatter, self).format(record) diff --git a/pyreindexer/tests/helpers/metadata.py b/pyreindexer/tests/helpers/metadata.py index 36a1622..2112a57 100644 --- a/pyreindexer/tests/helpers/metadata.py +++ b/pyreindexer/tests/helpers/metadata.py @@ -1,4 +1,4 @@ -from tests.helpers.log_helper import log_operation +from .log_helper import log_operation def put_metadata(namespace, key, value): diff --git a/pyreindexer/tests/helpers/namespace.py b/pyreindexer/tests/helpers/namespace.py index 04e6ee4..4a2d57a 100644 --- a/pyreindexer/tests/helpers/namespace.py +++ b/pyreindexer/tests/helpers/namespace.py @@ -1,6 +1,6 @@ import logging -from tests.helpers.log_helper import log_operation +from .log_helper import log_operation def create_namespace(database, namespace_name): @@ -24,14 +24,14 @@ def drop_namespace(database, namespace_name): db.namespace_drop(namespace_name) -def get_namespace_list(database): +def get_namespaces_list(database): """ Get list of namespaces in database """ log_operation.info("Get list of namespaces in database") db, db_name = database - namespace_list = db.namespaces_enum() - return namespace_list + namespaces_list = db.namespaces_enum() + return namespaces_list def get_ns_description(database, namespace): @@ -39,7 +39,7 @@ def get_ns_description(database, namespace): Get information about namespace in database """ db, namespace_name = namespace - namespace_list = get_namespace_list(database) + namespace_list = get_namespaces_list(database) log_operation.info(f"Get information about namespace {namespace_name} in database") - ns_entry = list(filter(lambda ns: ns['name'] == namespace_name, namespace_list)) + ns_entry = [ns for ns in namespace_list if ns["name"] == namespace_name] return ns_entry diff --git a/pyreindexer/tests/helpers/sql.py b/pyreindexer/tests/helpers/sql.py index d12821c..e44231b 100644 --- a/pyreindexer/tests/helpers/sql.py +++ b/pyreindexer/tests/helpers/sql.py @@ -1,8 +1,8 @@ -from tests.helpers.log_helper import log_operation +from .log_helper import log_operation def sql_query(namespace, query): db, namespace_name = namespace - item_list = list(db.select(query)) - log_operation.info(f"Execute SQL query: {query}, got result: {item_list}") - return item_list + items_list = list(db.select(query)) + log_operation.info(f"Execute SQL query: {query}, got result: {items_list}") + return items_list diff --git a/pyreindexer/tests/tests/test_database.py b/pyreindexer/tests/tests/test_database.py index 29e13db..92d25d1 100644 --- a/pyreindexer/tests/tests/test_database.py +++ b/pyreindexer/tests/tests/test_database.py @@ -1,22 +1,22 @@ -import pyreindexer from hamcrest import * -from tests.test_data.constants import special_namespaces, special_namespaces_cluster +import pyreindexer +from ..test_data.constants import special_namespaces, special_namespaces_cluster class TestCrudDb: def test_create_db(self): # Given ("Create empty database") - db_path = '/tmp/test_db' - db = pyreindexer.RxConnector('builtin://' + db_path) + db = pyreindexer.RxConnector('builtin:///tmp/test_db') # When ("Get namespaces list in created database") - namespace_list = db.namespaces_enum() + namespaces_list = db.namespaces_enum() # Then ("Check that database contains only special namespaces") - expect_namespaces = special_namespaces - if len(namespace_list) == len(special_namespaces_cluster): - expect_namespaces = special_namespaces_cluster - for namespace in expect_namespaces: - assert_that(namespace_list, has_item(has_entries("name", equal_to(namespace["name"]))), - "Database doesn't contain special namespaces") - assert_that(namespace_list, has_length(equal_to(len(expect_namespaces))), + if len(namespaces_list) == len(special_namespaces_cluster): + expected_namespaces = special_namespaces_cluster # v4 + else: + expected_namespaces = special_namespaces # v3 + assert_that(namespaces_list, has_length(len(expected_namespaces)), "Database doesn't contain special namespaces") + for namespace in expected_namespaces: + assert_that(namespaces_list, has_item(has_entries(name=namespace["name"])), + "Database doesn't contain special namespaces") diff --git a/pyreindexer/tests/tests/test_index.py b/pyreindexer/tests/tests/test_index.py index bc3b973..e65f60a 100644 --- a/pyreindexer/tests/tests/test_index.py +++ b/pyreindexer/tests/tests/test_index.py @@ -1,8 +1,8 @@ from hamcrest import * -from tests.helpers.index import * -from tests.helpers.namespace import get_ns_description -from tests.test_data.constants import index_definition, updated_index_definition +from ..helpers.index import * +from ..helpers.namespace import get_ns_description +from ..test_data.constants import index_definition, updated_index_definition class TestCrudIndexes: @@ -11,8 +11,7 @@ def test_initial_namespace_has_no_indexes(self, database, namespace): # When ("Get namespace information") ns_entry = get_ns_description(database, namespace) # Then ("Check that list of indexes in namespace is empty") - assert_that(ns_entry, has_item(has_entry("indexes", equal_to([]))), - "Index is not empty") + assert_that(ns_entry, has_item(has_entry("indexes", empty())), "Index is not empty") def test_create_index(self, database, namespace): # Given("Create namespace") @@ -20,7 +19,7 @@ def test_create_index(self, database, namespace): create_index(namespace, index_definition) # Then ("Check that index is added") ns_entry = get_ns_description(database, namespace) - assert_that(ns_entry, has_item(has_entry("indexes", has_item(equal_to(index_definition)))), + assert_that(ns_entry, has_item(has_entry("indexes", has_item(index_definition))), "Index wasn't created") drop_index(namespace, 'id') @@ -30,7 +29,7 @@ def test_update_index(self, database, namespace, index): update_index(namespace, updated_index_definition) # Then ("Check that index is updated") ns_entry = get_ns_description(database, namespace) - assert_that(ns_entry, has_item(has_entry("indexes", has_item(equal_to(updated_index_definition)))), + assert_that(ns_entry, has_item(has_entry("indexes", has_item(updated_index_definition))), "Index wasn't updated") def test_delete_index(self, database, namespace): @@ -40,8 +39,7 @@ def test_delete_index(self, database, namespace): drop_index(namespace, 'id') # Then ("Check that index is deleted") ns_entry = get_ns_description(database, namespace) - assert_that(ns_entry, has_item(has_entry("indexes", equal_to([]))), - "Index wasn't deleted") + assert_that(ns_entry, has_item(has_entry("indexes", empty())), "Index wasn't deleted") def test_cannot_add_index_with_same_name(self, database, namespace): # Given("Create namespace") @@ -49,17 +47,16 @@ def test_cannot_add_index_with_same_name(self, database, namespace): create_index(namespace, index_definition) # Then ("Check that we can't add index with the same name") assert_that(calling(create_index).with_args(namespace, updated_index_definition), - raises(Exception, matching=has_string(string_contains_in_order( - "Index", "already exists with different settings"))), + raises(Exception, pattern="Index '.*' already exists with different settings"), "Index with existing name was created") def test_cannot_update_not_existing_index_in_namespace(self, database, namespace): # Given ("Create namespace") - db, namespace_name = namespace + _, namespace_name = namespace # When ("Update index") # Then ("Check that we can't update index that was not created") assert_that(calling(update_index).with_args(namespace, index_definition), - raises(Exception, matching=has_string(f"Index 'id' not found in '{namespace_name}'")), + raises(Exception, pattern=f"Index 'id' not found in '{namespace_name}'"), "Not existing index was updated") def test_cannot_delete_not_existing_index_in_namespace(self, database, namespace): @@ -68,5 +65,5 @@ def test_cannot_delete_not_existing_index_in_namespace(self, database, namespace # Then ("Check that we can't delete index that was not created") index_name = 'id' assert_that(calling(drop_index).with_args(namespace, index_name), - raises(Exception, matching=has_string(f"Cannot remove index {index_name}: doesn't exist")), + raises(Exception, pattern=f"Cannot remove index {index_name}: doesn't exist"), "Not existing index was deleted") diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index d732051..c76d2ee 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -1,7 +1,7 @@ from hamcrest import * -from tests.helpers.items import * -from tests.test_data.constants import item_definition +from ..helpers.items import * +from ..test_data.constants import item_definition class TestCrudItems: @@ -11,8 +11,7 @@ def test_initial_namespace_has_no_items(self, namespace, index): # When ("Get namespace information") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) # Then ("Check that list of items in namespace is empty") - assert_that(select_result, has_length(0), "Item list is not empty") - assert_that(select_result, equal_to([]), "Item list is not empty") + assert_that(select_result, empty(), "Item list is not empty") def test_create_item_insert(self, namespace, index): # Given("Create namespace with index") @@ -21,12 +20,8 @@ def test_create_item_insert(self, namespace, index): insert_item(namespace, item_definition) # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(1), - "Item wasn't created") - assert_that(select_result, has_item(item_definition), - "Item wasn't created" - ) - delete_item(namespace, item_definition) + assert_that(select_result, has_length(1), "Item wasn't created") + assert_that(select_result, has_item(item_definition), "Item wasn't created") def test_create_item_insert_with_precepts(self, namespace, index): # Given("Create namespace with index") @@ -37,13 +32,9 @@ def test_create_item_insert_with_precepts(self, namespace, index): db.item_insert(namespace_name, {"id": 100, "field": "value"}, ["id=serial()"]) # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(number_items), - "Items wasn't created") - for i in range(number_items): - assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), - "Items wasn't created") + assert_that(select_result, has_length(number_items), "Items wasn't created") for i in range(number_items): - db.item_delete(namespace_name, {'id': i}) + assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), "Items wasn't created") def test_create_item_upsert(self, namespace, index): # Given("Create namespace with index") @@ -52,10 +43,8 @@ def test_create_item_upsert(self, namespace, index): upsert_item(namespace, item_definition) # Then ("Check that item is added") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(1), - "Item wasn't created") - assert_that(select_result, has_item(item_definition), - "Item wasn't created") + assert_that(select_result, has_length(1), "Item wasn't created") + assert_that(select_result, has_item(item_definition), "Item wasn't created") delete_item(namespace, item_definition) def test_update_item_upsert(self, namespace, index, item): @@ -66,10 +55,8 @@ def test_update_item_upsert(self, namespace, index, item): upsert_item(namespace, item_definition_updated) # Then ("Check that item is updated") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(1), - "Item wasn't updated") - assert_that(select_result, has_item(item_definition_updated), - "Item wasn't updated") + assert_that(select_result, has_length(1), "Item wasn't updated") + assert_that(select_result, has_item(item_definition_updated), "Item wasn't updated") def test_update_item_update(self, namespace, index, item): # Given("Create namespace with item") @@ -82,13 +69,11 @@ def test_update_item_update(self, namespace, index, item): assert_that(select_result, has_length(1), "Item wasn't updated") assert_that(select_result, has_item(item_definition_updated), "Item wasn't updated") - def test_delete_item(self, namespace, index): + def test_delete_item(self, namespace, index, item): # Given("Create namespace with item") db, namespace_name = namespace - insert_item(namespace, item_definition) # When ("Delete item") delete_item(namespace, item_definition) # Then ("Check that item is deleted") select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(0), "Item wasn't deleted") - assert_that(select_result, equal_to([]), "Item wasn't deleted") + assert_that(select_result, empty(), "Item wasn't deleted") diff --git a/pyreindexer/tests/tests/test_metadata.py b/pyreindexer/tests/tests/test_metadata.py index 22519f7..f3fa840 100644 --- a/pyreindexer/tests/tests/test_metadata.py +++ b/pyreindexer/tests/tests/test_metadata.py @@ -1,6 +1,6 @@ from hamcrest import * -from tests.helpers.metadata import * +from ..helpers.metadata import * class TestCrudMetadata: @@ -9,8 +9,7 @@ def test_initial_namespace_has_no_metadata(self, namespace): # When ("Get list of metadata keys") meta_list = get_metadata_keys(namespace) # Then ("Check that list of metadata is empty") - assert_that(meta_list, has_length(0), "Metadata is not empty") - assert_that(meta_list, equal_to([]), "Metadata is not empty") + assert_that(meta_list, empty(), "Metadata is not empty") def test_create_metadata(self, namespace): # Given("Create namespace") @@ -59,5 +58,4 @@ def test_delete_metadata(self, namespace, metadata): # When ("Get list of metadata keys") meta_list = get_metadata_keys(namespace) # Then ("Check that list of metadata is empty") - assert_that(meta_list, has_length(0), "Metadata is not empty") - assert_that(meta_list, equal_to([]), "Metadata is not empty") + assert_that(meta_list, empty(), "Metadata is not empty") diff --git a/pyreindexer/tests/tests/test_namespace.py b/pyreindexer/tests/tests/test_namespace.py index d18340d..37dfc8a 100644 --- a/pyreindexer/tests/tests/test_namespace.py +++ b/pyreindexer/tests/tests/test_namespace.py @@ -1,15 +1,16 @@ from hamcrest import * -from tests.helpers.namespace import * +from ..helpers.namespace import * class TestCrudNamespace: + def test_create_ns(self, database): # Given("Create namespace in empty database") namespace_name = 'test_ns' create_namespace(database, namespace_name) # When ("Get namespaces list in created database") - namespace_list = get_namespace_list(database) + namespace_list = get_namespaces_list(database) # Then ("Check that database contains created namespace") assert_that(namespace_list, has_item(has_entry("name", equal_to(namespace_name))), "Namespace wasn't created") @@ -22,7 +23,7 @@ def test_delete_ns(self, database): # When ("Delete namespace") drop_namespace(database, namespace_name) # Then ("Check that namespace was deleted") - namespace_list = get_namespace_list(database) + namespace_list = get_namespaces_list(database) assert_that(namespace_list, not_(has_item(has_entry("name", equal_to(namespace_name)))), "Namespace wasn't deleted") diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index cf9d0bf..338f82a 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -1,25 +1,25 @@ from hamcrest import * -from tests.helpers.sql import sql_query +from ..helpers.sql import sql_query class TestSqlQueries: def test_sql_select(self, namespace, index, item): # Given("Create namespace with item") db, namespace_name = namespace - item_definition = item # When ("Execute SQL query SELECT") - query = f'SELECT * FROM {namespace_name}' - item_list = sql_query(namespace, query) + query = f"SELECT * FROM {namespace_name}" + items_list = sql_query(namespace, query) # Then ("Check that selected item is in result") - assert_that(item_list, has_item(equal_to(item_definition)), "Can't SQL select data") + assert_that(items_list, equal_to([item]), "Can't SQL select data") def test_sql_select_with_join(self, namespace, second_namespace_for_join, index, items): # Given("Create two namespaces") db, namespace_name = namespace second_namespace_name, second_ns_item_definition_join = second_namespace_for_join # When ("Execute SQL query SELECT with JOIN") - query = f'SELECT id FROM {namespace_name} INNER JOIN {second_namespace_name} ON {namespace_name}.id = {second_namespace_name}.id' + query = f"SELECT id FROM {namespace_name} INNER JOIN {second_namespace_name} " \ + f"ON {namespace_name}.id = {second_namespace_name}.id" item_list = sql_query(namespace, query) # Then ("Check that selected item is in result") assert_that(item_list, @@ -30,19 +30,21 @@ def test_sql_select_with_condition(self, namespace, index, items): # Given("Create namespace with item") db, namespace_name = namespace # When ("Execute SQL query SELECT") - query = f'SELECT * FROM {namespace_name} WHERE id=3' - item_list = sql_query(namespace, query) + item_id = 3 + query = f"SELECT * FROM {namespace_name} WHERE id = {item_id}" + items_list = sql_query(namespace, query) # Then ("Check that selected item is in result") - assert_that(item_list, has_item(equal_to({'id': 3, 'val': 'testval3'})), "Can't SQL select data with condition") + assert_that(items_list, equal_to([items[item_id]]), "Can't SQL select data with condition") def test_sql_update(self, namespace, index, item): # Given("Create namespace with item") db, namespace_name = namespace # When ("Execute SQL query UPDATE") - query = f"UPDATE {namespace_name} SET \"val\" = 'new_val' WHERE id = 100" - item_list = sql_query(namespace, query) + item["val"] = "new_val" + query = f"UPDATE {namespace_name} SET \"val\" = '{item['val']}' WHERE id = 100" + items_list = sql_query(namespace, query) # Then ("Check that item is updated") - assert_that(item_list, has_item(equal_to({'id': 100, 'val': 'new_val'})), "Can't SQL update data") + assert_that(items_list, equal_to([item]), "Can't SQL update data") def test_sql_delete(self, namespace, index, item): # Given("Create namespace with item") @@ -53,16 +55,16 @@ def test_sql_delete(self, namespace, index, item): # Then ("Check that item is deleted") query_select = f"SELECT * FROM {namespace_name}" item_list = sql_query(namespace, query_select) - assert_that(item_list, equal_to([]), "Can't SQL delete data") + assert_that(item_list, empty(), "Can't SQL delete data") def test_sql_select_with_syntax_error(self, namespace, index, item): # Given("Create namespace with item") # When ("Execute SQL query SELECT with incorrect syntax") - query = 'SELECT *' + query = "SELECT *" # Then ("Check that selected item is in result") assert_that(calling(sql_query).with_args(namespace, query), - raises(Exception, matching=has_string(string_contains_in_order( - "Expected", "but found"))), "Error wasn't raised when syntax was incorrect") + raises(Exception, pattern="Expected .* but found"), + "Error wasn't raised when syntax was incorrect") def test_sql_select_with_aggregations(self, namespace, index, items): # Given("Create namespace with item") @@ -71,10 +73,11 @@ def test_sql_select_with_aggregations(self, namespace, index, items): for _ in range(5): db.item_insert(namespace_name, {"id": 100}, ["id=serial()"]) - select_result = db.select(f'SELECT min(id), max(id), avg(id) FROM {namespace_name}').get_agg_results() - assert_that(len(select_result), 3, "The aggregation result must contain 3 elements") + select_result = db.select(f"SELECT min(id), max(id), avg(id), sum(id) FROM {namespace_name}").get_agg_results() + assert_that(select_result, has_length(4), "The aggregation result does not contain all aggregations") - expected_values = {"min":1,"max":10,"avg":5.5} + ids = [i["id"] for i in items] + expected_values = {"min": min(ids), "max": max(ids), "avg": sum(ids) / len(items), "sum": sum(ids)} # Then ("Check that returned agg results are correct") for agg in select_result: From e6fee627b9b3e6ca16114ca3d96f57b93b12dd22 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 5 Nov 2024 17:35:17 +0300 Subject: [PATCH 046/125] Part XVI: start of implementation of query builder. Add Get(firstItem or '' and found flag) --- pyreindexer/query.py | 26 +++++++++++++++++++++++--- pyreindexer/query_results.py | 3 +-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 3d782ee..e84d924 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -803,12 +803,32 @@ def must_execute(self) -> QueryResults: ################################################################ // ToDo #func (q *Query) MustExecCtx(ctx context.Context) *Iterator { ################################################################ -#func (q *Query) Get() (item interface{}, found bool) { + + def get(self) -> (str, bool): + """Get will execute query, and return 1 string item + + # Returns: + (:tuple:string,bool): 1 string item and found flag + + # Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + """ + + if self.root is not None: + return self.get() + + selected_items = self.limit(1).must_execute() + for item in selected_items: + return item, True + + return '', False + ################################################################ // ToDo #func (q *Query) GetCtx(ctx context.Context) (item interface{}, found bool) { -################################################################ +################################################################ // ToDo - not actual #func (q *Query) GetJson() (json []byte, found bool) { -################################################################ // ToDo #func (q *Query) GetJsonCtx(ctx context.Context) (json []byte, found bool) { ################################################################ diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 3106422..7563037 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -46,8 +46,7 @@ def __next__(self): if self.pos < self.qres_iter_count: self.pos += 1 - self.err_code, self.err_msg, res = self.api.query_results_iterate( - self.qres_wrapper_ptr) + self.err_code, self.err_msg, res = self.api.query_results_iterate(self.qres_wrapper_ptr) if self.err_code: raise Exception(self.err_msg) return res From 4286c1b02bf9b34cde9a18f7cf1e238e6641416d Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 5 Nov 2024 18:14:07 +0300 Subject: [PATCH 047/125] Part XVII: clear code. Hide all context unimplemented functions --- pyreindexer/lib/include/query_wrapper.cc | 2 +- pyreindexer/query.py | 62 +++++++++--------------- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 7aa90ee..ff9f97c 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -231,7 +231,7 @@ void QueryWrapper::SetExpression(std::string_view field, std::string_view value) ser_.PutVarUint(1); // size ser_.PutVarUint(1); // is expression - ser_.PutVString(value); // ToDo q.putValue(value); + ser_.PutVString(value); } void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery) { diff --git a/pyreindexer/query.py b/pyreindexer/query.py index e84d924..fb68c8e 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -42,7 +42,6 @@ class Query(object): err_msg (string): The API error message root (:object:`Query`): The root query of the Reindexer query join_type (:enum:`JoinType`): Join type - # ToDo join_fields (list[string]): A list of fields (name as unique identification) to join join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects closed (bool): Whether the query is closed @@ -64,7 +63,6 @@ def __init__(self, api, query_wrapper_ptr: int): self.err_msg: str = '' self.root: Query = None self.join_type: JoinType = JoinType.LeftJoin - # ToDo self.join_fields: List[str] = [] self.join_queries: List[Query] = [] self.merged_queries: List[Query] = [] @@ -637,10 +635,6 @@ def with_rank(self) -> Query: self.api.with_rank(self.query_wrapper_ptr) return self -################################################################ ToDo -#func (q *Query) SetContext(ctx interface{}) *Query { -################################################################ - def execute(self) -> QueryResults: """Exec will execute query, and return slice of items @@ -660,13 +654,6 @@ def execute(self) -> QueryResults: self.__raise_on_error() return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) -################################################################ ToDo -#func (q *Query) ExecCtx(ctx context.Context) *Iterator { -################################################################ ToDo - not actual -#func (q *Query) ExecToJson(jsonRoots ...string) *JSONIterator { -#func (q *Query) ExecToJsonCtx(ctx context.Context, jsonRoots ...string) *JSONIterator { -################################################################ - def delete(self) -> int: """Delete will execute query, and delete items, matches query @@ -686,10 +673,6 @@ def delete(self) -> int: self.__raise_on_error() return number -################################################################ ToDo -#func (q *Query) DeleteCtx(ctx context.Context) (int, error) { -################################################################ - def set_object(self, field: str, values: List[simple_types]) -> Query: """Adds an update query to an object field for an update query @@ -780,10 +763,6 @@ def update(self) -> QueryResults: self.__raise_on_error() return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) -################################################################ // ToDo -#func (q *Query) UpdateCtx(ctx context.Context) *Iterator { -################################################################ - def must_execute(self) -> QueryResults: """Exec will execute query, and return slice of items, with status check @@ -800,10 +779,6 @@ def must_execute(self) -> QueryResults: result.status() return result -################################################################ // ToDo -#func (q *Query) MustExecCtx(ctx context.Context) *Iterator { -################################################################ - def get(self) -> (str, bool): """Get will execute query, and return 1 string item @@ -825,13 +800,6 @@ def get(self) -> (str, bool): return '', False -################################################################ // ToDo -#func (q *Query) GetCtx(ctx context.Context) (item interface{}, found bool) { -################################################################ // ToDo - not actual -#func (q *Query) GetJson() (json []byte, found bool) { -#func (q *Query) GetJsonCtx(ctx context.Context) (json []byte, found bool) { -################################################################ - def __join(self, query: Query, field: str, join_type: JoinType) -> Query: """Joins queries @@ -861,15 +829,34 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: query.join_type = join_type query.root = self self.join_queries.append(query) - # ToDo self.join_fields.append(field) - # ToDo self.join_handlers.append(None) return query def inner_join(self, query: Query, field: str) -> Query: + """InnerJoin joins 2 queries. + Items from the 1-st query are filtered by and expanded with the data from the 2-nd query + + # Arguments: + query (:obj:`Query`): Query object to left join + field (string): Joined field name. As unique identifier for the join between this query and `join_query`. + Parameter in order for InnerJoin to work: namespace of `query` contains `field` as one of its fields marked as `joined` + + # Returns: + (:obj:`Query`): Query object for further customizations + + """ + return self.__join(query, field, JoinType.InnerJoin) def join(self, query: Query, field: str) -> Query: - """Join is an alias for LeftJoin + """Join is an alias for LeftJoin. LeftJoin joins 2 queries. + Items from this query are expanded with the data from the `query` + + # Arguments: + query (:obj:`Query`): Query object to left join + field (string): Joined field name. As unique identifier for the join between this query and `join_query` + + # Returns: + (:obj:`Query`): Query object for further customizations """ @@ -880,7 +867,6 @@ def left_join(self, join_query: Query, field: str) -> Query: Items from this query are expanded with the data from the join_query. One of the conditions below must hold for `field` parameter in order for LeftJoin to work: namespace of `join_query` contains `field` as one of its fields marked as `joined` - # ToDo `this query` has a join handler (registered via `q.JoinHandler(...)` call) with the same `field` value # Arguments: query (:obj:`Query`): Query object to left join @@ -893,10 +879,6 @@ def left_join(self, join_query: Query, field: str) -> Query: return self.__join(join_query, field, JoinType.LeftJoin) -################################################################ ToDo -#func (q *Query) JoinHandler(field string, handler JoinHandler) *Query { -################################################################ - def merge(self, query: Query) -> Query: """Merge queries of the same type From f412d96c61147af12e5d5e2c7c882607d239c24b Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Tue, 5 Nov 2024 18:35:55 +0300 Subject: [PATCH 048/125] Prevent warning on build --- .github/workflows/deploy-workflow.yml | 1 + .github/workflows/test-deploy-workflow.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/deploy-workflow.yml b/.github/workflows/deploy-workflow.yml index 870bd89..7060e28 100644 --- a/.github/workflows/deploy-workflow.yml +++ b/.github/workflows/deploy-workflow.yml @@ -67,6 +67,7 @@ jobs: verbose: true # user: __token__ ## used as default password: pypi-${{ secrets.PYPI_TOKEN }} + attestations: false test: strategy: diff --git a/.github/workflows/test-deploy-workflow.yml b/.github/workflows/test-deploy-workflow.yml index 7e3ab37..1b3d607 100644 --- a/.github/workflows/test-deploy-workflow.yml +++ b/.github/workflows/test-deploy-workflow.yml @@ -67,6 +67,7 @@ jobs: # user: __token__ ## used as default password: pypi-${{ secrets.TESTPYPI_TOKEN }} repository-url: https://test.pypi.org/legacy/ + attestations: false test: strategy: From e57ed585577949b4938c3e1c5a0b016528e06121 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 10:16:18 +0300 Subject: [PATCH 049/125] Update comments --- README.md | 12 ++-- pyreindexer/lib/include/pyobjtools.cc | 2 +- pyreindexer/query.py | 99 +++++++++++++-------------- pyreindexer/rx_connector.py | 10 +-- pyreindexer/transaction.py | 14 ++-- 5 files changed, 67 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index f9e3ecb..0c96c55 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ __Raises:__ ```python RxConnector.new_transaction(self, namespace) ``` -Start a new transaction and return the transaction object to processing +Starts a new transaction and return the transaction object to processing __Arguments:__ @@ -432,7 +432,7 @@ __Attributes:__ ```python Transaction.commit(self) ``` -Commit a transaction +Commits a transaction __Raises:__ @@ -445,7 +445,7 @@ __Raises:__ ```python Transaction.delete(self, item_def) ``` -Delete an item from the transaction. +Deletes an item from the transaction. __Arguments:__ @@ -480,7 +480,7 @@ __Raises:__ ```python Transaction.rollback(self) ``` -Roll back a transaction +Rollbacks a transaction __Raises:__ @@ -493,7 +493,7 @@ __Raises:__ ```python Transaction.update(self, item_def, precepts=None) ``` -Update an item with its precepts to the transaction +Updates an item with its precepts to the transaction __Arguments:__ @@ -511,7 +511,7 @@ __Raises:__ ```python Transaction.upsert(self, item_def, precepts=None) ``` -Update an item with its precepts to the transaction. Creates the item if it not exists +Updates an item with its precepts to the transaction. Creates the item if it not exists __Arguments:__ diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 4db5f9d..1fd19dd 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -227,7 +227,7 @@ PyObject* pyValueFromJsonValue(const gason::JsonValue& value) { } break; case gason::JSON_EMPTY: - throw gason::Exception("Unexpected 'JSON_EMPTY' tag"); + throw gason::Exception("Unexpected `JSON_EMPTY` tag"); break; } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index fb68c8e..bc88a0b 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -44,7 +44,6 @@ class Query(object): join_type (:enum:`JoinType`): Join type join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects - closed (bool): Whether the query is closed """ @@ -67,7 +66,7 @@ def __init__(self, api, query_wrapper_ptr: int): self.merged_queries: List[Query] = [] def __del__(self): - """Free query memory + """Frees query memory """ @@ -87,7 +86,7 @@ def __raise_on_error(self): @staticmethod def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[simple_types]: - """Convert an input parameter to a list + """Converts an input parameter to a list # Arguments: param (Union[None, simple_types, list[simple_types]]): The input parameter @@ -102,7 +101,7 @@ def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[si return param def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: - """Add where condition to DB query with int args + """Adds where condition to DB query with args # Arguments: index (string): Field name used in condition clause @@ -125,13 +124,13 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ return self def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: - """Add sub query where condition to DB query with int args + """Adds sub-query where condition to DB query with args # Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): - Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index # Returns: (:obj:`Query`): Query object for further customizations @@ -148,7 +147,7 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_ return self def where_composite(self, index: str, condition: CondType, sub_query: Query) -> Query: - """Add where condition to DB query with interface args for composite indexes + """Adds where condition to DB query with interface args for composite indexes # Arguments: index (string): Field name used in condition clause @@ -164,9 +163,9 @@ def where_composite(self, index: str, condition: CondType, sub_query: Query) -> return self def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: - """Add where condition to DB query with UUID as string args. + """Adds where condition to DB query with UUID as string args. This function applies binary encoding to the UUID value. - 'index' MUST be declared as uuid index in this case + `index` MUST be declared as uuid index in this case # Arguments: index (string): Field name used in condition clause @@ -181,14 +180,14 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """ - keys = [] if keys is None else keys + params: list = self.__convert_to_list(keys) - self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, keys) + self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() return self def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: - """Add comparing two fields where condition to DB query + """Adds comparing two fields where condition to DB query # Arguments: first_field (string): First field name used in condition clause @@ -204,7 +203,7 @@ def where_between_fields(self, first_field: str, condition: CondType, second_fie return self def open_bracket(self) -> Query: - """Open bracket for where condition to DB query + """Opens bracket for where condition to DB query # Returns: (:obj:`Query`): Query object for further customizations @@ -219,7 +218,7 @@ def open_bracket(self) -> Query: return self def close_bracket(self) -> Query: - """CloseBracket - Close bracket for where condition to DB query + """Closes bracket for where condition to DB query # Returns: (:obj:`Query`): Query object for further customizations @@ -234,7 +233,7 @@ def close_bracket(self) -> Query: return self def match(self, index: str, keys: List[str]) -> Query: - """Add string EQ-condition to DB query with string args + """Adds string EQ-condition to DB query with string args # Arguments: index (string): Field name used in condition clause @@ -248,14 +247,14 @@ def match(self, index: str, keys: List[str]) -> Query: """ - keys = [] if keys is None else keys + params: list = self.__convert_to_list(keys) - self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, keys) + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, params) self.__raise_on_error() return self def dwithin(self, index: str, point: Point, distance: float) -> Query: - """Add DWithin condition to DB query + """Adds DWithin condition to DB query # Arguments: index (string): Field name used in condition clause @@ -271,8 +270,7 @@ def dwithin(self, index: str, point: Point, distance: float) -> Query: return self def distinct(self, index: str) -> Query: - """Performs distinct for a certain index. - Return only items with uniq value of field + """Performs distinct for a certain index. Return only items with uniq value of field # Arguments: index (string): Field name for distinct operation @@ -354,8 +352,7 @@ def __init__(self, query: Query): """Constructs a new Reindexer AggregateFacetRequest object # Arguments: - api (module): An API module for Reindexer calls - query_wrapper_ptr (int): A memory pointer to Reindexer query object + (:obj:`Query`): Query object for further customizations """ @@ -363,7 +360,7 @@ def __init__(self, query: Query): self.query_wrapper_ptr = query.query_wrapper_ptr def limit(self, limit: int) -> Query._AggregateFacet: - """Limit facet aggregation results + """Limits facet aggregation results # Arguments: limit (int): Limit of aggregation of facet @@ -377,7 +374,7 @@ def limit(self, limit: int) -> Query._AggregateFacet: return self def offset(self, offset: int) -> Query._AggregateFacet: - """Set offset of the facet aggregation results + """Sets offset of the facet aggregation results # Arguments: limit (int): Offset in facet aggregation results @@ -391,11 +388,11 @@ def offset(self, offset: int) -> Query._AggregateFacet: return self def sort(self, field: str, desc: bool) -> Query._AggregateFacet: - """Sort facets by field value + """Sorts facets by field value # Arguments: - field (str): Item field. Use field 'count' to sort by facet's count value - desc (bool): Sort in descending order + field (str): Item field. Use field `count` to sort by facet's count value + desc (bool): Sort in descending order flag # Returns: (:obj:`_AggregateFacet`): Facet object for further customizations @@ -406,12 +403,12 @@ def sort(self, field: str, desc: bool) -> Query._AggregateFacet: return self def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: - """Get fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data - column or 'count' and cut off by offset and limit. In order to support this functionality this method + """Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data + column or `count` and cut off by offset and limit. In order to support this functionality this method returns AggregationFacetRequest which has methods sort, limit and offset # Arguments: - fields (list[string]): Fields any data column name or 'count', fields should not be empty + fields (list[string]): Fields any data column name or `count`, fields should not be empty # Returns: (:obj:`_AggregateFacet`): Request object for further customizations @@ -425,14 +422,14 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: return self._AggregateFacet(self) def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_types]]=None) -> Query: - """Apply sort order to return from query items. If values argument specified, then items equal to values, + """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only # Arguments: index (string): The index name desc (bool): Sort in descending order keys (Union[None, simple_types, List[simple_types]]): - Value of index to match. For composite indexes keys must be list, with value of each subindex + Value of index to match. For composite indexes keys must be list, with value of each sub-index # Returns: (:obj:`Query`): Query object for further customizations @@ -449,7 +446,7 @@ def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_typ return self def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: - """Apply geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance + """Applies geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance between geometry field and point (ST_Distance) # Arguments: @@ -470,7 +467,7 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: return self.sort(request, desc) def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) -> Query: - """Apply geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance + """Applies geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance between 2 geometry fields (ST_Distance) # Arguments: @@ -529,7 +526,7 @@ def op_not(self) -> Query: return self def request_total(self, total_name: str= '') -> Query: - """Request total items calculation + """Requests total items calculation # Arguments: total_name (string, optional): Name to be requested @@ -543,7 +540,7 @@ def request_total(self, total_name: str= '') -> Query: return self def cached_total(self, total_name: str= '') -> Query: - """Request cached total items calculation + """Requests cached total items calculation # Arguments: total_name (string, optional): Name to be requested @@ -557,8 +554,7 @@ def cached_total(self, total_name: str= '') -> Query: return self def limit(self, limit_items: int) -> Query: - """Set a limit (count) of returned items. - Analog to sql LIMIT rowsNumber + """Sets a limit (count) of returned items. Analog to sql LIMIT rowsNumber # Arguments: limit_items (int): Number of rows to get from result set @@ -614,7 +610,7 @@ def strict(self, mode: StrictMode) -> Query: return self def explain(self) -> Query: - """Enable explain query + """Enables explain query # Returns: (:obj:`Query`): Query object for further customizations @@ -625,7 +621,7 @@ def explain(self) -> Query: return self def with_rank(self) -> Query: - """Output fulltext rank. Allowed only with fulltext query + """Outputs fulltext rank. Allowed only with fulltext query # Returns: (:obj:`Query`): Query object for further customizations @@ -636,7 +632,7 @@ def with_rank(self) -> Query: return self def execute(self) -> QueryResults: - """Exec will execute query, and return slice of items + """Executes a select query # Returns: (:obj:`QueryResults`): A QueryResults iterator @@ -655,7 +651,7 @@ def execute(self) -> QueryResults: return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) def delete(self) -> int: - """Delete will execute query, and delete items, matches query + """Executes a query, and delete items, matches query # Returns: (int): Number of deleted elements @@ -745,7 +741,7 @@ def expression(self, field: str, value: str) -> Query: return self def update(self) -> QueryResults: - """Update will execute query, and update fields in items, which matches query + """Executes update query, and update fields in items, which matches query # Returns: (:obj:`QueryResults`): A QueryResults iterator @@ -764,7 +760,7 @@ def update(self) -> QueryResults: return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) def must_execute(self) -> QueryResults: - """Exec will execute query, and return slice of items, with status check + """Executes a query, and update fields in items, which matches query, with status check # Returns: (:obj:`QueryResults`): A QueryResults iterator @@ -780,10 +776,10 @@ def must_execute(self) -> QueryResults: return result def get(self) -> (str, bool): - """Get will execute query, and return 1 string item + """Executes a query, and return 1 JSON item # Returns: - (:tuple:string,bool): 1 string item and found flag + (:tuple:string,bool): 1st string item and found flag # Raises: Exception: Raises with an error message when query is in an invalid state @@ -832,13 +828,14 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: return query def inner_join(self, query: Query, field: str) -> Query: - """InnerJoin joins 2 queries. + """Joins 2 queries. Items from the 1-st query are filtered by and expanded with the data from the 2-nd query # Arguments: query (:obj:`Query`): Query object to left join field (string): Joined field name. As unique identifier for the join between this query and `join_query`. - Parameter in order for InnerJoin to work: namespace of `query` contains `field` as one of its fields marked as `joined` + Parameter in order for InnerJoin to work: namespace of `query` contains `field` as one of its fields + marked as `joined` # Returns: (:obj:`Query`): Query object for further customizations @@ -848,7 +845,7 @@ def inner_join(self, query: Query, field: str) -> Query: return self.__join(query, field, JoinType.InnerJoin) def join(self, query: Query, field: str) -> Query: - """Join is an alias for LeftJoin. LeftJoin joins 2 queries. + """Join is an alias for LeftJoin. Joins 2 queries. Items from this query are expanded with the data from the `query` # Arguments: @@ -863,7 +860,7 @@ def join(self, query: Query, field: str) -> Query: return self.__join(query, field, JoinType.LeftJoin) def left_join(self, join_query: Query, field: str) -> Query: - """LeftJoin joins 2 queries. + """Joins 2 queries. Items from this query are expanded with the data from the join_query. One of the conditions below must hold for `field` parameter in order for LeftJoin to work: namespace of `join_query` contains `field` as one of its fields marked as `joined` @@ -880,7 +877,7 @@ def left_join(self, join_query: Query, field: str) -> Query: return self.__join(join_query, field, JoinType.LeftJoin) def merge(self, query: Query) -> Query: - """Merge queries of the same type + """Merges queries of the same type # Arguments: query (:obj:`Query`): Query object to merge diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index c050a9a..81a86eb 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -344,7 +344,7 @@ def select(self, query: str) -> QueryResults: return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) def new_transaction(self, namespace) -> Transaction: - """Start a new transaction and return the transaction object to processing + """Starts a new transaction and return the transaction object to processing # Arguments: namespace (string): A name of a namespace @@ -364,7 +364,7 @@ def new_transaction(self, namespace) -> Transaction: return Transaction(self.api, transaction_wrapper_ptr) def new_query(self, namespace: str) -> Query: - """Create a new query and return the query object to processing + """Creates a new query and return the query object to processing # Arguments: namespace (string): A name of a namespace @@ -383,7 +383,7 @@ def new_query(self, namespace: str) -> Query: return Query(self.api, query_wrapper_ptr) def select_query(self, query: Query) -> QueryResults: - """Executes an query and returns query results + """Executes a select query # Arguments: query (:obj:`Query`): An query object @@ -401,7 +401,7 @@ def select_query(self, query: Query) -> QueryResults: return query.execute() def delete_query(self, query: Query) -> int: - """Delete will execute query, and delete items, matches query + """Executes a query, and delete items, matches query # Arguments: query (:obj:`Query`): An query object @@ -419,7 +419,7 @@ def delete_query(self, query: Query) -> int: return query.delete() def update_query(self, query: Query) -> QueryResults: - """Update will execute query, and update fields in items, which matches query + """Executes update query, and update fields in items, which matches query # Arguments: query (:obj:`Query`): An query object diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index d401ac5..2d67943 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -24,7 +24,7 @@ def __init__(self, api, transaction_wrapper_ptr: int): self.err_msg = "" def __del__(self): - """Roll back a transaction if it was not previously stopped + """Rollbacks a transaction if it was not previously stopped """ @@ -73,7 +73,7 @@ def insert(self, item_def, precepts=None): self.__raise_on_error() def update(self, item_def, precepts=None): - """Update an item with its precepts to the transaction + """Updates an item with its precepts to the transaction # Arguments: item_def (dict): A dictionary of item definition @@ -91,7 +91,7 @@ def update(self, item_def, precepts=None): self.__raise_on_error() def upsert(self, item_def, precepts=None): - """Update an item with its precepts to the transaction. Creates the item if it not exists + """Updates an item with its precepts to the transaction. Creates the item if it not exists # Arguments: item_def (dict): A dictionary of item definition @@ -109,7 +109,7 @@ def upsert(self, item_def, precepts=None): self.__raise_on_error() def delete(self, item_def): - """Delete an item from the transaction + """Deletes an item from the transaction # Arguments: item_def (dict): A dictionary of item definition @@ -125,7 +125,7 @@ def delete(self, item_def): self.__raise_on_error() def commit(self): - """Apply changes + """Applies changes # Raises: Exception: Raises with an error message of API return if Transaction is over @@ -139,7 +139,7 @@ def commit(self): self.__raise_on_error() def commit_with_count(self) -> int: - """Apply changes and return the number of count of changed items + """Applies changes and return the number of count of changed items # Raises: Exception: Raises with an error message of API return if Transaction is over @@ -154,7 +154,7 @@ def commit_with_count(self) -> int: return count def rollback(self): - """Rollback changes + """Rollbacks changes # Raises: Exception: Raises with an error message of API return if Transaction is over From 4ab751e99242eced15b282e8c71b0d0c23c5d801 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 11:50:27 +0300 Subject: [PATCH 050/125] [ref] more test refactoring --- pyreindexer/tests/conftest.py | 63 ++++------ pyreindexer/tests/helpers/api.py | 141 ++++++++++++++++++++++ pyreindexer/tests/helpers/base_helper.py | 12 ++ pyreindexer/tests/helpers/index.py | 28 ----- pyreindexer/tests/helpers/items.py | 37 ------ pyreindexer/tests/helpers/log_helper.py | 6 +- pyreindexer/tests/helpers/metadata.py | 29 ----- pyreindexer/tests/helpers/namespace.py | 45 ------- pyreindexer/tests/helpers/sql.py | 8 -- pyreindexer/tests/tests/test_database.py | 7 +- pyreindexer/tests/tests/test_index.py | 46 ++++--- pyreindexer/tests/tests/test_items.py | 50 ++++---- pyreindexer/tests/tests/test_metadata.py | 34 +++--- pyreindexer/tests/tests/test_namespace.py | 38 +++--- pyreindexer/tests/tests/test_sql.py | 61 ++++------ 15 files changed, 285 insertions(+), 320 deletions(-) create mode 100644 pyreindexer/tests/helpers/api.py create mode 100644 pyreindexer/tests/helpers/base_helper.py delete mode 100644 pyreindexer/tests/helpers/index.py delete mode 100644 pyreindexer/tests/helpers/items.py delete mode 100644 pyreindexer/tests/helpers/metadata.py delete mode 100644 pyreindexer/tests/helpers/namespace.py delete mode 100644 pyreindexer/tests/helpers/sql.py diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 67545dc..90defaf 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -2,17 +2,13 @@ import pytest -from pyreindexer import RxConnector -from .helpers.index import * -from .helpers.items import * +from .helpers.api import ConnectorApi from .helpers.log_helper import log_fixture -from .helpers.metadata import put_metadata -from .helpers.namespace import * from .test_data.constants import index_definition, item_definition def pytest_addoption(parser): - parser.addoption("--mode", action="store", default='builtin', help='builtin or cproto') + parser.addoption("--mode", choices=["builtin", "cproto"], default="builtin", help="Connection mode") @pytest.fixture(scope="session", autouse=True) @@ -21,87 +17,80 @@ def log_setup(request): @pytest.fixture(scope="session") -def database(request): +def db(request): """ Create a database """ mode = request.config.getoption('--mode') db_name = 'test_db' - if mode == 'builtin': - prefix = 'builtin://tmp/' - elif mode == 'cproto': - prefix = 'cproto://127.0.0.1:6534/' - else: - raise ConnectionError - db = RxConnector(prefix + db_name) - yield db, db_name + prefix = "builtin://tmp/" if mode == "builtin" else "cproto://127.0.0.1:6534/" + db = ConnectorApi(f"{prefix}{db_name}") + yield db db.close() shutil.rmtree('tmp/', ignore_errors=True) @pytest.fixture(scope="function") -def namespace(database): +def namespace(db): """ Create a namespace """ - db, db_name = database ns_name = 'new_ns' - create_namespace(database, ns_name) - yield db, ns_name - drop_namespace(database, ns_name) + db.namespace.open(ns_name) + yield ns_name + db.namespace.drop(ns_name) @pytest.fixture(scope="function") -def index(namespace): +def index(db, namespace): """ Create an index to namespace """ - create_index(namespace, index_definition) + db.index.create(namespace, index_definition) yield - drop_index(namespace, 'id') + db.index.drop(namespace, "id") @pytest.fixture(scope="function") -def item(namespace): +def item(db, namespace): """ Create an item to namespace """ - insert_item(namespace, item_definition) + db.item.insert(namespace, item_definition) yield item_definition - delete_item(namespace, item_definition) + db.item.delete(namespace, item_definition) @pytest.fixture(scope="function") -def items(namespace): +def items(db, namespace): """ Create items to namespace """ items = [{"id": i, "val": f"testval{i}"} for i in range(10)] for item in items: - insert_item(namespace, item) + db.item.insert(namespace, item) yield items for item in items: - delete_item(namespace, item) + db.item.delete(namespace, item) @pytest.fixture(scope="function") -def metadata(namespace): +def metadata(db, namespace): """ Put metadata to namespace """ key, value = 'key', 'value' - put_metadata(namespace, key, value) + db.meta.put(namespace, key, value) yield key, value @pytest.fixture(scope="function") -def second_namespace_for_join(database): - db, db_name = database +def second_namespace_for_join(db): second_namespace_name = 'test_ns_for_join' - db.namespace_open(second_namespace_name) - db.index_add(second_namespace_name, index_definition) + db.namespace.open(second_namespace_name) + db.index.create(second_namespace_name, index_definition) second_ns_item_definition = {"id": 100, "second_ns_val": "second_ns_testval"} second_ns_item_definition_join = {"id": 1, "second_ns_val": "second_ns_testval_1"} - db.item_insert(second_namespace_name, second_ns_item_definition) - db.item_insert(second_namespace_name, second_ns_item_definition_join) + db.item.insert(second_namespace_name, second_ns_item_definition) + db.item.insert(second_namespace_name, second_ns_item_definition_join) yield second_namespace_name, second_ns_item_definition_join diff --git a/pyreindexer/tests/helpers/api.py b/pyreindexer/tests/helpers/api.py new file mode 100644 index 0000000..4c5de1f --- /dev/null +++ b/pyreindexer/tests/helpers/api.py @@ -0,0 +1,141 @@ +from pyreindexer import RxConnector +from pyreindexer.tests.helpers.log_helper import log_api + + +def make_request_and_response_log(method_description, request_msg, res=None) -> str: + return f"{method_description}\n\t[Request] => {request_msg}\n\t[Response] => {res}" + + +def api_method(func): + def wrapped(self, *args, **kwargs): + try: + method_description = func.__doc__.split("\n")[0] + except AttributeError: + raise RuntimeError("Api Method doesn't have a 'docstring' description") + + args_str = ", ".join(repr(a) for a in args) + kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) + request_msg = f"Called {func.__name__} with args ({args_str}) and kwargs ({kwargs_str})" + + r = func(self, *args, **kwargs) + log = make_request_and_response_log(method_description, request_msg, r) + log_api.info(log) + return r + + return wrapped + + +class ConnectorApi(RxConnector): + + def __init__(self, dsn): + super().__init__(dsn) + self.namespace = NamespaceApiMethods(self) + self.index = IndexesApiMethods(self) + self.item = ItemApiMethods(self) + self.query = QueryApiMethods(self) + self.meta = MetaApiMethods(self) + + +class NamespaceApiMethods: + def __init__(self, api): + self.api = api + + @api_method + def open(self, ns_name): + """ Open namespace """ + return self.api.namespace_open(ns_name) + + @api_method + def close(self, ns_name): + """ Close namespace """ + return self.api.namespace_close(ns_name) + + @api_method + def drop(self, ns_name): + """ Drop namespace """ + return self.api.namespace_drop(ns_name) + + @api_method + def enumerate(self, enum_not_opened=False): + """ Get namespaces list """ + return self.api.namespaces_enum(enum_not_opened) + + +class IndexesApiMethods: + def __init__(self, api): + self.api = api + + @api_method + def create(self, ns_name, index): + """ Add index """ + return self.api.index_add(ns_name, index) + + @api_method + def update(self, ns_name, index): + """ Update index """ + return self.api.index_update(ns_name, index) + + @api_method + def drop(self, ns_name, index_name): + """ Drop index """ + return self.api.index_drop(ns_name, index_name) + + +class ItemApiMethods: + def __init__(self, api): + self.api = api + + @api_method + def insert(self, ns_name, item, precepts=None): + """ Insert item """ + return self.api.item_insert(ns_name, item, precepts) + + @api_method + def upsert(self, ns_name, item, precepts=None): + """ Upsert item """ + return self.api.item_upsert(ns_name, item, precepts) + + @api_method + def update(self, ns_name, item, precepts=None): + """ Update item """ + return self.api.item_update(ns_name, item, precepts) + + @api_method + def delete(self, ns_name, item): + """ Delete item """ + return self.api.item_delete(ns_name, item) + + +class QueryApiMethods: + def __init__(self, api): + self.api = api + + @api_method + def sql(self, q): + """ Execute SQL query """ + return self.api.select(q) + + +class MetaApiMethods: + def __init__(self, api): + self.api = api + + @api_method + def put(self, ns_name, key, value): + """ Put meta with key and value """ + return self.api.meta_put(ns_name, key, value) + + @api_method + def get(self, ns_name, key): + """ Get meta by key """ + return self.api.meta_get(ns_name, key) + + @api_method + def enumerate(self, ns_name): + """ Get meta keys list """ + return self.api.meta_enum(ns_name) + + @api_method + def delete(self, ns_name, key): + """ Delete meta by key """ + return self.api.meta_delete(ns_name, key) diff --git a/pyreindexer/tests/helpers/base_helper.py b/pyreindexer/tests/helpers/base_helper.py new file mode 100644 index 0000000..e3a7bb2 --- /dev/null +++ b/pyreindexer/tests/helpers/base_helper.py @@ -0,0 +1,12 @@ +def get_ns_items(db, ns_name): + """ Get all items via sql query + """ + return list(db.query.sql(f"SELECT * FROM {ns_name}")) + + +def get_ns_description(db, ns_name): + """ Get information about namespace in database + """ + namespaces_list = db.namespace.enumerate() + ns_entry = [ns for ns in namespaces_list if ns["name"] == ns_name] + return ns_entry diff --git a/pyreindexer/tests/helpers/index.py b/pyreindexer/tests/helpers/index.py deleted file mode 100644 index f4f1a4e..0000000 --- a/pyreindexer/tests/helpers/index.py +++ /dev/null @@ -1,28 +0,0 @@ -from .log_helper import log_operation - - -def create_index(namespace, index_def): - """ - Create an index - """ - db, namespace_name = namespace - log_operation.info(f"Create an index to namespace '{namespace_name}', index={index_def}") - db.index_add(namespace_name, index_def) - - -def update_index(namespace, index_def): - """ - Update an index - """ - db, namespace_name = namespace - log_operation.info(f"Update an index to namespace '{namespace_name}', new index={index_def}") - db.index_update(namespace_name, index_def) - - -def drop_index(namespace, index_name): - """ - Drop index from namespace - """ - db, namespace_name = namespace - log_operation.info(f"Drop index from namespace '{namespace_name}', index name = '{index_name}'") - db.index_drop(namespace_name, index_name) diff --git a/pyreindexer/tests/helpers/items.py b/pyreindexer/tests/helpers/items.py deleted file mode 100644 index 53101b7..0000000 --- a/pyreindexer/tests/helpers/items.py +++ /dev/null @@ -1,37 +0,0 @@ -from .log_helper import log_operation - - -def insert_item(namespace, item_def): - """ - Insert item to namespace - """ - db, namespace_name = namespace - log_operation.info(f"Insert item: {item_def} to namespace {namespace_name}") - db.item_insert(namespace_name, item_def) - - -def upsert_item(namespace, item_def): - """ - Insert or update item to namespace - """ - db, namespace_name = namespace - log_operation.info(f"Upsert item: {item_def} to namespace {namespace_name}") - db.item_upsert(namespace_name, item_def) - - -def update_item(namespace, item_def): - """ - Update item to namespace - """ - db, namespace_name = namespace - log_operation.info(f"Update item: {item_def} to namespace {namespace_name}") - db.item_update(namespace_name, item_def) - - -def delete_item(namespace, item_def): - """ - Delete item from namespace - """ - db, namespace_name = namespace - log_operation.info(f"Delete item: {item_def} from namespace {namespace_name}") - db.item_delete(namespace_name, item_def) diff --git a/pyreindexer/tests/helpers/log_helper.py b/pyreindexer/tests/helpers/log_helper.py index fed533c..c4304b7 100644 --- a/pyreindexer/tests/helpers/log_helper.py +++ b/pyreindexer/tests/helpers/log_helper.py @@ -21,11 +21,11 @@ def format(self, record): # Create logger -log_operation = logging.getLogger('OPERATION') +log_api = logging.getLogger('API') log_fixture = logging.getLogger('FIXTURE') log_error = logging.getLogger('ERROR') -log_operation.setLevel(logging.INFO) +log_api.setLevel(logging.INFO) log_fixture.setLevel(logging.INFO) log_error.setLevel(logging.ERROR) @@ -41,5 +41,5 @@ def format(self, record): file_handler = logging.FileHandler(log_filename) file_handler.setLevel(logging.INFO) file_handler.setFormatter(formatter) -log_operation.addHandler(file_handler) +log_api.addHandler(file_handler) log_fixture.addHandler(file_handler) diff --git a/pyreindexer/tests/helpers/metadata.py b/pyreindexer/tests/helpers/metadata.py deleted file mode 100644 index 2112a57..0000000 --- a/pyreindexer/tests/helpers/metadata.py +++ /dev/null @@ -1,29 +0,0 @@ -from .log_helper import log_operation - - -def put_metadata(namespace, key, value): - db, namespace_name = namespace - log_operation.info(f"Put metadata '{key}: {value}' to namespace '{namespace_name}'") - db.meta_put(namespace_name, key, value) - return key, value - - -def get_metadata_keys(namespace): - db, namespace_name = namespace - log_operation.info("Get list of metadata keys") - meta_list = db.meta_enum(namespace_name) - return meta_list - - -def get_metadata_by_key(namespace, key): - db, namespace_name = namespace - value = db.meta_get(namespace_name, key) - log_operation.info(f"Get metadata value by key: '{key}: {value}'") - return value - - -def delete_metadata_by_key(namespace, key): - db, namespace_name = namespace - db.meta_delete(namespace_name, key) - log_operation.info(f"Delete metadata value from namespace '{namespace_name}' by key '{key}'") - return key diff --git a/pyreindexer/tests/helpers/namespace.py b/pyreindexer/tests/helpers/namespace.py deleted file mode 100644 index 4a2d57a..0000000 --- a/pyreindexer/tests/helpers/namespace.py +++ /dev/null @@ -1,45 +0,0 @@ -import logging - -from .log_helper import log_operation - - -def create_namespace(database, namespace_name): - """ - Create a namespace - """ - db, db_name = database - log_operation.info(f"Create a namespace with name '{namespace_name}' on database '{db_name}'") - try: - db.namespace_open(namespace_name) - except Exception as e: - logging.error(e) - - -def drop_namespace(database, namespace_name): - """ - Drop a namespace - """ - db, db_name = database - log_operation.info(f"Drop a namespace with name '{namespace_name}' on database '{db_name}'") - db.namespace_drop(namespace_name) - - -def get_namespaces_list(database): - """ - Get list of namespaces in database - """ - log_operation.info("Get list of namespaces in database") - db, db_name = database - namespaces_list = db.namespaces_enum() - return namespaces_list - - -def get_ns_description(database, namespace): - """ - Get information about namespace in database - """ - db, namespace_name = namespace - namespace_list = get_namespaces_list(database) - log_operation.info(f"Get information about namespace {namespace_name} in database") - ns_entry = [ns for ns in namespace_list if ns["name"] == namespace_name] - return ns_entry diff --git a/pyreindexer/tests/helpers/sql.py b/pyreindexer/tests/helpers/sql.py deleted file mode 100644 index e44231b..0000000 --- a/pyreindexer/tests/helpers/sql.py +++ /dev/null @@ -1,8 +0,0 @@ -from .log_helper import log_operation - - -def sql_query(namespace, query): - db, namespace_name = namespace - items_list = list(db.select(query)) - log_operation.info(f"Execute SQL query: {query}, got result: {items_list}") - return items_list diff --git a/pyreindexer/tests/tests/test_database.py b/pyreindexer/tests/tests/test_database.py index 92d25d1..82d6244 100644 --- a/pyreindexer/tests/tests/test_database.py +++ b/pyreindexer/tests/tests/test_database.py @@ -1,15 +1,12 @@ from hamcrest import * -import pyreindexer from ..test_data.constants import special_namespaces, special_namespaces_cluster class TestCrudDb: - def test_create_db(self): - # Given ("Create empty database") - db = pyreindexer.RxConnector('builtin:///tmp/test_db') + def test_create_db(self, db): # When ("Get namespaces list in created database") - namespaces_list = db.namespaces_enum() + namespaces_list = db.namespace.enumerate() # Then ("Check that database contains only special namespaces") if len(namespaces_list) == len(special_namespaces_cluster): expected_namespaces = special_namespaces_cluster # v4 diff --git a/pyreindexer/tests/tests/test_index.py b/pyreindexer/tests/tests/test_index.py index e65f60a..fe71729 100644 --- a/pyreindexer/tests/tests/test_index.py +++ b/pyreindexer/tests/tests/test_index.py @@ -1,69 +1,67 @@ from hamcrest import * -from ..helpers.index import * -from ..helpers.namespace import get_ns_description +from ..helpers.base_helper import get_ns_description from ..test_data.constants import index_definition, updated_index_definition class TestCrudIndexes: - def test_initial_namespace_has_no_indexes(self, database, namespace): + def test_initial_namespace_has_no_indexes(self, db, namespace): # Given("Create namespace") # When ("Get namespace information") - ns_entry = get_ns_description(database, namespace) + ns_entry = get_ns_description(db, namespace) # Then ("Check that list of indexes in namespace is empty") assert_that(ns_entry, has_item(has_entry("indexes", empty())), "Index is not empty") - def test_create_index(self, database, namespace): + def test_create_index(self, db, namespace): # Given("Create namespace") # When ("Add index") - create_index(namespace, index_definition) + db.index.create(namespace, index_definition) # Then ("Check that index is added") - ns_entry = get_ns_description(database, namespace) + ns_entry = get_ns_description(db, namespace) assert_that(ns_entry, has_item(has_entry("indexes", has_item(index_definition))), "Index wasn't created") - drop_index(namespace, 'id') + db.index.drop(namespace, 'id') - def test_update_index(self, database, namespace, index): + def test_update_index(self, db, namespace, index): # Given("Create namespace with index") # When ("Update index") - update_index(namespace, updated_index_definition) + db.index.update(namespace, updated_index_definition) # Then ("Check that index is updated") - ns_entry = get_ns_description(database, namespace) + ns_entry = get_ns_description(db, namespace) assert_that(ns_entry, has_item(has_entry("indexes", has_item(updated_index_definition))), "Index wasn't updated") - def test_delete_index(self, database, namespace): + def test_delete_index(self, db, namespace): # Given("Create namespace with index") - create_index(namespace, index_definition) + db.index.create(namespace, index_definition) # When ("Delete index") - drop_index(namespace, 'id') + db.index.drop(namespace, 'id') # Then ("Check that index is deleted") - ns_entry = get_ns_description(database, namespace) + ns_entry = get_ns_description(db, namespace) assert_that(ns_entry, has_item(has_entry("indexes", empty())), "Index wasn't deleted") - def test_cannot_add_index_with_same_name(self, database, namespace): + def test_cannot_add_index_with_same_name(self, db, namespace): # Given("Create namespace") # When ("Add index") - create_index(namespace, index_definition) + db.index.create(namespace, index_definition) # Then ("Check that we can't add index with the same name") - assert_that(calling(create_index).with_args(namespace, updated_index_definition), + assert_that(calling(db.index.create).with_args(namespace, updated_index_definition), raises(Exception, pattern="Index '.*' already exists with different settings"), "Index with existing name was created") - def test_cannot_update_not_existing_index_in_namespace(self, database, namespace): + def test_cannot_update_not_existing_index_in_namespace(self, db, namespace): # Given ("Create namespace") - _, namespace_name = namespace # When ("Update index") # Then ("Check that we can't update index that was not created") - assert_that(calling(update_index).with_args(namespace, index_definition), - raises(Exception, pattern=f"Index 'id' not found in '{namespace_name}'"), + assert_that(calling(db.index.update).with_args(namespace, index_definition), + raises(Exception, pattern=f"Index 'id' not found in '{namespace}'"), "Not existing index was updated") - def test_cannot_delete_not_existing_index_in_namespace(self, database, namespace): + def test_cannot_delete_not_existing_index_in_namespace(self, db, namespace): # Given ("Create namespace") # When ("Delete index") # Then ("Check that we can't delete index that was not created") index_name = 'id' - assert_that(calling(drop_index).with_args(namespace, index_name), + assert_that(calling(db.index.drop).with_args(namespace, index_name), raises(Exception, pattern=f"Cannot remove index {index_name}: doesn't exist"), "Not existing index was deleted") diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index c76d2ee..24c0646 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -1,79 +1,71 @@ from hamcrest import * -from ..helpers.items import * +from ..helpers.base_helper import get_ns_items from ..test_data.constants import item_definition class TestCrudItems: - def test_initial_namespace_has_no_items(self, namespace, index): + def test_initial_namespace_has_no_items(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Get namespace information") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) # Then ("Check that list of items in namespace is empty") assert_that(select_result, empty(), "Item list is not empty") - def test_create_item_insert(self, namespace, index): + def test_create_item_insert(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Insert item into namespace") - insert_item(namespace, item_definition) + db.item.insert(namespace, item_definition) # Then ("Check that item is added") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Item wasn't created") assert_that(select_result, has_item(item_definition), "Item wasn't created") - def test_create_item_insert_with_precepts(self, namespace, index): + def test_create_item_insert_with_precepts(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Insert items into namespace") number_items = 5 for _ in range(number_items): - db.item_insert(namespace_name, {"id": 100, "field": "value"}, ["id=serial()"]) + db.item_insert(namespace, {"id": 100, "field": "value"}, ["id=serial()"]) # Then ("Check that item is added") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(number_items), "Items wasn't created") for i in range(number_items): assert_that(select_result[i], equal_to({'id': i + 1, "field": "value"}), "Items wasn't created") - def test_create_item_upsert(self, namespace, index): + def test_create_item_upsert(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Upsert item into namespace") - upsert_item(namespace, item_definition) + db.item.upsert(namespace, item_definition) # Then ("Check that item is added") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Item wasn't created") assert_that(select_result, has_item(item_definition), "Item wasn't created") - delete_item(namespace, item_definition) - def test_update_item_upsert(self, namespace, index, item): + def test_update_item_upsert(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Upsert item") item_definition_updated = {'id': 100, 'val': "new_value"} - upsert_item(namespace, item_definition_updated) + db.item.upsert(namespace, item_definition_updated) # Then ("Check that item is updated") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Item wasn't updated") assert_that(select_result, has_item(item_definition_updated), "Item wasn't updated") - def test_update_item_update(self, namespace, index, item): + def test_update_item_update(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Update item") item_definition_updated = {'id': 100, 'val': "new_value"} - update_item(namespace, item_definition_updated) + db.item.update(namespace, item_definition_updated) # Then ("Check that item is updated") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Item wasn't updated") assert_that(select_result, has_item(item_definition_updated), "Item wasn't updated") - def test_delete_item(self, namespace, index, item): + def test_delete_item(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Delete item") - delete_item(namespace, item_definition) + db.item.delete(namespace, item_definition) # Then ("Check that item is deleted") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, empty(), "Item wasn't deleted") diff --git a/pyreindexer/tests/tests/test_metadata.py b/pyreindexer/tests/tests/test_metadata.py index f3fa840..f37b511 100644 --- a/pyreindexer/tests/tests/test_metadata.py +++ b/pyreindexer/tests/tests/test_metadata.py @@ -1,61 +1,59 @@ from hamcrest import * -from ..helpers.metadata import * - class TestCrudMetadata: - def test_initial_namespace_has_no_metadata(self, namespace): + def test_initial_namespace_has_no_metadata(self, db, namespace): # Given("Create namespace") # When ("Get list of metadata keys") - meta_list = get_metadata_keys(namespace) + meta_list = db.meta.enumerate(namespace) # Then ("Check that list of metadata is empty") assert_that(meta_list, empty(), "Metadata is not empty") - def test_create_metadata(self, namespace): + def test_create_metadata(self, db, namespace): # Given("Create namespace") # When ("Put metadata to namespace") key, value = 'key', 'value' - put_metadata(namespace, key, value) + db.meta.put(namespace, key, value) # Then ("Check that metadata was added") - meta_list = get_metadata_keys(namespace) + meta_list = db.meta.enumerate(namespace) assert_that(meta_list, has_length(1), "Metadata is not created") assert_that(meta_list, has_item(key), "Metadata is not created") - def test_get_metadata_by_key(self, namespace, metadata): + def test_get_metadata_by_key(self, db, namespace, metadata): # Given ("Put metadata to namespace") meta_key, meta_value = metadata # When ("Get metadata by key") - value = get_metadata_by_key(namespace, meta_key) + value = db.meta.get(namespace, meta_key) # Then ("Check that metadata was added") assert_that(value, equal_to(meta_value), "Can't get meta value by key") - def test_get_metadata_keys(self, namespace, metadata): + def test_get_metadata_keys(self, db, namespace, metadata): # Given ("Put metadata to namespace") meta_key, meta_value = metadata # When ("Get list of metadata keys") - meta_list = get_metadata_keys(namespace) + meta_list = db.meta.enumerate(namespace) # Then ("Check that metadata was added") assert_that(meta_list, has_item(meta_key), "Can't get list of meta keys") - def test_update_metadata(self, namespace, metadata): + def test_update_metadata(self, db, namespace, metadata): # Given ("Put metadata to namespace") meta_key, meta_value = metadata # When ("Update metadata") new_meta_value = 'new_value' - put_metadata(namespace, meta_key, new_meta_value) + db.meta.put(namespace, meta_key, new_meta_value) # Then ("Check that metadata was updated") - updated_value = get_metadata_by_key(namespace, meta_key) + updated_value = db.meta.get(namespace, meta_key) assert_that(updated_value, equal_to(new_meta_value), "Can't update metadata") - def test_delete_metadata(self, namespace, metadata): + def test_delete_metadata(self, db, namespace, metadata): # Given ("Put metadata to namespace") meta_key, meta_value = metadata # When ("Delete metadata") - delete_metadata_by_key(namespace, meta_key) + db.meta.delete(namespace, meta_key) # Then ("Check that metadata was removed") - read_value = get_metadata_by_key(namespace, meta_key) + read_value = db.meta.get(namespace, meta_key) assert_that(read_value, equal_to(''), "Can't delete metadata") # When ("Get list of metadata keys") - meta_list = get_metadata_keys(namespace) + meta_list = db.meta.enumerate(namespace) # Then ("Check that list of metadata is empty") assert_that(meta_list, empty(), "Metadata is not empty") diff --git a/pyreindexer/tests/tests/test_namespace.py b/pyreindexer/tests/tests/test_namespace.py index 37dfc8a..feee497 100644 --- a/pyreindexer/tests/tests/test_namespace.py +++ b/pyreindexer/tests/tests/test_namespace.py @@ -1,37 +1,33 @@ from hamcrest import * -from ..helpers.namespace import * - class TestCrudNamespace: - def test_create_ns(self, database): + def test_create_ns(self, db): # Given("Create namespace in empty database") - namespace_name = 'test_ns' - create_namespace(database, namespace_name) + namespace_name = 'test_ns1' + db.namespace.open(namespace_name) # When ("Get namespaces list in created database") - namespace_list = get_namespaces_list(database) + namespace_list = db.namespace.enumerate() # Then ("Check that database contains created namespace") - assert_that(namespace_list, has_item(has_entry("name", equal_to(namespace_name))), + assert_that(namespace_list, has_item(has_entries(name=namespace_name)), "Namespace wasn't created") - drop_namespace(database, namespace_name) + db.namespace.drop(namespace_name) - def test_delete_ns(self, database): - # Given("Create namespace") - namespace_name = 'test_ns' - create_namespace(database, namespace_name) + def test_delete_ns(self, db): + # Given("Create namespace in empty database") + namespace_name = 'test_ns2' + db.namespace.open(namespace_name) # When ("Delete namespace") - drop_namespace(database, namespace_name) + db.namespace.drop(namespace_name) # Then ("Check that namespace was deleted") - namespace_list = get_namespaces_list(database) - assert_that(namespace_list, not_(has_item(has_entry("name", equal_to(namespace_name)))), + namespace_list = db.namespace.enumerate() + assert_that(namespace_list, not_(has_item(has_entries(name=namespace_name))), "Namespace wasn't deleted") - def test_cannot_delete_ns_not_created(self, database): - # Given ("Create empty database") + def test_cannot_delete_ns_not_created(self, db): + # Given ("Created empty database") # Then ("Check that we cannot delete namespace that does not exist") namespace_name = 'test_ns' - assert_that(calling(drop_namespace).with_args(database, namespace_name), raises(Exception, matching=has_string( - f"Namespace '{namespace_name}' does not exist"))) - - + assert_that(calling(db.namespace.drop).with_args(namespace_name), + raises(Exception, pattern=f"Namespace '{namespace_name}' does not exist")) diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index 338f82a..d370220 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -1,79 +1,68 @@ from hamcrest import * -from ..helpers.sql import sql_query - class TestSqlQueries: - def test_sql_select(self, namespace, index, item): + def test_sql_select(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Execute SQL query SELECT") - query = f"SELECT * FROM {namespace_name}" - items_list = sql_query(namespace, query) + query = f"SELECT * FROM {namespace}" + items_list = list(db.query.sql(query)) # Then ("Check that selected item is in result") assert_that(items_list, equal_to([item]), "Can't SQL select data") - def test_sql_select_with_join(self, namespace, second_namespace_for_join, index, items): + def test_sql_select_with_join(self, db, namespace, second_namespace_for_join, index, items): # Given("Create two namespaces") - db, namespace_name = namespace - second_namespace_name, second_ns_item_definition_join = second_namespace_for_join + second_namespace, second_ns_item_definition_join = second_namespace_for_join # When ("Execute SQL query SELECT with JOIN") - query = f"SELECT id FROM {namespace_name} INNER JOIN {second_namespace_name} " \ - f"ON {namespace_name}.id = {second_namespace_name}.id" - item_list = sql_query(namespace, query) + query = f"SELECT id FROM {namespace} INNER JOIN {second_namespace} " \ + f"ON {namespace}.id = {second_namespace}.id" + item_list = list(db.query.sql(query)) # Then ("Check that selected item is in result") - assert_that(item_list, - has_item(equal_to({'id': 1, f'joined_{second_namespace_name}': [second_ns_item_definition_join]})), + item_with_joined = {'id': 1, f'joined_{second_namespace}': [second_ns_item_definition_join]} + assert_that(item_list, equal_to([item_with_joined]), "Can't SQL select data with JOIN") - def test_sql_select_with_condition(self, namespace, index, items): + def test_sql_select_with_condition(self, db, namespace, index, items): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Execute SQL query SELECT") item_id = 3 - query = f"SELECT * FROM {namespace_name} WHERE id = {item_id}" - items_list = sql_query(namespace, query) + query = f"SELECT * FROM {namespace} WHERE id = {item_id}" + items_list = list(db.query.sql(query)) # Then ("Check that selected item is in result") assert_that(items_list, equal_to([items[item_id]]), "Can't SQL select data with condition") - def test_sql_update(self, namespace, index, item): + def test_sql_update(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Execute SQL query UPDATE") item["val"] = "new_val" - query = f"UPDATE {namespace_name} SET \"val\" = '{item['val']}' WHERE id = 100" - items_list = sql_query(namespace, query) + query = f"UPDATE {namespace} SET \"val\" = '{item['val']}' WHERE id = 100" + items_list = list(db.query.sql(query)) # Then ("Check that item is updated") assert_that(items_list, equal_to([item]), "Can't SQL update data") - def test_sql_delete(self, namespace, index, item): + def test_sql_delete(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Execute SQL query DELETE") - query_delete = f"DELETE FROM {namespace_name} WHERE id = 100" - sql_query(namespace, query_delete) + query_delete = f"DELETE FROM {namespace} WHERE id = 100" + db.query.sql(query_delete) # Then ("Check that item is deleted") - query_select = f"SELECT * FROM {namespace_name}" - item_list = sql_query(namespace, query_select) + query_select = f"SELECT * FROM {namespace}" + item_list = list(db.query.sql(query_select)) assert_that(item_list, empty(), "Can't SQL delete data") - def test_sql_select_with_syntax_error(self, namespace, index, item): + def test_sql_select_with_syntax_error(self, db, namespace, index, item): # Given("Create namespace with item") # When ("Execute SQL query SELECT with incorrect syntax") query = "SELECT *" # Then ("Check that selected item is in result") - assert_that(calling(sql_query).with_args(namespace, query), + assert_that(calling(db.query.sql).with_args(query), raises(Exception, pattern="Expected .* but found"), "Error wasn't raised when syntax was incorrect") - def test_sql_select_with_aggregations(self, namespace, index, items): + def test_sql_select_with_aggregations(self, db, namespace, index, items): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Insert items into namespace") - for _ in range(5): - db.item_insert(namespace_name, {"id": 100}, ["id=serial()"]) - - select_result = db.select(f"SELECT min(id), max(id), avg(id), sum(id) FROM {namespace_name}").get_agg_results() + select_result = db.query.sql(f"SELECT min(id), max(id), avg(id), sum(id) FROM {namespace}").get_agg_results() assert_that(select_result, has_length(4), "The aggregation result does not contain all aggregations") ids = [i["id"] for i in items] From 5f242fe1c4f453a210f1b19ffe8c273e6a55d856 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 12:47:26 +0300 Subject: [PATCH 051/125] [fix] bump setup version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index adff914..abe6291 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.2.38', + version='0.2.39', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', author_email='igtulm@gmail.com', From 7fca19eacc9e5b268218db987bc156a82b88bfcf Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 13:40:20 +0300 Subject: [PATCH 052/125] [fix] fix imports for ci --- pyreindexer/tests/conftest.py | 6 +++--- pyreindexer/tests/tests/test_database.py | 2 +- pyreindexer/tests/tests/test_index.py | 4 ++-- pyreindexer/tests/tests/test_items.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 90defaf..acfe24c 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -2,9 +2,9 @@ import pytest -from .helpers.api import ConnectorApi -from .helpers.log_helper import log_fixture -from .test_data.constants import index_definition, item_definition +from pyreindexer.tests.helpers.api import ConnectorApi +from pyreindexer.tests.helpers.log_helper import log_fixture +from pyreindexer.tests.test_data.constants import index_definition, item_definition def pytest_addoption(parser): diff --git a/pyreindexer/tests/tests/test_database.py b/pyreindexer/tests/tests/test_database.py index 82d6244..e90a5d0 100644 --- a/pyreindexer/tests/tests/test_database.py +++ b/pyreindexer/tests/tests/test_database.py @@ -1,6 +1,6 @@ from hamcrest import * -from ..test_data.constants import special_namespaces, special_namespaces_cluster +from pyreindexer.tests.test_data.constants import special_namespaces, special_namespaces_cluster class TestCrudDb: diff --git a/pyreindexer/tests/tests/test_index.py b/pyreindexer/tests/tests/test_index.py index fe71729..11c866f 100644 --- a/pyreindexer/tests/tests/test_index.py +++ b/pyreindexer/tests/tests/test_index.py @@ -1,7 +1,7 @@ from hamcrest import * -from ..helpers.base_helper import get_ns_description -from ..test_data.constants import index_definition, updated_index_definition +from pyreindexer.tests.helpers.base_helper import get_ns_description +from pyreindexer.tests.test_data.constants import index_definition, updated_index_definition class TestCrudIndexes: diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index 24c0646..719259f 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -1,7 +1,7 @@ from hamcrest import * -from ..helpers.base_helper import get_ns_items -from ..test_data.constants import item_definition +from pyreindexer.tests.helpers.base_helper import get_ns_items +from pyreindexer.tests.test_data.constants import item_definition class TestCrudItems: From 35c2fd0b312fc0a4f52109cdc77b531a3891553d Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 14:00:16 +0300 Subject: [PATCH 053/125] [fix] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abe6291..9c0abb3 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.2.39', + version='0.2.40', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', author_email='igtulm@gmail.com', From e57fa7aee369b50056346309b60347c3cbfb2d40 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 14:58:52 +0300 Subject: [PATCH 054/125] [fix] update package_data --- setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 9c0abb3..4abc6f4 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.2.40', + version='0.2.41', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', author_email='igtulm@gmail.com', @@ -91,11 +91,8 @@ def build_cmake(self, ext): 'tests/tests/test_namespace.py', 'tests/tests/test_metadata.py', 'tests/tests/__init__.py', - 'tests/helpers/namespace.py', - 'tests/helpers/sql.py', - 'tests/helpers/items.py', - 'tests/helpers/index.py', - 'tests/helpers/metadata.py', + 'tests/helpers/api.py', + 'tests/helpers/base_helper.py', 'tests/helpers/log_helper.py', 'tests/helpers/__init__.py', 'tests/__init__.py' From 20b70d95fa12d2926be2ddbea0bc16d94dea1ac1 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 15:59:36 +0300 Subject: [PATCH 055/125] [fix] fix imports and package_data --- pyreindexer/tests/conftest.py | 6 ++--- pyreindexer/tests/helpers/api.py | 2 +- pyreindexer/tests/tests/test_database.py | 2 +- pyreindexer/tests/tests/test_index.py | 4 ++-- pyreindexer/tests/tests/test_items.py | 4 ++-- setup.py | 29 +++++------------------- 6 files changed, 15 insertions(+), 32 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index acfe24c..4c8deb4 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -2,9 +2,9 @@ import pytest -from pyreindexer.tests.helpers.api import ConnectorApi -from pyreindexer.tests.helpers.log_helper import log_fixture -from pyreindexer.tests.test_data.constants import index_definition, item_definition +from tests.helpers.api import ConnectorApi +from tests.helpers.log_helper import log_fixture +from tests.test_data.constants import index_definition, item_definition def pytest_addoption(parser): diff --git a/pyreindexer/tests/helpers/api.py b/pyreindexer/tests/helpers/api.py index 4c5de1f..3d534d0 100644 --- a/pyreindexer/tests/helpers/api.py +++ b/pyreindexer/tests/helpers/api.py @@ -1,5 +1,5 @@ from pyreindexer import RxConnector -from pyreindexer.tests.helpers.log_helper import log_api +from tests.helpers.log_helper import log_api def make_request_and_response_log(method_description, request_msg, res=None) -> str: diff --git a/pyreindexer/tests/tests/test_database.py b/pyreindexer/tests/tests/test_database.py index e90a5d0..76cf519 100644 --- a/pyreindexer/tests/tests/test_database.py +++ b/pyreindexer/tests/tests/test_database.py @@ -1,6 +1,6 @@ from hamcrest import * -from pyreindexer.tests.test_data.constants import special_namespaces, special_namespaces_cluster +from tests.test_data.constants import special_namespaces, special_namespaces_cluster class TestCrudDb: diff --git a/pyreindexer/tests/tests/test_index.py b/pyreindexer/tests/tests/test_index.py index 11c866f..5887e75 100644 --- a/pyreindexer/tests/tests/test_index.py +++ b/pyreindexer/tests/tests/test_index.py @@ -1,7 +1,7 @@ from hamcrest import * -from pyreindexer.tests.helpers.base_helper import get_ns_description -from pyreindexer.tests.test_data.constants import index_definition, updated_index_definition +from tests.helpers.base_helper import get_ns_description +from tests.test_data.constants import index_definition, updated_index_definition class TestCrudIndexes: diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index 719259f..151d1d3 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -1,7 +1,7 @@ from hamcrest import * -from pyreindexer.tests.helpers.base_helper import get_ns_items -from pyreindexer.tests.test_data.constants import item_definition +from tests.helpers.base_helper import get_ns_items +from tests.test_data.constants import item_definition class TestCrudItems: diff --git a/setup.py b/setup.py index 4abc6f4..6488ef1 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.2.41', + version='0.2.42', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', author_email='igtulm@gmail.com', @@ -73,29 +73,12 @@ def build_cmake(self, ext): keywords=["reindexer", "in-memory-database", "database", "python", "connector"], package_data={'pyreindexer': [ 'CMakeLists.txt', - 'lib/include/pyobjtools.h', - 'lib/include/pyobjtools.cc', - 'lib/include/queryresults_wrapper.h', - 'lib/src/rawpyreindexer.h', - 'lib/src/rawpyreindexer.cc', - 'lib/src/reindexerinterface.h', - 'lib/src/reindexerinterface.cc', + 'lib/include/*.h', + 'lib/include/*.cc', + 'lib/src/*.h', + 'lib/src/*.cc', 'example/main.py', - 'tests/conftest.py', - 'tests/test_data/constants.py', - 'tests/test_data/__init__.py', - 'tests/tests/test_sql.py', - 'tests/tests/test_index.py', - 'tests/tests/test_items.py', - 'tests/tests/test_database.py', - 'tests/tests/test_namespace.py', - 'tests/tests/test_metadata.py', - 'tests/tests/__init__.py', - 'tests/helpers/api.py', - 'tests/helpers/base_helper.py', - 'tests/helpers/log_helper.py', - 'tests/helpers/__init__.py', - 'tests/__init__.py' + 'tests/**/*.py' ]}, python_requires=">=3.6,<3.13", test_suite='tests', From 5351f9772d3073ba5bc700881d1ade5b30e9db46 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 16:23:02 +0300 Subject: [PATCH 056/125] Part XVIII: add using fetch count --- pyreindexer/lib/include/query_wrapper.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index ff9f97c..aec34ed 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -58,7 +58,6 @@ void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const s ++queriesCount_; } - void QueryWrapper::WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField) { ser_.PutVarUint(QueryItemType::QueryBetweenFieldsCondition); ser_.PutVarUint(nextOperation_); @@ -185,6 +184,7 @@ reindexer::JoinedQuery QueryWrapper::createJoinedQuery(JoinType joinType, reinde void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) { for (auto joinQuery : joinQueries) { auto jq = createJoinedQuery(joinQuery->joinType_, joinQuery->ser_); + jq.Limit(joinQuery->fetchCount_); query.AddJoinQuery(std::move(jq)); } } @@ -192,14 +192,16 @@ void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query QueryWrapper::prepareQuery() { reindexer::Serializer ser = prepareQueryData(ser_); auto query = reindexer::Query::Deserialize(ser); + query.Limit(fetchCount_); addJoinQueries(joinQueries_, query); for (auto mergedQuery : mergedQueries_) { auto mq = createJoinedQuery(JoinType::Merge, mergedQuery->ser_); query.Merge(std::move(mq)); + mq.Limit(mergedQuery->fetchCount_); - addJoinQueries(mergedQuery->joinQueries_, query); + addJoinQueries(mergedQuery->joinQueries_, mq); } return query; From c629d18ac1045c7dee42b448f3318909f04f025a Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 17:09:16 +0300 Subject: [PATCH 057/125] [ref] small code ref --- pyreindexer/point.py | 2 +- pyreindexer/query.py | 44 ++++++++++++++++-------------- pyreindexer/query_results.py | 14 +++++----- pyreindexer/raiser_mixin.py | 2 +- pyreindexer/rx_connector.py | 11 ++++---- pyreindexer/tests/helpers/items.py | 0 6 files changed, 39 insertions(+), 34 deletions(-) delete mode 100644 pyreindexer/tests/helpers/items.py diff --git a/pyreindexer/point.py b/pyreindexer/point.py index ee3affc..df5b9b6 100644 --- a/pyreindexer/point.py +++ b/pyreindexer/point.py @@ -1,4 +1,4 @@ -class Point(object): +class Point: """An object representing the context of a Reindexer 2D point # Attributes: diff --git a/pyreindexer/query.py b/pyreindexer/query.py index bc88a0b..fe9ef6e 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,8 +1,11 @@ from __future__ import annotations -from typing import List, Union + from enum import Enum -from pyreindexer.query_results import QueryResults +from typing import List, Union + from pyreindexer.point import Point +from pyreindexer.query_results import QueryResults + class CondType(Enum): CondAny = 0 @@ -18,21 +21,25 @@ class CondType(Enum): CondLike = 10 CondDWithin = 11 + class StrictMode(Enum): NotSet = 0 Empty = 1 Names = 2 Indexes = 3 + class JoinType(Enum): LeftJoin = 0 InnerJoin = 1 OrInnerJoin = 2 Merge = 3 + simple_types = Union[int, str, bool, float] -class Query(object): + +class Query: """An object representing the context of a Reindexer query # Attributes: @@ -100,7 +107,7 @@ def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[si param = param if isinstance(param, list) else [param] return param - def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds where condition to DB query with args # Arguments: @@ -123,7 +130,8 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ self.__raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def where_query(self, sub_query: Query, condition: CondType, + keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds sub-query where condition to DB query with args # Arguments: @@ -142,7 +150,8 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_ params: list = self.__convert_to_list(keys) - self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) + self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, + condition.value, params) self.__raise_on_error() return self @@ -339,7 +348,7 @@ def aggregate_max(self, index: str) -> Query: self.api.aggregate_max(self.query_wrapper_ptr, index) return self - class _AggregateFacet(object) : + class _AggregateFacet: """An object representing the context of a Reindexer aggregate facet # Attributes: @@ -421,7 +430,7 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: self.__raise_on_error() return self._AggregateFacet(self) - def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only @@ -459,11 +468,7 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: """ - request: str = "ST_Distance(" + index - request += ",ST_GeomFromText('point(" - request += "{:.10f}".format(point.x) + " " + "{:.10f}".format(point.y) - request += ")'))" - + request: str = f"ST_Distance({index},ST_GeomFromText('point({point.x:.10f} {point.y:.10f})'))" return self.sort(request, desc) def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) -> Query: @@ -483,8 +488,7 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) """ - request: str = 'ST_Distance(' + first_field + ',' + second_field + ')' - + request: str = f"ST_Distance({first_field},{second_field})" return self.sort(request, desc) def op_and(self) -> Query: @@ -525,7 +529,7 @@ def op_not(self) -> Query: self.api.op_not(self.query_wrapper_ptr) return self - def request_total(self, total_name: str= '') -> Query: + def request_total(self, total_name: str = '') -> Query: """Requests total items calculation # Arguments: @@ -539,7 +543,7 @@ def request_total(self, total_name: str= '') -> Query: self.api.request_total(self.query_wrapper_ptr, total_name) return self - def cached_total(self, total_name: str= '') -> Query: + def cached_total(self, total_name: str = '') -> Query: """Requests cached total items calculation # Arguments: @@ -643,7 +647,7 @@ def execute(self) -> QueryResults: """ - if self.root is not None : + if self.root is not None: return self.root.execute() self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select_query(self.query_wrapper_ptr) @@ -662,7 +666,7 @@ def delete(self) -> int: """ - if (self.root is not None) or (len(self.join_queries) > 0) : + if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Delete does not support joined queries") self.err_code, self.err_msg, number = self.api.delete_query(self.query_wrapper_ptr) @@ -752,7 +756,7 @@ def update(self) -> QueryResults: """ - if (self.root is not None) or (len(self.join_queries) > 0) : + if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Update does not support joined queries") self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.update_query(self.query_wrapper_ptr) diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 7563037..47c4d10 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,4 +1,7 @@ -class QueryResults(object): +from pyreindexer.raiser_mixin import RaiserMixin + + +class QueryResults(RaiserMixin): """QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer @@ -47,8 +50,7 @@ def __next__(self): if self.pos < self.qres_iter_count: self.pos += 1 self.err_code, self.err_msg, res = self.api.query_results_iterate(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.raise_on_error() return res else: del self @@ -70,8 +72,7 @@ def status(self): """ self.err_code, self.err_msg = self.api.query_results_status(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.raise_on_error() def count(self): """Returns a count of results @@ -103,6 +104,5 @@ def get_agg_results(self): """ self.err_code, self.err_msg, res = self.api.get_agg_results(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.raise_on_error() return res diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index daf7bb3..5b62d5f 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -1,4 +1,4 @@ -class RaiserMixin(object): +class RaiserMixin: """RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity """ diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 81a86eb..86dc309 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -1,8 +1,9 @@ -from typing import List, Dict -from pyreindexer.raiser_mixin import RaiserMixin +from typing import Dict, List + +from pyreindexer.query import Query from pyreindexer.query_results import QueryResults +from pyreindexer.raiser_mixin import RaiserMixin from pyreindexer.transaction import Transaction -from pyreindexer.query import Query class RxConnector(RaiserMixin): @@ -265,7 +266,7 @@ def meta_put(self, namespace, key, value) -> None: self.err_code, self.err_msg = self.api.meta_put(self.rx, namespace, key, value) self.raise_on_error() - def meta_get(self, namespace, key) -> str : + def meta_get(self, namespace, key) -> str: """Gets metadata from a storage of Reindexer by key specified # Arguments: @@ -303,7 +304,7 @@ def meta_delete(self, namespace, key) -> None: self.err_code, self.err_msg = self.api.meta_delete(self.rx, namespace, key) self.raise_on_error() - def meta_enum(self, namespace) -> List[str] : + def meta_enum(self, namespace) -> List[str]: """Gets a list of metadata keys from a storage of Reindexer # Arguments: diff --git a/pyreindexer/tests/helpers/items.py b/pyreindexer/tests/helpers/items.py deleted file mode 100644 index e69de29..0000000 From 4b40a64d534d55cc02132f4384569a10c7d03434 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 6 Nov 2024 17:09:35 +0300 Subject: [PATCH 058/125] [ref] ref tests with tx --- pyreindexer/tests/helpers/api.py | 52 +++++++- pyreindexer/tests/helpers/transaction.py | 79 ++---------- pyreindexer/tests/tests/test_transaction.py | 129 ++++++++------------ pyreindexer/transaction.py | 2 +- 4 files changed, 118 insertions(+), 144 deletions(-) diff --git a/pyreindexer/tests/helpers/api.py b/pyreindexer/tests/helpers/api.py index 3d534d0..71230cb 100644 --- a/pyreindexer/tests/helpers/api.py +++ b/pyreindexer/tests/helpers/api.py @@ -30,10 +30,11 @@ class ConnectorApi(RxConnector): def __init__(self, dsn): super().__init__(dsn) self.namespace = NamespaceApiMethods(self) - self.index = IndexesApiMethods(self) + self.index = IndexApiMethods(self) self.item = ItemApiMethods(self) self.query = QueryApiMethods(self) self.meta = MetaApiMethods(self) + self.tx = TransactionApiMethods(self) class NamespaceApiMethods: @@ -61,7 +62,7 @@ def enumerate(self, enum_not_opened=False): return self.api.namespaces_enum(enum_not_opened) -class IndexesApiMethods: +class IndexApiMethods: def __init__(self, api): self.api = api @@ -139,3 +140,50 @@ def enumerate(self, ns_name): def delete(self, ns_name, key): """ Delete meta by key """ return self.api.meta_delete(ns_name, key) + + +class TransactionApiMethods: + def __init__(self, api): + self.api = api + self.tx = None + + @api_method + def begin(self, ns_name): + """ Begin new transaction """ + self.tx = self.api.new_transaction(ns_name) + return self.tx + + @api_method + def commit(self): + """ Commit the transaction """ + return self.tx.commit() + + @api_method + def commit_with_count(self): + """ Commit the transaction and return the number of changed items """ + return self.tx.commit_with_count() + + @api_method + def rollback(self): + """ Rollback the transaction """ + return self.tx.rollback() + + @api_method + def item_insert(self, item, precepts=None): + """ Insert item into transaction """ + return self.tx.insert(item, precepts) + + @api_method + def item_upsert(self, item, precepts=None): + """ Upsert item into transaction """ + return self.tx.upsert(item, precepts) + + @api_method + def item_update(self, item, precepts=None): + """ Update item into transaction """ + return self.tx.update(item, precepts) + + @api_method + def item_delete(self, item): + """ Delete item from transaction """ + return self.tx.delete(item) diff --git a/pyreindexer/tests/helpers/transaction.py b/pyreindexer/tests/helpers/transaction.py index b7c39d7..cfe0997 100644 --- a/pyreindexer/tests/helpers/transaction.py +++ b/pyreindexer/tests/helpers/transaction.py @@ -1,87 +1,34 @@ -from tests.helpers.log_helper import log_operation - - -def insert_item_transaction(namespace, item_definition): +def insert_item_transaction(db, namespace, item_definition): """ Insert an item into namespace using transaction """ - db, namespace_name = namespace - log_operation.info(f"Insert item: {item_definition} to namespace {namespace_name} by transaction") - transaction = db.new_transaction(namespace_name) - transaction.insert(item_definition) + transaction = db.tx.begin(namespace) + transaction.item_insert(item_definition) transaction.commit() -def upsert_item_transaction(namespace, item_definition): +def upsert_item_transaction(db, namespace, item_definition): """ - Insert or update an item into namespace using transaction + Upsert or update an item into namespace using transaction """ - db, namespace_name = namespace - log_operation.info(f"Upsert item: {item_definition} to namespace {namespace_name} by transaction") - transaction = db.new_transaction(namespace_name) - transaction.upsert(item_definition) + transaction = db.tx.begin(namespace) + transaction.item_upsert(item_definition) transaction.commit() -def update_item_transaction(namespace, item_definition): +def update_item_transaction(db, namespace, item_definition): """ Update an item in namespace using transaction """ - db, namespace_name = namespace - log_operation.info(f"Update item: {item_definition} in namespace {namespace_name} by transaction") - transaction = db.new_transaction(namespace_name) - transaction.update(item_definition) + transaction = db.tx.begin(namespace) + transaction.item_update(item_definition) transaction.commit() -def delete_item_transaction(namespace, item_definition): +def delete_item_transaction(db, namespace, item_definition): """ Delete item from namespace using transaction """ - db, namespace_name = namespace - log_operation.info(f"Delete item: {item_definition} from namespace {namespace_name} by transaction") - transaction = db.new_transaction(namespace_name) - transaction.delete(item_definition) - transaction.commit() - - -def commit_transaction(transaction): - """ - Wrap a method call as a function (Commit) - """ + transaction = db.tx.begin(namespace) + transaction.item_delete(item_definition) transaction.commit() - - -def rollback_transaction(transaction): - """ - Wrap a method call as a function (Rollback) - """ - transaction.rollback() - - -def insert_transaction(transaction, item_def): - """ - Wrap a method call as a function (Insert) - """ - transaction.insert(item_def) - - -def update_transaction(transaction, item_def): - """ - Wrap a method call as a function (Update) - """ - transaction.update(item_def) - - -def upsert_transaction(transaction, item_def): - """ - Wrap a method call as a function (Upsert) - """ - transaction.upsert(item_def) - - -def delete_transaction(transaction, item_def): - """ - Wrap a method call as a function (Delete) - """ - transaction.delete(item_def) diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index db7181e..ed1c1f8 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -1,205 +1,184 @@ from hamcrest import * -from tests.helpers.items import * +from tests.helpers.base_helper import get_ns_items from tests.helpers.transaction import * from tests.test_data.constants import item_definition class TestCrudTransaction: - def test_negative_commit_after_rollback(self, namespace): + def test_negative_commit_after_rollback(self, db, namespace): # Given("Create namespace") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Rollback transaction") transaction.rollback() # Then ("Commit transaction") - assert_that(calling(commit_transaction).with_args(transaction), + assert_that(calling(transaction.commit).with_args(), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_rollback_after_commit(self, namespace): + def test_negative_rollback_after_commit(self, db, namespace): # Given("Create namespace") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Commit transaction") transaction.commit() # Then ("Rollback transaction") - assert_that(calling(rollback_transaction).with_args(transaction), + assert_that(calling(transaction.rollback).with_args(), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_insert_after_rollback(self, namespace, index): + def test_negative_insert_after_rollback(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Rollback transaction") transaction.rollback() # Then ("Insert transaction") - assert_that(calling(insert_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_insert).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_update_after_rollback(self, namespace, index): + def test_negative_update_after_rollback(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Rollback transaction") transaction.rollback() # Then ("Update transaction") - assert_that(calling(update_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_update).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_upsert_after_rollback(self, namespace, index): + def test_negative_upsert_after_rollback(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Rollback transaction") transaction.rollback() # Then ("Upsert transaction") - assert_that(calling(upsert_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_upsert).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_delete_after_rollback(self, namespace, index): + def test_negative_delete_after_rollback(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Rollback transaction") transaction.rollback() # Then ("Delete transaction") - assert_that(calling(delete_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_delete).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_insert_after_commit(self, namespace, index): + def test_negative_insert_after_commit(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Commit transaction") transaction.commit() # Then ("Insert transaction") - assert_that(calling(insert_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_insert).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_update_after_commit(self, namespace, index): + def test_negative_update_after_commit(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Commit transaction") transaction.commit() # Then ("Update transaction") - assert_that(calling(update_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_update).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_upsert_after_commit(self, namespace, index): + def test_negative_upsert_after_commit(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Commit transaction") transaction.commit() # Then ("Upsert transaction") - assert_that(calling(upsert_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_upsert).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_negative_delete_after_commit(self, namespace, index): + def test_negative_delete_after_commit(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Start new transaction") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) # Then ("Commit transaction") transaction.commit() # Then ("Delete transaction") - assert_that(calling(delete_transaction).with_args(transaction, item_definition), + assert_that(calling(transaction.item_delete).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) - def test_create_item_insert(self, namespace, index): + def test_create_item_insert(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Insert item into namespace") - insert_item_transaction(namespace, item_definition) + insert_item_transaction(db, namespace, item_definition) # Then ("Check that item is added") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Transaction: item wasn't created") assert_that(select_result, has_item(item_definition), "Transaction: item wasn't created") - delete_item(namespace, item_definition) - def test_create_item_insert_with_precepts(self, namespace, index): + def test_create_item_insert_with_precepts(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Insert items into namespace") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) number_items = 5 for i in range(number_items): transaction.insert({'id': 100, 'field': 'value' + str(100 + i)}, ['id=serial()']) count = transaction.commit_with_count() assert_that(count, equal_to(number_items), "Transaction: items wasn't created") # Then ("Check that item is added") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(number_items), "Transaction: items wasn't created") for i in range(number_items): assert_that(select_result[i], equal_to({'id': i + 1, 'field': 'value' + str(100 + i)}), "Transaction: items wasn't created") - def test_create_item_upsert(self, namespace, index): + def test_create_item_upsert(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Upsert item into namespace") - upsert_item_transaction(namespace, item_definition) + upsert_item_transaction(db, namespace, item_definition) # Then ("Check that item is added") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Transaction: item wasn't created") assert_that(select_result, has_item(item_definition), "Transaction: item wasn't created") - delete_item(namespace, item_definition) - def test_update_item_upsert(self, namespace, index, item): + def test_update_item_upsert(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Upsert item") item_definition_updated = {'id': 100, 'val': "new_value"} - upsert_item_transaction(namespace, item_definition_updated) + upsert_item_transaction(db, namespace, item_definition_updated) # Then ("Check that item is updated") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Transaction: item wasn't updated") assert_that(select_result, has_item(item_definition_updated), "Transaction: item wasn't updated") - def test_update_item_update(self, namespace, index, item): + def test_update_item_update(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Update item") item_definition_updated = {'id': 100, 'val': "new_value"} - update_item_transaction(namespace, item_definition_updated) + update_item_transaction(db, namespace, item_definition_updated) # Then ("Check that item is updated") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(1), "Transaction: item wasn't updated") assert_that(select_result, has_item(item_definition_updated), "Transaction: item wasn't updated") - def test_delete_item(self, namespace, index, item): + def test_delete_item(self, db, namespace, index, item): # Given("Create namespace with item") - db, namespace_name = namespace # When ("Delete item") - delete_item_transaction(namespace, item) + delete_item_transaction(db, namespace, item) # Then ("Check that item is deleted") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) - assert_that(select_result, has_length(0), "Transaction: item wasn't deleted") - assert_that(select_result, equal_to([]), "Transaction: item wasn't deleted") + select_result = get_ns_items(db, namespace) + assert_that(select_result, empty(), "Transaction: item wasn't deleted") - def test_rollback_transaction(self, namespace, index): + def test_rollback_transaction(self, db, namespace, index): # Given("Create namespace with index") - db, namespace_name = namespace # When ("Insert items into namespace") - transaction = db.new_transaction(namespace_name) + transaction = db.tx.begin(namespace) number_items = 5 for _ in range(number_items): transaction.insert({"id": 100, "field": "value"}) # Then ("Rollback transaction") transaction.rollback() # When ("Get namespace information") - select_result = list(db.select(f'SELECT * FROM {namespace_name}')) + select_result = get_ns_items(db, namespace) # Then ("Check that list of items in namespace is empty") - assert_that(select_result, has_length(0), "Transaction: item list is not empty") - assert_that(select_result, equal_to([]), "Transaction: item list is not empty") + assert_that(select_result, empty(), "Transaction: item list is not empty") diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 2d67943..1e75a6a 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,4 +1,4 @@ -class Transaction(object): +class Transaction: """An object representing the context of a Reindexer transaction # Attributes: From c255552dfc75a2379181f20b9cc7c3fe9782782f Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 18:34:23 +0300 Subject: [PATCH 059/125] Clear code after review --- pyreindexer/point.py | 2 +- pyreindexer/query.py | 8 ++++---- pyreindexer/query_results.py | 2 +- pyreindexer/raiser_mixin.py | 2 +- pyreindexer/transaction.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyreindexer/point.py b/pyreindexer/point.py index ee3affc..df5b9b6 100644 --- a/pyreindexer/point.py +++ b/pyreindexer/point.py @@ -1,4 +1,4 @@ -class Point(object): +class Point: """An object representing the context of a Reindexer 2D point # Attributes: diff --git a/pyreindexer/query.py b/pyreindexer/query.py index bc88a0b..9fd45ce 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -32,7 +32,7 @@ class JoinType(Enum): simple_types = Union[int, str, bool, float] -class Query(object): +class Query: """An object representing the context of a Reindexer query # Attributes: @@ -339,7 +339,7 @@ def aggregate_max(self, index: str) -> Query: self.api.aggregate_max(self.query_wrapper_ptr, index) return self - class _AggregateFacet(object) : + class _AggregateFacet: """An object representing the context of a Reindexer aggregate facet # Attributes: @@ -387,7 +387,7 @@ def offset(self, offset: int) -> Query._AggregateFacet: self.api.aggregation_offset(self.query_wrapper_ptr, offset) return self - def sort(self, field: str, desc: bool) -> Query._AggregateFacet: + def sort(self, field: str, desc: bool=False) -> Query._AggregateFacet: """Sorts facets by field value # Arguments: @@ -421,7 +421,7 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: self.__raise_on_error() return self._AggregateFacet(self) - def sort(self, index: str, desc: bool, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def sort(self, index: str, desc: bool=False, keys: Union[simple_types, List[simple_types]]=None) -> Query: """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 7563037..fb49714 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,4 +1,4 @@ -class QueryResults(object): +class QueryResults: """QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index daf7bb3..5b62d5f 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -1,4 +1,4 @@ -class RaiserMixin(object): +class RaiserMixin: """RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity """ diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 2d67943..1e75a6a 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,4 +1,4 @@ -class Transaction(object): +class Transaction: """An object representing the context of a Reindexer transaction # Attributes: From dc5a7d3a8d9753d2f26a10740c0f44e5509a6d6e Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 18:35:35 +0300 Subject: [PATCH 060/125] Part XVIII: rollback using fetch count. Fix execute method --- pyreindexer/lib/include/query_wrapper.cc | 33 ++++++++++++++++++++-- pyreindexer/lib/include/query_wrapper.h | 36 +++--------------------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index aec34ed..89968af 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -25,6 +25,36 @@ QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { ser_.PutVString(ns); } +void QueryWrapper::Where(std::string_view index, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QueryCondition); + ser_.PutVString(index); + ser_.PutVarUint(nextOperation_); + ser_.PutVarUint(condition); + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + ser_.PutVariant(key); + } + + nextOperation_ = OpType::OpAnd; + ++queriesCount_; +} + +void QueryWrapper::WhereQuery(QueryWrapper& query, CondType condition, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QuerySubQueryCondition); + ser_.PutVarUint(nextOperation_); + ser_.PutVString(query.ser_.Slice()); + ser_.PutVarUint(condition); + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + ser_.PutVariant(key); + } + + nextOperation_ = OpType::OpAnd; + ++queriesCount_; +} + void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); ser_.PutVarUint(nextOperation_); @@ -184,7 +214,6 @@ reindexer::JoinedQuery QueryWrapper::createJoinedQuery(JoinType joinType, reinde void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) { for (auto joinQuery : joinQueries) { auto jq = createJoinedQuery(joinQuery->joinType_, joinQuery->ser_); - jq.Limit(joinQuery->fetchCount_); query.AddJoinQuery(std::move(jq)); } } @@ -192,14 +221,12 @@ void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query QueryWrapper::prepareQuery() { reindexer::Serializer ser = prepareQueryData(ser_); auto query = reindexer::Query::Deserialize(ser); - query.Limit(fetchCount_); addJoinQueries(joinQueries_, query); for (auto mergedQuery : mergedQueries_) { auto mq = createJoinedQuery(JoinType::Merge, mergedQuery->ser_); query.Merge(std::move(mq)); - mq.Limit(mergedQuery->fetchCount_); addJoinQueries(mergedQuery->joinQueries_, mq); } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 17bd066..69d8c38 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -26,36 +26,8 @@ class QueryWrapper { public: QueryWrapper(DBInterface* db, std::string_view ns); - template - void Where(std::string_view index, CondType condition, const std::vector& keys) { - ser_.PutVarUint(QueryItemType::QueryCondition); - ser_.PutVString(index); - ser_.PutVarUint(nextOperation_); - ser_.PutVarUint(condition); - - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - putValue(key); - } - - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - } - template - void WhereQuery(QueryWrapper& query, CondType condition, const std::vector& keys) { - ser_.PutVarUint(QueryItemType::QuerySubQueryCondition); - ser_.PutVarUint(nextOperation_); - ser_.PutVString(query.ser_.Slice()); - ser_.PutVarUint(condition); - - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - putValue(key); - } - - nextOperation_ = OpType::OpAnd; - ++queriesCount_; - } + void Where(std::string_view index, CondType condition, const std::vector& keys); + void WhereQuery(QueryWrapper& query, CondType condition, const std::vector& keys); void WhereComposite(std::string_view index, CondType condition, QueryWrapper& query); void WhereUUID(std::string_view index, CondType condition, const std::vector& keys); @@ -147,8 +119,8 @@ class QueryWrapper { OpType nextOperation_{OpType::OpAnd}; unsigned queriesCount_{0}; std::deque openedBrackets_; - std::string totalName_; - int fetchCount_{1000}; + std::string totalName_; // ToDo now not used + int fetchCount_{100}; // ToDo now not used JoinType joinType_{JoinType::LeftJoin}; std::vector joinQueries_; std::vector mergedQueries_; From d5bb8682b89c05b2ca52da84422d608db0227037 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 18:37:42 +0300 Subject: [PATCH 061/125] Part XIX: remove Query aliases from connector (select\delete\update) --- pyreindexer/rx_connector.py | 54 ------------------------------------- 1 file changed, 54 deletions(-) diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 81a86eb..6870fda 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -382,60 +382,6 @@ def new_query(self, namespace: str) -> Query: self.raise_on_error() return Query(self.api, query_wrapper_ptr) - def select_query(self, query: Query) -> QueryResults: - """Executes a select query - - # Arguments: - query (:obj:`Query`): An query object - - # Returns: - (:obj:`QueryResults`): A QueryResults iterator - - # Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.raise_on_not_init() - return query.execute() - - def delete_query(self, query: Query) -> int: - """Executes a query, and delete items, matches query - - # Arguments: - query (:obj:`Query`): An query object - - # Returns: - (int): Number of deleted elements - - # Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.raise_on_not_init() - return query.delete() - - def update_query(self, query: Query) -> QueryResults: - """Executes update query, and update fields in items, which matches query - - # Arguments: - query (:obj:`Query`): An query object - - # Returns: - (:obj:`QueryResults`): A QueryResults iterator - - # Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code - - """ - - self.raise_on_not_init() - return query.update() - def _api_import(self, dsn): """Imports an API dynamically depending on protocol specified in dsn From 02908c5e5f1e029ef9dd70beb40476c46824c631 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 18:38:19 +0300 Subject: [PATCH 062/125] Simplify example code --- pyreindexer/example/main.py | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index c588d45..7164d18 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -1,5 +1,5 @@ from pyreindexer import RxConnector -from pyreindexer.query import CondType, StrictMode +from pyreindexer.query import CondType def create_index_example(db, namespace): @@ -68,7 +68,7 @@ def transaction_example(db, namespace, items_in_base): # update last one item, overwrite field 'value' item = items_in_base[items_count - 1] - item['value'] = 'the transaction was here' + item['value'] = 'transaction was here' transaction.update(item) # stop transaction and commit changes to namespace @@ -86,27 +86,18 @@ def transaction_example(db, namespace, items_in_base): def query_example(db, namespace): - (db.new_query(namespace) - .open_bracket() - .where_between_fields('fld1', CondType.CondEq, 'fld2') - .close_bracket() - .where('fld2', CondType.CondLe, [42]) - .limit(10) - .debug(1) - .request_total()) - - query = db.new_query(namespace).offset(1).cached_total() - (query.where('fld1', CondType.CondSet, ['s','t','o','p']) - .op_not() - .where('fld2', CondType.CondSet, [3.14]) - .where_query(db.new_query(namespace), CondType.CondSet, ['to','check']) - .explain() - .fetch_count(10)) - query.expression('fld1', 'array_remove(integer_array, [5,6,7,8]) || [1,2,3]').drop('fld2').strict(StrictMode.Names) - - db.new_query(namespace).sort_stfield_distance('fldGeom1', 'fldGeom2', False) - - db.new_query(namespace).aggregate_facet(['fld1', 'fld2']).limit(100) + selected_items = (db.new_query(namespace) + .where('value', CondType.CondEq, 'check') + .sort('id') + .limit(4) + .execute()) + + res_count = selected_items.count() + print('Query results count (limited): ', res_count) + + # disposable QueryResults iterator + for item in selected_items: + print('Item: ', item) def rx_example(): From 08941cffdc0b8d7207e69f6c15a22d3024fe3b53 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 6 Nov 2024 19:18:46 +0300 Subject: [PATCH 063/125] Remove `-Wold-style-cast` cmake flag --- pyreindexer/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyreindexer/CMakeLists.txt b/pyreindexer/CMakeLists.txt index 1bde9ee..6093c23 100644 --- a/pyreindexer/CMakeLists.txt +++ b/pyreindexer/CMakeLists.txt @@ -26,7 +26,7 @@ set(LIBSRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/src) set(RESOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/include) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wswitch-enum") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Werror -Wswitch-enum -Wno-unused-parameter -Wold-style-cast -fexceptions") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Werror -Wswitch-enum -Wno-unused-parameter -fexceptions") string (REPLACE "-O2" "-O3" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") file(GLOB_RECURSE SRCS ${RESOURCES_DIR}/*.cc ${LIBSRC_DIR}/*.cc) From e2a8ea88b990493591ed450e81e9e209a294632f Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Thu, 7 Nov 2024 11:19:12 +0300 Subject: [PATCH 064/125] [fix] fix tests with tx --- pyreindexer/tests/helpers/api.py | 77 ++++++++++++--------- pyreindexer/tests/helpers/log_helper.py | 8 +++ pyreindexer/tests/helpers/transaction.py | 8 +-- pyreindexer/tests/tests/test_transaction.py | 22 +++--- setup.py | 9 ++- 5 files changed, 70 insertions(+), 54 deletions(-) diff --git a/pyreindexer/tests/helpers/api.py b/pyreindexer/tests/helpers/api.py index 71230cb..b0df293 100644 --- a/pyreindexer/tests/helpers/api.py +++ b/pyreindexer/tests/helpers/api.py @@ -1,9 +1,10 @@ from pyreindexer import RxConnector +from query import Query from tests.helpers.log_helper import log_api def make_request_and_response_log(method_description, request_msg, res=None) -> str: - return f"{method_description}\n\t[Request] => {request_msg}\n\t[Response] => {res}" + return f"{method_description}\n\t[Request] => {request_msg}\n\t[Response] <= {res}" def api_method(func): @@ -116,6 +117,11 @@ def sql(self, q): """ Execute SQL query """ return self.api.select(q) + @api_method + def new(self, ns_name) -> Query: + """ Create a new query """ + return self.api.new_query(ns_name) + class MetaApiMethods: def __init__(self, api): @@ -145,45 +151,48 @@ def delete(self, ns_name, key): class TransactionApiMethods: def __init__(self, api): self.api = api - self.tx = None - @api_method - def begin(self, ns_name): - """ Begin new transaction """ - self.tx = self.api.new_transaction(ns_name) - return self.tx + class _TransactionApi: + def __init__(self, tx): + self.tx = tx - @api_method - def commit(self): - """ Commit the transaction """ - return self.tx.commit() + @api_method + def commit(self): + """ Commit the transaction """ + return self.tx.commit() - @api_method - def commit_with_count(self): - """ Commit the transaction and return the number of changed items """ - return self.tx.commit_with_count() + @api_method + def commit_with_count(self): + """ Commit the transaction and return the number of changed items """ + return self.tx.commit_with_count() - @api_method - def rollback(self): - """ Rollback the transaction """ - return self.tx.rollback() + @api_method + def rollback(self): + """ Rollback the transaction """ + return self.tx.rollback() - @api_method - def item_insert(self, item, precepts=None): - """ Insert item into transaction """ - return self.tx.insert(item, precepts) + @api_method + def insert_item(self, item, precepts=None): + """ Insert item into transaction """ + return self.tx.insert(item, precepts) - @api_method - def item_upsert(self, item, precepts=None): - """ Upsert item into transaction """ - return self.tx.upsert(item, precepts) + @api_method + def upsert_item(self, item, precepts=None): + """ Upsert item into transaction """ + return self.tx.upsert(item, precepts) - @api_method - def item_update(self, item, precepts=None): - """ Update item into transaction """ - return self.tx.update(item, precepts) + @api_method + def update_item(self, item, precepts=None): + """ Update item into transaction """ + return self.tx.update(item, precepts) + + @api_method + def delete_item(self, item): + """ Delete item from transaction """ + return self.tx.delete(item) @api_method - def item_delete(self, item): - """ Delete item from transaction """ - return self.tx.delete(item) + def begin(self, ns_name) -> "_TransactionApi": + """ Begin new transaction """ + tx = self.api.new_transaction(ns_name) + return self._TransactionApi(tx) diff --git a/pyreindexer/tests/helpers/log_helper.py b/pyreindexer/tests/helpers/log_helper.py index c4304b7..b7f6baf 100644 --- a/pyreindexer/tests/helpers/log_helper.py +++ b/pyreindexer/tests/helpers/log_helper.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import os +import sys from datetime import * @@ -43,3 +44,10 @@ def format(self, record): file_handler.setFormatter(formatter) log_api.addHandler(file_handler) log_fixture.addHandler(file_handler) + +# Log to console +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setLevel(logging.INFO) +console_handler.setFormatter(formatter) +log_api.addHandler(console_handler) +log_fixture.addHandler(console_handler) diff --git a/pyreindexer/tests/helpers/transaction.py b/pyreindexer/tests/helpers/transaction.py index cfe0997..0c3003f 100644 --- a/pyreindexer/tests/helpers/transaction.py +++ b/pyreindexer/tests/helpers/transaction.py @@ -3,7 +3,7 @@ def insert_item_transaction(db, namespace, item_definition): Insert an item into namespace using transaction """ transaction = db.tx.begin(namespace) - transaction.item_insert(item_definition) + transaction.insert_item(item_definition) transaction.commit() @@ -12,7 +12,7 @@ def upsert_item_transaction(db, namespace, item_definition): Upsert or update an item into namespace using transaction """ transaction = db.tx.begin(namespace) - transaction.item_upsert(item_definition) + transaction.upsert_item(item_definition) transaction.commit() @@ -21,7 +21,7 @@ def update_item_transaction(db, namespace, item_definition): Update an item in namespace using transaction """ transaction = db.tx.begin(namespace) - transaction.item_update(item_definition) + transaction.update_item(item_definition) transaction.commit() @@ -30,5 +30,5 @@ def delete_item_transaction(db, namespace, item_definition): Delete item from namespace using transaction """ transaction = db.tx.begin(namespace) - transaction.item_delete(item_definition) + transaction.delete_item(item_definition) transaction.commit() diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index ed1c1f8..12ad025 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -33,7 +33,7 @@ def test_negative_insert_after_rollback(self, db, namespace, index): # Then ("Rollback transaction") transaction.rollback() # Then ("Insert transaction") - assert_that(calling(transaction.item_insert).with_args(item_definition), + assert_that(calling(transaction.insert_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_update_after_rollback(self, db, namespace, index): @@ -43,7 +43,7 @@ def test_negative_update_after_rollback(self, db, namespace, index): # Then ("Rollback transaction") transaction.rollback() # Then ("Update transaction") - assert_that(calling(transaction.item_update).with_args(item_definition), + assert_that(calling(transaction.update_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_upsert_after_rollback(self, db, namespace, index): @@ -53,7 +53,7 @@ def test_negative_upsert_after_rollback(self, db, namespace, index): # Then ("Rollback transaction") transaction.rollback() # Then ("Upsert transaction") - assert_that(calling(transaction.item_upsert).with_args(item_definition), + assert_that(calling(transaction.upsert_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_delete_after_rollback(self, db, namespace, index): @@ -63,7 +63,7 @@ def test_negative_delete_after_rollback(self, db, namespace, index): # Then ("Rollback transaction") transaction.rollback() # Then ("Delete transaction") - assert_that(calling(transaction.item_delete).with_args(item_definition), + assert_that(calling(transaction.delete_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_insert_after_commit(self, db, namespace, index): @@ -73,7 +73,7 @@ def test_negative_insert_after_commit(self, db, namespace, index): # Then ("Commit transaction") transaction.commit() # Then ("Insert transaction") - assert_that(calling(transaction.item_insert).with_args(item_definition), + assert_that(calling(transaction.insert_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_update_after_commit(self, db, namespace, index): @@ -83,7 +83,7 @@ def test_negative_update_after_commit(self, db, namespace, index): # Then ("Commit transaction") transaction.commit() # Then ("Update transaction") - assert_that(calling(transaction.item_update).with_args(item_definition), + assert_that(calling(transaction.update_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_upsert_after_commit(self, db, namespace, index): @@ -93,7 +93,7 @@ def test_negative_upsert_after_commit(self, db, namespace, index): # Then ("Commit transaction") transaction.commit() # Then ("Upsert transaction") - assert_that(calling(transaction.item_upsert).with_args(item_definition), + assert_that(calling(transaction.upsert_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_negative_delete_after_commit(self, db, namespace, index): @@ -103,7 +103,7 @@ def test_negative_delete_after_commit(self, db, namespace, index): # Then ("Commit transaction") transaction.commit() # Then ("Delete transaction") - assert_that(calling(transaction.item_delete).with_args(item_definition), + assert_that(calling(transaction.delete_item).with_args(item_definition), raises(Exception, matching=has_string("Transaction is over"))) def test_create_item_insert(self, db, namespace, index): @@ -121,7 +121,7 @@ def test_create_item_insert_with_precepts(self, db, namespace, index): transaction = db.tx.begin(namespace) number_items = 5 for i in range(number_items): - transaction.insert({'id': 100, 'field': 'value' + str(100 + i)}, ['id=serial()']) + transaction.insert_item({'id': 100, 'field': f'value{100 + i}'}, ['id=serial()']) count = transaction.commit_with_count() assert_that(count, equal_to(number_items), "Transaction: items wasn't created") # Then ("Check that item is added") @@ -129,7 +129,7 @@ def test_create_item_insert_with_precepts(self, db, namespace, index): assert_that(select_result, has_length(number_items), "Transaction: items wasn't created") for i in range(number_items): assert_that(select_result[i], - equal_to({'id': i + 1, 'field': 'value' + str(100 + i)}), + equal_to({'id': i + 1, 'field': f'value{100 + i}'}), "Transaction: items wasn't created") def test_create_item_upsert(self, db, namespace, index): @@ -175,7 +175,7 @@ def test_rollback_transaction(self, db, namespace, index): transaction = db.tx.begin(namespace) number_items = 5 for _ in range(number_items): - transaction.insert({"id": 100, "field": "value"}) + transaction.insert_item({"id": 100, "field": "value"}) # Then ("Rollback transaction") transaction.rollback() # When ("Get namespace information") diff --git a/setup.py b/setup.py index 516a5b4..89cee20 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ import os import sys -from setuptools import setup, Extension +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig + if sys.version_info < (3, 6): raise RuntimeError('Require Python 3.6 or greater') @@ -74,10 +75,8 @@ def build_cmake(self, ext): keywords=["reindexer", "in-memory-database", "database", "python", "connector"], package_data={'pyreindexer': [ 'CMakeLists.txt', - 'lib/include/*.h', - 'lib/include/*.cc', - 'lib/src/*.h', - 'lib/src/*.cc', + 'lib/**/*.h', + 'lib/**/*.cc', 'example/main.py', 'tests/**/*.py' ]}, From 0c72d214c50006b14bc971116eae21d5f8d826f9 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Thu, 7 Nov 2024 11:21:00 +0300 Subject: [PATCH 065/125] add initial query tests --- pyreindexer/tests/tests/test_query.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pyreindexer/tests/tests/test_query.py diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py new file mode 100644 index 0000000..40cc52a --- /dev/null +++ b/pyreindexer/tests/tests/test_query.py @@ -0,0 +1,25 @@ +from hamcrest import * + +from query import CondType + + +class TestQuery: + def test_query_select_where(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # When ("Create new query") + query = db.query.new(namespace) + # When ("Make select query") + select_result = list(query.where("id", CondType.CondEq, 3).execute()) + # Then ("Check that selected item is in result") + assert_that(select_result, has_length(1), "Wrong query results") + assert_that(select_result, equal_to([items[3]]), "Wrong query results") + + def test_query_select_fields(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # When ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with select fields") + select_result = list(query.select(["id"]).must_execute()) + # Then ("Check that selected items is in result") + ids = [{"id": i["id"]} for i in items] + assert_that(select_result, equal_to(ids), "Wrong query results") From 19e881f1557014df85cc2717b6e3e1d72804580b Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 7 Nov 2024 12:56:53 +0300 Subject: [PATCH 066/125] Part XX: remove support of FetchCount from Query object --- pyreindexer/lib/include/query_wrapper.cc | 4 ---- pyreindexer/lib/include/query_wrapper.h | 3 --- pyreindexer/lib/src/rawpyreindexer.cc | 14 -------------- pyreindexer/lib/src/rawpyreindexer.h | 2 -- pyreindexer/query.py | 15 --------------- 5 files changed, 38 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 89968af..394bf42 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -302,10 +302,6 @@ void QueryWrapper::SelectFilter(const std::vector& fields) { } } -void QueryWrapper::FetchCount(int count) { - fetchCount_ = count; -} - void QueryWrapper::AddFunctions(const std::vector& functions) { for (const auto& function : functions) { ser_.PutVarUint(QueryItemType::QuerySelectFunction); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 69d8c38..4867bff 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -97,8 +97,6 @@ class QueryWrapper { void SelectFilter(const std::vector& fields); - void FetchCount(int count); - void AddFunctions(const std::vector& functions); void AddEqualPosition(const std::vector& equalPositions); @@ -120,7 +118,6 @@ class QueryWrapper { unsigned queriesCount_{0}; std::deque openedBrackets_; std::string totalName_; // ToDo now not used - int fetchCount_{100}; // ToDo now not used JoinType joinType_{JoinType::LeftJoin}; std::vector joinQueries_; std::vector mergedQueries_; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 716d0ca..6c4ad76 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1178,20 +1178,6 @@ static PyObject* SelectFilter(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* FetchCount(PyObject* self, PyObject* args) { - uintptr_t queryWrapperAddr = 0; - int count = 0; - if (!PyArg_ParseTuple(args, "ki", &queryWrapperAddr, &count)) { - return nullptr; - } - - auto query = getWrapper(queryWrapperAddr); - - query->FetchCount(count); - - Py_RETURN_NONE; -} - static PyObject* AddFunctions(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; PyObject* functionsList = nullptr; // borrowed ref after ParseTuple if passed diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index bc17dda..27736e4 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -94,7 +94,6 @@ static PyObject* Join(PyObject* self, PyObject* args); static PyObject* Merge(PyObject* self, PyObject* args); static PyObject* On(PyObject* self, PyObject* args); static PyObject* SelectFilter(PyObject* self, PyObject* args); -static PyObject* FetchCount(PyObject* self, PyObject* args); static PyObject* AddFunctions(PyObject* self, PyObject* args); static PyObject* AddEqualPosition(PyObject* self, PyObject* args); @@ -179,7 +178,6 @@ static PyMethodDef module_methods[] = { {"merge", Merge, METH_VARARGS, "merge 2 query"}, {"on", On, METH_VARARGS, "on specifies join condition"}, {"select_filter", SelectFilter, METH_VARARGS, "select add filter to fields of result's objects"}, - {"fetch_count", FetchCount, METH_VARARGS, "limit number of items"}, {"functions", AddFunctions, METH_VARARGS, "add sql-functions to query"}, {"equal_position", AddEqualPosition, METH_VARARGS, "add equal position fields"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 211818d..951f775 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -942,21 +942,6 @@ def select(self, fields: List[str]) -> Query: self.__raise_on_error() return self - def fetch_count(self, n: int) -> Query: - """Sets the number of items that will be fetched by one operation. - When n <= 0 query will fetch all results in one operation - - # Arguments: - n (int): Number of items - - # Returns: - (:obj:`Query`): Query object for further customizations - - """ - - self.api.fetch_count(self.query_wrapper_ptr, n) - return self - def functions(self, functions: List[str]) -> Query: """Adds sql-functions to query From 17009b877e8858c00fc14f6c6f70e53c448cedce Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Thu, 7 Nov 2024 12:58:09 +0300 Subject: [PATCH 067/125] Simplify code --- pyreindexer/rx_connector.py | 12 ++++++------ pyreindexer/transaction.py | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 03ffa35..8c37e16 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -47,6 +47,9 @@ def __del__(self): def close(self) -> None: """Closes an API instance with Reindexer resources freeing + # Raises: + Exception: Raises with an error message when Reindexer instance is not initialized yet + """ self._api_close() @@ -185,8 +188,7 @@ def item_insert(self, namespace, item_def, precepts=None) -> None: """ - if precepts is None: - precepts = [] + precepts = [] if precepts is None else precepts self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_insert(self.rx, namespace, item_def, precepts) self.raise_on_error() @@ -205,8 +207,7 @@ def item_update(self, namespace, item_def, precepts=None) -> None: """ - if precepts is None: - precepts = [] + precepts = [] if precepts is None else precepts self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_update(self.rx, namespace, item_def, precepts) self.raise_on_error() @@ -225,8 +226,7 @@ def item_upsert(self, namespace, item_def, precepts=None) -> None: """ - if precepts is None: - precepts = [] + precepts = [] if precepts is None else precepts self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_upsert(self.rx, namespace, item_def, precepts) self.raise_on_error() diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 1e75a6a..fd5f303 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -67,8 +67,7 @@ def insert(self, item_def, precepts=None): """ self.__raise_on_is_over() - if precepts is None: - precepts = [] + precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_insert_transaction(self.transaction_wrapper_ptr, item_def, precepts) self.__raise_on_error() From b3303be7fcfe7aae2eef9cb9d149c1b14917742b Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Thu, 7 Nov 2024 19:23:31 +0300 Subject: [PATCH 068/125] add new query tests --- pyreindexer/tests/conftest.py | 30 ++++- pyreindexer/tests/helpers/log_helper.py | 14 +- pyreindexer/tests/test_data/constants.py | 28 ++-- pyreindexer/tests/tests/test_items.py | 2 +- pyreindexer/tests/tests/test_query.py | 162 ++++++++++++++++++++++- 5 files changed, 205 insertions(+), 31 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 4c8deb4..7ef6ed5 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -4,7 +4,7 @@ from tests.helpers.api import ConnectorApi from tests.helpers.log_helper import log_fixture -from tests.test_data.constants import index_definition, item_definition +from tests.test_data.constants import composite_index_definition, index_definition, item_definition def pytest_addoption(parser): @@ -48,7 +48,30 @@ def index(db, namespace): """ db.index.create(namespace, index_definition) yield - db.index.drop(namespace, "id") + + +@pytest.fixture(scope="function") +def composite_index(db, namespace): + """ + Create indexes and composite index from them + """ + db.index.create(namespace, index_definition) + db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash"}) + db.index.create(namespace, composite_index_definition) + yield + + +@pytest.fixture(scope="function") +def rtree_index_and_items(db, namespace): + """ + Create rtree index and items + """ + db.index.create(namespace, {"name": "rtree", "json_paths": ["rtree"], "field_type": "point", + "index_type": "rtree", "rtree_type": "rstar"}) + items = [{"id": i, "rtree": [i, i]} for i in range(10)] + for item in items: + db.item.insert(namespace, item) + yield items @pytest.fixture(scope="function") @@ -58,7 +81,6 @@ def item(db, namespace): """ db.item.insert(namespace, item_definition) yield item_definition - db.item.delete(namespace, item_definition) @pytest.fixture(scope="function") @@ -70,8 +92,6 @@ def items(db, namespace): for item in items: db.item.insert(namespace, item) yield items - for item in items: - db.item.delete(namespace, item) @pytest.fixture(scope="function") diff --git a/pyreindexer/tests/helpers/log_helper.py b/pyreindexer/tests/helpers/log_helper.py index b7f6baf..e29ee3b 100644 --- a/pyreindexer/tests/helpers/log_helper.py +++ b/pyreindexer/tests/helpers/log_helper.py @@ -33,6 +33,13 @@ def format(self, record): # Log format formatter = OneLineExceptionFormatter('%(levelname)-5s [%(asctime)s] [%(name)s]: %(message)s') +# Log to console +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setLevel(logging.INFO) +console_handler.setFormatter(formatter) +log_api.addHandler(console_handler) +log_fixture.addHandler(console_handler) + # Save log to file log_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../logs/') if not os.path.exists(log_folder): @@ -44,10 +51,3 @@ def format(self, record): file_handler.setFormatter(formatter) log_api.addHandler(file_handler) log_fixture.addHandler(file_handler) - -# Log to console -console_handler = logging.StreamHandler(sys.stdout) -console_handler.setLevel(logging.INFO) -console_handler.setFormatter(formatter) -log_api.addHandler(console_handler) -log_fixture.addHandler(console_handler) diff --git a/pyreindexer/tests/test_data/constants.py b/pyreindexer/tests/test_data/constants.py index 71cc47c..8b7d3d2 100644 --- a/pyreindexer/tests/test_data/constants.py +++ b/pyreindexer/tests/test_data/constants.py @@ -9,12 +9,7 @@ "collate_mode": "none", "sort_order_letters": "", "expire_after": 0, - "config": { - - }, - "json_paths": [ - "id" - ] + "json_paths": ["id"] } updated_index_definition = { @@ -28,12 +23,18 @@ "collate_mode": "none", "sort_order_letters": "", "expire_after": 0, - "config": { + "json_paths": ["id_new"] +} - }, - "json_paths": [ - "id_new" - ] +composite_index_definition = { + "name": "comp_idx", + "field_type": "composite", + "index_type": "hash", + "is_pk": False, + "is_array": False, + "is_dense": False, + "is_sparse": False, + "json_paths": ["id", "val"] } special_namespaces = [{"name": "#namespaces"}, @@ -54,3 +55,8 @@ {"name": "#replicationstats"}] item_definition = {'id': 100, 'val': "testval"} + +AGGREGATE_FUNCTIONS_MATH = [(lambda x: max(x), "aggregate_max"), + (lambda x: min(x), "aggregate_min"), + (lambda x: sum(x), "aggregate_sum"), + (lambda x: sum(x) / len(x), "aggregate_avg")] diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index 151d1d3..c154a30 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -26,7 +26,7 @@ def test_create_item_insert_with_precepts(self, db, namespace, index): # When ("Insert items into namespace") number_items = 5 for _ in range(number_items): - db.item_insert(namespace, {"id": 100, "field": "value"}, ["id=serial()"]) + db.item.insert(namespace, {"id": 100, "field": "value"}, ["id=serial()"]) # Then ("Check that item is added") select_result = get_ns_items(db, namespace) assert_that(select_result, has_length(number_items), "Items wasn't created") diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 40cc52a..ef53abf 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -1,25 +1,173 @@ +import pytest from hamcrest import * +from point import Point from query import CondType +from tests.test_data.constants import AGGREGATE_FUNCTIONS_MATH -class TestQuery: +class TestQuerySelect: def test_query_select_where(self, db, namespace, index, items): # Given("Create namespace with index and items") - # When ("Create new query") + # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query") - select_result = list(query.where("id", CondType.CondEq, 3).execute()) + query_result = list(query.where("id", CondType.CondEq, 3).execute()) # Then ("Check that selected item is in result") - assert_that(select_result, has_length(1), "Wrong query results") - assert_that(select_result, equal_to([items[3]]), "Wrong query results") + assert_that(query_result, equal_to([items[3]]), "Wrong query results") def test_query_select_fields(self, db, namespace, index, items): # Given("Create namespace with index and items") - # When ("Create new query") + # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with select fields") select_result = list(query.select(["id"]).must_execute()) - # Then ("Check that selected items is in result") + # Then ("Check that selected items are in result") ids = [{"id": i["id"]} for i in items] assert_that(select_result, equal_to(ids), "Wrong query results") + + # TODO ??? build query + def test_query_select_where_query(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + # query = db.query.new(namespace).where("id", CondType.CondLt, 5) + # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) + query = db.query.new(namespace) + sub_query = db.query.new(namespace) + # When ("Make select query with where_query subquery") + query_result = list(query.where_query(sub_query, CondType.CondEq, 1).execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") + + # TODO ??? build query + def test_query_select_where_composite(self, db, namespace, composite_index, items): + # Given("Create namespace with composite index") + # Given ("Create new query") + query = db.query.new(namespace) + query_comp = db.query.new(namespace).where("id", CondType.CondEq, 1).where("val", CondType.CondEq, "testval1") + # When ("Make select query with where_composite") + query_result = list(query.where_composite("comp_idx", CondType.CondEq, query_comp).execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") + + def test_query_select_where_uuid(self, db, namespace, index): + # Given("Create namespace with index") + # Given ("Create uuid index") + db.index.create(namespace, {"name": "uuid", "json_paths": ["uuid"], "field_type": "uuid", "index_type": "hash"}) + # Given ("Create items") + items = [{"id": 0, "uuid": "f0dbda48-1b92-4aa1-b4dc-39b6867a202e"}, + {"id": 1, "uuid": "189005c5-106f-4582-9838-f7a5b18649fc"}, + {"id": 2, "uuid": "50005725-dd9d-4891-a9bc-7c7cb0ed2a3d"}] + for item in items: + db.item.insert(namespace, item) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with where_uuid") + item = items[1] + query_result = list(query.where_uuid("uuid", CondType.CondEq, [item["uuid"]]).execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([item]), "Wrong query results") + + def test_query_select_where_between_fields(self, db, namespace, index): + # Given("Create namespace with index") + # Given("Create items") + items = [{"id": 0, "age": 0.5}, {"id": 1, "age": 1}, {"id": 2, "age": 20}] + for item in items: + db.item.insert(namespace, item) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with where_between_fields") + query_result = list(query.where_between_fields("id", CondType.CondEq, "age").execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") + + def test_query_select_with_brackets_and_ops(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with brackets and different op") + query = (query.where("id", CondType.CondEq, 0) + .op_or().open_bracket() + .op_and().where("id", CondType.CondGe, 2) + .op_or().where("id", CondType.CondLt, 7) + .op_not().where("id", CondType.CondSet, [1, 6]) + + .op_and().open_bracket() + .op_not().where("id", CondType.CondRange, [5, 9]) + .close_bracket() + + .close_bracket()) + query_result = list(query.execute()) + # Then ("Check that selected items are in result") + expected_items = [items[i] for i in [0, 2, 3, 4]] + assert_that(query_result, equal_to(expected_items), "Wrong query results") + + def test_query_select_match(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create string index") + db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash"}) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with match") + query_result = list(query.match("val", ["testval1"]).execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") + + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with match and empty result") + query_result = list(query.match("val", ["testval"]).execute()) + # Then ("Check that result is empty") + assert_that(query_result, empty(), "Wrong query results") + + # TODO fix 'Process finished with exit code 139 (interrupted by signal 11:SIGSEGV)' + def test_query_select_dwithin(self, db, namespace, index, rtree_index_and_items): + # Given("Create namespace with rtree index and items") + items = rtree_index_and_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with dwithin") + query_result = list(query.dwithin("rtree", Point(3, 2.5), 1.5).execute()) + # Then ("Check that selected item is in result") + expected_items = [items[i] for i in [1, 2, 3]] # TODO change expected after err fix + assert_that(query_result, equal_to(expected_items), "Wrong query results") + + def test_query_select_limit_offset(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with limit and offset") + query_result = list(query.limit(3).offset(1).execute()) + # Then ("Check that selected items are in result") + expected_items = [items[i] for i in [1, 2, 3]] + assert_that(query_result, equal_to(expected_items), "Wrong query results") + + # TODO ??? get explain results + def test_query_select_explain(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with explain") + query_result = query.explain().execute() + # Then ("Check that selected items are in result") + assert_that(list(query_result), equal_to(items), "Wrong query results") + # Then ("Check that explain is in result") + assert_that(query_result, equal_to(1), "There is no explain in query results") + + +class TestQuerySelectAggregations: + @pytest.mark.parametrize("calculate, function_name", AGGREGATE_FUNCTIONS_MATH) + def test_query_select_aggregations_math(self, db, namespace, index, items, calculate, function_name): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + aggregation_query = getattr(query, function_name) + # When ("Make select query") + query_result = aggregation_query("id").execute() + # Then ("Check that selected item is in result") + assert_that(list(query_result), empty(), "Wrong query results") + # Then ("Check aggregation results") + ids_list = list(range(len(items))) + assert_that(query_result.get_agg_results(), + equal_to([{"value": calculate(ids_list), "type": function_name.split("_")[-1], "fields": ["id"]}]), + "Wrong aggregation results") From c4b8f081b1a7a7139666c365ca1aa031a2934930 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 8 Nov 2024 11:28:21 +0300 Subject: [PATCH 069/125] Part XXI: bug fix. Segmentation fault --- pyreindexer/lib/src/rawpyreindexer.cc | 2 +- pyreindexer/point.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 6c4ad76..0194a8a 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -795,7 +795,7 @@ static PyObject* DWithin(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; double x = 0, y = 0, distance = 0; - if (!PyArg_ParseTuple(args, "ksddd", &queryWrapperAddr, &x, &y, &distance)) { + if (!PyArg_ParseTuple(args, "ksddd", &queryWrapperAddr, &index, &x, &y, &distance)) { return nullptr; } diff --git a/pyreindexer/point.py b/pyreindexer/point.py index df5b9b6..e66574b 100644 --- a/pyreindexer/point.py +++ b/pyreindexer/point.py @@ -6,7 +6,7 @@ class Point: y (float): y coordinate of the point """ - def __init__(self, x, y): + def __init__(self, x: float, y: float): """Constructs a new Reindexer query object # Arguments: @@ -15,5 +15,5 @@ def __init__(self, x, y): """ - self.x = x - self.y = y + self.x: float = x + self.y: float = y From 3cf7a9a2fe1c052bf4b0c4c0fa037e08d1a4dc58 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Fri, 8 Nov 2024 15:53:36 +0300 Subject: [PATCH 070/125] add new query select tests --- pyreindexer/query.py | 22 ++- pyreindexer/tests/conftest.py | 85 ++++++-- pyreindexer/tests/helpers/api.py | 8 +- pyreindexer/tests/tests/test_query.py | 266 +++++++++++++++++++++++++- pyreindexer/tests/tests/test_sql.py | 4 +- 5 files changed, 343 insertions(+), 42 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 951f775..5d09c30 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -107,7 +107,7 @@ def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[si param = param if isinstance(param, list) else [param] return param - def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds where condition to DB query with args # Arguments: @@ -130,7 +130,8 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ self.__raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def where_query(self, sub_query: Query, condition: CondType, + keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds sub-query where condition to DB query with args # Arguments: @@ -149,7 +150,8 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_ params: list = self.__convert_to_list(keys) - self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) + self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, + condition.value, params) self.__raise_on_error() return self @@ -394,7 +396,7 @@ def offset(self, offset: int) -> Query._AggregateFacet: self.api.aggregation_offset(self.query_wrapper_ptr, offset) return self - def sort(self, field: str, desc: bool=False) -> Query._AggregateFacet: + def sort(self, field: str, desc: bool = False) -> Query._AggregateFacet: """Sorts facets by field value # Arguments: @@ -428,7 +430,7 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: self.__raise_on_error() return self._AggregateFacet(self) - def sort(self, index: str, desc: bool=False, keys: Union[simple_types, List[simple_types]]=None) -> Query: + def sort(self, index: str, desc: bool = False, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only @@ -527,7 +529,7 @@ def op_not(self) -> Query: self.api.op_not(self.query_wrapper_ptr) return self - def request_total(self, total_name: str= '') -> Query: + def request_total(self, total_name: str = '') -> Query: """Requests total items calculation # Arguments: @@ -541,7 +543,7 @@ def request_total(self, total_name: str= '') -> Query: self.api.request_total(self.query_wrapper_ptr, total_name) return self - def cached_total(self, total_name: str= '') -> Query: + def cached_total(self, total_name: str = '') -> Query: """Requests cached total items calculation # Arguments: @@ -645,7 +647,7 @@ def execute(self) -> QueryResults: """ - if self.root is not None : + if self.root is not None: return self.root.execute() self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select_query(self.query_wrapper_ptr) @@ -664,7 +666,7 @@ def delete(self) -> int: """ - if (self.root is not None) or (len(self.join_queries) > 0) : + if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Delete does not support joined queries") self.err_code, self.err_msg, number = self.api.delete_query(self.query_wrapper_ptr) @@ -754,7 +756,7 @@ def update(self) -> QueryResults: """ - if (self.root is not None) or (len(self.join_queries) > 0) : + if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Update does not support joined queries") self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.update_query(self.query_wrapper_ptr) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 7ef6ed5..45e5120 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -1,3 +1,4 @@ +import random import shutil import pytest @@ -30,7 +31,7 @@ def db(request): shutil.rmtree('tmp/', ignore_errors=True) -@pytest.fixture(scope="function") +@pytest.fixture def namespace(db): """ Create a namespace @@ -41,7 +42,7 @@ def namespace(db): db.namespace.drop(ns_name) -@pytest.fixture(scope="function") +@pytest.fixture def index(db, namespace): """ Create an index to namespace @@ -50,7 +51,39 @@ def index(db, namespace): yield -@pytest.fixture(scope="function") +@pytest.fixture +def item(db, namespace): + """ + Create an item to namespace + """ + db.item.insert(namespace, item_definition) + yield item_definition + + +@pytest.fixture +def items(db, namespace): + """ + Create items to namespace + """ + items = [{"id": i, "val": f"testval{i}"} for i in range(10)] + for item in items: + db.item.insert(namespace, item) + yield items + + +@pytest.fixture +def items_shuffled(db, namespace): + """ + Create items in random order + """ + items = [{"id": i, "val": f"testval{i}"} for i in range(5)] + random.shuffle(items) + for item in items: + db.item.insert(namespace, item) + yield items + + +@pytest.fixture def composite_index(db, namespace): """ Create indexes and composite index from them @@ -61,7 +94,7 @@ def composite_index(db, namespace): yield -@pytest.fixture(scope="function") +@pytest.fixture def rtree_index_and_items(db, namespace): """ Create rtree index and items @@ -74,27 +107,47 @@ def rtree_index_and_items(db, namespace): yield items -@pytest.fixture(scope="function") -def item(db, namespace): +@pytest.fixture +def ft_index_and_items(db, namespace): """ - Create an item to namespace + Create rtree index and items """ - db.item.insert(namespace, item_definition) - yield item_definition + db.index.create(namespace, {"name": "ft", "json_paths": ["ft"], "field_type": "string", "index_type": "text"}) + content = ["one word", "sword two", "three work 333"] + items = [{"id": i, "ft": c} for i, c in enumerate(content)] + for item in items: + db.item.insert(namespace, item) + yield items -@pytest.fixture(scope="function") -def items(db, namespace): +@pytest.fixture +def index_and_duplicate_items(db, namespace): """ - Create items to namespace + Create index and items with duplicate value """ - items = [{"id": i, "val": f"testval{i}"} for i in range(10)] + db.index.create(namespace, {"name": "idx", "json_paths": ["idx"], "field_type": "int", "index_type": "hash"}) + items = [{"id": 0, "idx": 0}, {"id": 1, "idx": 1}, {"id": 2, "idx": 1}, {"id": 3, "idx": 3}] + for item in items: + db.item.insert(namespace, item) + yield items + + +@pytest.fixture +def array_indexes_and_items(db, namespace): + """ + Create array indexes and items + """ + db.index.create(namespace, {"name": "arr1", "json_paths": ["arr1"], "field_type": "int", + "index_type": "tree", "is_array": True}) + db.index.create(namespace, {"name": "arr2", "json_paths": ["arr2"], "field_type": "int", + "index_type": "tree", "is_array": True}) + items = [{"id": i, "arr1": [i, i % 2], "arr2": [i % 2, i]} for i in range(5)] for item in items: db.item.insert(namespace, item) yield items -@pytest.fixture(scope="function") +@pytest.fixture def metadata(db, namespace): """ Put metadata to namespace @@ -104,8 +157,8 @@ def metadata(db, namespace): yield key, value -@pytest.fixture(scope="function") -def second_namespace_for_join(db): +@pytest.fixture +def second_namespace(db): second_namespace_name = 'test_ns_for_join' db.namespace.open(second_namespace_name) db.index.create(second_namespace_name, index_definition) diff --git a/pyreindexer/tests/helpers/api.py b/pyreindexer/tests/helpers/api.py index b0df293..26d0c77 100644 --- a/pyreindexer/tests/helpers/api.py +++ b/pyreindexer/tests/helpers/api.py @@ -3,8 +3,8 @@ from tests.helpers.log_helper import log_api -def make_request_and_response_log(method_description, request_msg, res=None) -> str: - return f"{method_description}\n\t[Request] => {request_msg}\n\t[Response] <= {res}" +def make_request_and_response_log(method_description, call_msg, res=None) -> str: + return f"{method_description}\n\t[Call] => {call_msg}\n\t[Return] <= {res}" def api_method(func): @@ -16,10 +16,10 @@ def wrapped(self, *args, **kwargs): args_str = ", ".join(repr(a) for a in args) kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) - request_msg = f"Called {func.__name__} with args ({args_str}) and kwargs ({kwargs_str})" + call_msg = f"args ({args_str}), kwargs ({kwargs_str})" r = func(self, *args, **kwargs) - log = make_request_and_response_log(method_description, request_msg, r) + log = make_request_and_response_log(method_description, call_msg, r) log_api.info(log) return r diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index ef53abf..4f0425d 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -2,7 +2,7 @@ from hamcrest import * from point import Point -from query import CondType +from query import CondType, StrictMode from tests.test_data.constants import AGGREGATE_FUNCTIONS_MATH @@ -26,6 +26,24 @@ def test_query_select_fields(self, db, namespace, index, items): ids = [{"id": i["id"]} for i in items] assert_that(select_result, equal_to(ids), "Wrong query results") + def test_query_select_get_true(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with get") + query_result = list(query.get()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[0], True]), "Wrong query results") + + def test_query_select_get_false(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with get") + query_result = list(query.where("id", CondType.CondEq, 99).get()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to(["", False]), "Wrong query results") + # TODO ??? build query def test_query_select_where_query(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -132,6 +150,17 @@ def test_query_select_dwithin(self, db, namespace, index, rtree_index_and_items) expected_items = [items[i] for i in [1, 2, 3]] # TODO change expected after err fix assert_that(query_result, equal_to(expected_items), "Wrong query results") + def test_query_select_equal_position(self, db, namespace, index, array_indexes_and_items): + # Given("Create namespace with array indexes and items") + items = array_indexes_and_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with equal_position") + query.where("arr1", CondType.CondEq, 1).where("arr2", CondType.CondEq, 1) + query_result = list(query.equal_position(["arr1", "arr2"]).execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") + def test_query_select_limit_offset(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") @@ -142,7 +171,57 @@ def test_query_select_limit_offset(self, db, namespace, index, items): expected_items = [items[i] for i in [1, 2, 3]] assert_that(query_result, equal_to(expected_items), "Wrong query results") - # TODO ??? get explain results + @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) + def test_query_select_request_total(self, db, namespace, index, items, func_total): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with limit and offset") + query_result = getattr(query, func_total)("id").execute() + # Then ("Check that selected items are in result") + assert_that(list(query_result), equal_to(items), "Wrong query results") + # Then ("Check that total is in result") + # TODO: need get_explain_results in query_results + # assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") + + def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): + # Given("Create namespace with ft index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query") + query_result = list(query.where("ft", CondType.CondEq, "word~").with_rank().execute()) + # Then ("Check that selected item is in result with rank") + for res_item in query_result: + assert_that(res_item, has_entries("id", is_(int), "ft", is_(str), "rank()", is_(float)), + "Wrong query results") + + def test_query_select_functions(self, db, namespace, index, ft_index_and_items): + # Given("Create namespace with ft index and items") + ft_index_and_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query") + query.where("ft", CondType.CondEq, "word~") + query_result = list(query.functions(["ft=highlight(<,>)"]).execute()) + # Then ("Check that selected item is in result and highlighted") + query_results_ft = [i["ft"] for i in query_result] + expected_ft_content = ["one ", " two", "three 333"] + assert_that(query_results_ft, contains_inanyorder(*expected_ft_content), "Wrong query results") + + # TODO self.api.merge(query) takes 4 arguments (but 1 given - query) + def test_query_select_merge(self, db, namespace, index, items, second_namespace): + # Given("Create namespace with index and items") + # Given("Create second namespace with index and items") + second_ns_name, item2 = second_namespace + # Given ("Create new query") + query1 = db.query.new(namespace).where("id", CondType.CondEq, 2) + # Given ("Create second query") + query2 = db.query.new(second_ns_name).where("id", CondType.CondEq, 1) + # When ("Make select query with merge") + query_result = list(query1.merge(query2).execute()) + # Then ("Check that selected item is in result with merge applied") + assert_that(query_result, equal_to([items[2], item2]), "Wrong query results") + def test_query_select_explain(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") @@ -152,22 +231,189 @@ def test_query_select_explain(self, db, namespace, index, items): # Then ("Check that selected items are in result") assert_that(list(query_result), equal_to(items), "Wrong query results") # Then ("Check that explain is in result") - assert_that(query_result, equal_to(1), "There is no explain in query results") - + # TODO: need get_explain_results in query_results + # assert_that(query_result.get_explain_results(), equal_to(""), "There is no explain in query results") -class TestQuerySelectAggregations: - @pytest.mark.parametrize("calculate, function_name", AGGREGATE_FUNCTIONS_MATH) - def test_query_select_aggregations_math(self, db, namespace, index, items, calculate, function_name): + # TODO add debug enums + @pytest.mark.parametrize("debug_level", [StrictMode.NotSet]) + def test_query_select_debug(self, db, namespace, index, items, debug_level): # Given("Create namespace with index and items") # Given ("Create new query") query = db.query.new(namespace) - aggregation_query = getattr(query, function_name) # When ("Make select query") - query_result = aggregation_query("id").execute() + query_result = list(query.debug(debug_level).where("id", CondType.CondEq, 1).execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") + + @pytest.mark.parametrize("strict_mode", [StrictMode.NotSet, StrictMode.Empty]) + def test_query_select_strict_mode_none_and_empty(self, db, namespace, index, items, strict_mode): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with strict mode") + query_result = list(query.strict(strict_mode).where("rand", CondType.CondEq, 1).must_execute()) # Then ("Check that selected item is in result") + assert_that(query_result, empty(), "Wrong query results") + + # TODO must be err + def test_query_select_strict_mode_names(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with strict mode") + query.strict(StrictMode.Names).where("rand", CondType.CondEq, 1) + err_msg = f"Current query strict mode allows aggregate existing fields only. " \ + f"There are no fields with name 'rand' in namespace '{namespace}'" + assert_that(calling(query.must_execute).with_args(), + raises(Exception, pattern=err_msg), + "Error wasn't raised while strict mode violated") + + # TODO must be err + def test_query_select_strict_mode_indexes(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with strict mode") + query.strict(StrictMode.Indexes).where("rand", CondType.CondEq, 1) + err_msg = f"Current query strict mode allows aggregate index fields only. " \ + f"There are no indexes with name 'rand' in namespace '{namespace}'" + assert_that(calling(query.must_execute).with_args(), + raises(Exception, pattern=err_msg), + "Error wasn't raised while strict mode violated") + + +class TestQuerySelectAggregations: + @pytest.mark.parametrize("calculate, aggregate_func", AGGREGATE_FUNCTIONS_MATH) + def test_query_select_aggregations_math(self, db, namespace, index, items, calculate, aggregate_func): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with aggregations") + query_result = getattr(query, aggregate_func)("id").execute() + # Then ("Check that result is empty") assert_that(list(query_result), empty(), "Wrong query results") # Then ("Check aggregation results") ids_list = list(range(len(items))) + expected_agg_result = [{"value": calculate(ids_list), "type": aggregate_func.split("_")[-1], "fields": ["id"]}] + assert_that(query_result.get_agg_results(), equal_to(expected_agg_result), "Wrong aggregation results") + + def test_query_select_distinct(self, db, namespace, index, index_and_duplicate_items): + # Given("Create namespace with index and duplicate items") + items = index_and_duplicate_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with distinct") + query_result = query.distinct("idx").execute() + # Then ("Check that only distinct items is in result") + expected_ids = [0, 1, 3] + expected_items = [items[i] for i in expected_ids] + assert_that(list(query_result), equal_to(expected_items), "Wrong query results") + # Then ("Check aggregation results") + expected_ids_str = [str(i) for i in expected_ids] assert_that(query_result.get_agg_results(), - equal_to([{"value": calculate(ids_list), "type": function_name.split("_")[-1], "fields": ["id"]}]), + has_item(has_entries(type="distinct", fields=["idx"], + distincts=contains_inanyorder(*expected_ids_str))), "Wrong aggregation results") + + def test_query_select_facet(self, db, namespace, index, index_and_duplicate_items): + # Given("Create namespace with index and duplicate items") + items = index_and_duplicate_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with facet") + query.aggregate_facet(["idx"]) # TODO объединить в цепочку? + query_result = query.execute() + # Then ("Check that result is empty") + assert_that(list(query_result), empty(), "Wrong query results") + # Then ("Check aggregation results") + expected_facets = [{"count": 1, "values": ["0"]}, {"count": 2, "values": ["1"]}, {"count": 1, "values": ["3"]}] + assert_that(query_result.get_agg_results(), + has_item(has_entries(type="facet", fields=["idx"], + facets=contains_inanyorder(*expected_facets))), + "Wrong aggregation results") + + def test_query_select_facet_sort_offset_limit(self, db, namespace, index, index_and_duplicate_items): + # Given("Create namespace with index and duplicate items") + items = index_and_duplicate_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with facet") + query.aggregate_facet(["id", "idx"]).sort("id", True).limit(2).offset(1) # TODO объединить в цепочку? + query_result = query.execute() + # Then ("Check that result is empty") + assert_that(list(query_result), empty(), "Wrong query results") + # Then ("Check aggregation results") + expected_facets = [{"count": 1, "values": ["2", "1"]}, {"count": 1, "values": ["1", "1"]}] + assert_that(query_result.get_agg_results(), + has_item(has_entries(type="facet", fields=["id", "idx"], + facets=contains_inanyorder(*expected_facets))), + "Wrong aggregation results") + + +class TestQuerySelectSort: + @pytest.mark.parametrize("is_reversed", [False, True]) + def test_query_select_sort(self, db, namespace, index, items_shuffled, is_reversed): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with sort") + query_result = list(query.sort("id", is_reversed).execute()) + # Then ("Check that selected items are sorted") + expected_items = sorted(items_shuffled, key=lambda x: x["id"], reverse=is_reversed) + assert_that(query_result, equal_to(expected_items)) + + # TODO fix forced_values + @pytest.mark.parametrize("forced_values, expected_ids", [ + (4, [4, 0, 1, 2, 3]), + ([1, 3], [1, 3, 0, 2, 4]) + ]) + @pytest.mark.parametrize("is_reversed", [False, True]) + def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, + is_reversed): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with sort") + query_result = list(query.sort("id", is_reversed, forced_values).execute()) # TODO exit code 255 + # Then ("Check that selected items are sorted") + expected_items = [items_shuffled[i] for i in expected_ids] + if is_reversed: + expected_items.reverse() + assert_that(query_result, equal_to(expected_items)) + + # TODO wip + @pytest.mark.parametrize("is_reversed", [False, True]) + def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): + # Given("Create namespace with rtree index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with sort point distance") + query_result = list(query.sort_stpoint_distance("id", Point(1, 2), is_reversed).execute()) + # Then ("Check that selected items are sorted") + expected_items = sorted(rtree_index_and_items, key=lambda x: x["id"], reverse=is_reversed) + assert_that(query_result, equal_to(expected_items)) + + # TODO wip + @pytest.mark.parametrize("is_reversed", [False, True]) + def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): + # Given("Create namespace with index") + # Given("Create 2 rtree indexes and items") + db.index.create(namespace, {"name": "rtree1", "json_paths": ["rtree1"], "field_type": "point", + "index_type": "rtree", "rtree_type": "rstar"}) + db.index.create(namespace, {"name": "rtree2", "json_paths": ["rtree2"], "field_type": "point", + "index_type": "rtree", "rtree_type": "rstar"}) + items = [{"id": i, "rtree1": [i - 1, i + 1], "rtree2": [i + 1, i + 0.5]} for i in range(5)] + for item in items: + db.item.insert(namespace, item) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with sort fields distance") + query_result = list(query.sort_stfield_distance("rtree1", "rtree2", is_reversed).execute()) + # Then ("Check that selected items are sorted") + expected_items = sorted(items, key=lambda x: x["id"], reverse=is_reversed) + assert_that(query_result, equal_to(expected_items)) + + +class TestQuerySelectJoin: + def test_query_select_join(self, db, namespace, index): + pass diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index d370220..8a4394c 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -10,9 +10,9 @@ def test_sql_select(self, db, namespace, index, item): # Then ("Check that selected item is in result") assert_that(items_list, equal_to([item]), "Can't SQL select data") - def test_sql_select_with_join(self, db, namespace, second_namespace_for_join, index, items): + def test_sql_select_with_join(self, db, namespace, second_namespace, index, items): # Given("Create two namespaces") - second_namespace, second_ns_item_definition_join = second_namespace_for_join + second_namespace, second_ns_item_definition_join = second_namespace # When ("Execute SQL query SELECT with JOIN") query = f"SELECT id FROM {namespace} INNER JOIN {second_namespace} " \ f"ON {namespace}.id = {second_namespace}.id" From edf2f4123b472ecf6b1906a79f9948ff13af0f62 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 8 Nov 2024 17:38:40 +0300 Subject: [PATCH 071/125] Part XXII: update readme generation (pydoc_markdown) --- README.md | 1342 ++++++++++++++++++++++++++----- pydoc-markdown.yml | 21 + pyreindexer/index_definition.py | 2 +- pyreindexer/point.py | 4 +- pyreindexer/query.py | 229 +++--- pyreindexer/query_results.py | 10 +- pyreindexer/raiser_mixin.py | 4 +- pyreindexer/rx_connector.py | 100 +-- pyreindexer/transaction.py | 30 +- readmegen.sh | 5 +- 10 files changed, 1344 insertions(+), 403 deletions(-) create mode 100644 pydoc-markdown.yml diff --git a/README.md b/README.md index 0c96c55..f868d33 100644 --- a/README.md +++ b/README.md @@ -1,378 +1,393 @@ -

pyreindexer

+ +# pyreindexer.rx\_connector -The pyreindexer module provides a connector and its auxiliary tools for interaction with Reindexer. + -

pyreindexer.rx_connector

- - -

RxConnector

+## RxConnector Objects ```python -RxConnector(self, dsn) +class RxConnector(RaiserMixin) ``` -RxConnector provides a binding to Reindexer upon two shared libraries (hereinafter - APIs): 'rawpyreindexerb.so' and 'rawpyreindexerc.so'. -The first one is aimed to a builtin way usage. That API embeds Reindexer so it could be used right in-place as is. -The second one acts as a lightweight client which establishes a connection to Reindexer server via RPC. -The APIs interfaces are completely the same. -__Attributes:__ +RxConnector provides a binding to Reindexer upon two shared libraries (hereinafter - APIs): 'rawpyreindexerb.so' + and 'rawpyreindexerc.so'. The first one is aimed to a builtin way usage. That API embeds Reindexer, so it could + be used right in-place as is. The second one acts as a lightweight client which establishes a connection to + Reindexer server via RPC. The APIs interfaces are completely the same. +#### Attributes: api (module): An API module loaded dynamically for Reindexer calls rx (int): A memory pointer to Reindexer instance err_code (int): The API error code err_msg (string): The API error message + -

close

+### close ```python -RxConnector.close(self) +def close() -> None ``` -Closes an API instance with Reindexer resources freeing -__Returns:__ +Closes an API instance with Reindexer resources freeing - None +#### Raises: + Exception: Raises with an error message when Reindexer instance is not initialized yet + -

namespace_open

+### namespace\_open ```python -RxConnector.namespace_open(self, namespace) +def namespace_open(namespace) -> None ``` -Opens a namespace specified or creates a namespace if it does not exist -__Arguments:__ +Opens a namespace specified or creates a namespace if it does not exist +#### Arguments: namespace (string): A name of a namespace -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

namespace_close

+### namespace\_close ```python -RxConnector.namespace_close(self, namespace) +def namespace_close(namespace) -> None ``` -Closes a namespace specified -__Arguments:__ +Closes a namespace specified +#### Arguments: namespace (string): A name of a namespace -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

namespace_drop

+### namespace\_drop ```python -RxConnector.namespace_drop(self, namespace) +def namespace_drop(namespace) -> None ``` -Drops a namespace specified -__Arguments:__ +Drops a namespace specified +#### Arguments: namespace (string): A name of a namespace -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

namespaces_enum

+### namespaces\_enum ```python -RxConnector.namespaces_enum(self, enum_not_opened=False) +def namespaces_enum(enum_not_opened=False) -> List[Dict[str, str]] ``` -Gets a list of namespaces available -__Arguments:__ +Gets a list of namespaces available +#### Arguments: enum_not_opened (bool, optional): An enumeration mode flag. If it is set then closed namespaces are in result list too. Defaults to False -__Returns:__ - +#### Returns: (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

index_add

+### index\_add ```python -RxConnector.index_add(self, namespace, index_def) +def index_add(namespace, index_def) -> None ``` -Adds an index to the namespace specified -__Arguments:__ +Adds an index to the namespace specified +#### Arguments: namespace (string): A name of a namespace - index_def (dict): A dictionary of index definiton - -__Raises:__ + index_def (dict): A dictionary of index definition +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

index_update

+### index\_update ```python -RxConnector.index_update(self, namespace, index_def) +def index_update(namespace, index_def) -> None ``` -Updates an index in the namespace specified -__Arguments:__ +Updates an index in the namespace specified +#### Arguments: namespace (string): A name of a namespace - index_def (dict): A dictionary of index definiton - -__Raises:__ + index_def (dict): A dictionary of index definition +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

index_drop

+### index\_drop ```python -RxConnector.index_drop(self, namespace, index_name) +def index_drop(namespace, index_name) -> None ``` -Drops an index from the namespace specified -__Arguments:__ +Drops an index from the namespace specified +#### Arguments: namespace (string): A name of a namespace index_name (string): A name of an index -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

item_insert

+### item\_insert ```python -RxConnector.item_insert(self, namespace, item_def, precepts=[]) +def item_insert(namespace, item_def, precepts=None) -> None ``` -Inserts an item with its precepts to the namespace specified. -__Arguments:__ +Inserts an item with its precepts to the namespace specified +#### Arguments: namespace (string): A name of a namespace - item_def (dict): A dictionary of item definiton - precepts (:obj:`list` of :obj:`str`): A dictionary of index definiton - -__Raises:__ + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

item_update

+### item\_update ```python -RxConnector.item_update(self, namespace, item_def, precepts=[]) +def item_update(namespace, item_def, precepts=None) -> None ``` -Updates an item with its precepts in the namespace specified. -__Arguments:__ +Updates an item with its precepts in the namespace specified +#### Arguments: namespace (string): A name of a namespace - item_def (dict): A dictionary of item definiton - precepts (:obj:`list` of :obj:`str`): A dictionary of index definiton - -__Raises:__ + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

item_upsert

+### item\_upsert ```python -RxConnector.item_upsert(self, namespace, item_def, precepts=[]) +def item_upsert(namespace, item_def, precepts=None) -> None ``` -Updates an item with its precepts in the namespace specified. Creates the item if it not exists -__Arguments:__ +Updates an item with its precepts in the namespace specified. Creates the item if it not exists +#### Arguments: namespace (string): A name of a namespace - item_def (dict): A dictionary of item definiton - precepts (:obj:`list` of :obj:`str`): A dictionary of index definiton - -__Raises:__ + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

item_delete

+### item\_delete ```python -RxConnector.item_delete(self, namespace, item_def) +def item_delete(namespace, item_def) -> None ``` -Deletes an item from the namespace specified -__Arguments:__ +Deletes an item from the namespace specified +#### Arguments: namespace (string): A name of a namespace - item_def (dict): A dictionary of item definiton - -__Raises:__ + item_def (dict): A dictionary of item definition +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

meta_put

+### meta\_put ```python -RxConnector.meta_put(self, namespace, key, value) +def meta_put(namespace, key, value) -> None ``` -Puts metadata to a storage of Reindexer by key -__Arguments:__ +Puts metadata to a storage of Reindexer by key +#### Arguments: namespace (string): A name of a namespace key (string): A key in a storage of Reindexer for metadata keeping value (string): A metadata for storage -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

meta_get

+### meta\_get ```python -RxConnector.meta_get(self, namespace, key) +def meta_get(namespace, key) -> str ``` -Gets metadata from a storage of Reindexer by key specified -__Arguments:__ +Gets metadata from a storage of Reindexer by key specified +#### Arguments: namespace (string): A name of a namespace key (string): A key in a storage of Reindexer where metadata is kept -__Returns:__ - +#### Returns: string: A metadata value -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

meta_delete

+### meta\_delete ```python -RxConnector.meta_delete(self, namespace, key) +def meta_delete(namespace, key) -> None ``` -Deletes metadata from a storage of Reindexer by key specified -__Arguments:__ +Deletes metadata from a storage of Reindexer by key specified +#### Arguments: namespace (string): A name of a namespace key (string): A key in a storage of Reindexer where metadata is kept -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

meta_enum

+### meta\_enum ```python -RxConnector.meta_enum(self, namespace) +def meta_enum(namespace) -> List[str] ``` -Gets a list of metadata keys from a storage of Reindexer -__Arguments:__ +Gets a list of metadata keys from a storage of Reindexer +#### Arguments: namespace (string): A name of a namespace -__Returns:__ - +#### Returns: (:obj:`list` of :obj:`str`): A list of all metadata keys -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

select

+### select ```python -RxConnector.select(self, query) +def select(query: str) -> QueryResults ``` -Executes an SQL query and returns query results -__Arguments:__ +Executes an SQL query and returns query results +#### Arguments: query (string): An SQL query -__Returns:__ - +#### Returns: (:obj:`QueryResults`): A QueryResults iterator -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

new_transaction

+### new\_transaction ```python -RxConnector.new_transaction(self, namespace) +def new_transaction(namespace) -> Transaction ``` -Starts a new transaction and return the transaction object to processing -__Arguments:__ +Starts a new transaction and return the transaction object to processing +#### Arguments: namespace (string): A name of a namespace -__Returns:__ - +#### Returns: (:obj:`Transaction`): A new transaction -__Raises:__ - +#### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code + -

pyreindexer.query_results

+### new\_query +```python +def new_query(namespace: str) -> Query +``` + +Creates a new query and return the query object to processing + +#### Arguments: + namespace (string): A name of a namespace + +#### Returns: + (:obj:`Query`): A new query + +#### Raises: + Exception: Raises with an error message when Reindexer instance is not initialized yet -

QueryResults

+ + +# pyreindexer.query\_results + + + +## QueryResults Objects ```python -QueryResults(self, api, qres_wrapper_ptr, qres_iter_count) +class QueryResults() ``` -QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. -When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer -__Attributes:__ +QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. + When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer +#### Attributes: api (module): An API module for Reindexer calls err_code (int): The API error code err_msg (string): The API error message @@ -380,173 +395,1074 @@ __Attributes:__ qres_iter_count (int): A count of results for iterations pos (int): The current result position in iterator + -

count

+### status ```python -QueryResults.count(self) +def status() ``` -Returns a count of results -__Returns__ +Check status - `int`: A count of results +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + -

get_agg_results

+### count ```python -QueryResults.get_agg_results(self) +def count() ``` -Returns aggregation results for the current query -__Returns__ +Returns a count of results - (:obj:`dict`): Dictionary with all results for the current query +# Returns + int: A count of results + + + +### get\_agg\_results + +```python +def get_agg_results() +``` + +Returns aggregation results for the current query -__Raises__ +# Returns + (:obj:`dict`): Dictionary with all results for the current query +#### Raises: Exception: Raises with an error message of API return on non-zero error code + -

pyreindexer.transaction

+# pyreindexer.transaction + -

Transaction

+## Transaction Objects ```python -Transaction(self, api, transaction_wrapper_ptr) +class Transaction() ``` -An object representing the context of a Reindexer transaction -__Attributes:__ +An object representing the context of a Reindexer transaction +#### Attributes: api (module): An API module for Reindexer calls transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object err_code (int): The API error code err_msg (string): The API error message + -

commit

+### insert ```python -Transaction.commit(self) +def insert(item_def, precepts=None) ``` -Commits a transaction -__Raises:__ +Inserts an item with its precepts to the transaction + +#### Arguments: + item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition +#### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code + -

delete

+### update ```python -Transaction.delete(self, item_def) +def update(item_def, precepts=None) ``` -Deletes an item from the transaction. -__Arguments:__ +Updates an item with its precepts to the transaction +#### Arguments: item_def (dict): A dictionary of item definition + precepts (:obj:`list` of :obj:`str`): A dictionary of index definition -__Raises:__ - +#### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code + -

insert

+### upsert ```python -Transaction.insert(self, item_def, precepts=None) +def upsert(item_def, precepts=None) ``` -Inserts an item with its precepts to the transaction -__Arguments:__ +Updates an item with its precepts to the transaction. Creates the item if it not exists +#### Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition -__Raises:__ +#### Raises: + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + +### delete + +```python +def delete(item_def) +``` + +Deletes an item from the transaction + +#### Arguments: + item_def (dict): A dictionary of item definition + +#### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code + -

rollback

+### commit ```python -Transaction.rollback(self) +def commit() ``` -Rollbacks a transaction -__Raises:__ +Applies changes +#### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code + -

update

+### commit\_with\_count ```python -Transaction.update(self, item_def, precepts=None) +def commit_with_count() -> int ``` -Updates an item with its precepts to the transaction -__Arguments:__ +Applies changes and return the number of count of changed items - item_def (dict): A dictionary of item definition - precepts (:obj:`list` of :obj:`str`): A dictionary of index definition +#### Raises: + Exception: Raises with an error message of API return if Transaction is over + Exception: Raises with an error message of API return on non-zero error code + + + +### rollback + +```python +def rollback() +``` -__Raises:__ +Rollbacks changes +#### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code + + +# pyreindexer.point -

upsert

+ + +## Point Objects ```python -Transaction.upsert(self, item_def, precepts=None) +class Point() ``` -Updates an item with its precepts to the transaction. Creates the item if it not exists -__Arguments:__ +An object representing the context of a Reindexer 2D point - item_def (dict): A dictionary of item definition - precepts (:obj:`list` of :obj:`str`): A dictionary of index definition +#### Attributes: + x (float): x coordinate of the point + y (float): y coordinate of the point -__Raises:__ + - Exception: Raises with an error message of API return if Transaction is over +# pyreindexer.query + + + +## Query Objects + +```python +class Query() +``` + +An object representing the context of a Reindexer query + +#### Attributes: + api (module): An API module for Reindexer calls + query_wrapper_ptr (int): A memory pointer to Reindexer query object + err_code (int): The API error code + err_msg (string): The API error message + root (:object:`Query`): The root query of the Reindexer query + join_type (:enum:`JoinType`): Join type + join_queries (list[:object:`Query`]): The list of join Reindexer query objects + merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects + + + +### where + +```python +def where(index: str, + condition: CondType, + keys: Union[simple_types, List[simple_types]] = None) -> Query +``` + +Adds where condition to DB query with args + +#### Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (Union[None, simple_types, list[simple_types]]): + Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: Exception: Raises with an error message of API return on non-zero error code + + +### where\_query + +```python +def where_query(sub_query: Query, + condition: CondType, + keys: Union[simple_types, List[simple_types]] = None) -> Query +``` + +Adds sub-query where condition to DB query with args + +#### Arguments: + sub_query (:obj:`Query`): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (Union[None, simple_types, list[simple_types]]): + Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index -

pyreindexer.index_definition

+#### Returns: + (:obj:`Query`): Query object for further customizations +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + -

IndexDefinition

+### where\_composite ```python -IndexDefinition(dict) +def where_composite(index: str, condition: CondType, + sub_query: Query) -> Query ``` -IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. -NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. -__Arguments:__ +Adds where condition to DB query with interface args for composite indexes + +#### Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + sub_query (:obj:`Query`): Field name used in condition clause + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### where\_uuid + +```python +def where_uuid(index: str, condition: CondType, keys: List[str]) -> Query +``` + +Adds where condition to DB query with UUID as string args. + This function applies binary encoding to the UUID value. + `index` MUST be declared as uuid index in this case + +#### Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### where\_between\_fields + +```python +def where_between_fields(first_field: str, condition: CondType, + second_field: str) -> Query +``` + +Adds comparing two fields where condition to DB query + +#### Arguments: + first_field (string): First field name used in condition clause + condition (:enum:`CondType`): Type of condition + second_field (string): Second field name used in condition clause + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### open\_bracket + +```python +def open_bracket() -> Query +``` + +Opens bracket for where condition to DB query + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### close\_bracket + +```python +def close_bracket() -> Query +``` + +Closes bracket for where condition to DB query + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### match + +```python +def match(index: str, keys: List[str]) -> Query +``` + +Adds string EQ-condition to DB query with string args + +#### Arguments: + index (string): Field name used in condition clause + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### dwithin + +```python +def dwithin(index: str, point: Point, distance: float) -> Query +``` + +Adds DWithin condition to DB query + +#### Arguments: + index (string): Field name used in condition clause + point (:obj:`Point`): Point object used in condition clause + distance (float): Distance in meters between point + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### distinct + +```python +def distinct(index: str) -> Query +``` + +Performs distinct for a certain index. Return only items with uniq value of field + +#### Arguments: + index (string): Field name for distinct operation + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### aggregate\_sum + +```python +def aggregate_sum(index: str) -> Query +``` + +Performs a summation of values for a specified index + +#### Arguments: + index (string): Field name for sum operation + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### aggregate\_avg + +```python +def aggregate_avg(index: str) -> Query +``` + +Finds for the average at the specified index + +#### Arguments: + index (string): Field name for sum operation + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### aggregate\_min + +```python +def aggregate_min(index: str) -> Query +``` + +Finds for the minimum at the specified index + +#### Arguments: + index (string): Field name for sum operation + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### aggregate\_max + +```python +def aggregate_max(index: str) -> Query +``` + +Finds for the maximum at the specified index + +#### Arguments: + index (string): Field name for sum operation + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### aggregate\_facet + +```python +def aggregate_facet(fields: List[str]) -> Query._AggregateFacet +``` + +Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data + column or `count` and cut off by offset and limit. In order to support this functionality this method + returns AggregationFacetRequest which has methods sort, limit and offset + +#### Arguments: + fields (list[string]): Fields any data column name or `count`, fields should not be empty + +#### Returns: + (:obj:`_AggregateFacet`): Request object for further customizations + + + +### sort + +```python +def sort(index: str, + desc: bool = False, + keys: Union[simple_types, List[simple_types]] = None) -> Query +``` + +Applies sort order to return from query items. If values argument specified, then items equal to values, + if found will be placed in the top positions. Forced sort is support for the first sorting field only + +#### Arguments: + index (string): The index name + desc (bool): Sort in descending order + keys (Union[None, simple_types, List[simple_types]]): + Value of index to match. For composite indexes keys must be list, with value of each sub-index + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### sort\_stpoint\_distance + +```python +def sort_stpoint_distance(index: str, point: Point, desc: bool) -> Query +``` + +Applies geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance + between geometry field and point (ST_Distance) + +#### Arguments: + index (string): The index name + point (:obj:`Point`): Point object used in sorting operation + desc (bool): Sort in descending order + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### sort\_stfield\_distance + +```python +def sort_stfield_distance(first_field: str, second_field: str, + desc: bool) -> Query +``` + +Applies geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance + between 2 geometry fields (ST_Distance) + +#### Arguments: + first_field (string): First field name used in condition + second_field (string): Second field name used in condition + desc (bool): Sort in descending order + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### op\_and + +```python +def op_and() -> Query +``` + +Next condition will be added with AND. + This is the default operation for WHERE statement. Do not have to be called explicitly in user's code. + Used in DSL conversion + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### op\_or + +```python +def op_or() -> Query +``` + +Next condition will be added with OR. + Implements short-circuiting: + if the previous condition is successful the next will not be evaluated, but except Join conditions + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### op\_not + +```python +def op_not() -> Query +``` + +Next condition will be added with NOT AND. + Implements short-circuiting: if the previous condition is failed the next will not be evaluated + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### request\_total + +```python +def request_total(total_name: str = '') -> Query +``` + +Requests total items calculation + +#### Arguments: + total_name (string, optional): Name to be requested + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### cached\_total + +```python +def cached_total(total_name: str = '') -> Query +``` + +Requests cached total items calculation + +#### Arguments: + total_name (string, optional): Name to be requested + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### limit + +```python +def limit(limit_items: int) -> Query +``` + +Sets a limit (count) of returned items. Analog to sql LIMIT rowsNumber + +#### Arguments: + limit_items (int): Number of rows to get from result set + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### offset + +```python +def offset(start_offset: int) -> Query +``` + +Sets the number of the first selected row from result query + +#### Arguments: + limit_items (int): Index of the first row to get from result set + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### debug + +```python +def debug(level: int) -> Query +``` + +Changes debug level + +#### Arguments: + level (int): Debug level + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### strict + +```python +def strict(mode: StrictMode) -> Query +``` + +Changes strict mode + +#### Arguments: + mode (:enum:`StrictMode`): Strict mode + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### explain + +```python +def explain() -> Query +``` + +Enables explain query + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### with\_rank + +```python +def with_rank() -> Query +``` + +Outputs fulltext rank. Allowed only with fulltext query + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### execute + +```python +def execute() -> QueryResults +``` + +Executes a select query + +#### Returns: + (:obj:`QueryResults`): A QueryResults iterator + +#### Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + + +### delete + +```python +def delete() -> int +``` + +Executes a query, and delete items, matches query + +#### Returns: + (int): Number of deleted elements + +#### Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + + +### set\_object + +```python +def set_object(field: str, values: List[simple_types]) -> Query +``` + +Adds an update query to an object field for an update query + +#### Arguments: + field (string): Field name + values (list[simple_types]): List of values to add + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### set + +```python +def set(field: str, values: List[simple_types]) -> Query +``` + +Adds a field update request to the update request + +#### Arguments: + field (string): Field name + values (list[simple_types]): List of values to add + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### drop + +```python +def drop(index: str) -> Query +``` + +Drops a value for a field + +#### Arguments: + index (string): Field name for drop operation + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### expression + +```python +def expression(field: str, value: str) -> Query +``` + +Updates indexed field by arithmetical expression + +#### Arguments: + field (string): Field name + value (string): New value expression for field + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### update + +```python +def update() -> QueryResults +``` + +Executes update query, and update fields in items, which matches query + +#### Returns: + (:obj:`QueryResults`): A QueryResults iterator + +#### Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + + +### must\_execute + +```python +def must_execute() -> QueryResults +``` + +Executes a query, and update fields in items, which matches query, with status check + +#### Returns: + (:obj:`QueryResults`): A QueryResults iterator + +#### Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + + +### get + +```python +def get() -> (str, bool) +``` + +Executes a query, and return 1 JSON item + +#### Returns: + (:tuple:string,bool): 1st string item and found flag + +#### Raises: + Exception: Raises with an error message when query is in an invalid state + Exception: Raises with an error message of API return on non-zero error code + + + +### inner\_join + +```python +def inner_join(query: Query, field: str) -> Query +``` + +Joins 2 queries. + Items from the 1-st query are filtered by and expanded with the data from the 2-nd query + +#### Arguments: + query (:obj:`Query`): Query object to left join + field (string): Joined field name. As unique identifier for the join between this query and `join_query`. + Parameter in order for InnerJoin to work: namespace of `query` contains `field` as one of its fields + marked as `joined` + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### join + +```python +def join(query: Query, field: str) -> Query +``` + +Join is an alias for LeftJoin. Joins 2 queries. + Items from this query are expanded with the data from the `query` + +#### Arguments: + query (:obj:`Query`): Query object to left join + field (string): Joined field name. As unique identifier for the join between this query and `join_query` + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### left\_join + +```python +def left_join(join_query: Query, field: str) -> Query +``` + +Joins 2 queries. + Items from this query are expanded with the data from the join_query. + One of the conditions below must hold for `field` parameter in order for LeftJoin to work: + namespace of `join_query` contains `field` as one of its fields marked as `joined` + +#### Arguments: + query (:obj:`Query`): Query object to left join + field (string): Joined field name. As unique identifier for the join between this query and `join_query` + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### merge + +```python +def merge(query: Query) -> Query +``` + +Merges queries of the same type + +#### Arguments: + query (:obj:`Query`): Query object to merge + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### on + +```python +def on(index: str, condition: CondType, join_index: str) -> Query +``` + +On specifies join condition + +#### Arguments: + index (string): Field name from `Query` namespace should be used during join + condition (:enum:`CondType`): Type of condition, specifies how `Query` will be joined with the latest join query issued on `Query` (e.g. `EQ`/`GT`/`SET`/...) + join_index (string): Index-field name from namespace for the latest join query issued on `Query` should be used during join + +#### Returns: + (:obj:`Query`): Query object for further customizations + + + +### select + +```python +def select(fields: List[str]) -> Query +``` + +Sets list of columns in this namespace to be finally selected. + The columns should be specified in the same case as the jsonpaths corresponding to them. + Non-existent fields and fields in the wrong case are ignored. + If there are no fields in this list that meet these conditions, then the filter works as "*" + +#### Arguments: + fields (list[string]): List of columns to be selected + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### functions + +```python +def functions(functions: List[str]) -> Query +``` + +Adds sql-functions to query + +#### Arguments: + functions (list[string]): Functions declaration + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +### equal\_position + +```python +def equal_position(equal_position: List[str]) -> Query +``` + +Adds equal position fields to arrays queries + +#### Arguments: + equal_poses (list[string]): Equal position fields to arrays queries + +#### Returns: + (:obj:`Query`): Query object for further customizations + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + + + +# pyreindexer.index\_definition + + + +## IndexDefinition Objects + +```python +class IndexDefinition(dict) +``` + +IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. +NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. + +#### Arguments: + name (str): An index name. + json_paths (:obj:`list` of :obj:`str`): A name for mapping a value to a json field. + field_type (str): A type of field. Possible values are: `int`, `int64`, `double`, `string`, `bool`, `composite`. + index_type (str): An index type. Possible values are: `hash`, `tree`, `text`, `-`. + is_pk (bool): True if a field is a primary key. + is_array (bool): True if an index is an array. + is_dense (bool): True if an index is dense. reduce index size. Saves 8 bytes per unique key value for 'hash' + and 'tree' index types. + For '-' index type saves 4-8 bytes per each element. Useful for indexes with high selectivity, + but for tree and hash indexes with low selectivity could + significantly decrease update performance. + is_sparse (bool): True if a value of an index may be not presented. + collate_mode (str): Sets an order of values by collate mode. Possible values are: + `none`, `ascii`, `utf8`, `numeric`, `custom`. + sort_order_letters (str): Order for a sort sequence for a custom collate mode. + config (dict): A config for a fulltext engine. + [More](https://github.com/Restream/reindexer/blob/master/fulltext.md). - name (str): An index name. - json_paths (:obj:`list` of :obj:`str`): A name for mapping a value to a json field. - field_type (str): A type of a field. Possible values are: `int`, `int64`, `double`, `string`, `bool`, `composite`. - index_type (str): An index type. Possible values are: `hash`, `tree`, `text`, `-`. - is_pk (bool): True if a field is a primary key. - is_array (bool): True if an index is an array. - is_dense (bool): True if an index is dense. reduce index size. Saves 8 bytes per unique key value for 'hash' and 'tree' index types. - For '-' index type saves 4-8 bytes per each element. Useful for indexes with high selectivity, but for tree and hash indexes with low selectivity could - significantly decrease update performance. - is_sparse (bool): True if a value of an index may be not presented. - collate_mode (str): Sets an order of values by collate mode. Possible values are: `none`, `ascii`, `utf8`, `numeric`, `custom`. - sort_order_letters (str): Order for a sort sequence for a custom collate mode. - config (dict): A config for a fulltext engine. [More](https://github.com/Restream/reindexer/blob/master/fulltext.md). diff --git a/pydoc-markdown.yml b/pydoc-markdown.yml new file mode 100644 index 0000000..41c1e53 --- /dev/null +++ b/pydoc-markdown.yml @@ -0,0 +1,21 @@ +loaders: + - type: python + packages: + - pyreindexer.rx_connector + - pyreindexer.query_results + - pyreindexer.transaction + - pyreindexer.point + - pyreindexer.query + - pyreindexer.index_definition +processors: + - type: filter + expression: not name.startswith('_') and default() +renderer: + type: markdown + filename: README.md + header_level_by_type: + Class: 2 + Function: 3 + Method: 3 + Module: 1 + Data: 3 diff --git a/pyreindexer/index_definition.py b/pyreindexer/index_definition.py index ce031e3..fc3547e 100644 --- a/pyreindexer/index_definition.py +++ b/pyreindexer/index_definition.py @@ -8,7 +8,7 @@ class IndexDefinition(dict): """IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently. NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY. - # Arguments: + #### Arguments: name (str): An index name. json_paths (:obj:`list` of :obj:`str`): A name for mapping a value to a json field. field_type (str): A type of field. Possible values are: `int`, `int64`, `double`, `string`, `bool`, `composite`. diff --git a/pyreindexer/point.py b/pyreindexer/point.py index e66574b..276f7dc 100644 --- a/pyreindexer/point.py +++ b/pyreindexer/point.py @@ -1,7 +1,7 @@ class Point: """An object representing the context of a Reindexer 2D point - # Attributes: + #### Attributes: x (float): x coordinate of the point y (float): y coordinate of the point """ @@ -9,7 +9,7 @@ class Point: def __init__(self, x: float, y: float): """Constructs a new Reindexer query object - # Arguments: + #### Arguments: x (float): x coordinate of the point y (float): y coordinate of the point diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 5d09c30..a7f76a2 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -42,7 +42,7 @@ class JoinType(Enum): class Query: """An object representing the context of a Reindexer query - # Attributes: + #### Attributes: api (module): An API module for Reindexer calls query_wrapper_ptr (int): A memory pointer to Reindexer query object err_code (int): The API error code @@ -57,7 +57,7 @@ class Query: def __init__(self, api, query_wrapper_ptr: int): """Constructs a new Reindexer query object - # Arguments: + #### Arguments: api (module): An API module for Reindexer calls query_wrapper_ptr (int): A memory pointer to Reindexer query object @@ -83,7 +83,7 @@ def __del__(self): def __raise_on_error(self): """Checks if there is an error code and raises with an error message - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -95,10 +95,10 @@ def __raise_on_error(self): def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[simple_types]: """Converts an input parameter to a list - # Arguments: + #### Arguments: param (Union[None, simple_types, list[simple_types]]): The input parameter - # Returns: + #### Returns: List[Union[int, bool, float, str]]: Always converted to a list """ @@ -110,16 +110,16 @@ def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[si def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds where condition to DB query with args - # Arguments: + #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -134,16 +134,16 @@ def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds sub-query where condition to DB query with args - # Arguments: + #### Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -158,12 +158,12 @@ def where_query(self, sub_query: Query, condition: CondType, def where_composite(self, index: str, condition: CondType, sub_query: Query) -> Query: """Adds where condition to DB query with interface args for composite indexes - # Arguments: + #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition sub_query (:obj:`Query`): Field name used in condition clause - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -176,15 +176,15 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: This function applies binary encoding to the UUID value. `index` MUST be declared as uuid index in this case - # Arguments: + #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -198,12 +198,12 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: def where_between_fields(self, first_field: str, condition: CondType, second_field: str) -> Query: """Adds comparing two fields where condition to DB query - # Arguments: + #### Arguments: first_field (string): First field name used in condition clause condition (:enum:`CondType`): Type of condition second_field (string): Second field name used in condition clause - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -214,10 +214,10 @@ def where_between_fields(self, first_field: str, condition: CondType, second_fie def open_bracket(self) -> Query: """Opens bracket for where condition to DB query - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -229,10 +229,10 @@ def open_bracket(self) -> Query: def close_bracket(self) -> Query: """Closes bracket for where condition to DB query - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -244,14 +244,14 @@ def close_bracket(self) -> Query: def match(self, index: str, keys: List[str]) -> Query: """Adds string EQ-condition to DB query with string args - # Arguments: + #### Arguments: index (string): Field name used in condition clause keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -265,12 +265,12 @@ def match(self, index: str, keys: List[str]) -> Query: def dwithin(self, index: str, point: Point, distance: float) -> Query: """Adds DWithin condition to DB query - # Arguments: + #### Arguments: index (string): Field name used in condition clause point (:obj:`Point`): Point object used in condition clause distance (float): Distance in meters between point - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -281,10 +281,10 @@ def dwithin(self, index: str, point: Point, distance: float) -> Query: def distinct(self, index: str) -> Query: """Performs distinct for a certain index. Return only items with uniq value of field - # Arguments: + #### Arguments: index (string): Field name for distinct operation - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -295,10 +295,10 @@ def distinct(self, index: str) -> Query: def aggregate_sum(self, index: str) -> Query: """Performs a summation of values for a specified index - # Arguments: + #### Arguments: index (string): Field name for sum operation - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -309,10 +309,10 @@ def aggregate_sum(self, index: str) -> Query: def aggregate_avg(self, index: str) -> Query: """Finds for the average at the specified index - # Arguments: + #### Arguments: index (string): Field name for sum operation - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -323,10 +323,10 @@ def aggregate_avg(self, index: str) -> Query: def aggregate_min(self, index: str) -> Query: """Finds for the minimum at the specified index - # Arguments: + #### Arguments: index (string): Field name for sum operation - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -337,10 +337,10 @@ def aggregate_min(self, index: str) -> Query: def aggregate_max(self, index: str) -> Query: """Finds for the maximum at the specified index - # Arguments: + #### Arguments: index (string): Field name for sum operation - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -351,7 +351,7 @@ def aggregate_max(self, index: str) -> Query: class _AggregateFacet: """An object representing the context of a Reindexer aggregate facet - # Attributes: + #### Attributes: api (module): An API module for Reindexer calls query_wrapper_ptr (int): A memory pointer to Reindexer query object @@ -360,7 +360,7 @@ class _AggregateFacet: def __init__(self, query: Query): """Constructs a new Reindexer AggregateFacetRequest object - # Arguments: + #### Arguments: (:obj:`Query`): Query object for further customizations """ @@ -371,10 +371,10 @@ def __init__(self, query: Query): def limit(self, limit: int) -> Query._AggregateFacet: """Limits facet aggregation results - # Arguments: + #### Arguments: limit (int): Limit of aggregation of facet - # Returns: + #### Returns: (:obj:`_AggregateFacet`): Facet object for further customizations """ @@ -385,10 +385,10 @@ def limit(self, limit: int) -> Query._AggregateFacet: def offset(self, offset: int) -> Query._AggregateFacet: """Sets offset of the facet aggregation results - # Arguments: + #### Arguments: limit (int): Offset in facet aggregation results - # Returns: + #### Returns: (:obj:`_AggregateFacet`): Facet object for further customizations """ @@ -399,11 +399,11 @@ def offset(self, offset: int) -> Query._AggregateFacet: def sort(self, field: str, desc: bool = False) -> Query._AggregateFacet: """Sorts facets by field value - # Arguments: + #### Arguments: field (str): Item field. Use field `count` to sort by facet's count value desc (bool): Sort in descending order flag - # Returns: + #### Returns: (:obj:`_AggregateFacet`): Facet object for further customizations """ @@ -416,10 +416,10 @@ def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: column or `count` and cut off by offset and limit. In order to support this functionality this method returns AggregationFacetRequest which has methods sort, limit and offset - # Arguments: + #### Arguments: fields (list[string]): Fields any data column name or `count`, fields should not be empty - # Returns: + #### Returns: (:obj:`_AggregateFacet`): Request object for further customizations """ @@ -434,16 +434,16 @@ def sort(self, index: str, desc: bool = False, keys: Union[simple_types, List[si """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only - # Arguments: + #### Arguments: index (string): The index name desc (bool): Sort in descending order keys (Union[None, simple_types, List[simple_types]]): Value of index to match. For composite indexes keys must be list, with value of each sub-index - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -458,12 +458,12 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: """Applies geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance between geometry field and point (ST_Distance) - # Arguments: + #### Arguments: index (string): The index name point (:obj:`Point`): Point object used in sorting operation desc (bool): Sort in descending order - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -475,15 +475,15 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) """Applies geometry sort order to return from query items. Wrapper for geometry sorting by shortest distance between 2 geometry fields (ST_Distance) - # Arguments: + #### Arguments: first_field (string): First field name used in condition second_field (string): Second field name used in condition desc (bool): Sort in descending order - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -496,7 +496,7 @@ def op_and(self) -> Query: This is the default operation for WHERE statement. Do not have to be called explicitly in user's code. Used in DSL conversion - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -509,7 +509,7 @@ def op_or(self) -> Query: Implements short-circuiting: if the previous condition is successful the next will not be evaluated, but except Join conditions - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -521,7 +521,7 @@ def op_not(self) -> Query: """Next condition will be added with NOT AND. Implements short-circuiting: if the previous condition is failed the next will not be evaluated - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -532,10 +532,10 @@ def op_not(self) -> Query: def request_total(self, total_name: str = '') -> Query: """Requests total items calculation - # Arguments: + #### Arguments: total_name (string, optional): Name to be requested - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -546,10 +546,10 @@ def request_total(self, total_name: str = '') -> Query: def cached_total(self, total_name: str = '') -> Query: """Requests cached total items calculation - # Arguments: + #### Arguments: total_name (string, optional): Name to be requested - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -560,10 +560,10 @@ def cached_total(self, total_name: str = '') -> Query: def limit(self, limit_items: int) -> Query: """Sets a limit (count) of returned items. Analog to sql LIMIT rowsNumber - # Arguments: + #### Arguments: limit_items (int): Number of rows to get from result set - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -574,10 +574,10 @@ def limit(self, limit_items: int) -> Query: def offset(self, start_offset: int) -> Query: """Sets the number of the first selected row from result query - # Arguments: + #### Arguments: limit_items (int): Index of the first row to get from result set - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -588,10 +588,10 @@ def offset(self, start_offset: int) -> Query: def debug(self, level: int) -> Query: """Changes debug level - # Arguments: + #### Arguments: level (int): Debug level - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -602,10 +602,10 @@ def debug(self, level: int) -> Query: def strict(self, mode: StrictMode) -> Query: """Changes strict mode - # Arguments: + #### Arguments: mode (:enum:`StrictMode`): Strict mode - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -616,7 +616,7 @@ def strict(self, mode: StrictMode) -> Query: def explain(self) -> Query: """Enables explain query - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -627,7 +627,7 @@ def explain(self) -> Query: def with_rank(self) -> Query: """Outputs fulltext rank. Allowed only with fulltext query - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -638,10 +638,10 @@ def with_rank(self) -> Query: def execute(self) -> QueryResults: """Executes a select query - # Returns: + #### Returns: (:obj:`QueryResults`): A QueryResults iterator - # Raises: + #### Raises: Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code @@ -657,10 +657,10 @@ def execute(self) -> QueryResults: def delete(self) -> int: """Executes a query, and delete items, matches query - # Returns: + #### Returns: (int): Number of deleted elements - # Raises: + #### Raises: Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code @@ -676,14 +676,14 @@ def delete(self) -> int: def set_object(self, field: str, values: List[simple_types]) -> Query: """Adds an update query to an object field for an update query - # Arguments: + #### Arguments: field (string): Field name values (list[simple_types]): List of values to add - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -697,14 +697,14 @@ def set_object(self, field: str, values: List[simple_types]) -> Query: def set(self, field: str, values: List[simple_types]) -> Query: """Adds a field update request to the update request - # Arguments: + #### Arguments: field (string): Field name values (list[simple_types]): List of values to add - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -718,10 +718,10 @@ def set(self, field: str, values: List[simple_types]) -> Query: def drop(self, index: str) -> Query: """Drops a value for a field - # Arguments: + #### Arguments: index (string): Field name for drop operation - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -732,11 +732,11 @@ def drop(self, index: str) -> Query: def expression(self, field: str, value: str) -> Query: """Updates indexed field by arithmetical expression - # Arguments: + #### Arguments: field (string): Field name value (string): New value expression for field - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -747,10 +747,10 @@ def expression(self, field: str, value: str) -> Query: def update(self) -> QueryResults: """Executes update query, and update fields in items, which matches query - # Returns: + #### Returns: (:obj:`QueryResults`): A QueryResults iterator - # Raises: + #### Raises: Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code @@ -766,10 +766,10 @@ def update(self) -> QueryResults: def must_execute(self) -> QueryResults: """Executes a query, and update fields in items, which matches query, with status check - # Returns: + #### Returns: (:obj:`QueryResults`): A QueryResults iterator - # Raises: + #### Raises: Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code @@ -782,10 +782,10 @@ def must_execute(self) -> QueryResults: def get(self) -> (str, bool): """Executes a query, and return 1 JSON item - # Returns: + #### Returns: (:tuple:string,bool): 1st string item and found flag - # Raises: + #### Raises: Exception: Raises with an error message when query is in an invalid state Exception: Raises with an error message of API return on non-zero error code @@ -803,15 +803,15 @@ def get(self) -> (str, bool): def __join(self, query: Query, field: str, join_type: JoinType) -> Query: """Joins queries - # Arguments: + #### Arguments: query (:obj:`Query`): Query object to join field (string): Joined field name type (:enum:`JoinType`): Join type - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message when query is in an invalid state """ @@ -835,13 +835,13 @@ def inner_join(self, query: Query, field: str) -> Query: """Joins 2 queries. Items from the 1-st query are filtered by and expanded with the data from the 2-nd query - # Arguments: + #### Arguments: query (:obj:`Query`): Query object to left join field (string): Joined field name. As unique identifier for the join between this query and `join_query`. Parameter in order for InnerJoin to work: namespace of `query` contains `field` as one of its fields marked as `joined` - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -852,11 +852,11 @@ def join(self, query: Query, field: str) -> Query: """Join is an alias for LeftJoin. Joins 2 queries. Items from this query are expanded with the data from the `query` - # Arguments: + #### Arguments: query (:obj:`Query`): Query object to left join field (string): Joined field name. As unique identifier for the join between this query and `join_query` - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -869,11 +869,11 @@ def left_join(self, join_query: Query, field: str) -> Query: One of the conditions below must hold for `field` parameter in order for LeftJoin to work: namespace of `join_query` contains `field` as one of its fields marked as `joined` - # Arguments: + #### Arguments: query (:obj:`Query`): Query object to left join field (string): Joined field name. As unique identifier for the join between this query and `join_query` - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -883,10 +883,10 @@ def left_join(self, join_query: Query, field: str) -> Query: def merge(self, query: Query) -> Query: """Merges queries of the same type - # Arguments: + #### Arguments: query (:obj:`Query`): Query object to merge - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -905,12 +905,12 @@ def merge(self, query: Query) -> Query: def on(self, index: str, condition: CondType, join_index: str) -> Query: """On specifies join condition - # Arguments: + #### Arguments: index (string): Field name from `Query` namespace should be used during join condition (:enum:`CondType`): Type of condition, specifies how `Query` will be joined with the latest join query issued on `Query` (e.g. `EQ`/`GT`/`SET`/...) join_index (string): Index-field name from namespace for the latest join query issued on `Query` should be used during join - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations """ @@ -927,13 +927,13 @@ def select(self, fields: List[str]) -> Query: Non-existent fields and fields in the wrong case are ignored. If there are no fields in this list that meet these conditions, then the filter works as "*" - # Arguments: + #### Arguments: fields (list[string]): List of columns to be selected - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -947,14 +947,15 @@ def select(self, fields: List[str]) -> Query: def functions(self, functions: List[str]) -> Query: """Adds sql-functions to query - # Arguments: + #### Arguments: functions (list[string]): Functions declaration - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code + """ functions = [] if functions is None else functions @@ -966,13 +967,13 @@ def functions(self, functions: List[str]) -> Query: def equal_position(self, equal_position: List[str]) -> Query: """Adds equal position fields to arrays queries - # Arguments: + #### Arguments: equal_poses (list[string]): Equal position fields to arrays queries - # Returns: + #### Returns: (:obj:`Query`): Query object for further customizations - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index fb49714..0e09c38 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -2,7 +2,7 @@ class QueryResults: """QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer - # Attributes: + #### Attributes: api (module): An API module for Reindexer calls err_code (int): The API error code err_msg (string): The API error message @@ -15,7 +15,7 @@ class QueryResults: def __init__(self, api, qres_wrapper_ptr, qres_iter_count): """Constructs a new Reindexer query results iterator object - # Arguments: + #### Arguments: api (module): An API module for Reindexer calls qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations @@ -39,7 +39,7 @@ def __iter__(self): def __next__(self): """Returns the next iteration result - # Raises: + #### Raises: StopIteration: Frees results on end of iterator and raises with iteration stop """ @@ -64,7 +64,7 @@ def __del__(self): def status(self): """Check status - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -97,7 +97,7 @@ def get_agg_results(self): # Returns (:obj:`dict`): Dictionary with all results for the current query - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index 5b62d5f..4918890 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -9,7 +9,7 @@ class RaiserMixin: def raise_on_error(self): """Checks if there is an error code and raises with an error message - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -20,7 +20,7 @@ def raise_on_error(self): def raise_on_not_init(self): """Checks if there is an error code and raises with an error message - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet """ diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 8c37e16..daa2ba6 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -12,7 +12,7 @@ class RxConnector(RaiserMixin): be used right in-place as is. The second one acts as a lightweight client which establishes a connection to Reindexer server via RPC. The APIs interfaces are completely the same. - # Attributes: + #### Attributes: api (module): An API module loaded dynamically for Reindexer calls rx (int): A memory pointer to Reindexer instance err_code (int): The API error code @@ -24,7 +24,7 @@ def __init__(self, dsn): """Constructs a new connector object. Initializes an error code and a Reindexer instance descriptor to zero - # Arguments: + #### Arguments: dsn (string): The connection string which contains a protocol Examples: 'builtin:///tmp/pyrx', 'cproto://127.0.0.1:6534/pyrx @@ -47,7 +47,7 @@ def __del__(self): def close(self) -> None: """Closes an API instance with Reindexer resources freeing - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet """ @@ -57,10 +57,10 @@ def close(self) -> None: def namespace_open(self, namespace) -> None: """Opens a namespace specified or creates a namespace if it does not exist - # Arguments: + #### Arguments: namespace (string): A name of a namespace - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -73,10 +73,10 @@ def namespace_open(self, namespace) -> None: def namespace_close(self, namespace) -> None: """Closes a namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -89,10 +89,10 @@ def namespace_close(self, namespace) -> None: def namespace_drop(self, namespace) -> None: """Drops a namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -105,14 +105,14 @@ def namespace_drop(self, namespace) -> None: def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: """Gets a list of namespaces available - # Arguments: + #### Arguments: enum_not_opened (bool, optional): An enumeration mode flag. If it is set then closed namespaces are in result list too. Defaults to False - # Returns: + #### Returns: (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -126,11 +126,11 @@ def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: def index_add(self, namespace, index_def) -> None: """Adds an index to the namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace index_def (dict): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -143,11 +143,11 @@ def index_add(self, namespace, index_def) -> None: def index_update(self, namespace, index_def) -> None: """Updates an index in the namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace index_def (dict): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -160,11 +160,11 @@ def index_update(self, namespace, index_def) -> None: def index_drop(self, namespace, index_name) -> None: """Drops an index from the namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace index_name (string): A name of an index - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -177,12 +177,12 @@ def index_drop(self, namespace, index_name) -> None: def item_insert(self, namespace, item_def, precepts=None) -> None: """Inserts an item with its precepts to the namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -196,12 +196,12 @@ def item_insert(self, namespace, item_def, precepts=None) -> None: def item_update(self, namespace, item_def, precepts=None) -> None: """Updates an item with its precepts in the namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -215,12 +215,12 @@ def item_update(self, namespace, item_def, precepts=None) -> None: def item_upsert(self, namespace, item_def, precepts=None) -> None: """Updates an item with its precepts in the namespace specified. Creates the item if it not exists - # Arguments: + #### Arguments: namespace (string): A name of a namespace item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -234,11 +234,11 @@ def item_upsert(self, namespace, item_def, precepts=None) -> None: def item_delete(self, namespace, item_def) -> None: """Deletes an item from the namespace specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace item_def (dict): A dictionary of item definition - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -251,12 +251,12 @@ def item_delete(self, namespace, item_def) -> None: def meta_put(self, namespace, key, value) -> None: """Puts metadata to a storage of Reindexer by key - # Arguments: + #### Arguments: namespace (string): A name of a namespace key (string): A key in a storage of Reindexer for metadata keeping value (string): A metadata for storage - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -269,14 +269,14 @@ def meta_put(self, namespace, key, value) -> None: def meta_get(self, namespace, key) -> str: """Gets metadata from a storage of Reindexer by key specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace key (string): A key in a storage of Reindexer where metadata is kept - # Returns: + #### Returns: string: A metadata value - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -290,11 +290,11 @@ def meta_get(self, namespace, key) -> str: def meta_delete(self, namespace, key) -> None: """Deletes metadata from a storage of Reindexer by key specified - # Arguments: + #### Arguments: namespace (string): A name of a namespace key (string): A key in a storage of Reindexer where metadata is kept - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -307,13 +307,13 @@ def meta_delete(self, namespace, key) -> None: def meta_enum(self, namespace) -> List[str]: """Gets a list of metadata keys from a storage of Reindexer - # Arguments: + #### Arguments: namespace (string): A name of a namespace - # Returns: + #### Returns: (:obj:`list` of :obj:`str`): A list of all metadata keys - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -327,13 +327,13 @@ def meta_enum(self, namespace) -> List[str]: def select(self, query: str) -> QueryResults: """Executes an SQL query and returns query results - # Arguments: + #### Arguments: query (string): An SQL query - # Returns: + #### Returns: (:obj:`QueryResults`): A QueryResults iterator - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -347,13 +347,13 @@ def select(self, query: str) -> QueryResults: def new_transaction(self, namespace) -> Transaction: """Starts a new transaction and return the transaction object to processing - # Arguments: + #### Arguments: namespace (string): A name of a namespace - # Returns: + #### Returns: (:obj:`Transaction`): A new transaction - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -367,13 +367,13 @@ def new_transaction(self, namespace) -> Transaction: def new_query(self, namespace: str) -> Query: """Creates a new query and return the query object to processing - # Arguments: + #### Arguments: namespace (string): A name of a namespace - # Returns: + #### Returns: (:obj:`Query`): A new query - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet """ @@ -386,10 +386,10 @@ def new_query(self, namespace: str) -> Query: def _api_import(self, dsn): """Imports an API dynamically depending on protocol specified in dsn - # Arguments: + #### Arguments: dsn (string): The connection string which contains a protocol - # Raises: + #### Raises: Exception: Raises an exception if a connection protocol is unrecognized """ @@ -406,10 +406,10 @@ def _api_init(self, dsn): """Initializes Reindexer instance and connects to a database specified in dsn Obtains a pointer to Reindexer instance - # Arguments: + #### Arguments: dsn (string): The connection string which contains a protocol - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet Exception: Raises with an error message of API return on non-zero error code @@ -423,7 +423,7 @@ def _api_init(self, dsn): def _api_close(self): """Destructs Reindexer instance correctly and resets memory pointer - # Raises: + #### Raises: Exception: Raises with an error message when Reindexer instance is not initialized yet """ diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index fd5f303..5053bcf 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,7 +1,7 @@ class Transaction: """An object representing the context of a Reindexer transaction - # Attributes: + #### Attributes: api (module): An API module for Reindexer calls transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object err_code (int): The API error code @@ -12,7 +12,7 @@ class Transaction: def __init__(self, api, transaction_wrapper_ptr: int): """Constructs a new Reindexer transaction object - # Arguments: + #### Arguments: api (module): An API module for Reindexer calls transaction_wrapper_ptr (int): A memory pointer to Reindexer transaction object @@ -34,7 +34,7 @@ def __del__(self): def __raise_on_error(self): """Checks if there is an error code and raises with an error message - # Raises: + #### Raises: Exception: Raises with an error message of API return on non-zero error code """ @@ -45,7 +45,7 @@ def __raise_on_error(self): def __raise_on_is_over(self): """Checks the state of a transaction and returns an error message when necessary - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over """ @@ -56,11 +56,11 @@ def __raise_on_is_over(self): def insert(self, item_def, precepts=None): """Inserts an item with its precepts to the transaction - # Arguments: + #### Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code @@ -74,11 +74,11 @@ def insert(self, item_def, precepts=None): def update(self, item_def, precepts=None): """Updates an item with its precepts to the transaction - # Arguments: + #### Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code @@ -92,11 +92,11 @@ def update(self, item_def, precepts=None): def upsert(self, item_def, precepts=None): """Updates an item with its precepts to the transaction. Creates the item if it not exists - # Arguments: + #### Arguments: item_def (dict): A dictionary of item definition precepts (:obj:`list` of :obj:`str`): A dictionary of index definition - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code @@ -110,10 +110,10 @@ def upsert(self, item_def, precepts=None): def delete(self, item_def): """Deletes an item from the transaction - # Arguments: + #### Arguments: item_def (dict): A dictionary of item definition - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code @@ -126,7 +126,7 @@ def delete(self, item_def): def commit(self): """Applies changes - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code @@ -140,7 +140,7 @@ def commit(self): def commit_with_count(self) -> int: """Applies changes and return the number of count of changed items - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code @@ -155,7 +155,7 @@ def commit_with_count(self) -> int: def rollback(self): """Rollbacks changes - # Raises: + #### Raises: Exception: Raises with an error message of API return if Transaction is over Exception: Raises with an error message of API return on non-zero error code diff --git a/readmegen.sh b/readmegen.sh index 64d852c..4108ab7 100755 --- a/readmegen.sh +++ b/readmegen.sh @@ -1,3 +1,6 @@ #!/bin/sh -pydocmd simple pyreindexer++ pyreindexer.rx_connector++ pyreindexer.query_results++ pyreindexer.transaction++ pyreindexer.index_definition++ > README.md +# to install: python -m pip install pydoc-markdown +# to configuration: see file pydoc-markdown.yml + +pydoc-markdown From 599410c28a2bbb9bfe9839759a7826aec015f047 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 8 Nov 2024 17:48:16 +0300 Subject: [PATCH 072/125] Part XXIII: enum for LogLevel --- README.md | 6 +++--- pyreindexer/query.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f868d33..b842816 100644 --- a/README.md +++ b/README.md @@ -1069,13 +1069,13 @@ Sets the number of the first selected row from result query ### debug ```python -def debug(level: int) -> Query +def debug(level: LogLevel) -> Query ``` -Changes debug level +Changes debug log level on server #### Arguments: - level (int): Debug level + level (:enum:`LogLevel`): Debug log level on server #### Returns: (:obj:`Query`): Query object for further customizations diff --git a/pyreindexer/query.py b/pyreindexer/query.py index a7f76a2..1a333ba 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -36,6 +36,14 @@ class JoinType(Enum): Merge = 3 +class LogLevel(Enum): + Off = 0 + Error = 1 + Warning = 2 + Info = 3 + Trace = 4 + + simple_types = Union[int, str, bool, float] @@ -585,18 +593,18 @@ def offset(self, start_offset: int) -> Query: self.api.offset(self.query_wrapper_ptr, start_offset) return self - def debug(self, level: int) -> Query: - """Changes debug level + def debug(self, level: LogLevel) -> Query: + """Changes debug log level on server #### Arguments: - level (int): Debug level + level (:enum:`LogLevel`): Debug log level on server #### Returns: (:obj:`Query`): Query object for further customizations """ - self.api.debug(self.query_wrapper_ptr, level) + self.api.debug(self.query_wrapper_ptr, level.value) return self def strict(self, mode: StrictMode) -> Query: From e4fc259e48a2276bbd41442c6e1f8cb6cd785db3 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 8 Nov 2024 20:15:15 +0300 Subject: [PATCH 073/125] Small updates --- pyreindexer/example/main.py | 24 ++++++++++++++++++++---- pyreindexer/query.py | 3 +++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 7164d18..5ab37bd 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -86,17 +86,33 @@ def transaction_example(db, namespace, items_in_base): def query_example(db, namespace): + any_items = (db.new_query(namespace) + .where('value', CondType.CondAny) + .sort('id') + .execute()) + print(f'Query results count (Any): {any_items.count()}') + for item in any_items: + print('Item: ', item) + selected_items = (db.new_query(namespace) .where('value', CondType.CondEq, 'check') .sort('id') .limit(4) .execute()) + print(f'Query results count (limited): {selected_items.count()}') + for item in selected_items: + print('Item: ', item) - res_count = selected_items.count() - print('Query results count (limited): ', res_count) + del_count = (db.new_query(namespace) + .where('name', CondType.CondEq, 'item_1') + .delete()) + print(f'Deleted count: {del_count}') - # disposable QueryResults iterator - for item in selected_items: + any_items = (db.new_query(namespace) + .where('value', CondType.CondAny) + .must_execute()) + print(f'Query results count (Any after delete): {any_items.count()}') + for item in any_items: print('Item: ', item) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 1a333ba..5e781b3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -134,6 +134,9 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ params: list = self.__convert_to_list(keys) + if condition == CondType.CondDWithin : + raise Exception("In this case, use a special method 'dwithin'") + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() return self From f38e721d0aff1e04b9fb78f46e66d4845b3a1c1a Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 11 Nov 2024 10:50:20 +0300 Subject: [PATCH 074/125] Part XXIV: Add support GetExplainResults --- pyreindexer/lib/include/queryresults_wrapper.h | 6 +++++- pyreindexer/lib/src/rawpyreindexer.cc | 13 +++++++++++++ pyreindexer/lib/src/rawpyreindexer.h | 2 ++ pyreindexer/query_results.py | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index 3d57600..b1ae39e 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -63,7 +63,11 @@ class QueryResultsWrapper { } } - const std::vector& GetAggregationResults() & { return qres_.GetAggregationResults(); } + const std::string& GetExplainResults() const& noexcept { return qres_.GetExplainResults(); } + const std::string& GetExplainResults() const&& = delete; + + const std::vector& GetAggregationResults() & + { return qres_.GetAggregationResults(); } const std::vector& GetAggregationResults() && = delete; private: diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 0194a8a..8479d01 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -488,6 +488,19 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { return res; } +static PyObject* GetExplainResults(PyObject* self, PyObject* args) { + uintptr_t qresWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "k", &qresWrapperAddr)) { + return nullptr; + } + + QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); + + const auto& explainResults = qresWrapper->GetExplainResults(); + + return Py_BuildValue("iss", errOK, "", explainResults.c_str()); +} + // transaction --------------------------------------------------------------------------------------------------------- static PyObject* NewTransaction(PyObject* self, PyObject* args) { diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 27736e4..5851039 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -43,6 +43,7 @@ static PyObject* QueryResultsWrapperStatus(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperIterate(PyObject* self, PyObject* args); static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args); static PyObject* GetAggregationResults(PyObject* self, PyObject* args); +static PyObject* GetExplainResults(PyObject* self, PyObject* args); // transaction (sync) static PyObject* NewTransaction(PyObject* self, PyObject* args); static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args); @@ -127,6 +128,7 @@ static PyMethodDef module_methods[] = { {"query_results_iterate", QueryResultsWrapperIterate, METH_VARARGS, "get query result"}, {"query_results_delete", QueryResultsWrapperDelete, METH_VARARGS, "free query results buffer"}, {"get_agg_results", GetAggregationResults, METH_VARARGS, "get aggregation results"}, + {"get_explain_results", GetExplainResults, METH_VARARGS, "get explain results"}, // transaction (sync) {"new_transaction", NewTransaction, METH_VARARGS, "start new transaction"}, {"item_insert_transaction", ItemInsertTransaction, METH_VARARGS, "item insert transaction"}, diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 0e09c38..f5492c0 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -106,3 +106,19 @@ def get_agg_results(self): if self.err_code: raise Exception(self.err_msg) return res + + def get_explain_results(self): + """Returns explain results for the current query + + # Returns + (string): Formatted string with explain of results for the current query + + #### Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg, res = self.api.get_explain_results(self.qres_wrapper_ptr) + if self.err_code: + raise Exception(self.err_msg) + return res From 192e1683829c9c06d83d8b19944537548019790d Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Mon, 11 Nov 2024 13:30:54 +0300 Subject: [PATCH 075/125] add query tests: update, delete --- pyreindexer/tests/conftest.py | 11 + pyreindexer/tests/helpers/base_helper.py | 7 + pyreindexer/tests/tests/test_query.py | 244 ++++++++++++++++++----- 3 files changed, 211 insertions(+), 51 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 45e5120..059c8f5 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -51,6 +51,17 @@ def index(db, namespace): yield +@pytest.fixture +def indexes(db, namespace): + """ + Create two indexes to namespace + """ + db.index.create(namespace, index_definition) + db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash", + "is_sparse": True}) + yield + + @pytest.fixture def item(db, namespace): """ diff --git a/pyreindexer/tests/helpers/base_helper.py b/pyreindexer/tests/helpers/base_helper.py index e3a7bb2..a049245 100644 --- a/pyreindexer/tests/helpers/base_helper.py +++ b/pyreindexer/tests/helpers/base_helper.py @@ -1,3 +1,6 @@ +import math + + def get_ns_items(db, ns_name): """ Get all items via sql query """ @@ -10,3 +13,7 @@ def get_ns_description(db, ns_name): namespaces_list = db.namespace.enumerate() ns_entry = [ns for ns in namespaces_list if ns["name"] == ns_name] return ns_entry + + +def calculate_distance(point1, point2): + return math.sqrt((point1[0] - point2[0]) ** 2 + (point1[1] - point2[1]) ** 2) diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 4f0425d..5676fde 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -1,8 +1,13 @@ +import copy +import json +import random + import pytest from hamcrest import * from point import Point -from query import CondType, StrictMode +from query import CondType, LogLevel, StrictMode +from tests.helpers.base_helper import calculate_distance, get_ns_items from tests.test_data.constants import AGGREGATE_FUNCTIONS_MATH @@ -16,6 +21,15 @@ def test_query_select_where(self, db, namespace, index, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[3]]), "Wrong query results") + def test_query_select_all(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query") + query_result = list(query.execute()) + # Then ("Check that all items are in result") + assert_that(query_result, equal_to(items), "Wrong query results") + def test_query_select_fields(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") @@ -44,27 +58,28 @@ def test_query_select_get_false(self, db, namespace, index, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to(["", False]), "Wrong query results") - # TODO ??? build query + # TODO does not work, ignore subquery results def test_query_select_where_query(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") # query = db.query.new(namespace).where("id", CondType.CondLt, 5) # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) - query = db.query.new(namespace) - sub_query = db.query.new(namespace) + query = db.query.new(namespace).where("id", CondType.CondGt, 0) + sub_query = db.query.new(namespace).select(["id"]).where("id", CondType.CondLt, 5) # When ("Make select query with where_query subquery") - query_result = list(query.where_query(sub_query, CondType.CondEq, 1).execute()) + query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) # Then ("Check that selected item is in result") - assert_that(query_result, equal_to([items[1]]), "Wrong query results") + expected_items = [items[i] for i in [2, 3, 4]] + assert_that(query_result, equal_to(expected_items), "Wrong query results") - # TODO ??? build query + # TODO how to build query with where_composite? need example def test_query_select_where_composite(self, db, namespace, composite_index, items): # Given("Create namespace with composite index") # Given ("Create new query") query = db.query.new(namespace) query_comp = db.query.new(namespace).where("id", CondType.CondEq, 1).where("val", CondType.CondEq, "testval1") # When ("Make select query with where_composite") - query_result = list(query.where_composite("comp_idx", CondType.CondEq, query_comp).execute()) + query_result = list(query.where_composite("comp_idx", CondType.CondEq, query_comp).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") @@ -82,7 +97,7 @@ def test_query_select_where_uuid(self, db, namespace, index): query = db.query.new(namespace) # When ("Make select query with where_uuid") item = items[1] - query_result = list(query.where_uuid("uuid", CondType.CondEq, [item["uuid"]]).execute()) + query_result = list(query.where_uuid("uuid", CondType.CondEq, [item["uuid"]]).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([item]), "Wrong query results") @@ -95,7 +110,7 @@ def test_query_select_where_between_fields(self, db, namespace, index): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with where_between_fields") - query_result = list(query.where_between_fields("id", CondType.CondEq, "age").execute()) + query_result = list(query.where_between_fields("id", CondType.CondEq, "age").must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") @@ -115,7 +130,7 @@ def test_query_select_with_brackets_and_ops(self, db, namespace, index, items): .close_bracket() .close_bracket()) - query_result = list(query.execute()) + query_result = list(query.must_execute()) # Then ("Check that selected items are in result") expected_items = [items[i] for i in [0, 2, 3, 4]] assert_that(query_result, equal_to(expected_items), "Wrong query results") @@ -127,27 +142,26 @@ def test_query_select_match(self, db, namespace, index, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with match") - query_result = list(query.match("val", ["testval1"]).execute()) + query_result = list(query.match("val", ["testval1"]).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with match and empty result") - query_result = list(query.match("val", ["testval"]).execute()) + query_result = list(query.match("val", ["testval"]).must_execute()) # Then ("Check that result is empty") assert_that(query_result, empty(), "Wrong query results") - # TODO fix 'Process finished with exit code 139 (interrupted by signal 11:SIGSEGV)' def test_query_select_dwithin(self, db, namespace, index, rtree_index_and_items): # Given("Create namespace with rtree index and items") items = rtree_index_and_items # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with dwithin") - query_result = list(query.dwithin("rtree", Point(3, 2.5), 1.5).execute()) + query_result = list(query.dwithin("rtree", Point(3, 2.5), 2.2).must_execute()) # Then ("Check that selected item is in result") - expected_items = [items[i] for i in [1, 2, 3]] # TODO change expected after err fix + expected_items = [items[i] for i in [2, 3, 4]] assert_that(query_result, equal_to(expected_items), "Wrong query results") def test_query_select_equal_position(self, db, namespace, index, array_indexes_and_items): @@ -157,7 +171,7 @@ def test_query_select_equal_position(self, db, namespace, index, array_indexes_a query = db.query.new(namespace) # When ("Make select query with equal_position") query.where("arr1", CondType.CondEq, 1).where("arr2", CondType.CondEq, 1) - query_result = list(query.equal_position(["arr1", "arr2"]).execute()) + query_result = list(query.equal_position(["arr1", "arr2"]).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") @@ -166,22 +180,22 @@ def test_query_select_limit_offset(self, db, namespace, index, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with limit and offset") - query_result = list(query.limit(3).offset(1).execute()) + query_result = list(query.limit(3).offset(1).must_execute()) # Then ("Check that selected items are in result") expected_items = [items[i] for i in [1, 2, 3]] assert_that(query_result, equal_to(expected_items), "Wrong query results") + @pytest.mark.skip(reason="need get_total in query_results") @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) def test_query_select_request_total(self, db, namespace, index, items, func_total): # Given("Create namespace with index and items") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with limit and offset") - query_result = getattr(query, func_total)("id").execute() + query_result = getattr(query, func_total)("id").must_execute() # Then ("Check that selected items are in result") assert_that(list(query_result), equal_to(items), "Wrong query results") # Then ("Check that total is in result") - # TODO: need get_explain_results in query_results # assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): @@ -189,7 +203,7 @@ def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query") - query_result = list(query.where("ft", CondType.CondEq, "word~").with_rank().execute()) + query_result = list(query.where("ft", CondType.CondEq, "word~").with_rank().must_execute()) # Then ("Check that selected item is in result with rank") for res_item in query_result: assert_that(res_item, has_entries("id", is_(int), "ft", is_(str), "rank()", is_(float)), @@ -197,12 +211,11 @@ def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): def test_query_select_functions(self, db, namespace, index, ft_index_and_items): # Given("Create namespace with ft index and items") - ft_index_and_items # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query") query.where("ft", CondType.CondEq, "word~") - query_result = list(query.functions(["ft=highlight(<,>)"]).execute()) + query_result = list(query.functions(["ft=highlight(<,>)"]).must_execute()) # Then ("Check that selected item is in result and highlighted") query_results_ft = [i["ft"] for i in query_result] expected_ft_content = ["one ", " two", "three 333"] @@ -212,13 +225,13 @@ def test_query_select_functions(self, db, namespace, index, ft_index_and_items): def test_query_select_merge(self, db, namespace, index, items, second_namespace): # Given("Create namespace with index and items") # Given("Create second namespace with index and items") - second_ns_name, item2 = second_namespace + second_namespace, item2 = second_namespace # Given ("Create new query") query1 = db.query.new(namespace).where("id", CondType.CondEq, 2) # Given ("Create second query") - query2 = db.query.new(second_ns_name).where("id", CondType.CondEq, 1) + query2 = db.query.new(second_namespace).where("id", CondType.CondEq, 1) # When ("Make select query with merge") - query_result = list(query1.merge(query2).execute()) + query_result = list(query1.merge(query2).must_execute()) # Then ("Check that selected item is in result with merge applied") assert_that(query_result, equal_to([items[2], item2]), "Wrong query results") @@ -227,21 +240,21 @@ def test_query_select_explain(self, db, namespace, index, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with explain") - query_result = query.explain().execute() + query_result = query.explain().must_execute() # Then ("Check that selected items are in result") assert_that(list(query_result), equal_to(items), "Wrong query results") # Then ("Check that explain is in result") - # TODO: need get_explain_results in query_results - # assert_that(query_result.get_explain_results(), equal_to(""), "There is no explain in query results") + explain_results = json.loads(query_result.get_explain_results()) + assert_that(explain_results, has_entries(selectors=is_(list), sort_index="-", total_us=is_(int)), + "Wrong explain results") - # TODO add debug enums - @pytest.mark.parametrize("debug_level", [StrictMode.NotSet]) + @pytest.mark.parametrize("debug_level", LogLevel) def test_query_select_debug(self, db, namespace, index, items, debug_level): # Given("Create namespace with index and items") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query") - query_result = list(query.debug(debug_level).where("id", CondType.CondEq, 1).execute()) + query_result = list(query.debug(debug_level).where("id", CondType.CondEq, 1).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") @@ -255,7 +268,7 @@ def test_query_select_strict_mode_none_and_empty(self, db, namespace, index, ite # Then ("Check that selected item is in result") assert_that(query_result, empty(), "Wrong query results") - # TODO must be err + # TODO must be err (see err_msg) def test_query_select_strict_mode_names(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") @@ -268,7 +281,7 @@ def test_query_select_strict_mode_names(self, db, namespace, index, items): raises(Exception, pattern=err_msg), "Error wasn't raised while strict mode violated") - # TODO must be err + # TODO must be err (see err_msg) def test_query_select_strict_mode_indexes(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") @@ -289,7 +302,7 @@ def test_query_select_aggregations_math(self, db, namespace, index, items, calcu # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with aggregations") - query_result = getattr(query, aggregate_func)("id").execute() + query_result = getattr(query, aggregate_func)("id").must_execute() # Then ("Check that result is empty") assert_that(list(query_result), empty(), "Wrong query results") # Then ("Check aggregation results") @@ -303,7 +316,7 @@ def test_query_select_distinct(self, db, namespace, index, index_and_duplicate_i # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with distinct") - query_result = query.distinct("idx").execute() + query_result = query.distinct("idx").must_execute() # Then ("Check that only distinct items is in result") expected_ids = [0, 1, 3] expected_items = [items[i] for i in expected_ids] @@ -321,8 +334,8 @@ def test_query_select_facet(self, db, namespace, index, index_and_duplicate_item # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with facet") - query.aggregate_facet(["idx"]) # TODO объединить в цепочку? - query_result = query.execute() + query.aggregate_facet(["idx"]) + query_result = query.must_execute() # Then ("Check that result is empty") assert_that(list(query_result), empty(), "Wrong query results") # Then ("Check aggregation results") @@ -338,8 +351,8 @@ def test_query_select_facet_sort_offset_limit(self, db, namespace, index, index_ # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with facet") - query.aggregate_facet(["id", "idx"]).sort("id", True).limit(2).offset(1) # TODO объединить в цепочку? - query_result = query.execute() + query.aggregate_facet(["id", "idx"]).sort("id", True).limit(2).offset(1) + query_result = query.must_execute() # Then ("Check that result is empty") assert_that(list(query_result), empty(), "Wrong query results") # Then ("Check aggregation results") @@ -357,12 +370,12 @@ def test_query_select_sort(self, db, namespace, index, items_shuffled, is_revers # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with sort") - query_result = list(query.sort("id", is_reversed).execute()) + query_result = list(query.sort("id", is_reversed).must_execute()) # Then ("Check that selected items are sorted") expected_items = sorted(items_shuffled, key=lambda x: x["id"], reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) - # TODO fix forced_values + # TODO exit code 255 @pytest.mark.parametrize("forced_values, expected_ids", [ (4, [4, 0, 1, 2, 3]), ([1, 3], [1, 3, 0, 2, 4]) @@ -374,26 +387,25 @@ def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, fo # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with sort") - query_result = list(query.sort("id", is_reversed, forced_values).execute()) # TODO exit code 255 + query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) # Then ("Check that selected items are sorted") expected_items = [items_shuffled[i] for i in expected_ids] if is_reversed: expected_items.reverse() assert_that(query_result, equal_to(expected_items)) - # TODO wip + # TODO exit code 134 (interrupted by signal 6:SIGABRT) @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): # Given("Create namespace with rtree index and items") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with sort point distance") - query_result = list(query.sort_stpoint_distance("id", Point(1, 2), is_reversed).execute()) + query_result = list(query.sort_stpoint_distance("id", Point(1, 2), is_reversed).must_execute()) # Then ("Check that selected items are sorted") expected_items = sorted(rtree_index_and_items, key=lambda x: x["id"], reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) - # TODO wip @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): # Given("Create namespace with index") @@ -402,18 +414,148 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): "index_type": "rtree", "rtree_type": "rstar"}) db.index.create(namespace, {"name": "rtree2", "json_paths": ["rtree2"], "field_type": "point", "index_type": "rtree", "rtree_type": "rstar"}) - items = [{"id": i, "rtree1": [i - 1, i + 1], "rtree2": [i + 1, i + 0.5]} for i in range(5)] + items = [{"id": i, "rtree1": [i, i ** 2], "rtree2": [i, i ** 3 - 4.5]} for i in range(1, 6)] for item in items: db.item.insert(namespace, item) # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with sort fields distance") - query_result = list(query.sort_stfield_distance("rtree1", "rtree2", is_reversed).execute()) + query_result = list(query.sort_stfield_distance("rtree1", "rtree2", is_reversed).must_execute()) # Then ("Check that selected items are sorted") - expected_items = sorted(items, key=lambda x: x["id"], reverse=is_reversed) + expected_items = sorted(items, key=lambda i: calculate_distance(i["rtree1"], i["rtree2"]), reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) class TestQuerySelectJoin: - def test_query_select_join(self, db, namespace, index): - pass + # TODO TypeError: 'CondType' object cannot be interpreted as an integer + def test_query_select_left_join(self, db, namespace, index, items, second_namespace): + # Given("Create two namespaces") + second_namespace, item2 = second_namespace + # Given ("Create two queries for join") + query1 = db.query.new(namespace) + query2 = db.query.new(second_namespace) + # When ("Make select query with join") + query_result = list(query1.join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) + # Then ("Check that joined item is in result") + item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} + items[1] = item_with_joined + assert_that(query_result, equal_to(items), "Wrong selected items with JOIN") + + # TODO TypeError: 'CondType' object cannot be interpreted as an integer + def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): + # Given("Create two namespaces") + second_namespace, item2 = second_namespace + # Given ("Create two queries for join") + query1 = db.query.new(namespace) + query2 = db.query.new(second_namespace) + # When ("Make select query with join") + query_result = list(query1.inner_join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) + # Then ("Check that joined item is in result") + item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} + assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") + + +class TestQueryUpdate: + # TODO exit code 134 (interrupted by signal 6:SIGABRT) + def test_query_update_set(self, db, namespace, indexes, items): + # Given("Create namespace with indexes and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update set query") + item = random.choice(items) + modified_item = copy.deepcopy(item) + modified_item["val"] = "modified" + query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() + # Then ("Check that item is updated") + assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") + # Then ("Check that items contain modified and do not contain original item") + items_after_update = get_ns_items(db, namespace) + assert_that(items_after_update, has_length(len(items)), "Wrong items count") + assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + + # TODO exit code 134 (interrupted by signal 6:SIGABRT) + def test_query_update_set_object(self, db, namespace, indexes): + # Given("Create namespace with index and items") + # Given ("Create nested index") + db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], + "field_type": "int", "index_type": "hash"}) + # Given ("Create items") + items = [{"id": i, "nested": {"field": i}} for i in range(3)] + for item in items: + db.item.insert(namespace, item) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update set_object query") + item = random.choice(items) + modified_item = copy.deepcopy(item) + modified_item["nested"]["field"] = 10 + query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() + # Then ("Check that item is updated") + assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") + # Then ("Check that items contain modified and do not contain original item") + items_after_update = get_ns_items(db, namespace) + assert_that(items_after_update, has_length(len(items)), "Wrong items count") + assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + + # TODO exit code 134 (interrupted by signal 6:SIGABRT) + def test_query_update_expression(self, db, namespace, indexes, items): + # Given("Create namespace with indexes and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update expression query") + item = random.choice(items) + modified_item = copy.deepcopy(item) + modified_item["val"] = abs(item["id"] - 20) + query_result = query.where("id", CondType.CondEq, item["id"]).expression("id", "abs(id-20)").update() + # Then ("Check that item is updated") + assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") + # Then ("Check that items contain modified and do not contain original item") + items_after_update = get_ns_items(db, namespace) + assert_that(items_after_update, has_length(len(items)), "Wrong items count") + assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + + def test_query_drop(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update drop query") + item = random.choice(items) + query_result = list(query.where("id", CondType.CondEq, item["id"]).drop("val").update()) + # Then ("Check that result contains item with dropped field") + del item["val"] + assert_that(query_result, equal_to([item]), "Wrong update query results after drop") + # Then ("Check that field was dropped for a chosen item") + items_after_drop = get_ns_items(db, namespace) + assert_that(items_after_drop, equal_to(items), "Wrong items after drop") + + def test_query_drop_all(self, db, namespace, indexes, items): + # Given("Create namespace with two indexes and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update drop query") + query_result = list(query.drop("val").update()) + # Then ("Check that result contains items with dropped field") + for item in items: + del item["val"] + assert_that(query_result, equal_to(items), "Wrong update query results after drop") + # Then ("Check that field was dropped") + items_after_drop = get_ns_items(db, namespace) + assert_that(items_after_drop, equal_to(items), "Wrong items after drop") + + +class TestQueryDelete: + def test_query_delete(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make delete query") + item = random.choice(items) + query_result = query.where("id", CondType.CondEq, item["id"]).delete() + # Then ("Check that chosen item was deleted") + assert_that(query_result, equal_to(1), "Wrong delete items count") + items_after_delete = get_ns_items(db, namespace) + assert_that(items_after_delete, has_length(len(items) - 1), "Wrong items count after delete") + assert_that(items_after_delete, not_(has_item(item)), "Deleted item is in namespace") From 603ac965bc2a747b67543cecec1319b3013ab171 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 11 Nov 2024 17:21:28 +0300 Subject: [PATCH 076/125] Update markdown --- README.md | 263 ++++++++++++++++++++++++----------- pydoc-markdown.yml | 8 +- pyreindexer/query_results.py | 6 +- 3 files changed, 195 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index b842816..3866cd5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,94 @@ +# The PyReindexer module provides a connector and its auxiliary tools for interaction with Reindexer + +* [pyreindexer.rx\_connector](#pyreindexer.rx_connector) + * [RxConnector](#pyreindexer.rx_connector.RxConnector) + * [close](#pyreindexer.rx_connector.RxConnector.close) + * [namespace\_open](#pyreindexer.rx_connector.RxConnector.namespace_open) + * [namespace\_close](#pyreindexer.rx_connector.RxConnector.namespace_close) + * [namespace\_drop](#pyreindexer.rx_connector.RxConnector.namespace_drop) + * [namespaces\_enum](#pyreindexer.rx_connector.RxConnector.namespaces_enum) + * [index\_add](#pyreindexer.rx_connector.RxConnector.index_add) + * [index\_update](#pyreindexer.rx_connector.RxConnector.index_update) + * [index\_drop](#pyreindexer.rx_connector.RxConnector.index_drop) + * [item\_insert](#pyreindexer.rx_connector.RxConnector.item_insert) + * [item\_update](#pyreindexer.rx_connector.RxConnector.item_update) + * [item\_upsert](#pyreindexer.rx_connector.RxConnector.item_upsert) + * [item\_delete](#pyreindexer.rx_connector.RxConnector.item_delete) + * [meta\_put](#pyreindexer.rx_connector.RxConnector.meta_put) + * [meta\_get](#pyreindexer.rx_connector.RxConnector.meta_get) + * [meta\_delete](#pyreindexer.rx_connector.RxConnector.meta_delete) + * [meta\_enum](#pyreindexer.rx_connector.RxConnector.meta_enum) + * [select](#pyreindexer.rx_connector.RxConnector.select) + * [new\_transaction](#pyreindexer.rx_connector.RxConnector.new_transaction) + * [new\_query](#pyreindexer.rx_connector.RxConnector.new_query) +* [pyreindexer.query\_results](#pyreindexer.query_results) + * [QueryResults](#pyreindexer.query_results.QueryResults) + * [status](#pyreindexer.query_results.QueryResults.status) + * [count](#pyreindexer.query_results.QueryResults.count) + * [get\_agg\_results](#pyreindexer.query_results.QueryResults.get_agg_results) + * [get\_explain\_results](#pyreindexer.query_results.QueryResults.get_explain_results) +* [pyreindexer.transaction](#pyreindexer.transaction) + * [Transaction](#pyreindexer.transaction.Transaction) + * [insert](#pyreindexer.transaction.Transaction.insert) + * [update](#pyreindexer.transaction.Transaction.update) + * [upsert](#pyreindexer.transaction.Transaction.upsert) + * [delete](#pyreindexer.transaction.Transaction.delete) + * [commit](#pyreindexer.transaction.Transaction.commit) + * [commit\_with\_count](#pyreindexer.transaction.Transaction.commit_with_count) + * [rollback](#pyreindexer.transaction.Transaction.rollback) +* [pyreindexer.point](#pyreindexer.point) + * [Point](#pyreindexer.point.Point) +* [pyreindexer.query](#pyreindexer.query) + * [Query](#pyreindexer.query.Query) + * [where](#pyreindexer.query.Query.where) + * [where\_query](#pyreindexer.query.Query.where_query) + * [where\_composite](#pyreindexer.query.Query.where_composite) + * [where\_uuid](#pyreindexer.query.Query.where_uuid) + * [where\_between\_fields](#pyreindexer.query.Query.where_between_fields) + * [open\_bracket](#pyreindexer.query.Query.open_bracket) + * [close\_bracket](#pyreindexer.query.Query.close_bracket) + * [match](#pyreindexer.query.Query.match) + * [dwithin](#pyreindexer.query.Query.dwithin) + * [distinct](#pyreindexer.query.Query.distinct) + * [aggregate\_sum](#pyreindexer.query.Query.aggregate_sum) + * [aggregate\_avg](#pyreindexer.query.Query.aggregate_avg) + * [aggregate\_min](#pyreindexer.query.Query.aggregate_min) + * [aggregate\_max](#pyreindexer.query.Query.aggregate_max) + * [aggregate\_facet](#pyreindexer.query.Query.aggregate_facet) + * [sort](#pyreindexer.query.Query.sort) + * [sort\_stpoint\_distance](#pyreindexer.query.Query.sort_stpoint_distance) + * [sort\_stfield\_distance](#pyreindexer.query.Query.sort_stfield_distance) + * [op\_and](#pyreindexer.query.Query.op_and) + * [op\_or](#pyreindexer.query.Query.op_or) + * [op\_not](#pyreindexer.query.Query.op_not) + * [request\_total](#pyreindexer.query.Query.request_total) + * [cached\_total](#pyreindexer.query.Query.cached_total) + * [limit](#pyreindexer.query.Query.limit) + * [offset](#pyreindexer.query.Query.offset) + * [debug](#pyreindexer.query.Query.debug) + * [strict](#pyreindexer.query.Query.strict) + * [explain](#pyreindexer.query.Query.explain) + * [with\_rank](#pyreindexer.query.Query.with_rank) + * [execute](#pyreindexer.query.Query.execute) + * [delete](#pyreindexer.query.Query.delete) + * [set\_object](#pyreindexer.query.Query.set_object) + * [set](#pyreindexer.query.Query.set) + * [drop](#pyreindexer.query.Query.drop) + * [expression](#pyreindexer.query.Query.expression) + * [update](#pyreindexer.query.Query.update) + * [must\_execute](#pyreindexer.query.Query.must_execute) + * [get](#pyreindexer.query.Query.get) + * [inner\_join](#pyreindexer.query.Query.inner_join) + * [join](#pyreindexer.query.Query.join) + * [left\_join](#pyreindexer.query.Query.left_join) + * [merge](#pyreindexer.query.Query.merge) + * [on](#pyreindexer.query.Query.on) + * [select](#pyreindexer.query.Query.select) + * [functions](#pyreindexer.query.Query.functions) + * [equal\_position](#pyreindexer.query.Query.equal_position) +* [pyreindexer.index\_definition](#pyreindexer.index_definition) + * [IndexDefinition](#pyreindexer.index_definition.IndexDefinition) + # pyreindexer.rx\_connector @@ -23,7 +114,7 @@ RxConnector provides a binding to Reindexer upon two shared libraries (hereinaft -### close +### RxConnector.close ```python def close() -> None @@ -36,7 +127,7 @@ Closes an API instance with Reindexer resources freeing -### namespace\_open +### RxConnector.namespace\_open ```python def namespace_open(namespace) -> None @@ -53,7 +144,7 @@ Opens a namespace specified or creates a namespace if it does not exist -### namespace\_close +### RxConnector.namespace\_close ```python def namespace_close(namespace) -> None @@ -70,7 +161,7 @@ Closes a namespace specified -### namespace\_drop +### RxConnector.namespace\_drop ```python def namespace_drop(namespace) -> None @@ -87,7 +178,7 @@ Drops a namespace specified -### namespaces\_enum +### RxConnector.namespaces\_enum ```python def namespaces_enum(enum_not_opened=False) -> List[Dict[str, str]] @@ -108,7 +199,7 @@ Gets a list of namespaces available -### index\_add +### RxConnector.index\_add ```python def index_add(namespace, index_def) -> None @@ -126,7 +217,7 @@ Adds an index to the namespace specified -### index\_update +### RxConnector.index\_update ```python def index_update(namespace, index_def) -> None @@ -144,7 +235,7 @@ Updates an index in the namespace specified -### index\_drop +### RxConnector.index\_drop ```python def index_drop(namespace, index_name) -> None @@ -162,7 +253,7 @@ Drops an index from the namespace specified -### item\_insert +### RxConnector.item\_insert ```python def item_insert(namespace, item_def, precepts=None) -> None @@ -181,7 +272,7 @@ Inserts an item with its precepts to the namespace specified -### item\_update +### RxConnector.item\_update ```python def item_update(namespace, item_def, precepts=None) -> None @@ -200,7 +291,7 @@ Updates an item with its precepts in the namespace specified -### item\_upsert +### RxConnector.item\_upsert ```python def item_upsert(namespace, item_def, precepts=None) -> None @@ -219,7 +310,7 @@ Updates an item with its precepts in the namespace specified. Creates the item i -### item\_delete +### RxConnector.item\_delete ```python def item_delete(namespace, item_def) -> None @@ -237,7 +328,7 @@ Deletes an item from the namespace specified -### meta\_put +### RxConnector.meta\_put ```python def meta_put(namespace, key, value) -> None @@ -256,7 +347,7 @@ Puts metadata to a storage of Reindexer by key -### meta\_get +### RxConnector.meta\_get ```python def meta_get(namespace, key) -> str @@ -277,7 +368,7 @@ Gets metadata from a storage of Reindexer by key specified -### meta\_delete +### RxConnector.meta\_delete ```python def meta_delete(namespace, key) -> None @@ -295,7 +386,7 @@ Deletes metadata from a storage of Reindexer by key specified -### meta\_enum +### RxConnector.meta\_enum ```python def meta_enum(namespace) -> List[str] @@ -315,7 +406,7 @@ Gets a list of metadata keys from a storage of Reindexer -### select +### RxConnector.select ```python def select(query: str) -> QueryResults @@ -335,7 +426,7 @@ Executes an SQL query and returns query results -### new\_transaction +### RxConnector.new\_transaction ```python def new_transaction(namespace) -> Transaction @@ -355,7 +446,7 @@ Starts a new transaction and return the transaction object to processing -### new\_query +### RxConnector.new\_query ```python def new_query(namespace: str) -> Query @@ -397,7 +488,7 @@ QueryResults is a disposable iterator of Reindexer results for such queries as S -### status +### QueryResults.status ```python def status() @@ -410,7 +501,7 @@ Check status -### count +### QueryResults.count ```python def count() @@ -418,12 +509,12 @@ def count() Returns a count of results -# Returns +#### Returns int: A count of results -### get\_agg\_results +### QueryResults.get\_agg\_results ```python def get_agg_results() @@ -431,12 +522,28 @@ def get_agg_results() Returns aggregation results for the current query -# Returns +#### Returns (:obj:`dict`): Dictionary with all results for the current query #### Raises: Exception: Raises with an error message of API return on non-zero error code + + +### QueryResults.get\_explain\_results + +```python +def get_explain_results() +``` + +Returns explain results for the current query + +#### Returns + (string): Formatted string with explain of results for the current query + +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + # pyreindexer.transaction @@ -459,7 +566,7 @@ An object representing the context of a Reindexer transaction -### insert +### Transaction.insert ```python def insert(item_def, precepts=None) @@ -477,7 +584,7 @@ Inserts an item with its precepts to the transaction -### update +### Transaction.update ```python def update(item_def, precepts=None) @@ -495,7 +602,7 @@ Updates an item with its precepts to the transaction -### upsert +### Transaction.upsert ```python def upsert(item_def, precepts=None) @@ -513,7 +620,7 @@ Updates an item with its precepts to the transaction. Creates the item if it not -### delete +### Transaction.delete ```python def delete(item_def) @@ -530,7 +637,7 @@ Deletes an item from the transaction -### commit +### Transaction.commit ```python def commit() @@ -544,7 +651,7 @@ Applies changes -### commit\_with\_count +### Transaction.commit\_with\_count ```python def commit_with_count() -> int @@ -558,7 +665,7 @@ Applies changes and return the number of count of changed items -### rollback +### Transaction.rollback ```python def rollback() @@ -607,14 +714,14 @@ An object representing the context of a Reindexer query query_wrapper_ptr (int): A memory pointer to Reindexer query object err_code (int): The API error code err_msg (string): The API error message - root (:object:`Query`): The root query of the Reindexer query + root (:object: Optional[`Query`]): The root query of the Reindexer query join_type (:enum:`JoinType`): Join type join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects -### where +### Query.where ```python def where(index: str, @@ -638,7 +745,7 @@ Adds where condition to DB query with args -### where\_query +### Query.where\_query ```python def where_query(sub_query: Query, @@ -662,7 +769,7 @@ Adds sub-query where condition to DB query with args -### where\_composite +### Query.where\_composite ```python def where_composite(index: str, condition: CondType, @@ -681,7 +788,7 @@ Adds where condition to DB query with interface args for composite indexes -### where\_uuid +### Query.where\_uuid ```python def where_uuid(index: str, condition: CondType, keys: List[str]) -> Query @@ -704,7 +811,7 @@ Adds where condition to DB query with UUID as string args. -### where\_between\_fields +### Query.where\_between\_fields ```python def where_between_fields(first_field: str, condition: CondType, @@ -723,7 +830,7 @@ Adds comparing two fields where condition to DB query -### open\_bracket +### Query.open\_bracket ```python def open_bracket() -> Query @@ -739,7 +846,7 @@ Opens bracket for where condition to DB query -### close\_bracket +### Query.close\_bracket ```python def close_bracket() -> Query @@ -755,7 +862,7 @@ Closes bracket for where condition to DB query -### match +### Query.match ```python def match(index: str, keys: List[str]) -> Query @@ -775,7 +882,7 @@ Adds string EQ-condition to DB query with string args -### dwithin +### Query.dwithin ```python def dwithin(index: str, point: Point, distance: float) -> Query @@ -793,7 +900,7 @@ Adds DWithin condition to DB query -### distinct +### Query.distinct ```python def distinct(index: str) -> Query @@ -809,7 +916,7 @@ Performs distinct for a certain index. Return only items with uniq value of fiel -### aggregate\_sum +### Query.aggregate\_sum ```python def aggregate_sum(index: str) -> Query @@ -825,7 +932,7 @@ Performs a summation of values for a specified index -### aggregate\_avg +### Query.aggregate\_avg ```python def aggregate_avg(index: str) -> Query @@ -841,7 +948,7 @@ Finds for the average at the specified index -### aggregate\_min +### Query.aggregate\_min ```python def aggregate_min(index: str) -> Query @@ -857,7 +964,7 @@ Finds for the minimum at the specified index -### aggregate\_max +### Query.aggregate\_max ```python def aggregate_max(index: str) -> Query @@ -873,7 +980,7 @@ Finds for the maximum at the specified index -### aggregate\_facet +### Query.aggregate\_facet ```python def aggregate_facet(fields: List[str]) -> Query._AggregateFacet @@ -891,7 +998,7 @@ Gets fields facet value. Applicable to multiple data fields and the result of th -### sort +### Query.sort ```python def sort(index: str, @@ -916,7 +1023,7 @@ Applies sort order to return from query items. If values argument specified, the -### sort\_stpoint\_distance +### Query.sort\_stpoint\_distance ```python def sort_stpoint_distance(index: str, point: Point, desc: bool) -> Query @@ -935,7 +1042,7 @@ Applies geometry sort order to return from query items. Wrapper for geometry sor -### sort\_stfield\_distance +### Query.sort\_stfield\_distance ```python def sort_stfield_distance(first_field: str, second_field: str, @@ -958,7 +1065,7 @@ Applies geometry sort order to return from query items. Wrapper for geometry sor -### op\_and +### Query.op\_and ```python def op_and() -> Query @@ -973,7 +1080,7 @@ Next condition will be added with AND. -### op\_or +### Query.op\_or ```python def op_or() -> Query @@ -988,7 +1095,7 @@ Next condition will be added with OR. -### op\_not +### Query.op\_not ```python def op_not() -> Query @@ -1002,7 +1109,7 @@ Next condition will be added with NOT AND. -### request\_total +### Query.request\_total ```python def request_total(total_name: str = '') -> Query @@ -1018,7 +1125,7 @@ Requests total items calculation -### cached\_total +### Query.cached\_total ```python def cached_total(total_name: str = '') -> Query @@ -1034,7 +1141,7 @@ Requests cached total items calculation -### limit +### Query.limit ```python def limit(limit_items: int) -> Query @@ -1050,7 +1157,7 @@ Sets a limit (count) of returned items. Analog to sql LIMIT rowsNumber -### offset +### Query.offset ```python def offset(start_offset: int) -> Query @@ -1066,7 +1173,7 @@ Sets the number of the first selected row from result query -### debug +### Query.debug ```python def debug(level: LogLevel) -> Query @@ -1082,7 +1189,7 @@ Changes debug log level on server -### strict +### Query.strict ```python def strict(mode: StrictMode) -> Query @@ -1098,7 +1205,7 @@ Changes strict mode -### explain +### Query.explain ```python def explain() -> Query @@ -1111,7 +1218,7 @@ Enables explain query -### with\_rank +### Query.with\_rank ```python def with_rank() -> Query @@ -1124,7 +1231,7 @@ Outputs fulltext rank. Allowed only with fulltext query -### execute +### Query.execute ```python def execute() -> QueryResults @@ -1141,7 +1248,7 @@ Executes a select query -### delete +### Query.delete ```python def delete() -> int @@ -1158,7 +1265,7 @@ Executes a query, and delete items, matches query -### set\_object +### Query.set\_object ```python def set_object(field: str, values: List[simple_types]) -> Query @@ -1178,7 +1285,7 @@ Adds an update query to an object field for an update query -### set +### Query.set ```python def set(field: str, values: List[simple_types]) -> Query @@ -1198,7 +1305,7 @@ Adds a field update request to the update request -### drop +### Query.drop ```python def drop(index: str) -> Query @@ -1214,7 +1321,7 @@ Drops a value for a field -### expression +### Query.expression ```python def expression(field: str, value: str) -> Query @@ -1231,7 +1338,7 @@ Updates indexed field by arithmetical expression -### update +### Query.update ```python def update() -> QueryResults @@ -1248,7 +1355,7 @@ Executes update query, and update fields in items, which matches query -### must\_execute +### Query.must\_execute ```python def must_execute() -> QueryResults @@ -1265,7 +1372,7 @@ Executes a query, and update fields in items, which matches query, with status c -### get +### Query.get ```python def get() -> (str, bool) @@ -1282,7 +1389,7 @@ Executes a query, and return 1 JSON item -### inner\_join +### Query.inner\_join ```python def inner_join(query: Query, field: str) -> Query @@ -1302,7 +1409,7 @@ Joins 2 queries. -### join +### Query.join ```python def join(query: Query, field: str) -> Query @@ -1320,7 +1427,7 @@ Join is an alias for LeftJoin. Joins 2 queries. -### left\_join +### Query.left\_join ```python def left_join(join_query: Query, field: str) -> Query @@ -1340,7 +1447,7 @@ Joins 2 queries. -### merge +### Query.merge ```python def merge(query: Query) -> Query @@ -1356,7 +1463,7 @@ Merges queries of the same type -### on +### Query.on ```python def on(index: str, condition: CondType, join_index: str) -> Query @@ -1374,7 +1481,7 @@ On specifies join condition -### select +### Query.select ```python def select(fields: List[str]) -> Query @@ -1396,7 +1503,7 @@ Sets list of columns in this namespace to be finally selected. -### functions +### Query.functions ```python def functions(functions: List[str]) -> Query @@ -1415,7 +1522,7 @@ Adds sql-functions to query -### equal\_position +### Query.equal\_position ```python def equal_position(equal_position: List[str]) -> Query diff --git a/pydoc-markdown.yml b/pydoc-markdown.yml index 41c1e53..33562ee 100644 --- a/pydoc-markdown.yml +++ b/pydoc-markdown.yml @@ -7,15 +7,21 @@ loaders: - pyreindexer.point - pyreindexer.query - pyreindexer.index_definition + processors: - type: filter expression: not name.startswith('_') and default() + renderer: type: markdown filename: README.md + add_method_class_prefix: true + render_toc: true + render_toc_title: The PyReindexer module provides a connector and its auxiliary tools for interaction with Reindexer header_level_by_type: + Module: 1 Class: 2 Function: 3 Method: 3 - Module: 1 Data: 3 + Variable: 3 diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index f5492c0..4c0995e 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -76,7 +76,7 @@ def status(self): def count(self): """Returns a count of results - # Returns + #### Returns int: A count of results """ @@ -94,7 +94,7 @@ def _close_iterator(self): def get_agg_results(self): """Returns aggregation results for the current query - # Returns + #### Returns (:obj:`dict`): Dictionary with all results for the current query #### Raises: @@ -110,7 +110,7 @@ def get_agg_results(self): def get_explain_results(self): """Returns explain results for the current query - # Returns + #### Returns (string): Formatted string with explain of results for the current query #### Raises: From 25533035f4e7865053165a423c051cb0e542f844 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 11 Nov 2024 20:15:42 +0300 Subject: [PATCH 077/125] Part XXV: Bug fixes --- README.md | 24 ++++-- pyreindexer/lib/include/query_wrapper.cc | 103 +++++++---------------- pyreindexer/lib/include/query_wrapper.h | 33 +------- pyreindexer/query.py | 30 ++++--- pyreindexer/tests/tests/test_query.py | 4 +- 5 files changed, 66 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 3866cd5..5bec17c 100644 --- a/README.md +++ b/README.md @@ -735,7 +735,8 @@ Adds where condition to DB query with args index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): - Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -759,7 +760,8 @@ Adds sub-query where condition to DB query with args sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): - Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index + Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -772,8 +774,10 @@ Adds sub-query where condition to DB query with args ### Query.where\_composite ```python -def where_composite(index: str, condition: CondType, - sub_query: Query) -> Query +def where_composite( + index: str, + condition: CondType, + keys: Union[simple_types, List[simple_types]] = None) -> Query ``` Adds where condition to DB query with interface args for composite indexes @@ -801,7 +805,8 @@ Adds where condition to DB query with UUID as string args. #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -872,7 +877,8 @@ Adds string EQ-condition to DB query with string args #### Arguments: index (string): Field name used in condition clause - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -986,9 +992,9 @@ Finds for the maximum at the specified index def aggregate_facet(fields: List[str]) -> Query._AggregateFacet ``` -Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data - column or `count` and cut off by offset and limit. In order to support this functionality this method - returns AggregationFacetRequest which has methods sort, limit and offset +Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted + by any data column or `count` and cut off by offset and limit. In order to support this functionality + this method returns AggregationFacetRequest which has methods sort, limit and offset #### Arguments: fields (list[string]): Fields any data column name or `count`, fields should not be empty diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 394bf42..8c22326 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -165,6 +165,18 @@ void QueryWrapper::Aggregation(const std::vector& fields) { } } +void QueryWrapper::Sort(std::string_view index, bool desc, const std::vector& keys) { + ser_.PutVarUint(QueryItemType::QuerySortIndex); + ser_.PutVString(index); + ser_.PutVarUint(desc? 1 : 0); + + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + ser_.PutVariant(key); + } +} + + void QueryWrapper::LogOp(OpType op) { switch (op) { case OpType::OpAnd: @@ -249,6 +261,24 @@ reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { return db_->UpdateQuery(query, qr); } +void QueryWrapper::SetObject(std::string_view field, const std::vector& values, QueryItemType type) { + ser_.PutVarUint(type); + ser_.PutVString(field); + if (type == QueryItemType::QueryUpdateObject) { + ser_.PutVarUint(values.size()); // values count + } + if (type != QueryItemType::QueryUpdateField) { + ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag + } + if (type != QueryItemType::QueryUpdateObject) { + ser_.PutVarUint(values.size()); // values count + } + for (const auto& value : values) { + ser_.PutVarUint(0); // function/value flag + ser_.PutVString(value); + } +} + void QueryWrapper::Drop(std::string_view field) { ser_.PutVarUint(QueryItemType::QueryDropField); ser_.PutVString(field); @@ -260,7 +290,7 @@ void QueryWrapper::SetExpression(std::string_view field, std::string_view value) ser_.PutVarUint(1); // size ser_.PutVarUint(1); // is expression - ser_.PutVString(value); + ser_.PutVariant(reindexer::Variant{value}); } void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery) { @@ -318,75 +348,4 @@ void QueryWrapper::AddEqualPosition(const std::vector& equalPositio } } - -template <> -void QueryWrapper::putValue(int8_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(value); -} -template <> -void QueryWrapper::putValue(uint8_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(int64_t(value)); -} -template <> -void QueryWrapper::putValue(int16_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(value); -} -template <> -void QueryWrapper::putValue(uint16_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(int64_t(value)); -} -template <> -void QueryWrapper::putValue(int32_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(value); -} -template <> -void QueryWrapper::putValue(uint32_t value) { - ser_.PutVarUint(VALUE_INT); - ser_.PutVarint(int64_t(value)); -} -template <> -void QueryWrapper::putValue(int64_t value) { - ser_.PutVarUint(VALUE_INT_64); - ser_.PutVarint(value); -} -template <> -void QueryWrapper::putValue(uint64_t value) { - ser_.PutVarUint(VALUE_INT_64); - ser_.PutVarint(value); -} -template <> -void QueryWrapper::putValue(std::string_view value) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(value); -} -template <> -void QueryWrapper::putValue(const std::string& value) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(value); -} -template <> -void QueryWrapper::putValue(bool value) { - ser_.PutVarUint(VALUE_BOOL); - ser_.PutBool(value); -} -template <> -void QueryWrapper::putValue(float value) { - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(value); -} -template <> -void QueryWrapper::putValue(double value) { - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(value); -} -template <> -void QueryWrapper::putValue(const reindexer::Variant& value) { - ser_.PutVariant(value); -} - } // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 4867bff..01a860c 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -42,17 +42,7 @@ class QueryWrapper { void Aggregation(const std::vector& fields); void AggregationSort(std::string_view field, bool desc); - template - void Sort(std::string_view index, bool desc, const std::vector& keys) { - ser_.PutVarUint(QueryItemType::QuerySortIndex); - ser_.PutVString(index); - ser_.PutVarUint(desc? 1 : 0); - - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - putValue(key); - } - } + void Sort(std::string_view index, bool desc, const std::vector& keys); void LogOp(OpType op); @@ -68,23 +58,7 @@ class QueryWrapper { reindexer::Error DeleteQuery(size_t& count); reindexer::Error UpdateQuery(QueryResultsWrapper& qr); - void SetObject(std::string_view field, const std::vector& values, QueryItemType type) { - ser_.PutVarUint(type); - ser_.PutVString(field); - if (type == QueryItemType::QueryUpdateObject) { - ser_.PutVarUint(values.size()); // values count - } - if (type != QueryItemType::QueryUpdateField) { - ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag - } - if (type != QueryItemType::QueryUpdateObject) { - ser_.PutVarUint(values.size()); // values count - } - for (const auto& value : values) { - ser_.PutVarUint(0); // function/value flag - putValue(value); - } - } + void SetObject(std::string_view field, const std::vector& values, QueryItemType type); void Drop(std::string_view field); @@ -103,9 +77,6 @@ class QueryWrapper { DBInterface* GetDB() const { return db_; } private: - template - void putValue(T) {} - reindexer::Serializer prepareQueryData(reindexer::WrSerializer& data); reindexer::JoinedQuery createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data); void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query); diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 5e781b3..cdde2eb 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,7 +1,7 @@ from __future__ import annotations from enum import Enum -from typing import List, Union +from typing import List, Union, Optional from pyreindexer.point import Point from pyreindexer.query_results import QueryResults @@ -55,7 +55,7 @@ class Query: query_wrapper_ptr (int): A memory pointer to Reindexer query object err_code (int): The API error code err_msg (string): The API error message - root (:object:`Query`): The root query of the Reindexer query + root (:object: Optional[`Query`]): The root query of the Reindexer query join_type (:enum:`JoinType`): Join type join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects @@ -75,7 +75,7 @@ def __init__(self, api, query_wrapper_ptr: int): self.query_wrapper_ptr: int = query_wrapper_ptr self.err_code: int = 0 self.err_msg: str = '' - self.root: Query = None + self.root: Optional[Query] = None self.join_type: JoinType = JoinType.LeftJoin self.join_queries: List[Query] = [] self.merged_queries: List[Query] = [] @@ -122,7 +122,8 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): - Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -149,7 +150,8 @@ def where_query(self, sub_query: Query, condition: CondType, sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition keys (Union[None, simple_types, list[simple_types]]): - Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index + Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -166,7 +168,8 @@ def where_query(self, sub_query: Query, condition: CondType, self.__raise_on_error() return self - def where_composite(self, index: str, condition: CondType, sub_query: Query) -> Query: + def where_composite(self, index: str, condition: CondType, + keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds where condition to DB query with interface args for composite indexes #### Arguments: @@ -179,8 +182,7 @@ def where_composite(self, index: str, condition: CondType, sub_query: Query) -> """ - self.api.where_composite(self.query_wrapper_ptr, index, condition.value, sub_query.query_wrapper_ptr) - return self + return self.api.where(self.query_wrapper_ptr, index, condition.value, keys) def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """Adds where condition to DB query with UUID as string args. @@ -190,7 +192,8 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -257,7 +260,8 @@ def match(self, index: str, keys: List[str]) -> Query: #### Arguments: index (string): Field name used in condition clause - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, with value of each subindex + keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -423,9 +427,9 @@ def sort(self, field: str, desc: bool = False) -> Query._AggregateFacet: return self def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: - """Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data - column or `count` and cut off by offset and limit. In order to support this functionality this method - returns AggregationFacetRequest which has methods sort, limit and offset + """Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted + by any data column or `count` and cut off by offset and limit. In order to support this functionality + this method returns AggregationFacetRequest which has methods sort, limit and offset #### Arguments: fields (list[string]): Fields any data column name or `count`, fields should not be empty diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 5676fde..5603e10 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -72,14 +72,12 @@ def test_query_select_where_query(self, db, namespace, index, items): expected_items = [items[i] for i in [2, 3, 4]] assert_that(query_result, equal_to(expected_items), "Wrong query results") - # TODO how to build query with where_composite? need example def test_query_select_where_composite(self, db, namespace, composite_index, items): # Given("Create namespace with composite index") # Given ("Create new query") query = db.query.new(namespace) - query_comp = db.query.new(namespace).where("id", CondType.CondEq, 1).where("val", CondType.CondEq, "testval1") # When ("Make select query with where_composite") - query_result = list(query.where_composite("comp_idx", CondType.CondEq, query_comp).must_execute()) + query_result = list(query.where_composite("comp_idx", CondType.CondEq, {1, "testval1"}).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") From fae7270f1bc04b2f68ba1d98c8ea9bd625e1deb7 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 12 Nov 2024 13:39:16 +0300 Subject: [PATCH 078/125] Part XXV: Bug fixes. Part II --- pyreindexer/lib/src/rawpyreindexer.cc | 2 +- pyreindexer/query.py | 67 +++-- pyreindexer/tests/tests/test_query.py | 380 +++++++++++++------------- 3 files changed, 231 insertions(+), 218 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 8479d01..3c7f01f 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1136,7 +1136,7 @@ static PyObject* Join(PyObject* self, PyObject* args) { static PyObject* Merge(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; uintptr_t queryWrapperAddrMerge = 0; - if (!PyArg_ParseTuple(args, "kIIk", &queryWrapperAddr, &queryWrapperAddrMerge)) { + if (!PyArg_ParseTuple(args, "kk", &queryWrapperAddr, &queryWrapperAddrMerge)) { return nullptr; } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index cdde2eb..01eb262 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -115,6 +115,22 @@ def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[si param = param if isinstance(param, list) else [param] return param + @staticmethod + def __convert_strs_to_list(param: tuple[str, ...]) -> List[str]: + """Converts an input parameter to a list + + #### Arguments: + param (list[*string]): The input parameter + + #### Returns: + List[string]: Always converted to a list + + """ + + param = [] if param is None else param + param = [item for item in param] + return param + def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds where condition to DB query with args @@ -142,6 +158,7 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ self.__raise_on_error() return self + def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds sub-query where condition to DB query with args @@ -182,9 +199,9 @@ def where_composite(self, index: str, condition: CondType, """ - return self.api.where(self.query_wrapper_ptr, index, condition.value, keys) + return self.where(index, condition, keys) - def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: + def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: """Adds where condition to DB query with UUID as string args. This function applies binary encoding to the UUID value. `index` MUST be declared as uuid index in this case @@ -192,7 +209,7 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -203,7 +220,7 @@ def where_uuid(self, index: str, condition: CondType, keys: List[str]) -> Query: """ - params: list = self.__convert_to_list(keys) + params: list = self.__convert_strs_to_list(keys) self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() @@ -255,12 +272,12 @@ def close_bracket(self) -> Query: self.__raise_on_error() return self - def match(self, index: str, keys: List[str]) -> Query: + def match(self, index: str, *keys: str) -> Query: """Adds string EQ-condition to DB query with string args #### Arguments: index (string): Field name used in condition clause - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -271,7 +288,7 @@ def match(self, index: str, keys: List[str]) -> Query: """ - params: list = self.__convert_to_list(keys) + params: list = self.__convert_strs_to_list(keys) self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, CondType.CondEq.value, params) self.__raise_on_error() @@ -426,22 +443,22 @@ def sort(self, field: str, desc: bool = False) -> Query._AggregateFacet: self.api.aggregation_sort(self.query_wrapper_ptr, field, desc) return self - def aggregate_facet(self, fields: List[str]) -> Query._AggregateFacet: + def aggregate_facet(self, *fields: str) -> Query._AggregateFacet: """Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted by any data column or `count` and cut off by offset and limit. In order to support this functionality this method returns AggregationFacetRequest which has methods sort, limit and offset #### Arguments: - fields (list[string]): Fields any data column name or `count`, fields should not be empty + fields (list[*string]): Fields any data column name or `count`, fields should not be empty #### Returns: (:obj:`_AggregateFacet`): Request object for further customizations """ - fields = [] if fields is None else fields + params: list = self.__convert_strs_to_list(fields) - self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, fields) + self.err_code, self.err_msg = self.api.aggregation(self.query_wrapper_ptr, params) self.__raise_on_error() return self._AggregateFacet(self) @@ -914,7 +931,7 @@ def merge(self, query: Query) -> Query: query.root = self self.merged_queries.append(query) - self.api.merge(query) + self.api.merge(self.query_wrapper_ptr, query.query_wrapper_ptr) return self def on(self, index: str, condition: CondType, join_index: str) -> Query: @@ -933,17 +950,17 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: if self.root is None: raise Exception("Can't join on root query") - self.api.on(self.query_wrapper_ptr, index, condition, join_index) + self.api.on(self.query_wrapper_ptr, index, condition.value, join_index) return self - def select(self, fields: List[str]) -> Query: + def select(self, *fields: str) -> Query: """Sets list of columns in this namespace to be finally selected. The columns should be specified in the same case as the jsonpaths corresponding to them. Non-existent fields and fields in the wrong case are ignored. If there are no fields in this list that meet these conditions, then the filter works as "*" #### Arguments: - fields (list[string]): List of columns to be selected + fields (list[*string]): List of columns to be selected #### Returns: (:obj:`Query`): Query object for further customizations @@ -953,17 +970,17 @@ def select(self, fields: List[str]) -> Query: """ - fields = [] if fields is None else fields + keys: list = self.__convert_strs_to_list(fields) - self.err_code, self.err_msg = self.api.select_filter(self.query_wrapper_ptr, fields) + self.err_code, self.err_msg = self.api.select_filter(self.query_wrapper_ptr, keys) self.__raise_on_error() return self - def functions(self, functions: List[str]) -> Query: + def functions(self, *functions: str) -> Query: """Adds sql-functions to query #### Arguments: - functions (list[string]): Functions declaration + functions (list[*string]): Functions declaration #### Returns: (:obj:`Query`): Query object for further customizations @@ -973,17 +990,17 @@ def functions(self, functions: List[str]) -> Query: """ - functions = [] if functions is None else functions + funcs: list = self.__convert_strs_to_list(functions) - self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, functions) + self.err_code, self.err_msg = self.api.functions(self.query_wrapper_ptr, funcs) self.__raise_on_error() return self - def equal_position(self, equal_position: List[str]) -> Query: + def equal_position(self, *equal_position: str) -> Query: """Adds equal position fields to arrays queries #### Arguments: - equal_poses (list[string]): Equal position fields to arrays queries + equal_poses (list[*string]): Equal position fields to arrays queries #### Returns: (:obj:`Query`): Query object for further customizations @@ -993,8 +1010,8 @@ def equal_position(self, equal_position: List[str]) -> Query: """ - equal_position = [] if equal_position is None else equal_position + equal_pos: list = self.__convert_strs_to_list(equal_position) - self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_position) + self.err_code, self.err_msg = self.api.equal_position(self.query_wrapper_ptr, equal_pos) self.__raise_on_error() return self diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 5603e10..9808b6e 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -35,7 +35,7 @@ def test_query_select_fields(self, db, namespace, index, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with select fields") - select_result = list(query.select(["id"]).must_execute()) + select_result = list(query.select("id").must_execute()) # Then ("Check that selected items are in result") ids = [{"id": i["id"]} for i in items] assert_that(select_result, equal_to(ids), "Wrong query results") @@ -58,28 +58,29 @@ def test_query_select_get_false(self, db, namespace, index, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to(["", False]), "Wrong query results") - # TODO does not work, ignore subquery results - def test_query_select_where_query(self, db, namespace, index, items): - # Given("Create namespace with index and items") - # Given ("Create new query") - # query = db.query.new(namespace).where("id", CondType.CondLt, 5) - # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) - query = db.query.new(namespace).where("id", CondType.CondGt, 0) - sub_query = db.query.new(namespace).select(["id"]).where("id", CondType.CondLt, 5) - # When ("Make select query with where_query subquery") - query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) - # Then ("Check that selected item is in result") - expected_items = [items[i] for i in [2, 3, 4]] - assert_that(query_result, equal_to(expected_items), "Wrong query results") - - def test_query_select_where_composite(self, db, namespace, composite_index, items): - # Given("Create namespace with composite index") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with where_composite") - query_result = list(query.where_composite("comp_idx", CondType.CondEq, {1, "testval1"}).must_execute()) - # Then ("Check that selected item is in result") - assert_that(query_result, equal_to([items[1]]), "Wrong query results") +# TODO does not work, ignore subquery results +# def test_query_select_where_query(self, db, namespace, index, items): +# # Given("Create namespace with index and items") +# # Given ("Create new query") +# # query = db.query.new(namespace).where("id", CondType.CondLt, 5) +# # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) +# query = db.query.new(namespace).where("id", CondType.CondGt, 0) +# sub_query = db.query.new(namespace).select(["id"]).where("id", CondType.CondLt, 5) +# # When ("Make select query with where_query subquery") +# query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) +# # Then ("Check that selected item is in result") +# expected_items = [items[i] for i in [2, 3, 4]] +# assert_that(query_result, equal_to(expected_items), "Wrong query results") + +# ToDo +# def test_query_select_where_composite(self, db, namespace, composite_index, items): +# # Given("Create namespace with composite index") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make select query with where_composite") +# query_result = list(query.where_composite("comp_idx", CondType.CondEq, [1, "testval1"]).must_execute()) +# # Then ("Check that selected item is in result") +# assert_that(query_result, equal_to([items[1]]), "Wrong query results") def test_query_select_where_uuid(self, db, namespace, index): # Given("Create namespace with index") @@ -95,7 +96,7 @@ def test_query_select_where_uuid(self, db, namespace, index): query = db.query.new(namespace) # When ("Make select query with where_uuid") item = items[1] - query_result = list(query.where_uuid("uuid", CondType.CondEq, [item["uuid"]]).must_execute()) + query_result = list(query.where_uuid("uuid", CondType.CondEq, item["uuid"]).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([item]), "Wrong query results") @@ -140,14 +141,14 @@ def test_query_select_match(self, db, namespace, index, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with match") - query_result = list(query.match("val", ["testval1"]).must_execute()) + query_result = list(query.match("val", "testval1").must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with match and empty result") - query_result = list(query.match("val", ["testval"]).must_execute()) + query_result = list(query.match("val", "testval").must_execute()) # Then ("Check that result is empty") assert_that(query_result, empty(), "Wrong query results") @@ -169,7 +170,7 @@ def test_query_select_equal_position(self, db, namespace, index, array_indexes_a query = db.query.new(namespace) # When ("Make select query with equal_position") query.where("arr1", CondType.CondEq, 1).where("arr2", CondType.CondEq, 1) - query_result = list(query.equal_position(["arr1", "arr2"]).must_execute()) + query_result = list(query.equal_position("arr1", "arr2").must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") @@ -183,18 +184,19 @@ def test_query_select_limit_offset(self, db, namespace, index, items): expected_items = [items[i] for i in [1, 2, 3]] assert_that(query_result, equal_to(expected_items), "Wrong query results") - @pytest.mark.skip(reason="need get_total in query_results") - @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) - def test_query_select_request_total(self, db, namespace, index, items, func_total): - # Given("Create namespace with index and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with limit and offset") - query_result = getattr(query, func_total)("id").must_execute() - # Then ("Check that selected items are in result") - assert_that(list(query_result), equal_to(items), "Wrong query results") - # Then ("Check that total is in result") - # assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") +# ToDo +# @pytest.mark.skip(reason="need get_total in query_results") +# @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) +# def test_query_select_request_total(self, db, namespace, index, items, func_total): +# # Given("Create namespace with index and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make select query with limit and offset") +# query_result = getattr(query, func_total)("id").must_execute() +# # Then ("Check that selected items are in result") +# assert_that(list(query_result), equal_to(items), "Wrong query results") +# # Then ("Check that total is in result") +# assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): # Given("Create namespace with ft index and items") @@ -213,13 +215,12 @@ def test_query_select_functions(self, db, namespace, index, ft_index_and_items): query = db.query.new(namespace) # When ("Make select query") query.where("ft", CondType.CondEq, "word~") - query_result = list(query.functions(["ft=highlight(<,>)"]).must_execute()) + query_result = list(query.functions("ft=highlight(<,>)").must_execute()) # Then ("Check that selected item is in result and highlighted") query_results_ft = [i["ft"] for i in query_result] expected_ft_content = ["one ", " two", "three 333"] assert_that(query_results_ft, contains_inanyorder(*expected_ft_content), "Wrong query results") - # TODO self.api.merge(query) takes 4 arguments (but 1 given - query) def test_query_select_merge(self, db, namespace, index, items, second_namespace): # Given("Create namespace with index and items") # Given("Create second namespace with index and items") @@ -266,32 +267,32 @@ def test_query_select_strict_mode_none_and_empty(self, db, namespace, index, ite # Then ("Check that selected item is in result") assert_that(query_result, empty(), "Wrong query results") - # TODO must be err (see err_msg) - def test_query_select_strict_mode_names(self, db, namespace, index, items): - # Given("Create namespace with index and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with strict mode") - query.strict(StrictMode.Names).where("rand", CondType.CondEq, 1) - err_msg = f"Current query strict mode allows aggregate existing fields only. " \ - f"There are no fields with name 'rand' in namespace '{namespace}'" - assert_that(calling(query.must_execute).with_args(), - raises(Exception, pattern=err_msg), - "Error wasn't raised while strict mode violated") - - # TODO must be err (see err_msg) - def test_query_select_strict_mode_indexes(self, db, namespace, index, items): - # Given("Create namespace with index and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with strict mode") - query.strict(StrictMode.Indexes).where("rand", CondType.CondEq, 1) - err_msg = f"Current query strict mode allows aggregate index fields only. " \ - f"There are no indexes with name 'rand' in namespace '{namespace}'" - assert_that(calling(query.must_execute).with_args(), - raises(Exception, pattern=err_msg), - "Error wasn't raised while strict mode violated") - +# TODO must be err (see err_msg) +# def test_query_select_strict_mode_names(self, db, namespace, index, items): +# # Given("Create namespace with index and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make select query with strict mode") +# query.strict(StrictMode.Names).where("rand", CondType.CondEq, 1) +# err_msg = f"Current query strict mode allows aggregate existing fields only. " \ +# f"There are no fields with name 'rand' in namespace '{namespace}'" +# assert_that(calling(query.must_execute).with_args(), +# raises(Exception, pattern=err_msg), +# "Error wasn't raised while strict mode violated") + + +# TODO must be err (see err_msg) +# def test_query_select_strict_mode_indexes(self, db, namespace, index, items): +# # Given("Create namespace with index and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make select query with strict mode") +# query.strict(StrictMode.Indexes).where("rand", CondType.CondEq, 1) +# err_msg = f"Current query strict mode allows aggregate index fields only. " \ +# f"There are no indexes with name 'rand' in namespace '{namespace}'" +# assert_that(calling(query.must_execute).with_args(), +# raises(Exception, pattern=err_msg), +# "Error wasn't raised while strict mode violated") class TestQuerySelectAggregations: @pytest.mark.parametrize("calculate, aggregate_func", AGGREGATE_FUNCTIONS_MATH) @@ -328,11 +329,11 @@ def test_query_select_distinct(self, db, namespace, index, index_and_duplicate_i def test_query_select_facet(self, db, namespace, index, index_and_duplicate_items): # Given("Create namespace with index and duplicate items") - items = index_and_duplicate_items + #items = index_and_duplicate_items # ToDo unused # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with facet") - query.aggregate_facet(["idx"]) + query.aggregate_facet("idx") query_result = query.must_execute() # Then ("Check that result is empty") assert_that(list(query_result), empty(), "Wrong query results") @@ -345,11 +346,11 @@ def test_query_select_facet(self, db, namespace, index, index_and_duplicate_item def test_query_select_facet_sort_offset_limit(self, db, namespace, index, index_and_duplicate_items): # Given("Create namespace with index and duplicate items") - items = index_and_duplicate_items + #items = index_and_duplicate_items # ToDo unused # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with facet") - query.aggregate_facet(["id", "idx"]).sort("id", True).limit(2).offset(1) + query.aggregate_facet("id", "idx").sort("id", True).limit(2).offset(1) query_result = query.must_execute() # Then ("Check that result is empty") assert_that(list(query_result), empty(), "Wrong query results") @@ -360,7 +361,6 @@ def test_query_select_facet_sort_offset_limit(self, db, namespace, index, index_ facets=contains_inanyorder(*expected_facets))), "Wrong aggregation results") - class TestQuerySelectSort: @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort(self, db, namespace, index, items_shuffled, is_reversed): @@ -373,36 +373,36 @@ def test_query_select_sort(self, db, namespace, index, items_shuffled, is_revers expected_items = sorted(items_shuffled, key=lambda x: x["id"], reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) - # TODO exit code 255 - @pytest.mark.parametrize("forced_values, expected_ids", [ - (4, [4, 0, 1, 2, 3]), - ([1, 3], [1, 3, 0, 2, 4]) - ]) - @pytest.mark.parametrize("is_reversed", [False, True]) - def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, - is_reversed): - # Given("Create namespace with index and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with sort") - query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) - # Then ("Check that selected items are sorted") - expected_items = [items_shuffled[i] for i in expected_ids] - if is_reversed: - expected_items.reverse() - assert_that(query_result, equal_to(expected_items)) - - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - @pytest.mark.parametrize("is_reversed", [False, True]) - def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): - # Given("Create namespace with rtree index and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with sort point distance") - query_result = list(query.sort_stpoint_distance("id", Point(1, 2), is_reversed).must_execute()) - # Then ("Check that selected items are sorted") - expected_items = sorted(rtree_index_and_items, key=lambda x: x["id"], reverse=is_reversed) - assert_that(query_result, equal_to(expected_items)) +# TODO exit code 255 +# @pytest.mark.parametrize("forced_values, expected_ids", [ +# (4, [4, 0, 1, 2, 3]), +# ([1, 3], [1, 3, 0, 2, 4]) +# ]) +# @pytest.mark.parametrize("is_reversed", [False, True]) +# def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, +# is_reversed): +# # Given("Create namespace with index and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make select query with sort") +# query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) +# # Then ("Check that selected items are sorted") +# expected_items = [items_shuffled[i] for i in expected_ids] +# if is_reversed: +# expected_items.reverse() +# assert_that(query_result, equal_to(expected_items)) + +# TODO exit code 134 (interrupted by signal 6:SIGABRT) +# @pytest.mark.parametrize("is_reversed", [False, True]) +# def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): +# # Given("Create namespace with rtree index and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make select query with sort point distance") +# query_result = list(query.sort_stpoint_distance("id", Point(1, 2), is_reversed).must_execute()) +# # Then ("Check that selected items are sorted") +# expected_items = sorted(rtree_index_and_items, key=lambda x: x["id"], reverse=is_reversed) +# assert_that(query_result, equal_to(expected_items)) @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): @@ -423,97 +423,94 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): expected_items = sorted(items, key=lambda i: calculate_distance(i["rtree1"], i["rtree2"]), reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) - -class TestQuerySelectJoin: - # TODO TypeError: 'CondType' object cannot be interpreted as an integer - def test_query_select_left_join(self, db, namespace, index, items, second_namespace): - # Given("Create two namespaces") - second_namespace, item2 = second_namespace - # Given ("Create two queries for join") - query1 = db.query.new(namespace) - query2 = db.query.new(second_namespace) - # When ("Make select query with join") - query_result = list(query1.join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) - # Then ("Check that joined item is in result") - item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} - items[1] = item_with_joined - assert_that(query_result, equal_to(items), "Wrong selected items with JOIN") - - # TODO TypeError: 'CondType' object cannot be interpreted as an integer - def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): - # Given("Create two namespaces") - second_namespace, item2 = second_namespace - # Given ("Create two queries for join") - query1 = db.query.new(namespace) - query2 = db.query.new(second_namespace) - # When ("Make select query with join") - query_result = list(query1.inner_join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) - # Then ("Check that joined item is in result") - item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} - assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") - +# ToDo +#class TestQuerySelectJoin: +# def test_query_select_left_join(self, db, namespace, index, items, second_namespace): +# # Given("Create two namespaces") +# second_namespace, item2 = second_namespace +# # Given ("Create two queries for join") +# query1 = db.query.new(namespace) +# query2 = db.query.new(second_namespace) +# # When ("Make select query with join") +# query_result = list(query1.join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) +# # Then ("Check that joined item is in result") +# item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} +# items[1] = item_with_joined +# assert_that(query_result, equal_to(items), "Wrong selected items with JOIN") + +# def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): +# # Given("Create two namespaces") +# second_namespace, item2 = second_namespace +# # Given ("Create two queries for join") +# query1 = db.query.new(namespace) +# query2 = db.query.new(second_namespace) +# # When ("Make select query with join") +# query_result = list(query1.inner_join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) +# # Then ("Check that joined item is in result") +# item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} +# assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") class TestQueryUpdate: - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - def test_query_update_set(self, db, namespace, indexes, items): - # Given("Create namespace with indexes and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make update set query") - item = random.choice(items) - modified_item = copy.deepcopy(item) - modified_item["val"] = "modified" - query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() - # Then ("Check that item is updated") - assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") - # Then ("Check that items contain modified and do not contain original item") - items_after_update = get_ns_items(db, namespace) - assert_that(items_after_update, has_length(len(items)), "Wrong items count") - assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") - assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") - - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - def test_query_update_set_object(self, db, namespace, indexes): - # Given("Create namespace with index and items") - # Given ("Create nested index") - db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], - "field_type": "int", "index_type": "hash"}) - # Given ("Create items") - items = [{"id": i, "nested": {"field": i}} for i in range(3)] - for item in items: - db.item.insert(namespace, item) - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make update set_object query") - item = random.choice(items) - modified_item = copy.deepcopy(item) - modified_item["nested"]["field"] = 10 - query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() - # Then ("Check that item is updated") - assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") - # Then ("Check that items contain modified and do not contain original item") - items_after_update = get_ns_items(db, namespace) - assert_that(items_after_update, has_length(len(items)), "Wrong items count") - assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") - assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") - - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - def test_query_update_expression(self, db, namespace, indexes, items): - # Given("Create namespace with indexes and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make update expression query") - item = random.choice(items) - modified_item = copy.deepcopy(item) - modified_item["val"] = abs(item["id"] - 20) - query_result = query.where("id", CondType.CondEq, item["id"]).expression("id", "abs(id-20)").update() - # Then ("Check that item is updated") - assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") - # Then ("Check that items contain modified and do not contain original item") - items_after_update = get_ns_items(db, namespace) - assert_that(items_after_update, has_length(len(items)), "Wrong items count") - assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") - assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") +# TODO exit code 134 (interrupted by signal 6:SIGABRT) +# def test_query_update_set(self, db, namespace, indexes, items): +# # Given("Create namespace with indexes and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make update set query") +# item = random.choice(items) +# modified_item = copy.deepcopy(item) +# modified_item["val"] = "modified" +# query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() +# # Then ("Check that item is updated") +# assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") +# # Then ("Check that items contain modified and do not contain original item") +# items_after_update = get_ns_items(db, namespace) +# assert_that(items_after_update, has_length(len(items)), "Wrong items count") +# assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") +# assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + +# TODO exit code 134 (interrupted by signal 6:SIGABRT) +# def test_query_update_set_object(self, db, namespace, indexes): +# # Given("Create namespace with index and items") +# # Given ("Create nested index") +# db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], +# "field_type": "int", "index_type": "hash"}) +# # Given ("Create items") +# items = [{"id": i, "nested": {"field": i}} for i in range(3)] +# for item in items: +# db.item.insert(namespace, item) +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make update set_object query") +# item = random.choice(items) +# modified_item = copy.deepcopy(item) +# modified_item["nested"]["field"] = 10 +# query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() +# # Then ("Check that item is updated") +# assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") +# # Then ("Check that items contain modified and do not contain original item") +# items_after_update = get_ns_items(db, namespace) +# assert_that(items_after_update, has_length(len(items)), "Wrong items count") +# assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") +# assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + +# TODO exit code 134 (interrupted by signal 6:SIGABRT) +# def test_query_update_expression(self, db, namespace, indexes, items): +# # Given("Create namespace with indexes and items") +# # Given ("Create new query") +# query = db.query.new(namespace) +# # When ("Make update expression query") +# item = random.choice(items) +# modified_item = copy.deepcopy(item) +# modified_item["val"] = abs(item["id"] - 20) +# query_result = query.where("id", CondType.CondEq, item["id"]).expression("id", "abs(id-20)").update() +# # Then ("Check that item is updated") +# assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") +# # Then ("Check that items contain modified and do not contain original item") +# items_after_update = get_ns_items(db, namespace) +# assert_that(items_after_update, has_length(len(items)), "Wrong items count") +# assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") +# assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") def test_query_drop(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -543,7 +540,6 @@ def test_query_drop_all(self, db, namespace, indexes, items): items_after_drop = get_ns_items(db, namespace) assert_that(items_after_drop, equal_to(items), "Wrong items after drop") - class TestQueryDelete: def test_query_delete(self, db, namespace, index, items): # Given("Create namespace with index and items") From c9e74da1127edd2df2e08befe988ce633fbd2263 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Tue, 12 Nov 2024 17:41:51 +0300 Subject: [PATCH 079/125] [fix] err for select and update, querytests --- pyreindexer/lib/src/rawpyreindexer.cc | 4 +- pyreindexer/query.py | 12 +- pyreindexer/raiser_mixin.py | 10 + pyreindexer/rx_connector.py | 56 ++---- pyreindexer/tests/tests/test_query.py | 271 +++++++++++++------------- pyreindexer/transaction.py | 35 ++-- 6 files changed, 192 insertions(+), 196 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 3c7f01f..5c2d2d7 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1030,10 +1030,10 @@ static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) Error err = errOK; switch (type) { case ExecuteType::Select: - query->SelectQuery(*qresWrapper); + err = query->SelectQuery(*qresWrapper); break; case ExecuteType::Update: - query->UpdateQuery(*qresWrapper); + err = query->UpdateQuery(*qresWrapper); break; default: assert(false); diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 01eb262..2ff2558 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,7 +1,7 @@ from __future__ import annotations from enum import Enum -from typing import List, Union, Optional +from typing import List, Optional, Union from pyreindexer.point import Point from pyreindexer.query_results import QueryResults @@ -151,14 +151,13 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ params: list = self.__convert_to_list(keys) - if condition == CondType.CondDWithin : + if condition == CondType.CondDWithin: raise Exception("In this case, use a special method 'dwithin'") self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds sub-query where condition to DB query with args @@ -185,21 +184,20 @@ def where_query(self, sub_query: Query, condition: CondType, self.__raise_on_error() return self - def where_composite(self, index: str, condition: CondType, - keys: Union[simple_types, List[simple_types]] = None) -> Query: + def where_composite(self, index: str, condition: CondType, *keys: simple_types) -> Query: """Adds where condition to DB query with interface args for composite indexes #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - sub_query (:obj:`Query`): Field name used in condition clause + keys (*simple_types): Values of composite index to be compared with (value of each sub-index) #### Returns: (:obj:`Query`): Query object for further customizations """ - return self.where(index, condition, keys) + return self.where(index, condition, *keys) def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: """Adds where condition to DB query with UUID as string args. diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index 4918890..045df65 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -27,3 +27,13 @@ def raise_on_not_init(self): if self.rx <= 0: raise Exception("Connection is not initialized") + + +def raise_if_error(func): + def wrapper(self, *args, **kwargs): + self.raise_on_not_init() + res = func(self, *args, **kwargs) + self.raise_on_error() + return res + + return wrapper diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index daa2ba6..6590541 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -2,7 +2,7 @@ from pyreindexer.query import Query from pyreindexer.query_results import QueryResults -from pyreindexer.raiser_mixin import RaiserMixin +from pyreindexer.raiser_mixin import RaiserMixin, raise_if_error from pyreindexer.transaction import Transaction @@ -54,6 +54,7 @@ def close(self) -> None: self._api_close() + @raise_if_error def namespace_open(self, namespace) -> None: """Opens a namespace specified or creates a namespace if it does not exist @@ -66,10 +67,9 @@ def namespace_open(self, namespace) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.namespace_open(self.rx, namespace) - self.raise_on_error() + @raise_if_error def namespace_close(self, namespace) -> None: """Closes a namespace specified @@ -82,10 +82,9 @@ def namespace_close(self, namespace) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.namespace_close(self.rx, namespace) - self.raise_on_error() + @raise_if_error def namespace_drop(self, namespace) -> None: """Drops a namespace specified @@ -98,10 +97,9 @@ def namespace_drop(self, namespace) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.namespace_drop(self.rx, namespace) - self.raise_on_error() + @raise_if_error def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: """Gets a list of namespaces available @@ -118,11 +116,10 @@ def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: """ - self.raise_on_not_init() self.err_code, self.err_msg, res = self.api.namespaces_enum(self.rx, enum_not_opened) - self.raise_on_error() return res + @raise_if_error def index_add(self, namespace, index_def) -> None: """Adds an index to the namespace specified @@ -136,10 +133,9 @@ def index_add(self, namespace, index_def) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.index_add(self.rx, namespace, index_def) - self.raise_on_error() + @raise_if_error def index_update(self, namespace, index_def) -> None: """Updates an index in the namespace specified @@ -153,10 +149,9 @@ def index_update(self, namespace, index_def) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.index_update(self.rx, namespace, index_def) - self.raise_on_error() + @raise_if_error def index_drop(self, namespace, index_name) -> None: """Drops an index from the namespace specified @@ -170,10 +165,9 @@ def index_drop(self, namespace, index_name) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.index_drop(self.rx, namespace, index_name) - self.raise_on_error() + @raise_if_error def item_insert(self, namespace, item_def, precepts=None) -> None: """Inserts an item with its precepts to the namespace specified @@ -189,10 +183,9 @@ def item_insert(self, namespace, item_def, precepts=None) -> None: """ precepts = [] if precepts is None else precepts - self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_insert(self.rx, namespace, item_def, precepts) - self.raise_on_error() + @raise_if_error def item_update(self, namespace, item_def, precepts=None) -> None: """Updates an item with its precepts in the namespace specified @@ -208,10 +201,9 @@ def item_update(self, namespace, item_def, precepts=None) -> None: """ precepts = [] if precepts is None else precepts - self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_update(self.rx, namespace, item_def, precepts) - self.raise_on_error() + @raise_if_error def item_upsert(self, namespace, item_def, precepts=None) -> None: """Updates an item with its precepts in the namespace specified. Creates the item if it not exists @@ -227,10 +219,9 @@ def item_upsert(self, namespace, item_def, precepts=None) -> None: """ precepts = [] if precepts is None else precepts - self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_upsert(self.rx, namespace, item_def, precepts) - self.raise_on_error() + @raise_if_error def item_delete(self, namespace, item_def) -> None: """Deletes an item from the namespace specified @@ -244,10 +235,9 @@ def item_delete(self, namespace, item_def) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.item_delete(self.rx, namespace, item_def) - self.raise_on_error() + @raise_if_error def meta_put(self, namespace, key, value) -> None: """Puts metadata to a storage of Reindexer by key @@ -262,10 +252,9 @@ def meta_put(self, namespace, key, value) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.meta_put(self.rx, namespace, key, value) - self.raise_on_error() + @raise_if_error def meta_get(self, namespace, key) -> str: """Gets metadata from a storage of Reindexer by key specified @@ -282,11 +271,10 @@ def meta_get(self, namespace, key) -> str: """ - self.raise_on_not_init() self.err_code, self.err_msg, res = self.api.meta_get(self.rx, namespace, key) - self.raise_on_error() return res + @raise_if_error def meta_delete(self, namespace, key) -> None: """Deletes metadata from a storage of Reindexer by key specified @@ -300,10 +288,9 @@ def meta_delete(self, namespace, key) -> None: """ - self.raise_on_not_init() self.err_code, self.err_msg = self.api.meta_delete(self.rx, namespace, key) - self.raise_on_error() + @raise_if_error def meta_enum(self, namespace) -> List[str]: """Gets a list of metadata keys from a storage of Reindexer @@ -319,11 +306,10 @@ def meta_enum(self, namespace) -> List[str]: """ - self.raise_on_not_init() self.err_code, self.err_msg, res = self.api.meta_enum(self.rx, namespace) - self.raise_on_error() return res + @raise_if_error def select(self, query: str) -> QueryResults: """Executes an SQL query and returns query results @@ -339,11 +325,10 @@ def select(self, query: str) -> QueryResults: """ - self.raise_on_not_init() self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select(self.rx, query) - self.raise_on_error() return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + @raise_if_error def new_transaction(self, namespace) -> Transaction: """Starts a new transaction and return the transaction object to processing @@ -359,11 +344,10 @@ def new_transaction(self, namespace) -> Transaction: """ - self.raise_on_not_init() self.err_code, self.err_msg, transaction_wrapper_ptr = self.api.new_transaction(self.rx, namespace) - self.raise_on_error() return Transaction(self.api, transaction_wrapper_ptr) + @raise_if_error def new_query(self, namespace: str) -> Query: """Creates a new query and return the query object to processing @@ -378,9 +362,7 @@ def new_query(self, namespace: str) -> Query: """ - self.raise_on_not_init() self.err_code, self.err_msg, query_wrapper_ptr = self.api.create_query(self.rx, namespace) - self.raise_on_error() return Query(self.api, query_wrapper_ptr) def _api_import(self, dsn): diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 9808b6e..8eda45a 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -58,29 +58,29 @@ def test_query_select_get_false(self, db, namespace, index, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to(["", False]), "Wrong query results") -# TODO does not work, ignore subquery results -# def test_query_select_where_query(self, db, namespace, index, items): -# # Given("Create namespace with index and items") -# # Given ("Create new query") -# # query = db.query.new(namespace).where("id", CondType.CondLt, 5) -# # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) -# query = db.query.new(namespace).where("id", CondType.CondGt, 0) -# sub_query = db.query.new(namespace).select(["id"]).where("id", CondType.CondLt, 5) -# # When ("Make select query with where_query subquery") -# query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) -# # Then ("Check that selected item is in result") -# expected_items = [items[i] for i in [2, 3, 4]] -# assert_that(query_result, equal_to(expected_items), "Wrong query results") - -# ToDo -# def test_query_select_where_composite(self, db, namespace, composite_index, items): -# # Given("Create namespace with composite index") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make select query with where_composite") -# query_result = list(query.where_composite("comp_idx", CondType.CondEq, [1, "testval1"]).must_execute()) -# # Then ("Check that selected item is in result") -# assert_that(query_result, equal_to([items[1]]), "Wrong query results") + # TODO does not work, ignore subquery results + # def test_query_select_where_query(self, db, namespace, index, items): + # # Given("Create namespace with index and items") + # # Given ("Create new query") + # # query = db.query.new(namespace).where("id", CondType.CondLt, 5) + # # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) + # query = db.query.new(namespace).where("id", CondType.CondGt, 0) + # sub_query = db.query.new(namespace).select(["id"]).where("id", CondType.CondLt, 5) + # # When ("Make select query with where_query subquery") + # query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) + # # Then ("Check that selected item is in result") + # expected_items = [items[i] for i in [2, 3, 4]] + # assert_that(query_result, equal_to(expected_items), "Wrong query results") + + # ToDo + # def test_query_select_where_composite(self, db, namespace, composite_index, items): + # # Given("Create namespace with composite index") + # # Given ("Create new query") + # query = db.query.new(namespace) + # # When ("Make select query with where_composite") + # query_result = list(query.where_composite("comp_idx", CondType.CondEq, 1, "testval1").must_execute()) + # # Then ("Check that selected item is in result") + # assert_that(query_result, equal_to([items[1]]), "Wrong query results") def test_query_select_where_uuid(self, db, namespace, index): # Given("Create namespace with index") @@ -184,19 +184,19 @@ def test_query_select_limit_offset(self, db, namespace, index, items): expected_items = [items[i] for i in [1, 2, 3]] assert_that(query_result, equal_to(expected_items), "Wrong query results") -# ToDo -# @pytest.mark.skip(reason="need get_total in query_results") -# @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) -# def test_query_select_request_total(self, db, namespace, index, items, func_total): -# # Given("Create namespace with index and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make select query with limit and offset") -# query_result = getattr(query, func_total)("id").must_execute() -# # Then ("Check that selected items are in result") -# assert_that(list(query_result), equal_to(items), "Wrong query results") -# # Then ("Check that total is in result") -# assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") + # ToDo + # @pytest.mark.skip(reason="need get_total in query_results") + # @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) + # def test_query_select_request_total(self, db, namespace, index, items, func_total): + # # Given("Create namespace with index and items") + # # Given ("Create new query") + # query = db.query.new(namespace) + # # When ("Make select query with limit and offset") + # query_result = getattr(query, func_total)("id").must_execute() + # # Then ("Check that selected items are in result") + # assert_that(list(query_result), equal_to(items), "Wrong query results") + # # Then ("Check that total is in result") + # assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): # Given("Create namespace with ft index and items") @@ -267,6 +267,7 @@ def test_query_select_strict_mode_none_and_empty(self, db, namespace, index, ite # Then ("Check that selected item is in result") assert_that(query_result, empty(), "Wrong query results") + # TODO must be err (see err_msg) # def test_query_select_strict_mode_names(self, db, namespace, index, items): # # Given("Create namespace with index and items") @@ -329,7 +330,6 @@ def test_query_select_distinct(self, db, namespace, index, index_and_duplicate_i def test_query_select_facet(self, db, namespace, index, index_and_duplicate_items): # Given("Create namespace with index and duplicate items") - #items = index_and_duplicate_items # ToDo unused # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with facet") @@ -346,7 +346,6 @@ def test_query_select_facet(self, db, namespace, index, index_and_duplicate_item def test_query_select_facet_sort_offset_limit(self, db, namespace, index, index_and_duplicate_items): # Given("Create namespace with index and duplicate items") - #items = index_and_duplicate_items # ToDo unused # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with facet") @@ -361,6 +360,7 @@ def test_query_select_facet_sort_offset_limit(self, db, namespace, index, index_ facets=contains_inanyorder(*expected_facets))), "Wrong aggregation results") + class TestQuerySelectSort: @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort(self, db, namespace, index, items_shuffled, is_reversed): @@ -373,36 +373,37 @@ def test_query_select_sort(self, db, namespace, index, items_shuffled, is_revers expected_items = sorted(items_shuffled, key=lambda x: x["id"], reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) -# TODO exit code 255 -# @pytest.mark.parametrize("forced_values, expected_ids", [ -# (4, [4, 0, 1, 2, 3]), -# ([1, 3], [1, 3, 0, 2, 4]) -# ]) -# @pytest.mark.parametrize("is_reversed", [False, True]) -# def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, -# is_reversed): -# # Given("Create namespace with index and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make select query with sort") -# query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) -# # Then ("Check that selected items are sorted") -# expected_items = [items_shuffled[i] for i in expected_ids] -# if is_reversed: -# expected_items.reverse() -# assert_that(query_result, equal_to(expected_items)) - -# TODO exit code 134 (interrupted by signal 6:SIGABRT) -# @pytest.mark.parametrize("is_reversed", [False, True]) -# def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): -# # Given("Create namespace with rtree index and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make select query with sort point distance") -# query_result = list(query.sort_stpoint_distance("id", Point(1, 2), is_reversed).must_execute()) -# # Then ("Check that selected items are sorted") -# expected_items = sorted(rtree_index_and_items, key=lambda x: x["id"], reverse=is_reversed) -# assert_that(query_result, equal_to(expected_items)) + # TODO exit code 255 + # @pytest.mark.parametrize("forced_values, expected_ids", [ + # (4, [4, 0, 1, 2, 3]), + # ([1, 3], [1, 3, 0, 2, 4]) + # ]) + # @pytest.mark.parametrize("is_reversed", [False, True]) + # def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, + # is_reversed): + # # Given("Create namespace with index and items") + # # Given ("Create new query") + # query = db.query.new(namespace) + # # When ("Make select query with sort") + # query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) + # # Then ("Check that selected items are sorted") + # expected_items = [items_shuffled[i] for i in expected_ids] + # if is_reversed: + # expected_items.reverse() + # assert_that(query_result, equal_to(expected_items)) + + @pytest.mark.parametrize("is_reversed", [False, True]) + def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): + # Given("Create namespace with rtree index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with sort point distance") + point = Point(1.1, 1.5) + query_result = list(query.sort_stpoint_distance("rtree", point, is_reversed).must_execute()) + # Then ("Check that selected items are sorted") + expected_items = sorted(rtree_index_and_items, key=lambda i: calculate_distance(i["rtree"], [point.x, point.y]), + reverse=is_reversed) + assert_that(query_result, equal_to(expected_items)) @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): @@ -423,21 +424,22 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): expected_items = sorted(items, key=lambda i: calculate_distance(i["rtree1"], i["rtree2"]), reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) + # ToDo -#class TestQuerySelectJoin: +# class TestQuerySelectJoin: # def test_query_select_left_join(self, db, namespace, index, items, second_namespace): # # Given("Create two namespaces") # second_namespace, item2 = second_namespace # # Given ("Create two queries for join") -# query1 = db.query.new(namespace) +# query1 = db.query.new(namespace).where("id", CondType.CondLt, 3) # query2 = db.query.new(second_namespace) # # When ("Make select query with join") -# query_result = list(query1.join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) +# query_result = list(query1.join(query2, "joined").on("id", CondType.CondEq, "id").must_execute()) # # Then ("Check that joined item is in result") -# item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} +# item_with_joined = {"id": 1, "joined": [item2]} # items[1] = item_with_joined -# assert_that(query_result, equal_to(items), "Wrong selected items with JOIN") - +# assert_that(query_result, equal_to(items[:3]), "Wrong selected items with JOIN") +# # def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): # # Given("Create two namespaces") # second_namespace, item2 = second_namespace @@ -451,66 +453,66 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): # assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") class TestQueryUpdate: -# TODO exit code 134 (interrupted by signal 6:SIGABRT) -# def test_query_update_set(self, db, namespace, indexes, items): -# # Given("Create namespace with indexes and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make update set query") -# item = random.choice(items) -# modified_item = copy.deepcopy(item) -# modified_item["val"] = "modified" -# query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() -# # Then ("Check that item is updated") -# assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") -# # Then ("Check that items contain modified and do not contain original item") -# items_after_update = get_ns_items(db, namespace) -# assert_that(items_after_update, has_length(len(items)), "Wrong items count") -# assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") -# assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") - -# TODO exit code 134 (interrupted by signal 6:SIGABRT) -# def test_query_update_set_object(self, db, namespace, indexes): -# # Given("Create namespace with index and items") -# # Given ("Create nested index") -# db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], -# "field_type": "int", "index_type": "hash"}) -# # Given ("Create items") -# items = [{"id": i, "nested": {"field": i}} for i in range(3)] -# for item in items: -# db.item.insert(namespace, item) -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make update set_object query") -# item = random.choice(items) -# modified_item = copy.deepcopy(item) -# modified_item["nested"]["field"] = 10 -# query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() -# # Then ("Check that item is updated") -# assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") -# # Then ("Check that items contain modified and do not contain original item") -# items_after_update = get_ns_items(db, namespace) -# assert_that(items_after_update, has_length(len(items)), "Wrong items count") -# assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") -# assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") - -# TODO exit code 134 (interrupted by signal 6:SIGABRT) -# def test_query_update_expression(self, db, namespace, indexes, items): -# # Given("Create namespace with indexes and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make update expression query") -# item = random.choice(items) -# modified_item = copy.deepcopy(item) -# modified_item["val"] = abs(item["id"] - 20) -# query_result = query.where("id", CondType.CondEq, item["id"]).expression("id", "abs(id-20)").update() -# # Then ("Check that item is updated") -# assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") -# # Then ("Check that items contain modified and do not contain original item") -# items_after_update = get_ns_items(db, namespace) -# assert_that(items_after_update, has_length(len(items)), "Wrong items count") -# assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") -# assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + # TODO exit code 134 (interrupted by signal 6:SIGABRT) + # def test_query_update_set(self, db, namespace, indexes, items): + # # Given("Create namespace with indexes and items") + # # Given ("Create new query") + # query = db.query.new(namespace) + # # When ("Make update set query") + # item = random.choice(items) + # modified_item = copy.deepcopy(item) + # modified_item["val"] = "modified" + # query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() + # # Then ("Check that item is updated") + # assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") + # # Then ("Check that items contain modified and do not contain original item") + # items_after_update = get_ns_items(db, namespace) + # assert_that(items_after_update, has_length(len(items)), "Wrong items count") + # assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + # assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + + # TODO exit code 134 (interrupted by signal 6:SIGABRT) + # def test_query_update_set_object(self, db, namespace, indexes): + # # Given("Create namespace with index and items") + # # Given ("Create nested index") + # db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], + # "field_type": "int", "index_type": "hash"}) + # # Given ("Create items") + # items = [{"id": i, "nested": {"field": i}} for i in range(3)] + # for item in items: + # db.item.insert(namespace, item) + # # Given ("Create new query") + # query = db.query.new(namespace) + # # When ("Make update set_object query") + # item = random.choice(items) + # modified_item = copy.deepcopy(item) + # modified_item["nested"]["field"] = 10 + # query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() + # # Then ("Check that item is updated") + # assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") + # # Then ("Check that items contain modified and do not contain original item") + # items_after_update = get_ns_items(db, namespace) + # assert_that(items_after_update, has_length(len(items)), "Wrong items count") + # assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + # assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + + # TODO exit code 134 (interrupted by signal 6:SIGABRT) + # def test_query_update_expression(self, db, namespace, indexes, items): + # # Given("Create namespace with indexes and items") + # # Given ("Create new query") + # query = db.query.new(namespace) + # # When ("Make update expression query") + # item = random.choice(items) + # modified_item = copy.deepcopy(item) + # modified_item["val"] = abs(item["id"] - 20) + # query_result = query.where("id", CondType.CondEq, item["id"]).expression("id", "abs(id-20)").update() + # # Then ("Check that item is updated") + # assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") + # # Then ("Check that items contain modified and do not contain original item") + # items_after_update = get_ns_items(db, namespace) + # assert_that(items_after_update, has_length(len(items)), "Wrong items count") + # assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + # assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") def test_query_drop(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -540,6 +542,7 @@ def test_query_drop_all(self, db, namespace, indexes, items): items_after_drop = get_ns_items(db, namespace) assert_that(items_after_drop, equal_to(items), "Wrong items after drop") + class TestQueryDelete: def test_query_delete(self, db, namespace, index, items): # Given("Create namespace with index and items") diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 5053bcf..618d764 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,3 +1,13 @@ +def raise_if_error(func): + def wrapper(self, *args, **kwargs): + self._raise_on_is_over() + res = func(self, *args, **kwargs) + self._raise_on_error() + return res + + return wrapper + + class Transaction: """An object representing the context of a Reindexer transaction @@ -31,7 +41,7 @@ def __del__(self): if self.transaction_wrapper_ptr > 0: _, _ = self.api.rollback_transaction(self.transaction_wrapper_ptr) - def __raise_on_error(self): + def _raise_on_error(self): """Checks if there is an error code and raises with an error message #### Raises: @@ -42,7 +52,7 @@ def __raise_on_error(self): if self.err_code: raise Exception(self.err_msg) - def __raise_on_is_over(self): + def _raise_on_is_over(self): """Checks the state of a transaction and returns an error message when necessary #### Raises: @@ -53,6 +63,7 @@ def __raise_on_is_over(self): if self.transaction_wrapper_ptr <= 0: raise Exception("Transaction is over") + @raise_if_error def insert(self, item_def, precepts=None): """Inserts an item with its precepts to the transaction @@ -66,11 +77,10 @@ def insert(self, item_def, precepts=None): """ - self.__raise_on_is_over() precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_insert_transaction(self.transaction_wrapper_ptr, item_def, precepts) - self.__raise_on_error() + @raise_if_error def update(self, item_def, precepts=None): """Updates an item with its precepts to the transaction @@ -84,11 +94,10 @@ def update(self, item_def, precepts=None): """ - self.__raise_on_is_over() precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_update_transaction(self.transaction_wrapper_ptr, item_def, precepts) - self.__raise_on_error() + @raise_if_error def upsert(self, item_def, precepts=None): """Updates an item with its precepts to the transaction. Creates the item if it not exists @@ -102,11 +111,10 @@ def upsert(self, item_def, precepts=None): """ - self.__raise_on_is_over() precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_upsert_transaction(self.transaction_wrapper_ptr, item_def, precepts) - self.__raise_on_error() + @raise_if_error def delete(self, item_def): """Deletes an item from the transaction @@ -119,10 +127,9 @@ def delete(self, item_def): """ - self.__raise_on_is_over() self.err_code, self.err_msg = self.api.item_delete_transaction(self.transaction_wrapper_ptr, item_def) - self.__raise_on_error() + @raise_if_error def commit(self): """Applies changes @@ -132,11 +139,10 @@ def commit(self): """ - self.__raise_on_is_over() self.err_code, self.err_msg, _ = self.api.commit_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 - self.__raise_on_error() + @raise_if_error def commit_with_count(self) -> int: """Applies changes and return the number of count of changed items @@ -146,12 +152,11 @@ def commit_with_count(self) -> int: """ - self.__raise_on_is_over() self.err_code, self.err_msg, count = self.api.commit_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 - self.__raise_on_error() return count + @raise_if_error def rollback(self): """Rollbacks changes @@ -161,7 +166,5 @@ def rollback(self): """ - self.__raise_on_is_over() self.err_code, self.err_msg = self.api.rollback_transaction(self.transaction_wrapper_ptr) self.transaction_wrapper_ptr = 0 - self.__raise_on_error() From 835b7dcb389452ede5b40f2866291894f09015b8 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Tue, 12 Nov 2024 19:28:47 +0300 Subject: [PATCH 080/125] [fix] more query test fixes --- pyreindexer/tests/conftest.py | 3 +- pyreindexer/tests/tests/test_query.py | 88 ++++++++++++++++----------- 2 files changed, 55 insertions(+), 36 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 059c8f5..69af770 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -57,8 +57,7 @@ def indexes(db, namespace): Create two indexes to namespace """ db.index.create(namespace, index_definition) - db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash", - "is_sparse": True}) + db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash"}) yield diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 8eda45a..f60310a 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -1,4 +1,3 @@ -import copy import json import random @@ -59,13 +58,13 @@ def test_query_select_get_false(self, db, namespace, index, items): assert_that(query_result, equal_to(["", False]), "Wrong query results") # TODO does not work, ignore subquery results - # def test_query_select_where_query(self, db, namespace, index, items): + # def test_query_select_where_query(self, db, namespace, index, items): # # Given("Create namespace with index and items") # # Given ("Create new query") # # query = db.query.new(namespace).where("id", CondType.CondLt, 5) # # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) # query = db.query.new(namespace).where("id", CondType.CondGt, 0) - # sub_query = db.query.new(namespace).select(["id"]).where("id", CondType.CondLt, 5) + # sub_query = db.query.new(namespace).select("id").where("id", CondType.CondLt, 5) # # When ("Make select query with where_query subquery") # query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) # # Then ("Check that selected item is in result") @@ -257,43 +256,39 @@ def test_query_select_debug(self, db, namespace, index, items, debug_level): # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") - @pytest.mark.parametrize("strict_mode", [StrictMode.NotSet, StrictMode.Empty]) - def test_query_select_strict_mode_none_and_empty(self, db, namespace, index, items, strict_mode): + def test_query_select_strict_mode_empty(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with strict mode") - query_result = list(query.strict(strict_mode).where("rand", CondType.CondEq, 1).must_execute()) + query_result = list(query.strict(StrictMode.Empty).where("rand", CondType.CondEq, 1).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, empty(), "Wrong query results") + @pytest.mark.parametrize("strict_mode", [StrictMode.NotSet, StrictMode.Names]) + def test_query_select_strict_mode_names_default(self, db, namespace, index, items, strict_mode): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with strict mode") + query.strict(strict_mode).where("rand", CondType.CondEq, 1) + err_msg = "Current query strict mode allows filtering by existing fields only. " \ + f"There are no fields with name 'rand' in namespace '{namespace}'" + assert_that(calling(query.execute).with_args(), + raises(Exception, pattern=err_msg)) + + def test_query_select_strict_mode_indexes(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with strict mode") + query.strict(StrictMode.Indexes).where("rand", CondType.CondEq, 1) + err_msg = "Current query strict mode allows filtering by indexes only. " \ + f"There are no indexes with name 'rand' in namespace '{namespace}'" + assert_that(calling(query.execute).with_args(), + raises(Exception, pattern=err_msg), + "Error wasn't raised while strict mode violated") -# TODO must be err (see err_msg) -# def test_query_select_strict_mode_names(self, db, namespace, index, items): -# # Given("Create namespace with index and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make select query with strict mode") -# query.strict(StrictMode.Names).where("rand", CondType.CondEq, 1) -# err_msg = f"Current query strict mode allows aggregate existing fields only. " \ -# f"There are no fields with name 'rand' in namespace '{namespace}'" -# assert_that(calling(query.must_execute).with_args(), -# raises(Exception, pattern=err_msg), -# "Error wasn't raised while strict mode violated") - - -# TODO must be err (see err_msg) -# def test_query_select_strict_mode_indexes(self, db, namespace, index, items): -# # Given("Create namespace with index and items") -# # Given ("Create new query") -# query = db.query.new(namespace) -# # When ("Make select query with strict mode") -# query.strict(StrictMode.Indexes).where("rand", CondType.CondEq, 1) -# err_msg = f"Current query strict mode allows aggregate index fields only. " \ -# f"There are no indexes with name 'rand' in namespace '{namespace}'" -# assert_that(calling(query.must_execute).with_args(), -# raises(Exception, pattern=err_msg), -# "Error wasn't raised while strict mode violated") class TestQuerySelectAggregations: @pytest.mark.parametrize("calculate, aggregate_func", AGGREGATE_FUNCTIONS_MATH) @@ -528,8 +523,11 @@ def test_query_drop(self, db, namespace, index, items): items_after_drop = get_ns_items(db, namespace) assert_that(items_after_drop, equal_to(items), "Wrong items after drop") - def test_query_drop_all(self, db, namespace, indexes, items): - # Given("Create namespace with two indexes and items") + def test_query_drop_all(self, db, namespace, index, items): + # Given("Create namespace with index") + # Given("Create sparse index") + db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash", + "is_sparse": True}) # Given ("Create new query") query = db.query.new(namespace) # When ("Make update drop query") @@ -542,6 +540,16 @@ def test_query_drop_all(self, db, namespace, indexes, items): items_after_drop = get_ns_items(db, namespace) assert_that(items_after_drop, equal_to(items), "Wrong items after drop") + def test_cannot_query_drop_not_sparse(self, db, namespace, indexes, items): + # Given("Create namespace with indexs and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Try to make update drop query with not sparse index") + item = random.choice(items) + assert_that(calling(query.drop("val").update).with_args(), + raises(Exception, + pattern="It's only possible to drop sparse or non-index fields via UPDATE statement!")) + class TestQueryDelete: def test_query_delete(self, db, namespace, index, items): @@ -556,3 +564,15 @@ def test_query_delete(self, db, namespace, index, items): items_after_delete = get_ns_items(db, namespace) assert_that(items_after_delete, has_length(len(items) - 1), "Wrong items count after delete") assert_that(items_after_delete, not_(has_item(item)), "Deleted item is in namespace") + + def test_query_delete_all(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make delete query") + item = random.choice(items) + query_result = query.delete() + # Then ("Check that chosen item was deleted") + assert_that(query_result, equal_to(10), "Wrong delete items count") + items_after_delete = get_ns_items(db, namespace) + assert_that(items_after_delete, empty(), "Wrong items count after delete") From f4946b594985d0475a7c69bd706669827aab7be1 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 13 Nov 2024 10:41:47 +0300 Subject: [PATCH 081/125] [fix] fix forced_sort test --- pyreindexer/tests/tests/test_query.py | 35 +++++++++++++-------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index f60310a..df1eab1 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -368,24 +368,23 @@ def test_query_select_sort(self, db, namespace, index, items_shuffled, is_revers expected_items = sorted(items_shuffled, key=lambda x: x["id"], reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) - # TODO exit code 255 - # @pytest.mark.parametrize("forced_values, expected_ids", [ - # (4, [4, 0, 1, 2, 3]), - # ([1, 3], [1, 3, 0, 2, 4]) - # ]) - # @pytest.mark.parametrize("is_reversed", [False, True]) - # def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, - # is_reversed): - # # Given("Create namespace with index and items") - # # Given ("Create new query") - # query = db.query.new(namespace) - # # When ("Make select query with sort") - # query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) - # # Then ("Check that selected items are sorted") - # expected_items = [items_shuffled[i] for i in expected_ids] - # if is_reversed: - # expected_items.reverse() - # assert_that(query_result, equal_to(expected_items)) + @pytest.mark.parametrize("forced_values, expected_ids", [ + (4, [4, 0, 1, 2, 3]), + ([1, 3], [1, 3, 0, 2, 4]) + ]) + @pytest.mark.parametrize("is_reversed", [False, True]) + def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, forced_values, expected_ids, + is_reversed): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with sort") + query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) + # Then ("Check that selected items are sorted") + expected_items = [item for i in expected_ids for item in items_shuffled if item["id"] == i] + if is_reversed: + expected_items.reverse() + assert_that(query_result, equal_to(expected_items)) @pytest.mark.parametrize("is_reversed", [False, True]) def test_query_select_sort_stpoint(self, db, namespace, index, rtree_index_and_items, is_reversed): From 49eb82ea56cc9644dfdb92dfd22e29d504b222b2 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 13 Nov 2024 17:07:41 +0300 Subject: [PATCH 082/125] Part XXV: Bug fixes. Part III: set & set_object --- pyreindexer/lib/include/query_wrapper.cc | 31 +++++---- pyreindexer/lib/include/query_wrapper.h | 3 +- pyreindexer/lib/src/rawpyreindexer.cc | 39 ++++++++--- pyreindexer/query.py | 5 +- pyreindexer/tests/tests/test_query.py | 86 ++++++++++++------------ 5 files changed, 97 insertions(+), 67 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 8c22326..6c9ac81 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -261,21 +261,27 @@ reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { return db_->UpdateQuery(query, qr); } -void QueryWrapper::SetObject(std::string_view field, const std::vector& values, QueryItemType type) { - ser_.PutVarUint(type); +void QueryWrapper::SetObject(std::string_view field, const std::vector& values) { + ser_.PutVarUint(QueryItemType::QueryUpdateObject); ser_.PutVString(field); - if (type == QueryItemType::QueryUpdateObject) { - ser_.PutVarUint(values.size()); // values count - } - if (type != QueryItemType::QueryUpdateField) { - ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag - } - if (type != QueryItemType::QueryUpdateObject) { - ser_.PutVarUint(values.size()); // values count - } + ser_.PutVarUint(values.size()); // values count + ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag for (const auto& value : values) { ser_.PutVarUint(0); // function/value flag + ser_.PutVarUint(TagType::TAG_STRING); // type ID ser_.PutVString(value); + break; + } +} + +void QueryWrapper::Set(std::string_view field, const std::vector& values) { + ser_.PutVarUint(QueryItemType::QueryUpdateFieldV2); + ser_.PutVString(field); + ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag + ser_.PutVarUint(values.size()); // values count + for (const auto& value : values) { + ser_.PutVarUint(0); // is expression + ser_.PutVariant(value); } } @@ -287,8 +293,7 @@ void QueryWrapper::Drop(std::string_view field) { void QueryWrapper::SetExpression(std::string_view field, std::string_view value) { ser_.PutVarUint(QueryItemType::QueryUpdateField); ser_.PutVString(field); - - ser_.PutVarUint(1); // size + ser_.PutVarUint(1); // values count ser_.PutVarUint(1); // is expression ser_.PutVariant(reindexer::Variant{value}); } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 01a860c..1df1537 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -58,7 +58,8 @@ class QueryWrapper { reindexer::Error DeleteQuery(size_t& count); reindexer::Error UpdateQuery(QueryResultsWrapper& qr); - void SetObject(std::string_view field, const std::vector& values, QueryItemType type); + void SetObject(std::string_view field, const std::vector& values); + void Set(std::string_view field, const std::vector& values); void Drop(std::string_view field); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 5c2d2d7..a1e56de 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1051,8 +1051,7 @@ static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) static PyObject* SelectQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Select); } static PyObject* UpdateQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Update); } -namespace { -static PyObject* setObject(PyObject* self, PyObject* args, QueryItemType type) { +static PyObject* SetObject(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* field = nullptr; PyObject* valuesList = nullptr; // borrowed ref after ParseTuple @@ -1075,18 +1074,42 @@ static PyObject* setObject(PyObject* self, PyObject* args, QueryItemType type) { Py_DECREF(valuesList); - if ((type == QueryItemType::QueryUpdateField) && (values.size() > 1)) { - type = QueryItemType::QueryUpdateFieldV2; + auto query = getWrapper(queryWrapperAddr); + + query->SetObject(field, values); + + return pyErr(errOK); +} + +static PyObject* Set(PyObject* self, PyObject* args) { + uintptr_t queryWrapperAddr = 0; + char* field = nullptr; + PyObject* valuesList = nullptr; // borrowed ref after ParseTuple + if (!PyArg_ParseTuple(args, "ksO!", &queryWrapperAddr, &field, &PyList_Type, &valuesList)) { + return nullptr; } + + Py_INCREF(valuesList); + + std::vector values; + if (valuesList != nullptr) { + try { + values = ParseListToVec(&valuesList); + } catch (const Error& err) { + Py_DECREF(valuesList); + + return pyErr(err); + } + } + + Py_DECREF(valuesList); + auto query = getWrapper(queryWrapperAddr); - query->SetObject(field, values, type); + query->Set(field, values); return pyErr(errOK); } -} // namespace -static PyObject* SetObject(PyObject* self, PyObject* args) { return setObject(self, args, QueryItemType::QueryUpdateObject); } -static PyObject* Set(PyObject* self, PyObject* args) { return setObject(self, args, QueryItemType::QueryUpdateField); } static PyObject* Drop(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 2ff2558..f580da3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -197,7 +197,8 @@ def where_composite(self, index: str, condition: CondType, *keys: simple_types) """ - return self.where(index, condition, *keys) + param = [key for key in keys] + return self.where(index, condition, param) def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: """Adds where condition to DB query with UUID as string args. @@ -498,7 +499,7 @@ def sort_stpoint_distance(self, index: str, point: Point, desc: bool) -> Query: """ - request: str = f"ST_Distance({index},ST_GeomFromText('point({point.x:.10f} {point.y:.10f})'))" + request: str = f"ST_Distance({index},ST_GeomFromText('point({point.x:.12f} {point.y:.12f})'))" return self.sort(request, desc) def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) -> Query: diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index df1eab1..2395068 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -1,3 +1,4 @@ +import copy import json import random @@ -419,7 +420,7 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): assert_that(query_result, equal_to(expected_items)) -# ToDo +# ToDo implement List joinedActors = QueryResult.getJoinedActors(); # class TestQuerySelectJoin: # def test_query_select_left_join(self, db, namespace, index, items, second_namespace): # # Given("Create two namespaces") @@ -446,49 +447,48 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): # item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} # assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") -class TestQueryUpdate: - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - # def test_query_update_set(self, db, namespace, indexes, items): - # # Given("Create namespace with indexes and items") - # # Given ("Create new query") - # query = db.query.new(namespace) - # # When ("Make update set query") - # item = random.choice(items) - # modified_item = copy.deepcopy(item) - # modified_item["val"] = "modified" - # query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() - # # Then ("Check that item is updated") - # assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") - # # Then ("Check that items contain modified and do not contain original item") - # items_after_update = get_ns_items(db, namespace) - # assert_that(items_after_update, has_length(len(items)), "Wrong items count") - # assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") - # assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - # def test_query_update_set_object(self, db, namespace, indexes): - # # Given("Create namespace with index and items") - # # Given ("Create nested index") - # db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], - # "field_type": "int", "index_type": "hash"}) - # # Given ("Create items") - # items = [{"id": i, "nested": {"field": i}} for i in range(3)] - # for item in items: - # db.item.insert(namespace, item) - # # Given ("Create new query") - # query = db.query.new(namespace) - # # When ("Make update set_object query") - # item = random.choice(items) - # modified_item = copy.deepcopy(item) - # modified_item["nested"]["field"] = 10 - # query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() - # # Then ("Check that item is updated") - # assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") - # # Then ("Check that items contain modified and do not contain original item") - # items_after_update = get_ns_items(db, namespace) - # assert_that(items_after_update, has_length(len(items)), "Wrong items count") - # assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") - # assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") +class TestQueryUpdate: + def test_query_update_set(self, db, namespace, indexes, items): + # Given("Create namespace with indexes and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update set query") + item = random.choice(items) + modified_item = copy.deepcopy(item) + modified_item["val"] = "modified" + query_result = query.where("id", CondType.CondEq, item["id"]).set("val", ["modified"]).update() + # Then ("Check that item is updated") + assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") + # Then ("Check that items contain modified and do not contain original item") + items_after_update = get_ns_items(db, namespace) + assert_that(items_after_update, has_length(len(items)), "Wrong items count") + assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + + def test_query_update_set_object(self, db, namespace, indexes): + # Given("Create namespace with index and items") + # Given ("Create nested index") + db.index.create(namespace, {"name": "idx", "json_paths": ["nested.field"], + "field_type": "int", "index_type": "hash"}) + # Given ("Create items") + items = [{"id": i, "nested": {"field": i}} for i in range(3)] + for item in items: + db.item.insert(namespace, item) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update set_object query") + item = random.choice(items) + modified_item = copy.deepcopy(item) + modified_item["nested"]["field"] = 10 + query_result = query.where("id", CondType.CondEq, item["id"]).set_object("nested", [{"field": 10}]).update() + # Then ("Check that item is updated") + assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set object") + # Then ("Check that items contain modified and do not contain original item") + items_after_update = get_ns_items(db, namespace) + assert_that(items_after_update, has_length(len(items)), "Wrong items count") + assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") # TODO exit code 134 (interrupted by signal 6:SIGABRT) # def test_query_update_expression(self, db, namespace, indexes, items): From 60d071a05b1586961e2088fc7478bef2dd071029 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 13 Nov 2024 18:02:56 +0300 Subject: [PATCH 083/125] add more query tests --- pyreindexer/tests/conftest.py | 23 ++++ pyreindexer/tests/tests/test_query.py | 132 +++++++++++++++----- pyreindexer/tests/tests/test_transaction.py | 4 +- 3 files changed, 129 insertions(+), 30 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 69af770..665d3fc 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -51,6 +51,16 @@ def index(db, namespace): yield +@pytest.fixture +def sparse_index(db, namespace): + """ + Create sparse index to namespace + """ + db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash", + "is_sparse": True}) + yield + + @pytest.fixture def indexes(db, namespace): """ @@ -130,6 +140,19 @@ def ft_index_and_items(db, namespace): yield items +@pytest.fixture +def array_index_and_items(db, namespace): + """ + Create rtree index and items + """ + db.index.create(namespace, {"name": "arr", "json_paths": ["arr"], "field_type": "int", "index_type": "tree", + "is_array": True}) + items = [{"id": i, "arr": [i, i + 1]} for i in range(5)] + for item in items: + db.item.insert(namespace, item) + yield items + + @pytest.fixture def index_and_duplicate_items(db, namespace): """ diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 2395068..6d91d62 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -21,6 +21,16 @@ def test_query_select_where(self, db, namespace, index, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[3]]), "Wrong query results") + def test_query_select_where_array(self, db, namespace, index, array_index_and_items): + # Given("Create namespace with array index and items") + items = array_index_and_items + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query") + query_result = list(query.where("arr", CondType.CondEq, [3]).must_execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[2], items[3]]), "Wrong query results") + def test_query_select_all(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") @@ -58,6 +68,37 @@ def test_query_select_get_false(self, db, namespace, index, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to(["", False]), "Wrong query results") + def test_query_select_where_cond_any(self, db, namespace, index, sparse_index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with cond any") + query_result = list(query.where("val", CondType.CondAny, None).must_execute()) + # Then ("Check that all items is in result") + assert_that(query_result, equal_to(items), "Wrong query results") + + # TODO Expected: <[{'id': -1}]> but: was <[{'id': 18446744073709551615}]> ?? + def test_query_select_where_cond_empty(self, db, namespace, index, sparse_index, items): + # Given("Create namespace with index and items") + # Given ("Create item without sparse field") + item_empty = {"id": -1} + db.item.insert(namespace, item_empty) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with cond empty") + query_result = list(query.where("val", CondType.CondEmpty, None).must_execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([item_empty]), "Wrong query results") + + def test_query_select_where_cond_like(self, db, namespace, indexes, items): + # Given("Create namespace with indexes and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with cond like") + query_result = list(query.where("val", CondType.CondLike, "test%").must_execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to(items), "Wrong query results") + # TODO does not work, ignore subquery results # def test_query_select_where_query(self, db, namespace, index, items): # # Given("Create namespace with index and items") @@ -141,9 +182,9 @@ def test_query_select_match(self, db, namespace, index, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with match") - query_result = list(query.match("val", "testval1").must_execute()) + query_result = list(query.match("val", "testval1", "testval2").must_execute()) # Then ("Check that selected item is in result") - assert_that(query_result, equal_to([items[1]]), "Wrong query results") + assert_that(query_result, equal_to([items[1], items[2]]), "Wrong query results") # Given ("Create new query") query = db.query.new(namespace) @@ -215,8 +256,8 @@ def test_query_select_functions(self, db, namespace, index, ft_index_and_items): query = db.query.new(namespace) # When ("Make select query") query.where("ft", CondType.CondEq, "word~") - query_result = list(query.functions("ft=highlight(<,>)").must_execute()) - # Then ("Check that selected item is in result and highlighted") + query_result = list(query.functions("ft=highlight(<,>)", "ft=highlight(!!,!)").must_execute()) + # Then ("Check that selected item is in result and highlighted (the first function is applied)") query_results_ft = [i["ft"] for i in query_result] expected_ft_content = ["one ", " two", "three 333"] assert_that(query_results_ft, contains_inanyorder(*expected_ft_content), "Wrong query results") @@ -287,8 +328,26 @@ def test_query_select_strict_mode_indexes(self, db, namespace, index, items): err_msg = "Current query strict mode allows filtering by indexes only. " \ f"There are no indexes with name 'rand' in namespace '{namespace}'" assert_that(calling(query.execute).with_args(), - raises(Exception, pattern=err_msg), - "Error wasn't raised while strict mode violated") + raises(Exception, pattern=err_msg)) + + # TODO "Exception: Query results contain WAL items. Query results from WAL must either be requested in JSON format or with client, supporting RAW items" + # (ExecToJson in GO) + def test_query_select_wal_gt(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with lsn gt") + query_result = query.where("#lsn", CondType.CondGt, 1).must_execute() # .execute_to_json() ?? + # TODO add lsn validation + + # TODO + def test_query_select_wal_any(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with lsn any") + query_result = query.where("#lsn", CondType.CondAny, None).must_execute() # .execute_to_json() ?? + # TODO add lsn validation class TestQuerySelectAggregations: @@ -369,6 +428,16 @@ def test_query_select_sort(self, db, namespace, index, items_shuffled, is_revers expected_items = sorted(items_shuffled, key=lambda x: x["id"], reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) + def test_query_select_sort_expression(self, db, namespace, index, items_shuffled): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with expression sort") + query_result = list(query.sort("abs(-2 * id) + 1.5").must_execute()) + # Then ("Check that items are sorted") + expected_items = sorted(items_shuffled, key=lambda x: x["id"]) + assert_that(query_result, equal_to(expected_items)) + @pytest.mark.parametrize("forced_values, expected_ids", [ (4, [4, 0, 1, 2, 3]), ([1, 3], [1, 3, 0, 2, 4]) @@ -379,7 +448,7 @@ def test_query_select_forced_sort(self, db, namespace, index, items_shuffled, fo # Given("Create namespace with index and items") # Given ("Create new query") query = db.query.new(namespace) - # When ("Make select query with sort") + # When ("Make select query with forced sort") query_result = list(query.sort("id", is_reversed, forced_values).must_execute()) # Then ("Check that selected items are sorted") expected_items = [item for i in expected_ids for item in items_shuffled if item["id"] == i] @@ -419,6 +488,17 @@ def test_query_select_sort_stfield(self, db, namespace, index, is_reversed): expected_items = sorted(items, key=lambda i: calculate_distance(i["rtree1"], i["rtree2"]), reverse=is_reversed) assert_that(query_result, equal_to(expected_items)) + def test_query_select_sort_strict_mode_indexes(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Try to select query with strict and sort non-existent index") + query.strict(StrictMode.Indexes).sort("rand") + err_msg = "Current query strict mode allows sort by index fields only. " \ + f"There are no indexes with name 'rand' in namespace '{namespace}'" + assert_that(calling(query.execute).with_args(), + raises(Exception, pattern=err_msg)) + # ToDo implement List joinedActors = QueryResult.getJoinedActors(); # class TestQuerySelectJoin: @@ -490,23 +570,22 @@ def test_query_update_set_object(self, db, namespace, indexes): assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") - # TODO exit code 134 (interrupted by signal 6:SIGABRT) - # def test_query_update_expression(self, db, namespace, indexes, items): - # # Given("Create namespace with indexes and items") - # # Given ("Create new query") - # query = db.query.new(namespace) - # # When ("Make update expression query") - # item = random.choice(items) - # modified_item = copy.deepcopy(item) - # modified_item["val"] = abs(item["id"] - 20) - # query_result = query.where("id", CondType.CondEq, item["id"]).expression("id", "abs(id-20)").update() - # # Then ("Check that item is updated") - # assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") - # # Then ("Check that items contain modified and do not contain original item") - # items_after_update = get_ns_items(db, namespace) - # assert_that(items_after_update, has_length(len(items)), "Wrong items count") - # assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") - # assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") + def test_query_update_expression(self, db, namespace, index, items): + # Given("Create namespace with indexes and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make update expression query") + item = random.choice(items) + modified_item = copy.deepcopy(item) + modified_item["field"] = item["id"] + 0.5 + query_result = query.where("id", CondType.CondEq, item["id"]).expression("field", "id + 0.5*serial()").update() + # Then ("Check that item is updated") + assert_that(list(query_result), equal_to([modified_item]), "Wrong update query results after set") + # Then ("Check that items contain modified and do not contain original item") + items_after_update = get_ns_items(db, namespace) + assert_that(items_after_update, has_length(len(items)), "Wrong items count") + assert_that(items_after_update, has_item(modified_item), "New updated item not is in namespace") + assert_that(items_after_update, not_(has_item(item)), "Old updated item is in namespace") def test_query_drop(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -522,11 +601,8 @@ def test_query_drop(self, db, namespace, index, items): items_after_drop = get_ns_items(db, namespace) assert_that(items_after_drop, equal_to(items), "Wrong items after drop") - def test_query_drop_all(self, db, namespace, index, items): + def test_query_drop_all(self, db, namespace, index, sparse_index, items): # Given("Create namespace with index") - # Given("Create sparse index") - db.index.create(namespace, {"name": "val", "json_paths": ["val"], "field_type": "string", "index_type": "hash", - "is_sparse": True}) # Given ("Create new query") query = db.query.new(namespace) # When ("Make update drop query") diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index 12ad025..1eea358 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -174,8 +174,8 @@ def test_rollback_transaction(self, db, namespace, index): # When ("Insert items into namespace") transaction = db.tx.begin(namespace) number_items = 5 - for _ in range(number_items): - transaction.insert_item({"id": 100, "field": "value"}) + for i in range(number_items): + transaction.insert_item({"id": i, "field": "value"}) # Then ("Rollback transaction") transaction.rollback() # When ("Get namespace information") From b0913906e899242a3cd5542e17479eb64d3af40c Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 13 Nov 2024 18:31:23 +0300 Subject: [PATCH 084/125] Simplify code --- pyreindexer/lib/include/query_wrapper.cc | 48 ++++++++--------- pyreindexer/lib/include/query_wrapper.h | 9 ++-- pyreindexer/lib/src/rawpyreindexer.cc | 68 ++++++++++-------------- pyreindexer/query.py | 3 +- 4 files changed, 58 insertions(+), 70 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 6c9ac81..f959452 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -246,9 +246,21 @@ reindexer::Query QueryWrapper::prepareQuery() { return query; } -reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { +reindexer::Error QueryWrapper::ExecuteQuery(ExecuteType type, QueryResultsWrapper& qr) { auto query = prepareQuery(); - return db_->SelectQuery(query, qr); + + Error err = errOK; + switch (type) { + case ExecuteType::Select: + err = db_->SelectQuery(query, qr); + break; + case ExecuteType::Update: + err = db_->UpdateQuery(query, qr); + break; + default: + assert(false); + } + return err; } reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { @@ -256,9 +268,16 @@ reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { return db_->DeleteQuery(query, count); } -reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { - auto query = prepareQuery(); - return db_->UpdateQuery(query, qr); +void QueryWrapper::Set(std::string_view field, const std::vector& values, + IsExpression isExpression) { + ser_.PutVarUint(QueryItemType::QueryUpdateFieldV2); + ser_.PutVString(field); + ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag + ser_.PutVarUint(values.size()); // values count + for (const auto& value : values) { + ser_.PutVarUint(isExpression == IsExpression::Yes? 1 : 0); // is expression + ser_.PutVariant(value); + } } void QueryWrapper::SetObject(std::string_view field, const std::vector& values) { @@ -274,30 +293,11 @@ void QueryWrapper::SetObject(std::string_view field, const std::vector& values) { - ser_.PutVarUint(QueryItemType::QueryUpdateFieldV2); - ser_.PutVString(field); - ser_.PutVarUint(values.size() > 1? 1 : 0); // is array flag - ser_.PutVarUint(values.size()); // values count - for (const auto& value : values) { - ser_.PutVarUint(0); // is expression - ser_.PutVariant(value); - } -} - void QueryWrapper::Drop(std::string_view field) { ser_.PutVarUint(QueryItemType::QueryDropField); ser_.PutVString(field); } -void QueryWrapper::SetExpression(std::string_view field, std::string_view value) { - ser_.PutVarUint(QueryItemType::QueryUpdateField); - ser_.PutVString(field); - ser_.PutVarUint(1); // values count - ser_.PutVarUint(1); // is expression - ser_.PutVariant(reindexer::Variant{value}); -} - void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery) { assert(joinQuery); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 1df1537..362fc38 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -54,17 +54,16 @@ class QueryWrapper { void Modifier(QueryItemType type); - reindexer::Error SelectQuery(QueryResultsWrapper& qr); + enum class ExecuteType { Select, Update }; + reindexer::Error ExecuteQuery(ExecuteType type, QueryResultsWrapper& qr); reindexer::Error DeleteQuery(size_t& count); - reindexer::Error UpdateQuery(QueryResultsWrapper& qr); + enum class IsExpression { Yes, No }; + void Set(std::string_view field, const std::vector& values, IsExpression isExpression); void SetObject(std::string_view field, const std::vector& values); - void Set(std::string_view field, const std::vector& values); void Drop(std::string_view field); - void SetExpression(std::string_view field, std::string_view value); - void Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery); void Merge(QueryWrapper* mergeQuery); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index a1e56de..1788c1a 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -15,8 +15,8 @@ using reindexer::WrSerializer; namespace { uintptr_t initReindexer() { - DBInterface* db = new DBInterface(); - return reinterpret_cast(db); + auto db = std::make_unique(); + return reinterpret_cast(db.release()); } DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } @@ -34,7 +34,7 @@ T* getWrapper(uintptr_t wrapperAddr) { } template -void wrapperDelete(uintptr_t wrapperAddr) { +void deleteWrapper(uintptr_t wrapperAddr) { T* queryWrapperPtr = getWrapper(wrapperAddr); delete queryWrapperPtr; } @@ -105,16 +105,16 @@ static PyObject* Select(PyObject* self, PyObject* args) { } auto db = getDB(rx); - auto qresWrapper = new QueryResultsWrapper(db); + auto qresWrapper = std::make_unique(db); auto err = qresWrapper->Select(query); if (!err.ok()) { - delete qresWrapper; - return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); } - return Py_BuildValue("iskI", err.code(), err.what().c_str(), reinterpret_cast(qresWrapper), qresWrapper->Count()); + auto count = qresWrapper->Count(); + return Py_BuildValue("iskI", err.code(), err.what().c_str(), + reinterpret_cast(qresWrapper.release()), count); } // namespace ---------------------------------------------------------------------------------------------------------- @@ -448,7 +448,7 @@ static PyObject* QueryResultsWrapperDelete(PyObject* self, PyObject* args) { return nullptr; } - wrapperDelete(qresWrapperAddr); + deleteWrapper(qresWrapperAddr); Py_RETURN_NONE; } @@ -511,15 +511,13 @@ static PyObject* NewTransaction(PyObject* self, PyObject* args) { } auto db = getDB(rx); - auto transaction = new TransactionWrapper(db); + auto transaction = std::make_unique(db); auto err = transaction->Start(ns); if (!err.ok()) { - delete transaction; - return Py_BuildValue("isk", err.code(), err.what().c_str(), 0); } - return Py_BuildValue("isk", err.code(), err.what().c_str(), reinterpret_cast(transaction)); + return Py_BuildValue("isk", err.code(), err.what().c_str(), reinterpret_cast(transaction.release())); } namespace { @@ -614,7 +612,7 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args) { size_t count = 0; auto err = transaction->Commit(count); - wrapperDelete(transactionWrapperAddr); + deleteWrapper(transactionWrapperAddr); return Py_BuildValue("isI", err.code(), err.what().c_str(), count); } @@ -630,7 +628,7 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); auto err = transaction->Rollback(); - wrapperDelete(transactionWrapperAddr); + deleteWrapper(transactionWrapperAddr); return pyErr(err); } @@ -645,8 +643,8 @@ static PyObject* CreateQuery(PyObject* self, PyObject* args) { } auto db = getDB(rx); - auto query = new QueryWrapper(db, ns); - return Py_BuildValue("isK", errOK, "", reinterpret_cast(query)); + auto query = std::make_unique(db, ns); + return Py_BuildValue("isK", errOK, "", reinterpret_cast(query.release())); } static PyObject* DestroyQuery(PyObject* self, PyObject* args) { @@ -655,7 +653,7 @@ static PyObject* DestroyQuery(PyObject* self, PyObject* args) { return nullptr; } - wrapperDelete(queryWrapperAddr); + deleteWrapper(queryWrapperAddr); Py_RETURN_NONE; } @@ -1017,8 +1015,7 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { } namespace { -enum class ExecuteType { Select, Update }; -static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) { +static PyObject* executeQuery(PyObject* self, PyObject* args, QueryWrapper::ExecuteType type) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; @@ -1026,30 +1023,23 @@ static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) auto query = getWrapper(queryWrapperAddr); - auto qresWrapper = new QueryResultsWrapper(query->GetDB()); - Error err = errOK; - switch (type) { - case ExecuteType::Select: - err = query->SelectQuery(*qresWrapper); - break; - case ExecuteType::Update: - err = query->UpdateQuery(*qresWrapper); - break; - default: - assert(false); - } - + auto qresWrapper = std::make_unique(query->GetDB()); + auto err = query->ExecuteQuery(type, *qresWrapper); if (!err.ok()) { - delete qresWrapper; - return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); } - return Py_BuildValue("iskI", err.code(), err.what().c_str(), reinterpret_cast(qresWrapper), qresWrapper->Count()); + auto count = qresWrapper->Count(); + return Py_BuildValue("iskI", err.code(), err.what().c_str(), + reinterpret_cast(qresWrapper.release()), count); } } // namespace -static PyObject* SelectQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Select); } -static PyObject* UpdateQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Update); } +static PyObject* SelectQuery(PyObject* self, PyObject* args) { + return executeQuery(self, args, QueryWrapper::ExecuteType::Select); +} +static PyObject* UpdateQuery(PyObject* self, PyObject* args) { + return executeQuery(self, args, QueryWrapper::ExecuteType::Update); +} static PyObject* SetObject(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -1106,7 +1096,7 @@ static PyObject* Set(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->Set(field, values); + query->Set(field, values, QueryWrapper::IsExpression::No); return pyErr(errOK); } @@ -1135,7 +1125,7 @@ static PyObject* SetExpression(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); - query->SetExpression(field, value); + query->Set(field, {reindexer::Variant{value}}, QueryWrapper::IsExpression::Yes); Py_RETURN_NONE; } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index f580da3..6d77860 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -128,8 +128,7 @@ def __convert_strs_to_list(param: tuple[str, ...]) -> List[str]: """ param = [] if param is None else param - param = [item for item in param] - return param + return list(param) def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: """Adds where condition to DB query with args From 46afcd59e32988d5e0af733068d771c2cd7df9c5 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 13 Nov 2024 19:46:46 +0300 Subject: [PATCH 085/125] Part XXV: Bug fixes. Part IV: expected: <[{'id': -1}]> but: was <[{'id': 18446744073709551615}]> --- pyreindexer/lib/include/pyobjtools.cc | 2 +- pyreindexer/tests/tests/test_items.py | 10 ++++++++++ pyreindexer/tests/tests/test_query.py | 1 - 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 1fd19dd..de26d44 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -186,7 +186,7 @@ PyObject* pyValueFromJsonValue(const gason::JsonValue& value) { switch (value.getTag()) { case gason::JSON_NUMBER: - pyValue = PyLong_FromSize_t(value.toNumber()); // new ref + pyValue = PyLong_FromLongLong(value.toNumber()); // new ref break; case gason::JSON_DOUBLE: pyValue = PyFloat_FromDouble(value.toDouble()); // new ref diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index c154a30..cc3ecbd 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -21,6 +21,16 @@ def test_create_item_insert(self, db, namespace, index): assert_that(select_result, has_length(1), "Item wasn't created") assert_that(select_result, has_item(item_definition), "Item wasn't created") + def test_create_item_insert_empty(self, db, namespace, index): + # Given("Create namespace with index") + # When ("Insert item into namespace") + item_empty = {"id": -1} + db.item.insert(namespace, item_empty) + # Then ("Check that item is added") + select_result = get_ns_items(db, namespace) + assert_that(select_result, has_length(1), "Item wasn't created") + assert_that(select_result, has_item(item_empty), "Item wasn't created") + def test_create_item_insert_with_precepts(self, db, namespace, index): # Given("Create namespace with index") # When ("Insert items into namespace") diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 6d91d62..e0ffbf6 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -77,7 +77,6 @@ def test_query_select_where_cond_any(self, db, namespace, index, sparse_index, i # Then ("Check that all items is in result") assert_that(query_result, equal_to(items), "Wrong query results") - # TODO Expected: <[{'id': -1}]> but: was <[{'id': 18446744073709551615}]> ?? def test_query_select_where_cond_empty(self, db, namespace, index, sparse_index, items): # Given("Create namespace with index and items") # Given ("Create item without sparse field") From 7bc415e9c69cde7ca0c98661aa8200ec98a137c1 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 13 Nov 2024 19:58:37 +0300 Subject: [PATCH 086/125] add more tests with negative --- pyreindexer/tests/tests/test_items.py | 6 ++++-- pyreindexer/tests/tests/test_query.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index cc3ecbd..28ae868 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -1,3 +1,4 @@ +import pytest from hamcrest import * from tests.helpers.base_helper import get_ns_items @@ -21,10 +22,11 @@ def test_create_item_insert(self, db, namespace, index): assert_that(select_result, has_length(1), "Item wasn't created") assert_that(select_result, has_item(item_definition), "Item wasn't created") - def test_create_item_insert_empty(self, db, namespace, index): + @pytest.mark.parametrize("value", [-1, -2.33]) + def test_create_item_insert_negative(self, db, namespace, index, value): # Given("Create namespace with index") # When ("Insert item into namespace") - item_empty = {"id": -1} + item_empty = {"id": -1, "field": value} db.item.insert(namespace, item_empty) # Then ("Check that item is added") select_result = get_ns_items(db, namespace) diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index e0ffbf6..9587bff 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -364,6 +364,24 @@ def test_query_select_aggregations_math(self, db, namespace, index, items, calcu expected_agg_result = [{"value": calculate(ids_list), "type": aggregate_func.split("_")[-1], "fields": ["id"]}] assert_that(query_result.get_agg_results(), equal_to(expected_agg_result), "Wrong aggregation results") + @pytest.mark.parametrize("calculate, aggregate_func", AGGREGATE_FUNCTIONS_MATH) + def test_query_select_aggregations_math_negative(self, db, namespace, index, calculate, aggregate_func): + # Given("Create namespace with index") + # Given("Create new items") + items = [{"id": -i} for i in range(5)] + for item in items: + db.item.insert(namespace, item) + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with aggregations") + query_result = getattr(query, aggregate_func)("id").must_execute() + # Then ("Check that result is empty") + assert_that(list(query_result), empty(), "Wrong query results") + # Then ("Check aggregation results") + ids_list = [i["id"] for i in items] + expected_agg_result = [{"value": calculate(ids_list), "type": aggregate_func.split("_")[-1], "fields": ["id"]}] + assert_that(query_result.get_agg_results(), equal_to(expected_agg_result), "Wrong aggregation results") + def test_query_select_distinct(self, db, namespace, index, index_and_duplicate_items): # Given("Create namespace with index and duplicate items") items = index_and_duplicate_items From 2164fcf59652be9cb03d7d47451453b41a818279 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 13 Nov 2024 20:01:14 +0300 Subject: [PATCH 087/125] add null to create items test --- pyreindexer/tests/tests/test_items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyreindexer/tests/tests/test_items.py b/pyreindexer/tests/tests/test_items.py index 28ae868..bd60577 100644 --- a/pyreindexer/tests/tests/test_items.py +++ b/pyreindexer/tests/tests/test_items.py @@ -26,7 +26,7 @@ def test_create_item_insert(self, db, namespace, index): def test_create_item_insert_negative(self, db, namespace, index, value): # Given("Create namespace with index") # When ("Insert item into namespace") - item_empty = {"id": -1, "field": value} + item_empty = {"id": -1, "field": value, "empty_field": None} db.item.insert(namespace, item_empty) # Then ("Check that item is added") select_result = get_ns_items(db, namespace) From 89a7465da7b2aa58cb4477a9d10dde7a2bf2e573 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 15 Nov 2024 10:25:57 +0300 Subject: [PATCH 088/125] Part XXVI: add return TotalCount --- pyreindexer/lib/include/query_wrapper.cc | 14 +++++----- pyreindexer/lib/include/query_wrapper.h | 7 ++--- .../lib/include/queryresults_wrapper.h | 7 ++++- pyreindexer/lib/src/rawpyreindexer.cc | 24 +++++++++-------- pyreindexer/lib/src/reindexerinterface.cc | 6 ++--- pyreindexer/query.py | 19 ++++++++------ pyreindexer/query_results.py | 26 ++++++++++++++----- pyreindexer/rx_connector.py | 4 +-- 8 files changed, 62 insertions(+), 45 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index f959452..874ae17 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -176,7 +176,6 @@ void QueryWrapper::Sort(std::string_view index, bool desc, const std::vector& joinQueries, reindexer::Query& query) { +void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) const { for (auto joinQuery : joinQueries) { auto jq = createJoinedQuery(joinQuery->joinType_, joinQuery->ser_); query.AddJoinQuery(std::move(jq)); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 362fc38..0780d25 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -46,7 +46,7 @@ class QueryWrapper { void LogOp(OpType op); - void Total(std::string_view totalName, CalcTotalMode mode); + void Total(CalcTotalMode mode); void AddValue(QueryItemType type, unsigned value); @@ -77,9 +77,7 @@ class QueryWrapper { DBInterface* GetDB() const { return db_; } private: - reindexer::Serializer prepareQueryData(reindexer::WrSerializer& data); - reindexer::JoinedQuery createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data); - void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query); + void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) const; reindexer::Query prepareQuery(); DBInterface* db_{nullptr}; @@ -88,7 +86,6 @@ class QueryWrapper { OpType nextOperation_{OpType::OpAnd}; unsigned queriesCount_{0}; std::deque openedBrackets_; - std::string totalName_; // ToDo now not used JoinType joinType_{JoinType::LeftJoin}; std::vector joinQueries_; std::vector mergedQueries_; diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index b1ae39e..5b84676 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -40,11 +40,16 @@ class QueryResultsWrapper { return it_.Status(); } - size_t Count() const { + size_t Count() const noexcept { assert(wrap_); return qres_.Count(); } + size_t TotalCount() const noexcept { + assert(wrap_); + return qres_.TotalCount(); + } + void GetItemJSON(reindexer::WrSerializer& wrser, bool withHdrLen) { assert(wrap_); it_.GetJSON(wrser, withHdrLen); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 1788c1a..4b8af34 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -43,7 +43,8 @@ PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { QueryResultsWrapper* qresWrapperPtr = getWrapper(qresWrapperAddr); WrSerializer wrSer; - qresWrapperPtr->GetItemJSON(wrSer, false); + static const bool withHeaderLen = false; + qresWrapperPtr->GetItemJSON(wrSer, withHeaderLen); qresWrapperPtr->Next(); PyObject* dictFromJson = nullptr; @@ -109,12 +110,13 @@ static PyObject* Select(PyObject* self, PyObject* args) { auto err = qresWrapper->Select(query); if (!err.ok()) { - return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); + return Py_BuildValue("iskII", err.code(), err.what().c_str(), 0, 0); } auto count = qresWrapper->Count(); - return Py_BuildValue("iskI", err.code(), err.what().c_str(), - reinterpret_cast(qresWrapper.release()), count); + auto totalCount = qresWrapper->TotalCount(); + return Py_BuildValue("iskII", err.code(), err.what().c_str(), + reinterpret_cast(qresWrapper.release()), count, totalCount); } // namespace ---------------------------------------------------------------------------------------------------------- @@ -952,14 +954,13 @@ static PyObject* Not(PyObject* self, PyObject* args) { return logOp(self, args, namespace { static PyObject* total(PyObject* self, PyObject* args, CalcTotalMode mode) { uintptr_t queryWrapperAddr = 0; - char* totalName = nullptr; - if (!PyArg_ParseTuple(args, "ks", &queryWrapperAddr, &totalName)) { + if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); - query->Total(totalName, mode); + query->Total(mode); Py_RETURN_NONE; } @@ -1026,12 +1027,13 @@ static PyObject* executeQuery(PyObject* self, PyObject* args, QueryWrapper::Exec auto qresWrapper = std::make_unique(query->GetDB()); auto err = query->ExecuteQuery(type, *qresWrapper); if (!err.ok()) { - return Py_BuildValue("iskI", err.code(), err.what().c_str(), 0, 0); + return Py_BuildValue("iskII", err.code(), err.what().c_str(), 0, 0, 0); } - auto count = qresWrapper->Count(); - return Py_BuildValue("iskI", err.code(), err.what().c_str(), - reinterpret_cast(qresWrapper.release()), count); + auto count = qresWrapper->Count(); + auto totalCount = qresWrapper->TotalCount(); + return Py_BuildValue("iskII", err.code(), err.what().c_str(), + reinterpret_cast(qresWrapper.release()), count, totalCount); } } // namespace static PyObject* SelectQuery(PyObject* self, PyObject* args) { diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 4dda981..dd6c6ac 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -94,9 +94,9 @@ Error ReindexerInterface::modify(reindexer::cl template Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& transaction, size_t& count) { - typename DBT::QueryResultsT qr; - auto err = db_.CommitTransaction(transaction, qr); - count = qr.Count(); + typename DBT::QueryResultsT qres; + auto err = db_.CommitTransaction(transaction, qres); + count = qres.Count(); return err; } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 6d77860..7554bb3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -559,7 +559,7 @@ def op_not(self) -> Query: self.api.op_not(self.query_wrapper_ptr) return self - def request_total(self, total_name: str = '') -> Query: + def request_total(self) -> Query: """Requests total items calculation #### Arguments: @@ -570,10 +570,10 @@ def request_total(self, total_name: str = '') -> Query: """ - self.api.request_total(self.query_wrapper_ptr, total_name) + self.api.request_total(self.query_wrapper_ptr) return self - def cached_total(self, total_name: str = '') -> Query: + def cached_total(self) -> Query: """Requests cached total items calculation #### Arguments: @@ -584,7 +584,7 @@ def cached_total(self, total_name: str = '') -> Query: """ - self.api.cached_total(self.query_wrapper_ptr, total_name) + self.api.cached_total(self.query_wrapper_ptr) return self def limit(self, limit_items: int) -> Query: @@ -680,9 +680,10 @@ def execute(self) -> QueryResults: if self.root is not None: return self.root.execute() - self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select_query(self.query_wrapper_ptr) + (self.err_code, self.err_msg, + wrapper_ptr, iter_count, total_count) = self.api.select_query(self.query_wrapper_ptr) self.__raise_on_error() - return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + return QueryResults(self.api, wrapper_ptr, iter_count, total_count) def delete(self) -> int: """Executes a query, and delete items, matches query @@ -789,9 +790,11 @@ def update(self) -> QueryResults: if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Update does not support joined queries") - self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.update_query(self.query_wrapper_ptr) + (self.err_code, self.err_msg, + wrapper_ptr, iter_count, total_count) = self.api.update_query(self.query_wrapper_ptr) self.__raise_on_error() - return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + return QueryResults(self.api, wrapper_ptr, iter_count, total_count) + def must_execute(self) -> QueryResults: """Executes a query, and update fields in items, which matches query, with status check diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 4c0995e..8a080e3 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -12,19 +12,21 @@ class QueryResults: """ - def __init__(self, api, qres_wrapper_ptr, qres_iter_count): + def __init__(self, api, qres_wrapper_ptr, qres_iter_count, qres_total_count): """Constructs a new Reindexer query results iterator object #### Arguments: api (module): An API module for Reindexer calls qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations + qres_total_count (int): A total\cached count of results """ self.api = api self.qres_wrapper_ptr = qres_wrapper_ptr self.qres_iter_count = qres_iter_count + self.qres_total_count = qres_total_count self.pos = 0 self.err_code = 0 self.err_msg = "" @@ -61,7 +63,7 @@ def __del__(self): self._close_iterator() - def status(self): + def status(self) -> None: """Check status #### Raises: @@ -73,8 +75,8 @@ def status(self): if self.err_code: raise Exception(self.err_msg) - def count(self): - """Returns a count of results + def count(self) -> int: + """Returns a count of results for iterations #### Returns int: A count of results @@ -83,7 +85,17 @@ def count(self): return self.qres_iter_count - def _close_iterator(self): + def total_count(self) -> int: + """Returns a total\cached count of results + + #### Returns + int: A total\cached count of results + + """ + + return self.qres_total_count + + def _close_iterator(self) -> None: """Frees query results for the current iterator """ @@ -91,7 +103,7 @@ def _close_iterator(self): self.qres_iter_count = 0 self.api.query_results_delete(self.qres_wrapper_ptr) - def get_agg_results(self): + def get_agg_results(self) -> dict: """Returns aggregation results for the current query #### Returns @@ -107,7 +119,7 @@ def get_agg_results(self): raise Exception(self.err_msg) return res - def get_explain_results(self): + def get_explain_results(self) -> str: """Returns explain results for the current query #### Returns diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 6590541..19612fc 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -325,8 +325,8 @@ def select(self, query: str) -> QueryResults: """ - self.err_code, self.err_msg, qres_wrapper_ptr, qres_iter_count = self.api.select(self.rx, query) - return QueryResults(self.api, qres_wrapper_ptr, qres_iter_count) + self.err_code, self.err_msg, wrapper_ptr, iter_count, total_count = self.api.select(self.rx, query) + return QueryResults(self.api, wrapper_ptr, iter_count, total_count) @raise_if_error def new_transaction(self, namespace) -> Transaction: From 26d70c2e4d8f188642f6ade79448abd51114b2c8 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 18 Nov 2024 11:51:05 +0300 Subject: [PATCH 089/125] Part XXVII: disable WAL-queries --- pyreindexer/lib/include/pyobjtools.cc | 21 ++++++++----- pyreindexer/lib/include/query_wrapper.cc | 7 +++-- pyreindexer/tests/tests/test_query.py | 40 +++++++++--------------- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index de26d44..1bcb633 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -9,7 +9,7 @@ void pyValueSerialize(PyObject** value, reindexer::WrSerializer& wrSer); void pyListSerialize(PyObject** list, reindexer::WrSerializer& wrSer) { if (!PyList_Check(*list)) { - throw reindexer::Error(errParseJson, std::string("List expected, got ") + Py_TYPE(*list)->tp_name); + throw reindexer::Error(ErrorCode::errParseJson, std::string("List expected, got ") + Py_TYPE(*list)->tp_name); } wrSer << '['; @@ -30,7 +30,8 @@ void pyListSerialize(PyObject** list, reindexer::WrSerializer& wrSer) { void pyDictSerialize(PyObject** dict, reindexer::WrSerializer& wrSer) { if (!PyDict_Check(*dict)) { - throw reindexer::Error(errParseJson, std::string("Dictionary expected, got ") + Py_TYPE(*dict)->tp_name); + throw reindexer::Error(ErrorCode::errParseJson, + std::string("Dictionary expected, got ") + Py_TYPE(*dict)->tp_name); } wrSer << '{'; @@ -81,7 +82,8 @@ void pyValueSerialize(PyObject** value, reindexer::WrSerializer& wrSer) { } else if (PyDict_Check(*value)) { pyDictSerialize(value, wrSer); } else { - throw reindexer::Error(errParseJson, std::string("Unable to parse value of type ") + Py_TYPE(*value)->tp_name); + throw reindexer::Error(ErrorCode::errParseJson, + std::string("Unable to parse value of type ") + Py_TYPE(*value)->tp_name); } } @@ -91,8 +93,9 @@ void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer) { } else if (PyList_Check(*obj) ) { pyListSerialize(obj, wrSer); } else { - throw reindexer::Error(errParseJson, - std::string("PyObject must be a dictionary or a list for JSON serializing, got ") + Py_TYPE(*obj)->tp_name); + throw reindexer::Error(ErrorCode::errParseJson, + std::string("PyObject must be a dictionary or a list for JSON serializing, got ") + + Py_TYPE(*obj)->tp_name); } } @@ -138,7 +141,8 @@ std::vector ParseStrListToStrVec(PyObject** list) { PyObject* item = PyList_GetItem(*list, i); if (!PyUnicode_Check(item)) { - throw reindexer::Error(errParseJson, std::string("String expected, got ") + Py_TYPE(item)->tp_name); + throw reindexer::Error(ErrorCode::errParseJson, + std::string("String expected, got ") + Py_TYPE(item)->tp_name); } result.push_back(PyUnicode_AsUTF8(item)); @@ -163,7 +167,8 @@ reindexer::Variant convert(PyObject** value) { } else if (PyUnicode_Check(*value)) { return reindexer::Variant(std::string_view(PyUnicode_AsUTF8(*value))); } else { - throw reindexer::Error(errParseJson, std::string("Unexpected type, got ") + Py_TYPE(*value)->tp_name); + throw reindexer::Error(ErrorCode::errParseJson, + std::string("Unexpected type, got ") + Py_TYPE(*value)->tp_name); } return {}; } @@ -240,7 +245,7 @@ PyObject* PyObjectFromJson(reindexer::span json) { auto root = parser.Parse(json); return pyValueFromJsonValue(root.value); // stolen ref } catch (const gason::Exception& ex) { - throw reindexer::Error(errParseJson, std::string("PyObjectFromJson: ") + ex.what()); + throw reindexer::Error(ErrorCode::errParseJson, std::string("PyObjectFromJson: ") + ex.what()); } } } // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 874ae17..b0e398a 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -112,11 +112,11 @@ Error QueryWrapper::OpenBracket() { reindexer::Error QueryWrapper::CloseBracket() { if (nextOperation_ != OpType::OpAnd) { - return reindexer::Error(errLogic, "Operation before close bracket"); + return reindexer::Error(ErrorCode::errLogic, "Operation before close bracket"); } if (openedBrackets_.empty()) { - return reindexer::Error(errLogic, "Close bracket before open it"); + return reindexer::Error(ErrorCode::errLogic, "Close bracket before open it"); } ser_.PutVarUint(QueryItemType::QueryCloseBracket); @@ -246,6 +246,9 @@ reindexer::Query QueryWrapper::prepareQuery() { reindexer::Error QueryWrapper::ExecuteQuery(ExecuteType type, QueryResultsWrapper& qr) { auto query = prepareQuery(); + if (query.IsWALQuery()) { + return reindexer::Error(ErrorCode::errQueryExec, "WAL queries are not supported"); + } Error err = errOK; switch (type) { diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 9587bff..8f8df4a 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -224,19 +224,17 @@ def test_query_select_limit_offset(self, db, namespace, index, items): expected_items = [items[i] for i in [1, 2, 3]] assert_that(query_result, equal_to(expected_items), "Wrong query results") - # ToDo - # @pytest.mark.skip(reason="need get_total in query_results") - # @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) - # def test_query_select_request_total(self, db, namespace, index, items, func_total): - # # Given("Create namespace with index and items") - # # Given ("Create new query") - # query = db.query.new(namespace) - # # When ("Make select query with limit and offset") - # query_result = getattr(query, func_total)("id").must_execute() - # # Then ("Check that selected items are in result") - # assert_that(list(query_result), equal_to(items), "Wrong query results") - # # Then ("Check that total is in result") - # assert_that(query_result.get_total_results(), equal_to(""), "There is no total in query results") + @pytest.mark.parametrize("func_total", ["request_total", "cached_total"]) + def test_query_select_request_total(self, db, namespace, index, items, func_total): + # Given("Create namespace with index and items") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with limit and offset") + query_result = getattr(query, func_total)().limit(1).must_execute() + # Then ("Check that total is in result") + assert_that(query_result.total_count(), equal_to(len(items)), "Unexpected total in query results") + assert_that(query_result.total_count(), not_(equal_to(query_result.count())), + "Unexpected total in query results") def test_query_select_with_rank(self, db, namespace, index, ft_index_and_items): # Given("Create namespace with ft index and items") @@ -329,24 +327,14 @@ def test_query_select_strict_mode_indexes(self, db, namespace, index, items): assert_that(calling(query.execute).with_args(), raises(Exception, pattern=err_msg)) - # TODO "Exception: Query results contain WAL items. Query results from WAL must either be requested in JSON format or with client, supporting RAW items" - # (ExecToJson in GO) - def test_query_select_wal_gt(self, db, namespace, index, items): - # Given("Create namespace with index and items") - # Given ("Create new query") - query = db.query.new(namespace) - # When ("Make select query with lsn gt") - query_result = query.where("#lsn", CondType.CondGt, 1).must_execute() # .execute_to_json() ?? - # TODO add lsn validation - - # TODO def test_query_select_wal_any(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query with lsn any") - query_result = query.where("#lsn", CondType.CondAny, None).must_execute() # .execute_to_json() ?? - # TODO add lsn validation + err_msg = "WAL queries are not supported" + query.where("#lsn", CondType.CondAny, None) + assert_that(calling(query.execute).with_args(), raises(Exception, pattern=err_msg)) class TestQuerySelectAggregations: From 7c0bc729da5846a012d5940c59e6b3b84d7021b7 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 18 Nov 2024 20:38:51 +0300 Subject: [PATCH 090/125] Part XXVII: disable WAL-queries. Catch unhandled exception --- pyreindexer/lib/include/query_wrapper.cc | 37 ++++++++++++++++-------- pyreindexer/lib/include/query_wrapper.h | 2 +- pyreindexer/query_results.py | 4 +-- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index b0e398a..481fb7d 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -228,29 +228,38 @@ void QueryWrapper::addJoinQueries(const std::vector& joinQueries, } } -reindexer::Query QueryWrapper::prepareQuery() { - reindexer::Serializer ser = prepareQueryData(ser_); - auto query = reindexer::Query::Deserialize(ser); +reindexer::Error QueryWrapper::prepareQuery(reindexer::Query& query) { + reindexer::Error error = errOK; + try { + reindexer::Serializer ser = prepareQueryData(ser_); + query = reindexer::Query::Deserialize(ser); - addJoinQueries(joinQueries_, query); + addJoinQueries(joinQueries_, query); - for (auto mergedQuery : mergedQueries_) { - auto mq = createJoinedQuery(JoinType::Merge, mergedQuery->ser_); - query.Merge(std::move(mq)); + for (auto mergedQuery : mergedQueries_) { + auto mq = createJoinedQuery(JoinType::Merge, mergedQuery->ser_); + query.Merge(std::move(mq)); - addJoinQueries(mergedQuery->joinQueries_, mq); + addJoinQueries(mergedQuery->joinQueries_, mq); + } + } catch (const reindexer::Error& err) { + error = err; } - return query; + return error; } reindexer::Error QueryWrapper::ExecuteQuery(ExecuteType type, QueryResultsWrapper& qr) { - auto query = prepareQuery(); + reindexer::Query query; + auto err = prepareQuery(query); + if (!err.ok()) { + return err; + } + if (query.IsWALQuery()) { return reindexer::Error(ErrorCode::errQueryExec, "WAL queries are not supported"); } - Error err = errOK; switch (type) { case ExecuteType::Select: err = db_->SelectQuery(query, qr); @@ -265,7 +274,11 @@ reindexer::Error QueryWrapper::ExecuteQuery(ExecuteType type, QueryResultsWrappe } reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { - auto query = prepareQuery(); + reindexer::Query query; + auto err = prepareQuery(query); + if (!err.ok()) { + return err; + } return db_->DeleteQuery(query, count); } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 0780d25..c121ca2 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -78,7 +78,7 @@ class QueryWrapper { private: void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) const; - reindexer::Query prepareQuery(); + reindexer::Error prepareQuery(reindexer::Query& query); DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 8a080e3..119755f 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -86,10 +86,10 @@ def count(self) -> int: return self.qres_iter_count def total_count(self) -> int: - """Returns a total\cached count of results + """Returns a total or cached count of results #### Returns - int: A total\cached count of results + int: A total or cached count of results """ From 4480decf4291ad3ed0e6582461232aa8f1dcfc56 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 19 Nov 2024 20:57:51 +0300 Subject: [PATCH 091/125] Part XXVIII: Return JOIN result. Part --- pyreindexer/lib/src/reindexerinterface.cc | 10 +++++++--- pyreindexer/lib/src/reindexerinterface.h | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index dd6c6ac..ac35861 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -1,10 +1,14 @@ #include "reindexerinterface.h" #include "client/cororeindexer.h" #include "core/reindexer.h" +#include "core/type_consts.h" #include "queryresults_wrapper.h" #include "transaction_wrapper.h" namespace pyreindexer { +namespace { + const int QRESULTS_FLAGS = kResultsJson | kResultsWithRank | kResultsWithJoined; +} template <> ReindexerInterface::ReindexerInterface() {} @@ -55,7 +59,7 @@ ReindexerInterface::~ReindexerInterface() { template Error ReindexerInterface::Select(const std::string& query, QueryResultsWrapper& result) { return execute([this, query, &result] { - typename DBT::QueryResultsT qres; + typename DBT::QueryResultsT qres(QRESULTS_FLAGS); auto res = select(query, qres); result.Wrap(std::move(qres)); return res; @@ -94,7 +98,7 @@ Error ReindexerInterface::modify(reindexer::cl template Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& transaction, size_t& count) { - typename DBT::QueryResultsT qres; + typename DBT::QueryResultsT qres(QRESULTS_FLAGS); auto err = db_.CommitTransaction(transaction, qres); count = qres.Count(); return err; @@ -118,7 +122,7 @@ Error ReindexerInterface::deleteQuery(const reindexer::Query& query, size_t template Error ReindexerInterface::updateQuery(const reindexer::Query& query, QueryResultsWrapper& result) { - typename DBT::QueryResultsT qres; + typename DBT::QueryResultsT qres(QRESULTS_FLAGS); auto err = db_.Update(query, qres); result.Wrap(std::move(qres)); return err; diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 96d7d4d..ed5afd2 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -43,7 +43,7 @@ class GenericCommand : public ICommand { private: CallableT command_; - Error err_; + Error err_{errOK}; std::atomic_bool executed_{false}; }; From 9a4abfb585420e8539979e2b170d26594533a6a1 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 20 Nov 2024 13:50:47 +0300 Subject: [PATCH 092/125] Part XXIX: where_query check --- README.md | 79 +++++++++++++++++------- pydoc-markdown.yml | 2 + pyreindexer/lib/include/query_wrapper.cc | 4 +- pyreindexer/lib/include/query_wrapper.h | 4 +- pyreindexer/lib/src/rawpyreindexer.cc | 8 +-- pyreindexer/lib/src/rawpyreindexer.h | 8 +-- pyreindexer/query.py | 20 +++++- pyreindexer/tests/tests/test_query.py | 43 +++++++++---- 8 files changed, 117 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 5bec17c..4a873f7 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ * [QueryResults](#pyreindexer.query_results.QueryResults) * [status](#pyreindexer.query_results.QueryResults.status) * [count](#pyreindexer.query_results.QueryResults.count) + * [total\_count](#pyreindexer.query_results.QueryResults.total_count) * [get\_agg\_results](#pyreindexer.query_results.QueryResults.get_agg_results) * [get\_explain\_results](#pyreindexer.query_results.QueryResults.get_explain_results) * [pyreindexer.transaction](#pyreindexer.transaction) @@ -42,6 +43,7 @@ * [Query](#pyreindexer.query.Query) * [where](#pyreindexer.query.Query.where) * [where\_query](#pyreindexer.query.Query.where_query) + * [where\_subquery](#pyreindexer.query.Query.where_subquery) * [where\_composite](#pyreindexer.query.Query.where_composite) * [where\_uuid](#pyreindexer.query.Query.where_uuid) * [where\_between\_fields](#pyreindexer.query.Query.where_between_fields) @@ -491,7 +493,7 @@ QueryResults is a disposable iterator of Reindexer results for such queries as S ### QueryResults.status ```python -def status() +def status() -> None ``` Check status @@ -504,20 +506,33 @@ Check status ### QueryResults.count ```python -def count() +def count() -> int ``` -Returns a count of results +Returns a count of results for iterations #### Returns int: A count of results + + +### QueryResults.total\_count + +```python +def total_count() -> int +``` + +Returns a total or cached count of results + +#### Returns + int: A total or cached count of results + ### QueryResults.get\_agg\_results ```python -def get_agg_results() +def get_agg_results() -> dict ``` Returns aggregation results for the current query @@ -533,7 +548,7 @@ Returns aggregation results for the current query ### QueryResults.get\_explain\_results ```python -def get_explain_results() +def get_explain_results() -> str ``` Returns explain results for the current query @@ -769,15 +784,31 @@ Adds sub-query where condition to DB query with args #### Raises: Exception: Raises with an error message of API return on non-zero error code + + +### Query.where\_subquery + +```python +def where_subquery(index: str, condition: CondType, sub_query: Query) -> Query +``` + +Adds sub-query where condition to DB query + +#### Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + sub_query (:obj:`Query`): Field name used in condition clause + +#### Returns: + (:obj:`Query`): Query object for further customizations + ### Query.where\_composite ```python -def where_composite( - index: str, - condition: CondType, - keys: Union[simple_types, List[simple_types]] = None) -> Query +def where_composite(index: str, condition: CondType, *keys: + simple_types) -> Query ``` Adds where condition to DB query with interface args for composite indexes @@ -785,7 +816,7 @@ Adds where condition to DB query with interface args for composite indexes #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - sub_query (:obj:`Query`): Field name used in condition clause + keys (*simple_types): Values of composite index to be compared with (value of each sub-index) #### Returns: (:obj:`Query`): Query object for further customizations @@ -795,7 +826,7 @@ Adds where condition to DB query with interface args for composite indexes ### Query.where\_uuid ```python -def where_uuid(index: str, condition: CondType, keys: List[str]) -> Query +def where_uuid(index: str, condition: CondType, *keys: str) -> Query ``` Adds where condition to DB query with UUID as string args. @@ -805,7 +836,7 @@ Adds where condition to DB query with UUID as string args. #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -870,14 +901,14 @@ Closes bracket for where condition to DB query ### Query.match ```python -def match(index: str, keys: List[str]) -> Query +def match(index: str, *keys: str) -> Query ``` Adds string EQ-condition to DB query with string args #### Arguments: index (string): Field name used in condition clause - keys (list[string]): Value of index to be compared with. For composite indexes keys must be list, + keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -989,7 +1020,7 @@ Finds for the maximum at the specified index ### Query.aggregate\_facet ```python -def aggregate_facet(fields: List[str]) -> Query._AggregateFacet +def aggregate_facet(*fields: str) -> Query._AggregateFacet ``` Gets fields facet value. Applicable to multiple data fields and the result of that could be sorted @@ -997,7 +1028,7 @@ Gets fields facet value. Applicable to multiple data fields and the result of th this method returns AggregationFacetRequest which has methods sort, limit and offset #### Arguments: - fields (list[string]): Fields any data column name or `count`, fields should not be empty + fields (list[*string]): Fields any data column name or `count`, fields should not be empty #### Returns: (:obj:`_AggregateFacet`): Request object for further customizations @@ -1118,7 +1149,7 @@ Next condition will be added with NOT AND. ### Query.request\_total ```python -def request_total(total_name: str = '') -> Query +def request_total() -> Query ``` Requests total items calculation @@ -1134,7 +1165,7 @@ Requests total items calculation ### Query.cached\_total ```python -def cached_total(total_name: str = '') -> Query +def cached_total() -> Query ``` Requests cached total items calculation @@ -1490,7 +1521,7 @@ On specifies join condition ### Query.select ```python -def select(fields: List[str]) -> Query +def select(*fields: str) -> Query ``` Sets list of columns in this namespace to be finally selected. @@ -1499,7 +1530,7 @@ Sets list of columns in this namespace to be finally selected. If there are no fields in this list that meet these conditions, then the filter works as "*" #### Arguments: - fields (list[string]): List of columns to be selected + fields (list[*string]): List of columns to be selected #### Returns: (:obj:`Query`): Query object for further customizations @@ -1512,13 +1543,13 @@ Sets list of columns in this namespace to be finally selected. ### Query.functions ```python -def functions(functions: List[str]) -> Query +def functions(*functions: str) -> Query ``` Adds sql-functions to query #### Arguments: - functions (list[string]): Functions declaration + functions (list[*string]): Functions declaration #### Returns: (:obj:`Query`): Query object for further customizations @@ -1531,13 +1562,13 @@ Adds sql-functions to query ### Query.equal\_position ```python -def equal_position(equal_position: List[str]) -> Query +def equal_position(*equal_position: str) -> Query ``` Adds equal position fields to arrays queries #### Arguments: - equal_poses (list[string]): Equal position fields to arrays queries + equal_poses (list[*string]): Equal position fields to arrays queries #### Returns: (:obj:`Query`): Query object for further customizations diff --git a/pydoc-markdown.yml b/pydoc-markdown.yml index 33562ee..bdd4de0 100644 --- a/pydoc-markdown.yml +++ b/pydoc-markdown.yml @@ -16,6 +16,8 @@ renderer: type: markdown filename: README.md add_method_class_prefix: true + classdef_with_decorators: false + signature_with_decorators: false render_toc: true render_toc_title: The PyReindexer module provides a connector and its auxiliary tools for interaction with Reindexer header_level_by_type: diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 481fb7d..77f1a66 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -40,7 +40,7 @@ void QueryWrapper::Where(std::string_view index, CondType condition, const std:: ++queriesCount_; } -void QueryWrapper::WhereQuery(QueryWrapper& query, CondType condition, const std::vector& keys) { +void QueryWrapper::WhereSubQuery(QueryWrapper& query, CondType condition, const std::vector& keys) { ser_.PutVarUint(QueryItemType::QuerySubQueryCondition); ser_.PutVarUint(nextOperation_); ser_.PutVString(query.ser_.Slice()); @@ -55,7 +55,7 @@ void QueryWrapper::WhereQuery(QueryWrapper& query, CondType condition, const std ++queriesCount_; } -void QueryWrapper::WhereComposite(std::string_view index, CondType condition, QueryWrapper& query) { +void QueryWrapper::WhereFieldSubQuery(std::string_view index, CondType condition, QueryWrapper& query) { ser_.PutVarUint(QueryItemType::QueryFieldSubQueryCondition); ser_.PutVarUint(nextOperation_); ser_.PutVString(index); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index c121ca2..8d6a943 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -27,8 +27,8 @@ class QueryWrapper { QueryWrapper(DBInterface* db, std::string_view ns); void Where(std::string_view index, CondType condition, const std::vector& keys); - void WhereQuery(QueryWrapper& query, CondType condition, const std::vector& keys); - void WhereComposite(std::string_view index, CondType condition, QueryWrapper& query); + void WhereSubQuery(QueryWrapper& query, CondType condition, const std::vector& keys); + void WhereFieldSubQuery(std::string_view index, CondType condition, QueryWrapper& query); void WhereUUID(std::string_view index, CondType condition, const std::vector& keys); void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 4b8af34..55dd20c 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -691,7 +691,7 @@ static PyObject* Where(PyObject* self, PyObject* args) { return pyErr(errOK); } -static PyObject* WhereQuery(PyObject* self, PyObject* args) { +static PyObject* WhereSubQuery(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; uintptr_t subQueryWrapperAddr = 0; unsigned condition = 0; @@ -718,12 +718,12 @@ static PyObject* WhereQuery(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); auto subQuery = getWrapper(subQueryWrapperAddr); - query->WhereQuery(*subQuery, CondType(condition), keys); + query->WhereSubQuery(*subQuery, CondType(condition), keys); return pyErr(errOK); } -static PyObject* WhereComposite(PyObject* self, PyObject* args) { +static PyObject* WhereFieldSubQuery(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned condition = 0; @@ -735,7 +735,7 @@ static PyObject* WhereComposite(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); auto subQuery = getWrapper(subQueryWrapperAddr); - query->WhereComposite(index, CondType(condition), *subQuery); + query->WhereFieldSubQuery(index, CondType(condition), *subQuery); Py_RETURN_NONE; } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 5851039..87b6f61 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -56,8 +56,8 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args); static PyObject* CreateQuery(PyObject* self, PyObject* args); static PyObject* DestroyQuery(PyObject* self, PyObject* args); static PyObject* Where(PyObject* self, PyObject* args); -static PyObject* WhereQuery(PyObject* self, PyObject* args); -static PyObject* WhereComposite(PyObject* self, PyObject* args); +static PyObject* WhereSubQuery(PyObject* self, PyObject* args); +static PyObject* WhereFieldSubQuery(PyObject* self, PyObject* args); static PyObject* WhereUUID(PyObject* self, PyObject* args); static PyObject* WhereBetweenFields(PyObject* self, PyObject* args); static PyObject* OpenBracket(PyObject* self, PyObject* args); @@ -141,8 +141,8 @@ static PyMethodDef module_methods[] = { {"create_query", CreateQuery, METH_VARARGS, "create new query"}, {"destroy_query", DestroyQuery, METH_VARARGS, "delete query object. Free query object memory"}, {"where", Where, METH_VARARGS, "add where condition with args"}, - {"where_query", WhereQuery, METH_VARARGS, "add sub-query where condition"}, - {"where_composite", WhereComposite, METH_VARARGS, "add where condition for composite indexes"}, + {"where_subquery", WhereSubQuery, METH_VARARGS, "add sub-query where condition"}, + {"where_field_subquery", WhereFieldSubQuery, METH_VARARGS, "add where condition for sub-query"}, {"where_uuid", WhereUUID, METH_VARARGS, "add where condition with UUIDs"}, {"where_between_fields", WhereBetweenFields, METH_VARARGS, "add comparing two fields where condition"}, {"open_bracket", OpenBracket, METH_VARARGS, "open bracket for where condition"}, diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 7554bb3..50a8eaf 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -178,11 +178,27 @@ def where_query(self, sub_query: Query, condition: CondType, params: list = self.__convert_to_list(keys) - self.err_code, self.err_msg = self.api.where_query(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, - condition.value, params) + self.err_code, self.err_msg = self.api.where_subquery(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, + condition.value, params) self.__raise_on_error() return self + def where_subquery(self, index: str, condition: CondType, sub_query: Query) -> Query: + """Adds sub-query where condition to DB query + + #### Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + sub_query (:obj:`Query`): Field name used in condition clause + + #### Returns: + (:obj:`Query`): Query object for further customizations + + """ + + self.api.where_field_subquery(self.query_wrapper_ptr, index, condition.value, sub_query.query_wrapper_ptr) + return self + def where_composite(self, index: str, condition: CondType, *keys: simple_types) -> Query: """Adds where condition to DB query with interface args for composite indexes diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 8f8df4a..376d169 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -98,19 +98,36 @@ def test_query_select_where_cond_like(self, db, namespace, indexes, items): # Then ("Check that selected item is in result") assert_that(query_result, equal_to(items), "Wrong query results") - # TODO does not work, ignore subquery results - # def test_query_select_where_query(self, db, namespace, index, items): - # # Given("Create namespace with index and items") - # # Given ("Create new query") - # # query = db.query.new(namespace).where("id", CondType.CondLt, 5) - # # sub_query = db.query.new(namespace).where("id", CondType.CondGt, 0) - # query = db.query.new(namespace).where("id", CondType.CondGt, 0) - # sub_query = db.query.new(namespace).select("id").where("id", CondType.CondLt, 5) - # # When ("Make select query with where_query subquery") - # query_result = list(query.where_query(sub_query, CondType.CondGe, 2).must_execute()) - # # Then ("Check that selected item is in result") - # expected_items = [items[i] for i in [2, 3, 4]] - # assert_that(query_result, equal_to(expected_items), "Wrong query results") + def test_query_select_where_query_field(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + sub_query = (db.query.new(namespace) + .select("id") + .where("id", CondType.CondLt, 5) + .where("id", CondType.CondGe, 2)) + # When ("Make select query with where_subquery") + query_result = list(db.query.new(namespace).where_subquery('id', CondType.CondSet, sub_query).execute()) + # Then ("Check that selected item is in result") + expected_items = [items[i] for i in [2, 3, 4]] + assert_that(query_result, equal_to(expected_items), "Wrong query results") + + def test_query_select_where_query_all(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + sub_query = db.query.new(namespace).select("id").where("id", CondType.CondLt, 5) + # When ("Make select query with where_query and subquery") + query_result = list(db.query.new(namespace).where_query(sub_query, CondType.CondGe, 2).execute()) + # Then ("Check that all items selected in result") + assert_that(query_result, equal_to(items), "Wrong query results") + + def test_query_select_where_query_empty(self, db, namespace, index, items): + # Given("Create namespace with index and items") + # Given ("Create new query") + sub_query = db.query.new(namespace).select("id").where("id", CondType.CondEq, 5) + # When ("Make select query with where_query and subquery") + query_result = list(db.query.new(namespace).where_query(sub_query, CondType.CondLe, 2).execute()) + # Then ("Check that result is empty") + assert_that(query_result, empty(), "Wrong query results") # ToDo # def test_query_select_where_composite(self, db, namespace, composite_index, items): From c57663f7997a43f3d3ce9e864d0d565b6c68ce31 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 21 Nov 2024 14:41:09 +0300 Subject: [PATCH 093/125] Part XXX: where_composite fixes. Part --- pyreindexer/lib/include/pyobjtools.cc | 22 ++++++++++-- pyreindexer/lib/include/query_wrapper.cc | 43 ++++++++++++------------ pyreindexer/lib/include/query_wrapper.h | 1 + 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index 1bcb633..dde4ab1 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -90,7 +90,7 @@ void pyValueSerialize(PyObject** value, reindexer::WrSerializer& wrSer) { void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer) { if (PyDict_Check(*obj)) { pyDictSerialize(obj, wrSer); - } else if (PyList_Check(*obj) ) { + } else if (PyList_Check(*obj)) { pyListSerialize(obj, wrSer); } else { throw reindexer::Error(ErrorCode::errParseJson, @@ -117,7 +117,7 @@ std::vector PyObjectToJson(PyObject** obj) { wrSer.Reset(); } } - } else if (PyList_Check(*obj) ) { + } else if (PyList_Check(*obj)) { Py_ssize_t sz = PyList_Size(*obj); for (Py_ssize_t i = 0; i < sz; ++i) { PyObject* value = PyList_GetItem(*obj, i); @@ -166,6 +166,24 @@ reindexer::Variant convert(PyObject** value) { return reindexer::Variant(int64_t(PyLong_AsLong(*value))); } else if (PyUnicode_Check(*value)) { return reindexer::Variant(std::string_view(PyUnicode_AsUTF8(*value))); + } else if (PyList_Check(*value)) { + auto size = PyList_Size(*value); + reindexer::VariantArray array; + array.reserve(size); + for (Py_ssize_t i = 0; i < size; ++i) { + auto item = PyList_GetItem(*value, i); + array.push_back(convert(&item)); + } + return reindexer::Variant{array}; + } else if (PyTuple_Check(*value)){ + auto size = PyTuple_Size(*value); + reindexer::VariantArray array; + array.reserve(size); + for (Py_ssize_t i = 0; i < size; ++i) { + auto item = PyTuple_GetItem(*value, i); + array.push_back(convert(&item)); + } + return reindexer::Variant{array}; } else { throw reindexer::Error(ErrorCode::errParseJson, std::string("Unexpected type, got ") + Py_TYPE(*value)->tp_name); diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 77f1a66..7c77dc1 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -6,18 +6,17 @@ #include "core/keyvalue/uuid.h" namespace pyreindexer { - namespace { - const int VALUE_INT_64 = 0; - const int VALUE_DOUBLE = 1; - const int VALUE_STRING = 2; - const int VALUE_BOOL = 3; - const int VALUE_NULL = 4; - const int VALUE_INT = 8; - const int VALUE_UNDEFINED = 9; - const int VALUE_COMPOSITE = 10; - const int VALUE_TUPLE = 11; - const int VALUE_UUID = 12; +const int VALUE_INT_64 = 0; +const int VALUE_DOUBLE = 1; +const int VALUE_STRING = 2; +const int VALUE_BOOL = 3; +const int VALUE_NULL = 4; +const int VALUE_INT = 8; +const int VALUE_UNDEFINED = 9; +const int VALUE_COMPOSITE = 10; +const int VALUE_TUPLE = 11; +const int VALUE_UUID = 12; } // namespace QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { @@ -36,6 +35,8 @@ void QueryWrapper::Where(std::string_view index, CondType condition, const std:: ser_.PutVariant(key); } + putKeys(keys); + nextOperation_ = OpType::OpAnd; ++queriesCount_; } @@ -45,11 +46,7 @@ void QueryWrapper::WhereSubQuery(QueryWrapper& query, CondType condition, const ser_.PutVarUint(nextOperation_); ser_.PutVString(query.ser_.Slice()); ser_.PutVarUint(condition); - - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - ser_.PutVariant(key); - } + putKeys(keys); nextOperation_ = OpType::OpAnd; ++queriesCount_; @@ -169,11 +166,7 @@ void QueryWrapper::Sort(std::string_view index, bool desc, const std::vector 1? 1 : 0); // is array flag for (const auto& value : values) { ser_.PutVarUint(0); // function/value flag - ser_.PutVarUint(TagType::TAG_STRING); // type ID + ser_.PutVarUint(VALUE_STRING); // type ID ser_.PutVString(value); break; } @@ -367,4 +360,10 @@ void QueryWrapper::AddEqualPosition(const std::vector& equalPositio } } +void QueryWrapper::putKeys(const std::vector& keys) { + ser_.PutVarUint(keys.size()); + for (const auto& key : keys) { + ser_.PutVariant(key); + } +} } // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 8d6a943..afee1e3 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -79,6 +79,7 @@ class QueryWrapper { private: void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) const; reindexer::Error prepareQuery(reindexer::Query& query); + void putKeys(const std::vector& keys); DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; From fdcfe30a92cc37d5f5e435d4a3622f49019c094e Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 21 Nov 2024 15:21:34 +0300 Subject: [PATCH 094/125] Part XXX: where_composite fixes. Part --- pyreindexer/query.py | 73 +++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 50a8eaf..e409d15 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -100,43 +100,43 @@ def __raise_on_error(self): raise Exception(self.err_msg) @staticmethod - def __convert_to_list(param: Union[simple_types, List[simple_types]]) -> List[simple_types]: - """Converts an input parameter to a list + def __convert_to_list(param: tuple[list[simple_types], ...]) -> list[list[simple_types]]: + """Converts an input parameter to a list of lists #### Arguments: - param (Union[None, simple_types, list[simple_types]]): The input parameter + param (list[simple_types], ...): The input parameter #### Returns: - List[Union[int, bool, float, str]]: Always converted to a list + list[list[union[int, bool, float, str]]]: Always converted to a list of lists """ - param = [] if param is None else param - param = param if isinstance(param, list) else [param] - return param + result = list(list()) if param is None else param + result = result if isinstance(result, list) else list(result) + return result @staticmethod - def __convert_strs_to_list(param: tuple[str, ...]) -> List[str]: + def __convert_strs_to_list(param: tuple[str, ...]) -> list[str]: """Converts an input parameter to a list #### Arguments: param (list[*string]): The input parameter #### Returns: - List[string]: Always converted to a list + list[string]: Always converted to a list """ - param = [] if param is None else param - return list(param) + result = list() if param is None else list(param) + return result - def where(self, index: str, condition: CondType, keys: Union[simple_types, List[simple_types]] = None) -> Query: + def __where(self, index: str, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: """Adds where condition to DB query with args #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (Union[None, simple_types, list[simple_types]]): + keys (list[simple_types], ...): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -148,23 +148,44 @@ def where(self, index: str, condition: CondType, keys: Union[simple_types, List[ """ - params: list = self.__convert_to_list(keys) - if condition == CondType.CondDWithin: raise Exception("In this case, use a special method 'dwithin'") + params = self.__convert_to_list(keys) + print('===================== keys: ' + str(keys)) # ToDo + print('===================== params: ' + str(params)) # ToDo + self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() return self - def where_query(self, sub_query: Query, condition: CondType, - keys: Union[simple_types, List[simple_types]] = None) -> Query: + def where(self, index: str, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: + """Adds where condition to DB query with args + + #### Arguments: + index (string): Field name used in condition clause + condition (:enum:`CondType`): Type of condition + keys (*list[simple_types]): + Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index + + #### Returns: + (:obj:`Query`): Query object for further customizations + + #### Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + return self.__where(index, condition, keys) + + def where_query(self, sub_query: Query, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: """Adds sub-query where condition to DB query with args #### Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (Union[None, simple_types, list[simple_types]]): + keys (*list[simple_types]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -176,7 +197,7 @@ def where_query(self, sub_query: Query, condition: CondType, """ - params: list = self.__convert_to_list(keys) + params = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.where_subquery(self.query_wrapper_ptr, sub_query.query_wrapper_ptr, condition.value, params) @@ -199,7 +220,7 @@ def where_subquery(self, index: str, condition: CondType, sub_query: Query) -> Q self.api.where_field_subquery(self.query_wrapper_ptr, index, condition.value, sub_query.query_wrapper_ptr) return self - def where_composite(self, index: str, condition: CondType, *keys: simple_types) -> Query: + def where_composite(self, index: str, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: """Adds where condition to DB query with interface args for composite indexes #### Arguments: @@ -210,10 +231,12 @@ def where_composite(self, index: str, condition: CondType, *keys: simple_types) #### Returns: (:obj:`Query`): Query object for further customizations + #### Raises: + Exception: Raises with an error message of API return on non-zero error code + """ - param = [key for key in keys] - return self.where(index, condition, param) + return self.__where(index, condition, keys) def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: """Adds where condition to DB query with UUID as string args. @@ -476,7 +499,7 @@ def aggregate_facet(self, *fields: str) -> Query._AggregateFacet: self.__raise_on_error() return self._AggregateFacet(self) - def sort(self, index: str, desc: bool = False, keys: Union[simple_types, List[simple_types]] = None) -> Query: + def sort(self, index: str, desc: bool = False, keys: tuple[list[simple_types], ...] = None) -> Query: """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only @@ -494,7 +517,7 @@ def sort(self, index: str, desc: bool = False, keys: Union[simple_types, List[si """ - params: list = self.__convert_to_list(keys) + params = self.__convert_to_list(keys) self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, params) self.__raise_on_error() @@ -806,7 +829,7 @@ def update(self) -> QueryResults: if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Update does not support joined queries") - (self.err_code, self.err_msg, + (self.err_code, self.err_msg, _, wrapper_ptr, iter_count, total_count) = self.api.update_query(self.query_wrapper_ptr) self.__raise_on_error() return QueryResults(self.api, wrapper_ptr, iter_count, total_count) From 49ddd645276b2e9173203467d3ab40f3e31b20fa Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 21 Nov 2024 16:46:17 +0300 Subject: [PATCH 095/125] Part XXX: where_composite fixes. Part. Fix --- pyreindexer/lib/include/query_wrapper.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 7c77dc1..86508da 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -29,12 +29,6 @@ void QueryWrapper::Where(std::string_view index, CondType condition, const std:: ser_.PutVString(index); ser_.PutVarUint(nextOperation_); ser_.PutVarUint(condition); - - ser_.PutVarUint(keys.size()); - for (const auto& key : keys) { - ser_.PutVariant(key); - } - putKeys(keys); nextOperation_ = OpType::OpAnd; From cfce0141c478ab9af863143c845326e2ba8b6c0a Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 21 Nov 2024 21:09:15 +0300 Subject: [PATCH 096/125] Part XXX: where_composite done --- README.md | 60 ++++++++++-------- pyreindexer/query.py | 88 +++++++++++++++++---------- pyreindexer/tests/tests/test_query.py | 17 +++--- 3 files changed, 99 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 4a873f7..be6028f 100644 --- a/README.md +++ b/README.md @@ -739,9 +739,8 @@ An object representing the context of a Reindexer query ### Query.where ```python -def where(index: str, - condition: CondType, - keys: Union[simple_types, List[simple_types]] = None) -> Query +def where(index: str, condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query ``` Adds where condition to DB query with args @@ -749,7 +748,7 @@ Adds where condition to DB query with args #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (Union[None, simple_types, list[simple_types]]): + keys (list[simple_types], ...): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -764,9 +763,9 @@ Adds where condition to DB query with args ### Query.where\_query ```python -def where_query(sub_query: Query, - condition: CondType, - keys: Union[simple_types, List[simple_types]] = None) -> Query +def where_query( + sub_query: Query, condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query ``` Adds sub-query where condition to DB query with args @@ -774,7 +773,7 @@ Adds sub-query where condition to DB query with args #### Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (Union[None, simple_types, list[simple_types]]): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -807,8 +806,8 @@ Adds sub-query where condition to DB query ### Query.where\_composite ```python -def where_composite(index: str, condition: CondType, *keys: - simple_types) -> Query +def where_composite(index: str, condition: CondType, + keys: tuple[list[simple_types], ...]) -> Query ``` Adds where condition to DB query with interface args for composite indexes @@ -816,11 +815,20 @@ Adds where condition to DB query with interface args for composite indexes #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (*simple_types): Values of composite index to be compared with (value of each sub-index) + keys (list[simple_types], ...): Values of composite index to be compared with (value of each sub-index). + Supported variants: + ([1, "test1"], [2, "test2"]) + [[1, "test1"], [2, "test2"]]) + ([1, "testval1"], ) + [[1, "testval1"]] + (1, "testval1") #### Returns: (:obj:`Query`): Query object for further customizations +#### Raises: + Exception: Raises with an error message of API return on non-zero error code + ### Query.where\_uuid @@ -836,8 +844,8 @@ Adds where condition to DB query with UUID as string args. #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, - with value of each sub-index + keys (*string): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -908,8 +916,8 @@ Adds string EQ-condition to DB query with string args #### Arguments: index (string): Field name used in condition clause - keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, - with value of each sub-index + keys (*string): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -1028,7 +1036,7 @@ Gets fields facet value. Applicable to multiple data fields and the result of th this method returns AggregationFacetRequest which has methods sort, limit and offset #### Arguments: - fields (list[*string]): Fields any data column name or `count`, fields should not be empty + fields (*string): Fields any data column name or `count`, fields should not be empty #### Returns: (:obj:`_AggregateFacet`): Request object for further customizations @@ -1038,9 +1046,11 @@ Gets fields facet value. Applicable to multiple data fields and the result of th ### Query.sort ```python -def sort(index: str, - desc: bool = False, - keys: Union[simple_types, List[simple_types]] = None) -> Query +def sort( + index: str, + desc: bool = False, + keys: Union[simple_types, tuple[list[simple_types], + ...]] = None) -> Query ``` Applies sort order to return from query items. If values argument specified, then items equal to values, @@ -1049,7 +1059,7 @@ Applies sort order to return from query items. If values argument specified, the #### Arguments: index (string): The index name desc (bool): Sort in descending order - keys (Union[None, simple_types, List[simple_types]]): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to match. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -1305,7 +1315,7 @@ Executes a query, and delete items, matches query ### Query.set\_object ```python -def set_object(field: str, values: List[simple_types]) -> Query +def set_object(field: str, values: list[simple_types]) -> Query ``` Adds an update query to an object field for an update query @@ -1325,7 +1335,7 @@ Adds an update query to an object field for an update query ### Query.set ```python -def set(field: str, values: List[simple_types]) -> Query +def set(field: str, values: list[simple_types]) -> Query ``` Adds a field update request to the update request @@ -1530,7 +1540,7 @@ Sets list of columns in this namespace to be finally selected. If there are no fields in this list that meet these conditions, then the filter works as "*" #### Arguments: - fields (list[*string]): List of columns to be selected + fields (*string): List of columns to be selected #### Returns: (:obj:`Query`): Query object for further customizations @@ -1549,7 +1559,7 @@ def functions(*functions: str) -> Query Adds sql-functions to query #### Arguments: - functions (list[*string]): Functions declaration + functions (*string): Functions declaration #### Returns: (:obj:`Query`): Query object for further customizations @@ -1568,7 +1578,7 @@ def equal_position(*equal_position: str) -> Query Adds equal position fields to arrays queries #### Arguments: - equal_poses (list[*string]): Equal position fields to arrays queries + equal_poses (*string): Equal position fields to arrays queries #### Returns: (:obj:`Query`): Query object for further customizations diff --git a/pyreindexer/query.py b/pyreindexer/query.py index e409d15..a60a8ed 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import Iterable from enum import Enum -from typing import List, Optional, Union +from typing import Optional, Union from pyreindexer.point import Point from pyreindexer.query_results import QueryResults @@ -77,8 +78,8 @@ def __init__(self, api, query_wrapper_ptr: int): self.err_msg: str = '' self.root: Optional[Query] = None self.join_type: JoinType = JoinType.LeftJoin - self.join_queries: List[Query] = [] - self.merged_queries: List[Query] = [] + self.join_queries: list[Query] = [] + self.merged_queries: list[Query] = [] def __del__(self): """Frees query memory @@ -100,27 +101,42 @@ def __raise_on_error(self): raise Exception(self.err_msg) @staticmethod - def __convert_to_list(param: tuple[list[simple_types], ...]) -> list[list[simple_types]]: + def __convert_to_list(param: Union[simple_types, tuple[list[simple_types], ...]]) -> list: """Converts an input parameter to a list of lists #### Arguments: - param (list[simple_types], ...): The input parameter + param (union[simple_types, (list[simple_types], ...)]): The input parameter #### Returns: - list[list[union[int, bool, float, str]]]: Always converted to a list of lists + list[union[union[int, bool, float, str], list[union[int, bool, float, str]]]: + Always converted to a list of lists """ - result = list(list()) if param is None else param - result = result if isinstance(result, list) else list(result) - return result + if param is None: + return list() + + if isinstance(param, str) or not isinstance(param, Iterable): + result: list = [param] + return result + + if isinstance(param, list): + return param + + res = param + if not isinstance(res, list): + res = list(res) + if len(res) == 0 or (len(res) > 0 and not isinstance(res[0], list)): + wrap : list = [res] + res = wrap + return res @staticmethod def __convert_strs_to_list(param: tuple[str, ...]) -> list[str]: """Converts an input parameter to a list #### Arguments: - param (list[*string]): The input parameter + param (tuple[string, ...]): The input parameter #### Returns: list[string]: Always converted to a list @@ -130,13 +146,14 @@ def __convert_strs_to_list(param: tuple[str, ...]) -> list[str]: result = list() if param is None else list(param) return result - def __where(self, index: str, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: + def __where(self, index: str, condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query: """Adds where condition to DB query with args #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[simple_types], ...): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -152,20 +169,19 @@ def __where(self, index: str, condition: CondType, keys: tuple[list[simple_types raise Exception("In this case, use a special method 'dwithin'") params = self.__convert_to_list(keys) - print('===================== keys: ' + str(keys)) # ToDo - print('===================== params: ' + str(params)) # ToDo self.err_code, self.err_msg = self.api.where(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() return self - def where(self, index: str, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: + def where(self, index: str, condition: CondType, + keys: Union[simple_types, tuple[list[simple_types],...]]) -> Query: """Adds where condition to DB query with args #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (*list[simple_types]): + keys (list[simple_types], ...): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -179,13 +195,14 @@ def where(self, index: str, condition: CondType, keys: tuple[list[simple_types], return self.__where(index, condition, keys) - def where_query(self, sub_query: Query, condition: CondType, keys: tuple[list[simple_types], ...]) -> Query: + def where_query(self, sub_query: Query, condition: CondType, + keys: Union[simple_types, tuple[list[simple_types],...]]) -> Query: """Adds sub-query where condition to DB query with args #### Arguments: sub_query (:obj:`Query`): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (*list[simple_types]): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -226,7 +243,13 @@ def where_composite(self, index: str, condition: CondType, keys: tuple[list[simp #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (*simple_types): Values of composite index to be compared with (value of each sub-index) + keys (list[simple_types], ...): Values of composite index to be compared with (value of each sub-index). + Supported variants: + ([1, "test1"], [2, "test2"]) + [[1, "test1"], [2, "test2"]]) + ([1, "testval1"], ) + [[1, "testval1"]] + (1, "testval1") #### Returns: (:obj:`Query`): Query object for further customizations @@ -246,8 +269,8 @@ def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, - with value of each sub-index + keys (*string): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -314,8 +337,8 @@ def match(self, index: str, *keys: str) -> Query: #### Arguments: index (string): Field name used in condition clause - keys (list[*string]): Value of index to be compared with. For composite indexes keys must be list, - with value of each sub-index + keys (*string): Value of index to be compared with. For composite indexes keys must be list, + with value of each sub-index #### Returns: (:obj:`Query`): Query object for further customizations @@ -486,7 +509,7 @@ def aggregate_facet(self, *fields: str) -> Query._AggregateFacet: this method returns AggregationFacetRequest which has methods sort, limit and offset #### Arguments: - fields (list[*string]): Fields any data column name or `count`, fields should not be empty + fields (*string): Fields any data column name or `count`, fields should not be empty #### Returns: (:obj:`_AggregateFacet`): Request object for further customizations @@ -499,14 +522,15 @@ def aggregate_facet(self, *fields: str) -> Query._AggregateFacet: self.__raise_on_error() return self._AggregateFacet(self) - def sort(self, index: str, desc: bool = False, keys: tuple[list[simple_types], ...] = None) -> Query: + def sort(self, index: str, desc: bool = False, + keys: Union[simple_types, tuple[list[simple_types],...]] = None) -> Query: """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only #### Arguments: index (string): The index name desc (bool): Sort in descending order - keys (Union[None, simple_types, List[simple_types]]): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to match. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -743,7 +767,7 @@ def delete(self) -> int: self.__raise_on_error() return number - def set_object(self, field: str, values: List[simple_types]) -> Query: + def set_object(self, field: str, values: list[simple_types]) -> Query: """Adds an update query to an object field for an update query #### Arguments: @@ -764,7 +788,7 @@ def set_object(self, field: str, values: List[simple_types]) -> Query: self.__raise_on_error() return self - def set(self, field: str, values: List[simple_types]) -> Query: + def set(self, field: str, values: list[simple_types]) -> Query: """Adds a field update request to the update request #### Arguments: @@ -829,7 +853,7 @@ def update(self) -> QueryResults: if (self.root is not None) or (len(self.join_queries) > 0): raise Exception("Update does not support joined queries") - (self.err_code, self.err_msg, _, + (self.err_code, self.err_msg, wrapper_ptr, iter_count, total_count) = self.api.update_query(self.query_wrapper_ptr) self.__raise_on_error() return QueryResults(self.api, wrapper_ptr, iter_count, total_count) @@ -1000,7 +1024,7 @@ def select(self, *fields: str) -> Query: If there are no fields in this list that meet these conditions, then the filter works as "*" #### Arguments: - fields (list[*string]): List of columns to be selected + fields (*string): List of columns to be selected #### Returns: (:obj:`Query`): Query object for further customizations @@ -1020,7 +1044,7 @@ def functions(self, *functions: str) -> Query: """Adds sql-functions to query #### Arguments: - functions (list[*string]): Functions declaration + functions (*string): Functions declaration #### Returns: (:obj:`Query`): Query object for further customizations @@ -1040,7 +1064,7 @@ def equal_position(self, *equal_position: str) -> Query: """Adds equal position fields to arrays queries #### Arguments: - equal_poses (list[*string]): Equal position fields to arrays queries + equal_poses (*string): Equal position fields to arrays queries #### Returns: (:obj:`Query`): Query object for further customizations diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 376d169..6b59a64 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -129,15 +129,14 @@ def test_query_select_where_query_empty(self, db, namespace, index, items): # Then ("Check that result is empty") assert_that(query_result, empty(), "Wrong query results") - # ToDo - # def test_query_select_where_composite(self, db, namespace, composite_index, items): - # # Given("Create namespace with composite index") - # # Given ("Create new query") - # query = db.query.new(namespace) - # # When ("Make select query with where_composite") - # query_result = list(query.where_composite("comp_idx", CondType.CondEq, 1, "testval1").must_execute()) - # # Then ("Check that selected item is in result") - # assert_that(query_result, equal_to([items[1]]), "Wrong query results") + def test_query_select_where_composite(self, db, namespace, composite_index, items): + # Given("Create namespace with composite index") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with where_composite") + query_result = list(query.where_composite("comp_idx", CondType.CondEq, (1, "testval1")).must_execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1]]), "Wrong query results") def test_query_select_where_uuid(self, db, namespace, index): # Given("Create namespace with index") From 437856297fadd62d684d1f2ad013790571014d69 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 25 Nov 2024 15:54:30 +0300 Subject: [PATCH 097/125] Part XXXI: join support/ Part --- pyreindexer/lib/include/query_wrapper.cc | 16 ++++++++++++---- pyreindexer/lib/include/query_wrapper.h | 2 +- pyreindexer/lib/src/rawpyreindexer.cc | 5 ++--- pyreindexer/query.py | 8 +++----- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 86508da..110fcca 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -231,6 +231,10 @@ reindexer::Error QueryWrapper::prepareQuery(reindexer::Query& query) { } } catch (const reindexer::Error& err) { error = err; + } catch (const std::exception& ex) { + error = reindexer::Error(ErrorCode::errQueryExec, ex.what()); + } catch (...) { + error = reindexer::Error(ErrorCode::errQueryExec, "Internal error"); } return error; @@ -299,7 +303,7 @@ void QueryWrapper::Drop(std::string_view field) { ser_.PutVString(field); } -void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery) { +void QueryWrapper::Join(JoinType type, QueryWrapper* joinQuery) { assert(joinQuery); joinType_ = type; @@ -307,10 +311,14 @@ void QueryWrapper::Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* jo nextOperation_ = OpType::OpAnd; joinType_ = JoinType::OrInnerJoin; } - ser_.PutVarUint(QueryJoinCondition); - ser_.PutVarUint(joinType_); - ser_.PutVarUint(joinQueryIndex); + if (joinType_ != JoinType::LeftJoin) { + ser_.PutVarUint(QueryJoinCondition); + ser_.PutVarUint(joinType_); + ser_.PutVarUint(joinQueries_.size()); + } + + joinQuery->joinType_ = type; joinQueries_.push_back(joinQuery); } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index afee1e3..f249b22 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -64,7 +64,7 @@ class QueryWrapper { void Drop(std::string_view field); - void Join(JoinType type, unsigned joinQueryIndex, QueryWrapper* joinQuery); + void Join(JoinType type, QueryWrapper* joinQuery); void Merge(QueryWrapper* mergeQuery); reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 55dd20c..158b09c 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1135,15 +1135,14 @@ static PyObject* SetExpression(PyObject* self, PyObject* args) { static PyObject* Join(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; unsigned type = 0; - unsigned index = 0; uintptr_t queryWrapperAddrJoin = 0; - if (!PyArg_ParseTuple(args, "kIIk", &queryWrapperAddr, &type, &index, &queryWrapperAddrJoin)) { + if (!PyArg_ParseTuple(args, "kIk", &queryWrapperAddr, &type, &queryWrapperAddrJoin)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); auto queryJoin = getWrapper(queryWrapperAddrJoin); - query->Join(JoinType(type), index, queryJoin); + query->Join(JoinType(type), queryJoin); Py_RETURN_NONE; } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index a60a8ed..b8fbf17 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -181,7 +181,7 @@ def where(self, index: str, condition: CondType, #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[simple_types], ...): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -858,7 +858,6 @@ def update(self) -> QueryResults: self.__raise_on_error() return QueryResults(self.api, wrapper_ptr, iter_count, total_count) - def must_execute(self) -> QueryResults: """Executes a query, and update fields in items, which matches query, with status check @@ -918,9 +917,8 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: if query.root is not None: raise Exception("Query.join call on already joined query. You should create new Query") - if join_type is not JoinType.LeftJoin: - # index of join query - self.api.join(self.query_wrapper_ptr, join_type.value, len(self.join_queries), query.query_wrapper_ptr) + # index of join query + self.api.join(self.query_wrapper_ptr, join_type.value, query.query_wrapper_ptr) query.join_type = join_type query.root = self From bbf4b411b1ce66acb317fa2cfbeb0a1a841d8e40 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 25 Nov 2024 20:31:25 +0300 Subject: [PATCH 098/125] Part XXXI: join support. Part --- pyreindexer/lib/include/query_wrapper.cc | 34 ++++++++-------- pyreindexer/lib/include/query_wrapper.h | 2 +- pyreindexer/tests/tests/test_query.py | 51 ++++++++++++------------ 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 110fcca..7a28258 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -195,40 +195,40 @@ void QueryWrapper::Modifier(QueryItemType type) { } namespace { -reindexer::Serializer prepareQueryData(reindexer::WrSerializer& data) { - reindexer::WrSerializer buffer; +void serializeQuery(reindexer::WrSerializer& data, reindexer::WrSerializer& buffer) { buffer.Write(data.Slice()); // do full copy of query data buffer.PutVarUint(QueryItemType::QueryEnd); // close query data - return {buffer.Buf(), buffer.Len()}; } -reindexer::JoinedQuery createJoinedQuery(JoinType joinType, reindexer::WrSerializer& data) { - reindexer::Serializer jser = prepareQueryData(data); - return {joinType, reindexer::Query::Deserialize(jser)}; +void serializeJoinQuery(JoinType type, reindexer::WrSerializer& data, reindexer::WrSerializer& buffer) { + buffer.PutVarUint(type); + serializeQuery(data, buffer); } -} // namespace +} // namespace -void QueryWrapper::addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) const { - for (auto joinQuery : joinQueries) { - auto jq = createJoinedQuery(joinQuery->joinType_, joinQuery->ser_); - query.AddJoinQuery(std::move(jq)); +void QueryWrapper::addJoinQueries(const std::vector& queries, reindexer::WrSerializer& buffer) const { + for (auto query : queries) { + serializeJoinQuery(query->joinType_, query->ser_, buffer); } } reindexer::Error QueryWrapper::prepareQuery(reindexer::Query& query) { reindexer::Error error = errOK; try { - reindexer::Serializer ser = prepareQueryData(ser_); - query = reindexer::Query::Deserialize(ser); + // current query (root) + reindexer::WrSerializer buffer; + serializeQuery(ser_, buffer); - addJoinQueries(joinQueries_, query); + addJoinQueries(joinQueries_, buffer); for (auto mergedQuery : mergedQueries_) { - auto mq = createJoinedQuery(JoinType::Merge, mergedQuery->ser_); - query.Merge(std::move(mq)); + serializeJoinQuery(JoinType::Merge, mergedQuery->ser_, buffer); - addJoinQueries(mergedQuery->joinQueries_, mq); + addJoinQueries(mergedQuery->joinQueries_, buffer); } + + reindexer::Serializer fullQueryData{buffer.Buf(), buffer.Len()}; + query = reindexer::Query::Deserialize(fullQueryData); } catch (const reindexer::Error& err) { error = err; } catch (const std::exception& ex) { diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index f249b22..c3f65f5 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -77,7 +77,7 @@ class QueryWrapper { DBInterface* GetDB() const { return db_; } private: - void addJoinQueries(const std::vector& joinQueries, reindexer::Query& query) const; + void addJoinQueries(const std::vector& queries, reindexer::WrSerializer& buffer) const; reindexer::Error prepareQuery(reindexer::Query& query); void putKeys(const std::vector& keys); diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 6b59a64..3bfaf7a 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -521,32 +521,31 @@ def test_query_select_sort_strict_mode_indexes(self, db, namespace, index, items raises(Exception, pattern=err_msg)) -# ToDo implement List joinedActors = QueryResult.getJoinedActors(); -# class TestQuerySelectJoin: -# def test_query_select_left_join(self, db, namespace, index, items, second_namespace): -# # Given("Create two namespaces") -# second_namespace, item2 = second_namespace -# # Given ("Create two queries for join") -# query1 = db.query.new(namespace).where("id", CondType.CondLt, 3) -# query2 = db.query.new(second_namespace) -# # When ("Make select query with join") -# query_result = list(query1.join(query2, "joined").on("id", CondType.CondEq, "id").must_execute()) -# # Then ("Check that joined item is in result") -# item_with_joined = {"id": 1, "joined": [item2]} -# items[1] = item_with_joined -# assert_that(query_result, equal_to(items[:3]), "Wrong selected items with JOIN") -# -# def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): -# # Given("Create two namespaces") -# second_namespace, item2 = second_namespace -# # Given ("Create two queries for join") -# query1 = db.query.new(namespace) -# query2 = db.query.new(second_namespace) -# # When ("Make select query with join") -# query_result = list(query1.inner_join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) -# # Then ("Check that joined item is in result") -# item_with_joined = {'id': 1, f'joined_{second_namespace}': [item2]} -# assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") +class TestQuerySelectJoin: + def test_query_select_left_join(self, db, namespace, index, items, second_namespace): + # Given("Create two namespaces") + second_namespace, item2 = second_namespace + # Given ("Create two queries for join") + query1 = db.query.new(namespace).where("id", CondType.CondLt, 3) + query2 = db.query.new(second_namespace) + # When ("Make select query with join") + query_result = list(query1.join(query2, "joined").on("id", CondType.CondEq, "id").must_execute()) + # Then ("Check that joined item is in result") + item_with_joined = {'id': 1, 'val': 'testval1', f'joined_{second_namespace}': [item2]} + items[1] = item_with_joined + assert_that(query_result, equal_to(items[:3]), "Wrong selected items with JOIN") + + def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): + # Given("Create two namespaces") + second_namespace, item2 = second_namespace + # Given ("Create two queries for join") + query1 = db.query.new(namespace) + query2 = db.query.new(second_namespace) + # When ("Make select query with join") + query_result = list(query1.inner_join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) + # Then ("Check that joined item is in result") + item_with_joined = {'id': 1, 'val': 'testval1', f'joined_{second_namespace}': [item2]} + assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") class TestQueryUpdate: From 9ec593fd1e505915ef16e9cb08a7e474b15b984d Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 26 Nov 2024 10:40:24 +0300 Subject: [PATCH 099/125] Part XXXI: join support. Simplify code --- pyreindexer/lib/include/query_wrapper.cc | 2 +- pyreindexer/query.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 7a28258..914ddf6 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -318,7 +318,7 @@ void QueryWrapper::Join(JoinType type, QueryWrapper* joinQuery) { ser_.PutVarUint(joinQueries_.size()); } - joinQuery->joinType_ = type; + joinQuery->joinType_ = joinType_; joinQueries_.push_back(joinQuery); } diff --git a/pyreindexer/query.py b/pyreindexer/query.py index b8fbf17..f918de3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -57,7 +57,6 @@ class Query: err_code (int): The API error code err_msg (string): The API error message root (:object: Optional[`Query`]): The root query of the Reindexer query - join_type (:enum:`JoinType`): Join type join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects @@ -77,7 +76,6 @@ def __init__(self, api, query_wrapper_ptr: int): self.err_code: int = 0 self.err_msg: str = '' self.root: Optional[Query] = None - self.join_type: JoinType = JoinType.LeftJoin self.join_queries: list[Query] = [] self.merged_queries: list[Query] = [] @@ -920,7 +918,6 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: # index of join query self.api.join(self.query_wrapper_ptr, join_type.value, query.query_wrapper_ptr) - query.join_type = join_type query.root = self self.join_queries.append(query) return query From 6910083dddf805de8b9d96d7bac7c0064075f9ef Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 27 Nov 2024 11:15:45 +0300 Subject: [PATCH 100/125] add join tests --- pyreindexer/tests/conftest.py | 30 ++++-- pyreindexer/tests/tests/test_query.py | 141 ++++++++++++++++++++++---- pyreindexer/tests/tests/test_sql.py | 5 +- 3 files changed, 147 insertions(+), 29 deletions(-) diff --git a/pyreindexer/tests/conftest.py b/pyreindexer/tests/conftest.py index 665d3fc..23d075e 100644 --- a/pyreindexer/tests/conftest.py +++ b/pyreindexer/tests/conftest.py @@ -192,11 +192,29 @@ def metadata(db, namespace): @pytest.fixture def second_namespace(db): - second_namespace_name = 'test_ns_for_join' + second_namespace_name = "test_ns_for_join" db.namespace.open(second_namespace_name) db.index.create(second_namespace_name, index_definition) - second_ns_item_definition = {"id": 100, "second_ns_val": "second_ns_testval"} - second_ns_item_definition_join = {"id": 1, "second_ns_val": "second_ns_testval_1"} - db.item.insert(second_namespace_name, second_ns_item_definition) - db.item.insert(second_namespace_name, second_ns_item_definition_join) - yield second_namespace_name, second_ns_item_definition_join + yield second_namespace_name + db.namespace.drop(second_namespace_name) + + +@pytest.fixture +def second_item(db, second_namespace): + """ + Create item for the second namespace + """ + item = {"id": 1, "second_ns_val": "second_ns_testval_1"} + db.item.insert(second_namespace, item) + yield item + + +@pytest.fixture +def second_items(db, second_namespace): + """ + Create more items for the second namespace + """ + items = [{"id": i, "second_ns_val": f"second_ns_testval_{i}"} for i in range(1, 6)] + for item in items: + db.item.insert(second_namespace, item) + yield items diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 3bfaf7a..8d76010 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -27,9 +27,10 @@ def test_query_select_where_array(self, db, namespace, index, array_index_and_it # Given ("Create new query") query = db.query.new(namespace) # When ("Make select query") - query_result = list(query.where("arr", CondType.CondEq, [3]).must_execute()) + query_result = list(query.where("arr", CondType.CondEq, [2, 3]).must_execute()) # Then ("Check that selected item is in result") - assert_that(query_result, equal_to([items[2], items[3]]), "Wrong query results") + expected_items = [items[i] for i in [1, 2, 3]] + assert_that(query_result, equal_to(expected_items), "Wrong query results") def test_query_select_all(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -102,9 +103,9 @@ def test_query_select_where_query_field(self, db, namespace, index, items): # Given("Create namespace with index and items") # Given ("Create new query") sub_query = (db.query.new(namespace) - .select("id") - .where("id", CondType.CondLt, 5) - .where("id", CondType.CondGe, 2)) + .select("id") + .where("id", CondType.CondLt, 5) + .where("id", CondType.CondGe, 2)) # When ("Make select query with where_subquery") query_result = list(db.query.new(namespace).where_subquery('id', CondType.CondSet, sub_query).execute()) # Then ("Check that selected item is in result") @@ -138,6 +139,20 @@ def test_query_select_where_composite(self, db, namespace, composite_index, item # Then ("Check that selected item is in result") assert_that(query_result, equal_to([items[1]]), "Wrong query results") + @pytest.mark.parametrize("value", [ + [[1, "testval1"], [2, "testval2"]], + [(1, "testval1"), (2, "testval2")], + ([1, "testval1"], [2, "testval2"]) + ]) + def test_query_select_where_composite_multiple_results(self, db, namespace, composite_index, items, value): + # Given("Create namespace with composite index") + # Given ("Create new query") + query = db.query.new(namespace) + # When ("Make select query with where_composite") + query_result = list(query.where_composite("comp_idx", CondType.CondEq, value).must_execute()) + # Then ("Check that selected item is in result") + assert_that(query_result, equal_to([items[1], items[2]]), "Wrong query results") + def test_query_select_where_uuid(self, db, namespace, index): # Given("Create namespace with index") # Given ("Create uuid index") @@ -275,10 +290,8 @@ def test_query_select_functions(self, db, namespace, index, ft_index_and_items): expected_ft_content = ["one ", " two", "three 333"] assert_that(query_results_ft, contains_inanyorder(*expected_ft_content), "Wrong query results") - def test_query_select_merge(self, db, namespace, index, items, second_namespace): - # Given("Create namespace with index and items") - # Given("Create second namespace with index and items") - second_namespace, item2 = second_namespace + def test_query_select_merge(self, db, namespace, index, items, second_namespace, second_item): + # Given("Create two namespaces with index and items") # Given ("Create new query") query1 = db.query.new(namespace).where("id", CondType.CondEq, 2) # Given ("Create second query") @@ -286,7 +299,7 @@ def test_query_select_merge(self, db, namespace, index, items, second_namespace) # When ("Make select query with merge") query_result = list(query1.merge(query2).must_execute()) # Then ("Check that selected item is in result with merge applied") - assert_that(query_result, equal_to([items[2], item2]), "Wrong query results") + assert_that(query_result, equal_to([items[2], second_item]), "Wrong query results") def test_query_select_explain(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -522,29 +535,118 @@ def test_query_select_sort_strict_mode_indexes(self, db, namespace, index, items class TestQuerySelectJoin: - def test_query_select_left_join(self, db, namespace, index, items, second_namespace): - # Given("Create two namespaces") - second_namespace, item2 = second_namespace + def test_query_select_left_join(self, db, namespace, index, items, second_namespace, second_item): + # Given("Create two namespaces with index and items") # Given ("Create two queries for join") query1 = db.query.new(namespace).where("id", CondType.CondLt, 3) query2 = db.query.new(second_namespace) # When ("Make select query with join") query_result = list(query1.join(query2, "joined").on("id", CondType.CondEq, "id").must_execute()) # Then ("Check that joined item is in result") - item_with_joined = {'id': 1, 'val': 'testval1', f'joined_{second_namespace}': [item2]} + item_with_joined = {"id": 1, "val": "testval1", f"joined_{second_namespace}": [second_item]} items[1] = item_with_joined assert_that(query_result, equal_to(items[:3]), "Wrong selected items with JOIN") - def test_query_select_inner_join(self, db, namespace, index, items, second_namespace): - # Given("Create two namespaces") - second_namespace, item2 = second_namespace + def test_query_select_inner_join(self, db, namespace, index, items, second_namespace, second_item): + # Given("Create two namespaces with index and items") # Given ("Create two queries for join") query1 = db.query.new(namespace) query2 = db.query.new(second_namespace) # When ("Make select query with join") - query_result = list(query1.inner_join(query2, "id").on("id", CondType.CondEq, "id").must_execute()) + query_result = list(query1.inner_join(query2, "joined").on("id", CondType.CondEq, "id").must_execute()) # Then ("Check that joined item is in result") - item_with_joined = {'id': 1, 'val': 'testval1', f'joined_{second_namespace}': [item2]} + item_with_joined = {"id": 1, "val": "testval1", f"joined_{second_namespace}": [second_item]} + assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") + + def test_query_select_left_and_inner_join(self, db, namespace, index, items, second_namespace, second_items): + # Given("Create two namespaces with index and items") + # Given ("Create join query 1") + query1 = db.query.new(namespace).where("id", CondType.CondRange, [0, 5]) + query2 = db.query.new(second_namespace).where("id", CondType.CondGe, 2) + join_query1 = query1.left_join(query2, "joined").on("id", CondType.CondEq, "id") + query3 = db.query.new(second_namespace).where("id", CondType.CondRange, [0, 2]) + join_query2 = join_query1.inner_join(query3, "joined").on("id", CondType.CondEq, "id") + # When ("Make select query with join") + query_result = list(join_query2.must_execute()) + # Then ("Check that joined items are in result") + expected_items = [ + {"id": 1, "val": "testval1", f"joined_2_{second_namespace}": [second_items[0]]}, + {"id": 2, "val": "testval2", f"joined_1_{second_namespace}": [second_items[1]], + f"joined_2_{second_namespace}": [second_items[1]]} + ] + assert_that(query_result, equal_to(expected_items), "Wrong selected items with JOIN") + + def test_cannot_query_select_join_without_on(self, db, namespace, index, items, second_namespace, second_item): + # Given("Create two namespaces with index and items") + # Given ("Create two queries for join") + query1 = db.query.new(namespace) + query2 = db.query.new(second_namespace) + # When ("Try to ake select query join without on") + assert_that(calling(query1.inner_join(query2, "joined").execute).with_args(), + raises(Exception, pattern="Join without ON conditions")) + + def test_query_select_merge_with_joins(self, db, namespace, index, items, second_namespace, second_item): + # Given("Create two namespaces with index and items") + # Given ("Create join query 1") + query11 = db.query.new(namespace) + query12 = db.query.new(second_namespace) + join_query1 = query11.inner_join(query12, "joined").on("id", CondType.CondEq, "id") + # Given ("Create join query 2") + query21 = db.query.new(namespace).where("id", CondType.CondSet, [0, 1, 3]) + query22 = db.query.new(second_namespace) + join_query2 = query21.join(query22, "joined").on("id", CondType.CondEq, "id") + # When ("Make select query with merge") + query_result = list(join_query1.merge(join_query2).must_execute()) + # Then ("Check that selected items are in result with join and merge applied") + item_with_joined = {"id": 1, "val": "testval1", f"joined_{second_namespace}": [second_item]} + expected_items = [item_with_joined, items[0], item_with_joined, items[3]] + assert_that(query_result, equal_to(expected_items), "Wrong query results") + + def test_query_select_join_with_merges(self, db, namespace, index, items, second_namespace, second_items): + # Given("Create two namespaces with index and items") + # Given ("Create merge query 1") + query11 = db.query.new(namespace).where("id", CondType.CondSet, [2, 3]) + query12 = db.query.new(second_namespace).where("id", CondType.CondEq, 5) + merge_query1 = query11.merge(query12) + # Given ("Create merge query 2") + query21 = db.query.new(second_namespace).where("id", CondType.CondLt, 4).where("id", CondType.CondGe, 2) + query22 = db.query.new(namespace).where("id", CondType.CondRange, [3, 5]) + merge_query2 = query21.merge(query22) + # When ("Make select query with join") + query_result = list(merge_query1.inner_join(merge_query2, "joined").on("id", CondType.CondEq, "id").execute()) + # Then ("Check that selected items are in result with join and merge applied") + expected_items = [{"id": 2, "val": "testval2", + f"joined_{second_namespace}": [{"id": 2, "second_ns_val": "second_ns_testval_2"}]}, + {"id": 3, "val": "testval3", + f"joined_{second_namespace}": [{"id": 3, "second_ns_val": "second_ns_testval_3"}]}, + {"id": 5, "second_ns_val": "second_ns_testval_5"}] + assert_that(query_result, equal_to(expected_items), "Wrong query results") + + def test_query_select_sort_and_inner_join(self, db, namespace, index, items, second_namespace, second_items): + # Given("Create two namespaces with index and items") + # Given ("Create two queries for join") + query1 = db.query.new(namespace).sort("id", True) + query2 = db.query.new(second_namespace) + # When ("Make select query with sort and join") + query_result = list(query1.inner_join(query2, "joined").on("id", CondType.CondEq, "id").must_execute()) + # Then ("Check that joined items are in sorted result") + expected_items = [{"id": i, "val": f"testval{i}", f"joined_{second_namespace}": [second_items[i - 1]]} + for i in range(5, 0, -1)] + assert_that(query_result, equal_to(expected_items), "Wrong selected items with JOIN") + + def test_query_select_inner_join_sort_joined(self, db, namespace, index, items, second_namespace): + # Given("Create two namespaces with index and items") + items_2 = [{"id": 1, "age": 1}, {"id": 3, "age": 1}, {"id": 2, "age": 1}] + [db.item.insert(second_namespace, item) for item in items_2] + # Given ("Create two queries for join") + query1 = db.query.new(namespace) + query2 = db.query.new(second_namespace) + # When ("Make select query with join and sort") + query_result = list(query1.inner_join(query2, "joined").on("id", CondType.CondEq, "age") + .sort("id").must_execute()) + # Then ("Check that sorted joined items are in result") + sorted_items2 = sorted(items_2, key=lambda x: x["id"]) + item_with_joined = {"id": 1, "val": "testval1", f"joined_{second_namespace}": sorted_items2} assert_that(query_result, equal_to([item_with_joined]), "Wrong selected items with JOIN") @@ -640,7 +742,6 @@ def test_cannot_query_drop_not_sparse(self, db, namespace, indexes, items): # Given ("Create new query") query = db.query.new(namespace) # When ("Try to make update drop query with not sparse index") - item = random.choice(items) assert_that(calling(query.drop("val").update).with_args(), raises(Exception, pattern="It's only possible to drop sparse or non-index fields via UPDATE statement!")) diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index 8a4394c..b186a9d 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -10,15 +10,14 @@ def test_sql_select(self, db, namespace, index, item): # Then ("Check that selected item is in result") assert_that(items_list, equal_to([item]), "Can't SQL select data") - def test_sql_select_with_join(self, db, namespace, second_namespace, index, items): + def test_sql_select_with_join(self, db, namespace, index, items, second_namespace, second_item): # Given("Create two namespaces") - second_namespace, second_ns_item_definition_join = second_namespace # When ("Execute SQL query SELECT with JOIN") query = f"SELECT id FROM {namespace} INNER JOIN {second_namespace} " \ f"ON {namespace}.id = {second_namespace}.id" item_list = list(db.query.sql(query)) # Then ("Check that selected item is in result") - item_with_joined = {'id': 1, f'joined_{second_namespace}': [second_ns_item_definition_join]} + item_with_joined = {'id': 1, f'joined_{second_namespace}': [second_item]} assert_that(item_list, equal_to([item_with_joined]), "Can't SQL select data with JOIN") From 0d09c11e17fd1515eb3c9761098f572fbe01bc72 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Wed, 27 Nov 2024 13:07:26 +0300 Subject: [PATCH 101/125] Part XXXII: fix v.4 build --- .../lib/include/queryresults_wrapper.h | 38 ++++++++++--------- pyreindexer/lib/include/transaction_wrapper.h | 20 +++++----- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index 5b84676..4a980b1 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -27,8 +27,7 @@ class QueryResultsWrapper { void Wrap(QueryResultsT&& qres) { qres_ = std::move(qres); - it_ = qres_.begin(); - wrap_ = true; + it_ = qres_->begin(); } Error Select(const std::string& query) { @@ -36,50 +35,55 @@ class QueryResultsWrapper { } Error Status() { - assert(wrap_); + assert(qres_.has_value()); return it_.Status(); } size_t Count() const noexcept { - assert(wrap_); - return qres_.Count(); + assert(qres_.has_value()); + return qres_->Count(); } size_t TotalCount() const noexcept { - assert(wrap_); - return qres_.TotalCount(); + assert(qres_.has_value()); + return qres_->TotalCount(); } void GetItemJSON(reindexer::WrSerializer& wrser, bool withHdrLen) { - assert(wrap_); + assert(qres_.has_value()); it_.GetJSON(wrser, withHdrLen); } void Next() { - assert(wrap_); + assert(qres_.has_value()); db_->FetchResults(*this); } void FetchResults() { - assert(wrap_); + assert(qres_.has_value()); ++it_; - if (it_ == qres_.end()) { - it_ = qres_.begin(); + if (it_ == qres_->end()) { + it_ = qres_->begin(); } } - const std::string& GetExplainResults() const& noexcept { return qres_.GetExplainResults(); } - const std::string& GetExplainResults() const&& = delete; + const std::string& GetExplainResults() & noexcept { + assert(qres_.has_value()); + return qres_->GetExplainResults(); + } + const std::string& GetExplainResults() && = delete; const std::vector& GetAggregationResults() & - { return qres_.GetAggregationResults(); } + { + assert(qres_.has_value()); + return qres_->GetAggregationResults(); + } const std::vector& GetAggregationResults() && = delete; private: DBInterface* db_{nullptr}; - QueryResultsT qres_; + std::optional qres_; QueryResultsT::Iterator it_; - bool wrap_{false}; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/include/transaction_wrapper.h b/pyreindexer/lib/include/transaction_wrapper.h index cc0a9c3..0356a82 100644 --- a/pyreindexer/lib/include/transaction_wrapper.h +++ b/pyreindexer/lib/include/transaction_wrapper.h @@ -30,7 +30,6 @@ class TransactionWrapper { void Wrap(TransactionT&& transaction) { transaction_ = std::move(transaction); - wrap_ = true; } Error Start(std::string_view ns) { @@ -38,29 +37,28 @@ class TransactionWrapper { } ItemT NewItem() { - assert(wrap_); - return db_->NewItem(transaction_); + assert(transaction_.has_value()); + return db_->NewItem(*transaction_); } Error Modify(ItemT&& item, ItemModifyMode mode) { - assert(wrap_); - return db_->Modify(transaction_, std::move(item), mode); + assert(transaction_.has_value()); + return db_->Modify(*transaction_, std::move(item), mode); } Error Commit(size_t& count) { - assert(wrap_); - return db_->CommitTransaction(transaction_, count); + assert(transaction_.has_value()); + return db_->CommitTransaction(*transaction_, count); } Error Rollback() { - assert(wrap_); - return db_->RollbackTransaction(transaction_); + assert(transaction_.has_value()); + return db_->RollbackTransaction(*transaction_); } private: DBInterface* db_{nullptr}; - TransactionT transaction_; - bool wrap_{false}; + std::optional transaction_; }; } // namespace pyreindexer From ca7916e558b45983f95361ff11a5441f21a990ae Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 29 Nov 2024 15:17:13 +0300 Subject: [PATCH 102/125] Corrections and fixes --- README.md | 4 +-- pyreindexer/lib/include/query_wrapper.cc | 31 ++++--------------- .../lib/include/queryresults_wrapper.h | 2 +- pyreindexer/lib/src/reindexerinterface.cc | 2 +- pyreindexer/query.py | 4 ++- pyreindexer/query_results.py | 2 +- setup.py | 1 - 7 files changed, 14 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index be6028f..712b7ee 100644 --- a/README.md +++ b/README.md @@ -730,7 +730,6 @@ An object representing the context of a Reindexer query err_code (int): The API error code err_msg (string): The API error message root (:object: Optional[`Query`]): The root query of the Reindexer query - join_type (:enum:`JoinType`): Join type join_queries (list[:object:`Query`]): The list of join Reindexer query objects merged_queries (list[:object:`Query`]): The list of merged Reindexer query objects @@ -748,7 +747,7 @@ Adds where condition to DB query with args #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (list[simple_types], ...): + keys (union[simple_types, (list[simple_types], ...)]): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index @@ -1329,6 +1328,7 @@ Adds an update query to an object field for an update query #### Raises: Exception: Raises with an error message of API return on non-zero error code + Exception: Raises with an error message if no values are specified diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 914ddf6..76ecd5e 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -6,19 +6,6 @@ #include "core/keyvalue/uuid.h" namespace pyreindexer { -namespace { -const int VALUE_INT_64 = 0; -const int VALUE_DOUBLE = 1; -const int VALUE_STRING = 2; -const int VALUE_BOOL = 3; -const int VALUE_NULL = 4; -const int VALUE_INT = 8; -const int VALUE_UNDEFINED = 9; -const int VALUE_COMPOSITE = 10; -const int VALUE_TUPLE = 11; -const int VALUE_UUID = 12; -} // namespace - QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { assert(db_); ser_.PutVString(ns); @@ -67,11 +54,9 @@ void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const s for (const auto& key : keys) { try { auto uuid = reindexer::Uuid(key); - ser_.PutVarUint(VALUE_UUID); - ser_.PutUuid(uuid); + ser_.PutVariant(reindexer::Variant(uuid)); } catch (const Error& err) { - ser_.PutVarUint(VALUE_STRING); - ser_.PutVString(key); + ser_.PutVariant(reindexer::Variant(key)); } } @@ -123,12 +108,9 @@ void QueryWrapper::DWithin(std::string_view index, double x, double y, double di ser_.PutVarUint(CondType::CondDWithin); ser_.PutVarUint(3); - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(x); - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(y); - ser_.PutVarUint(VALUE_DOUBLE); - ser_.PutDouble(distance); + ser_.PutVariant(reindexer::Variant(x)); + ser_.PutVariant(reindexer::Variant(y)); + ser_.PutVariant(reindexer::Variant(distance)); nextOperation_ = OpType::OpAnd; ++queriesCount_; @@ -292,8 +274,7 @@ void QueryWrapper::SetObject(std::string_view field, const std::vector 1? 1 : 0); // is array flag for (const auto& value : values) { ser_.PutVarUint(0); // function/value flag - ser_.PutVarUint(VALUE_STRING); // type ID - ser_.PutVString(value); + ser_.PutVariant(reindexer::Variant(value)); break; } } diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index 4a980b1..bd0a653 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -21,7 +21,7 @@ using QueryResultsT = reindexer::QueryResults; class QueryResultsWrapper { public: - QueryResultsWrapper(DBInterface* db) : db_{db}, qres_{kResultsJson} { + QueryResultsWrapper(DBInterface* db) : db_{db} { assert(db_); } diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index ac35861..3f5b1fa 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -106,7 +106,7 @@ Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& tra template Error ReindexerInterface::selectQuery(const reindexer::Query& query, QueryResultsWrapper& result) { - typename DBT::QueryResultsT qres; + typename DBT::QueryResultsT qres(QRESULTS_FLAGS); auto err = db_.Select(query, qres); result.Wrap(std::move(qres)); return err; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index f918de3..c89c6cb 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -777,10 +777,12 @@ def set_object(self, field: str, values: list[simple_types]) -> Query: #### Raises: Exception: Raises with an error message of API return on non-zero error code + Exception: Raises with an error message if no values are specified """ - values = [] if values is None else values + if values is None: + raise Exception("A required parameter is not specified. `values` can't be None") self.err_code, self.err_msg = self.api.set_object(self.query_wrapper_ptr, field, values) self.__raise_on_error() diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 119755f..67fce7b 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -19,7 +19,7 @@ def __init__(self, api, qres_wrapper_ptr, qres_iter_count, qres_total_count): api (module): An API module for Reindexer calls qres_wrapper_ptr (int): A memory pointer to Reindexer iterator object qres_iter_count (int): A count of results for iterations - qres_total_count (int): A total\cached count of results + qres_total_count (int): A total or cached count of results """ diff --git a/setup.py b/setup.py index 89cee20..e857b34 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ def build_cmake(self, ext): version='0.2.42', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', - author_email='igtulm@gmail.com', maintainer='Reindexer Team', maintainer_email='contactus@reindexer.io', url='https://github.com/Restream/reindexer-py', From c23fb3349c938302de4ca3ff6a7f169dfd94a11e Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Fri, 29 Nov 2024 17:21:14 +0300 Subject: [PATCH 103/125] Simplify code, correct style --- pyreindexer/lib/include/query_wrapper.cc | 5 - pyreindexer/lib/include/query_wrapper.h | 2 - pyreindexer/lib/src/rawpyreindexer.cc | 250 +++++++++-------------- pyreindexer/lib/src/rawpyreindexer.h | 16 +- pyreindexer/query_results.py | 23 ++- 5 files changed, 125 insertions(+), 171 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 76ecd5e..4fd929f 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -167,11 +167,6 @@ void QueryWrapper::AddValue(QueryItemType type, unsigned value) { ser_.PutVarUint(value); } -void QueryWrapper::Strict(StrictMode mode) { - ser_.PutVarUint(QueryItemType::QueryStrictMode); - ser_.PutVarUint(mode); -} - void QueryWrapper::Modifier(QueryItemType type) { ser_.PutVarUint(type); } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index c3f65f5..b694b50 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -50,8 +50,6 @@ class QueryWrapper { void AddValue(QueryItemType type, unsigned value); - void Strict(StrictMode mode); - void Modifier(QueryItemType type); enum class ExecuteType { Select, Update }; diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 158b09c..248845f 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -19,15 +19,6 @@ uintptr_t initReindexer() { return reinterpret_cast(db.release()); } -DBInterface* getDB(uintptr_t rx) { return reinterpret_cast(rx); } - -void destroyReindexer(uintptr_t rx) { - DBInterface* db = getDB(rx); - delete db; -} - -PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } - template T* getWrapper(uintptr_t wrapperAddr) { return reinterpret_cast(wrapperAddr); @@ -39,13 +30,15 @@ void deleteWrapper(uintptr_t wrapperAddr) { delete queryWrapperPtr; } +PyObject* pyErr(const Error& err) { return Py_BuildValue("is", err.code(), err.what().c_str()); } + PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { - QueryResultsWrapper* qresWrapperPtr = getWrapper(qresWrapperAddr); + QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); WrSerializer wrSer; static const bool withHeaderLen = false; - qresWrapperPtr->GetItemJSON(wrSer, withHeaderLen); - qresWrapperPtr->Next(); + qresWrapper->GetItemJSON(wrSer, withHeaderLen); + qresWrapper->Next(); PyObject* dictFromJson = nullptr; try { @@ -82,7 +75,7 @@ static PyObject* Destroy(PyObject* self, PyObject* args) { return nullptr; } - destroyReindexer(rx); + deleteWrapper(rx); Py_RETURN_NONE; } @@ -94,7 +87,7 @@ static PyObject* Connect(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->Connect(dsn); + auto err = getWrapper(rx)->Connect(dsn); return pyErr(err); } @@ -105,7 +98,7 @@ static PyObject* Select(PyObject* self, PyObject* args) { return nullptr; } - auto db = getDB(rx); + auto db = getWrapper(rx); auto qresWrapper = std::make_unique(db); auto err = qresWrapper->Select(query); @@ -128,7 +121,7 @@ static PyObject* NamespaceOpen(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->OpenNamespace(ns); + auto err = getWrapper(rx)->OpenNamespace(ns); return pyErr(err); } @@ -139,7 +132,7 @@ static PyObject* NamespaceClose(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->CloseNamespace(ns); + auto err = getWrapper(rx)->CloseNamespace(ns); return pyErr(err); } @@ -150,7 +143,7 @@ static PyObject* NamespaceDrop(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->DropNamespace(ns); + auto err = getWrapper(rx)->DropNamespace(ns); return pyErr(err); } @@ -162,7 +155,7 @@ static PyObject* EnumNamespaces(PyObject* self, PyObject* args) { } std::vector nsDefs; - auto err = getDB(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); + auto err = getWrapper(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); if (!err.ok()) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); } @@ -225,7 +218,7 @@ static PyObject* IndexAdd(PyObject* self, PyObject* args) { IndexDef indexDef; auto err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); if (err.ok()) { - err = getDB(rx)->AddIndex(ns, indexDef); + err = getWrapper(rx)->AddIndex(ns, indexDef); } return pyErr(err); } @@ -255,7 +248,7 @@ static PyObject* IndexUpdate(PyObject* self, PyObject* args) { IndexDef indexDef; auto err = indexDef.FromJSON(reindexer::giftStr(wrSer.Slice())); if (err.ok()) { - err = getDB(rx)->UpdateIndex(ns, indexDef); + err = getWrapper(rx)->UpdateIndex(ns, indexDef); } return pyErr(err); } @@ -267,7 +260,7 @@ static PyObject* IndexDrop(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->DropIndex(ns, IndexDef(indexName)); + auto err = getWrapper(rx)->DropIndex(ns, IndexDef(indexName)); return pyErr(err); } @@ -277,8 +270,8 @@ namespace { PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t rx = 0; char* ns = nullptr; - PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple - PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple + PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "ksO!|O!", &rx, &ns, &PyDict_Type, &itemDefDict, &PyList_Type, &preceptsList)) { return nullptr; } @@ -286,7 +279,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { Py_INCREF(itemDefDict); Py_XINCREF(preceptsList); - auto item = getDB(rx)->NewItem(ns); + auto item = getWrapper(rx)->NewItem(ns); auto err = item.Status(); if (!err.ok()) { return pyErr(err); @@ -331,20 +324,19 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { switch (mode) { case ModeInsert: - err = getDB(rx)->Insert(ns, item); + err = getWrapper(rx)->Insert(ns, item); break; case ModeUpdate: - err = getDB(rx)->Update(ns, item); + err = getWrapper(rx)->Update(ns, item); break; case ModeUpsert: - err = getDB(rx)->Upsert(ns, item); + err = getWrapper(rx)->Upsert(ns, item); break; case ModeDelete: - err = getDB(rx)->Delete(ns, item); + err = getWrapper(rx)->Delete(ns, item); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown item modify mode"); - return nullptr; + err = reindexer::Error(ErrorCode::errLogic, "Unknown item modify mode"); } return pyErr(err); @@ -364,7 +356,7 @@ static PyObject* PutMeta(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->PutMeta(ns, key, value); + auto err = getWrapper(rx)->PutMeta(ns, key, value); return pyErr(err); } @@ -376,7 +368,7 @@ static PyObject* GetMeta(PyObject* self, PyObject* args) { } std::string value; - auto err = getDB(rx)->GetMeta(ns, key, value); + auto err = getWrapper(rx)->GetMeta(ns, key, value); return Py_BuildValue("iss", err.code(), err.what().c_str(), value.c_str()); } @@ -387,7 +379,7 @@ static PyObject* DeleteMeta(PyObject* self, PyObject* args) { return nullptr; } - auto err = getDB(rx)->DeleteMeta(ns, key); + auto err = getWrapper(rx)->DeleteMeta(ns, key); return pyErr(err); } @@ -399,7 +391,7 @@ static PyObject* EnumMeta(PyObject* self, PyObject* args) { } std::vector keys; - auto err = getDB(rx)->EnumMeta(ns, keys); + auto err = getWrapper(rx)->EnumMeta(ns, keys); if (!err.ok()) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); } @@ -430,8 +422,7 @@ static PyObject* QueryResultsWrapperStatus(PyObject* self, PyObject* args) { return nullptr; } - QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); - auto err = qresWrapper->Status(); + auto err = getWrapper(qresWrapperAddr)->Status(); return pyErr(err); } @@ -461,9 +452,7 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args) { return nullptr; } - QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); - - const auto& aggResults = qresWrapper->GetAggregationResults(); + const auto& aggResults = getWrapper(qresWrapperAddr)->GetAggregationResults(); WrSerializer wrSer; wrSer << "["; for (size_t i = 0; i < aggResults.size(); ++i) { @@ -496,9 +485,7 @@ static PyObject* GetExplainResults(PyObject* self, PyObject* args) { return nullptr; } - QueryResultsWrapper* qresWrapper = getWrapper(qresWrapperAddr); - - const auto& explainResults = qresWrapper->GetExplainResults(); + const auto& explainResults = getWrapper(qresWrapperAddr)->GetExplainResults(); return Py_BuildValue("iss", errOK, "", explainResults.c_str()); } @@ -512,7 +499,7 @@ static PyObject* NewTransaction(PyObject* self, PyObject* args) { return nullptr; } - auto db = getDB(rx); + auto db = getWrapper(rx); auto transaction = std::make_unique(db); auto err = transaction->Start(ns); if (!err.ok()) { @@ -523,24 +510,24 @@ static PyObject* NewTransaction(PyObject* self, PyObject* args) { } namespace { -PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { +PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) { uintptr_t transactionWrapperAddr = 0; - PyObject* itemDefDict = nullptr; // borrowed ref after ParseTuple - PyObject* preceptsList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "kO!|O!", &transactionWrapperAddr, &PyDict_Type, &itemDefDict, &PyList_Type, &preceptsList)) { + PyObject* defDict = nullptr; // borrowed ref after ParseTuple + PyObject* precepts = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kO!|O!", &transactionWrapperAddr, &PyDict_Type, &defDict, &PyList_Type, &precepts)) { return nullptr; } - Py_INCREF(itemDefDict); - Py_XINCREF(preceptsList); + Py_INCREF(defDict); + Py_XINCREF(precepts); auto transaction = getWrapper(transactionWrapperAddr); auto item = transaction->NewItem(); auto err = item.Status(); if (!err.ok()) { - Py_DECREF(itemDefDict); - Py_XDECREF(preceptsList); + Py_DECREF(defDict); + Py_XDECREF(precepts); return pyErr(err); } @@ -548,31 +535,31 @@ PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode m WrSerializer wrSer; try { - PyObjectToJson(&itemDefDict, wrSer); + PyObjectToJson(&defDict, wrSer); } catch (const Error& err) { - Py_DECREF(itemDefDict); - Py_XDECREF(preceptsList); + Py_DECREF(defDict); + Py_XDECREF(precepts); return pyErr(err); } - Py_DECREF(itemDefDict); + Py_DECREF(defDict); char* json = const_cast(wrSer.c_str()); err = item.Unsafe().FromJSON(json, 0, mode == ModeDelete); if (!err.ok()) { - Py_XDECREF(preceptsList); + Py_XDECREF(precepts); return pyErr(err); } - if (preceptsList != nullptr && mode != ModeDelete) { + if (precepts != nullptr && mode != ModeDelete) { std::vector itemPrecepts; try { - itemPrecepts = ParseStrListToStrVec(&preceptsList); + itemPrecepts = ParseStrListToStrVec(&precepts); } catch (const Error& err) { - Py_DECREF(preceptsList); + Py_DECREF(precepts); return pyErr(err); } @@ -580,7 +567,7 @@ PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode m item.SetPrecepts(itemPrecepts); } - Py_XDECREF(preceptsList); + Py_XDECREF(precepts); switch (mode) { case ModeInsert: @@ -590,17 +577,16 @@ PyObject* itemModifyTransaction(PyObject* self, PyObject* args, ItemModifyMode m err = transaction->Modify(std::move(item), mode); return pyErr(err); default: - PyErr_SetString(PyExc_RuntimeError, "Unknown item modify transaction mode"); - return nullptr; + return pyErr(reindexer::Error(ErrorCode::errLogic, "Unknown item modify transaction mode")); } return nullptr; } } // namespace -static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeInsert); } -static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpdate); } -static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeUpsert); } -static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args) { return itemModifyTransaction(self, args, ModeDelete); } +static PyObject* InsertTransaction(PyObject* self, PyObject* args) { return modifyTransaction(self, args, ModeInsert); } +static PyObject* UpdateTransaction(PyObject* self, PyObject* args) { return modifyTransaction(self, args, ModeUpdate); } +static PyObject* UpsertTransaction(PyObject* self, PyObject* args) { return modifyTransaction(self, args, ModeUpsert); } +static PyObject* DeleteTransaction(PyObject* self, PyObject* args) { return modifyTransaction(self, args, ModeDelete); } static PyObject* CommitTransaction(PyObject* self, PyObject* args) { uintptr_t transactionWrapperAddr = 0; @@ -608,11 +594,9 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return nullptr; } - auto transaction = getWrapper(transactionWrapperAddr); - assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); size_t count = 0; - auto err = transaction->Commit(count); + auto err = getWrapper(transactionWrapperAddr)->Commit(count); deleteWrapper(transactionWrapperAddr); @@ -625,10 +609,8 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return nullptr; } - auto transaction = getWrapper(transactionWrapperAddr); - assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); - auto err = transaction->Rollback(); + auto err = getWrapper(transactionWrapperAddr)->Rollback(); deleteWrapper(transactionWrapperAddr); @@ -644,7 +626,7 @@ static PyObject* CreateQuery(PyObject* self, PyObject* args) { return nullptr; } - auto db = getDB(rx); + auto db = getWrapper(rx); auto query = std::make_unique(db, ns); return Py_BuildValue("isK", errOK, "", reinterpret_cast(query.release())); } @@ -664,7 +646,7 @@ static PyObject* Where(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { return nullptr; } @@ -684,9 +666,7 @@ static PyObject* Where(PyObject* self, PyObject* args) { Py_XDECREF(keysList); - auto query = getWrapper(queryWrapperAddr); - - query->Where(index, CondType(condition), keys); + getWrapper(queryWrapperAddr)->Where(index, CondType(condition), keys); return pyErr(errOK); } @@ -694,9 +674,9 @@ static PyObject* Where(PyObject* self, PyObject* args) { static PyObject* WhereSubQuery(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; uintptr_t subQueryWrapperAddr = 0; - unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "kkIO!", &queryWrapperAddr, &subQueryWrapperAddr, &condition, &PyList_Type, &keysList)) { + unsigned cond = 0; + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "kkIO!", &queryWrapperAddr, &subQueryWrapperAddr, &cond, &PyList_Type, &keysList)) { return nullptr; } @@ -718,7 +698,7 @@ static PyObject* WhereSubQuery(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); auto subQuery = getWrapper(subQueryWrapperAddr); - query->WhereSubQuery(*subQuery, CondType(condition), keys); + query->WhereSubQuery(*subQuery, CondType(cond), keys); return pyErr(errOK); } @@ -744,7 +724,7 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned condition = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &condition, &PyList_Type, &keysList)) { return nullptr; } @@ -764,9 +744,7 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { Py_XDECREF(keysList); - auto query = getWrapper(queryWrapperAddr); - - query->WhereUUID(index, CondType(condition), keys); + getWrapper(queryWrapperAddr)->WhereUUID(index, CondType(condition), keys); return pyErr(errOK); } @@ -780,9 +758,7 @@ static PyObject* WhereBetweenFields(PyObject* self, PyObject* args) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->WhereBetweenFields(first_field, CondType(condition), second_field); + getWrapper(queryWrapperAddr)->WhereBetweenFields(first_field, CondType(condition), second_field); Py_RETURN_NONE; } @@ -812,9 +788,7 @@ static PyObject* DWithin(PyObject* self, PyObject* args) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->DWithin(index, x, y, distance); + getWrapper(queryWrapperAddr)->DWithin(index, x, y, distance); Py_RETURN_NONE; } @@ -827,14 +801,14 @@ PyObject* aggregate(PyObject* self, PyObject* args, AggType type) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->Aggregate(field, type); + getWrapper(queryWrapperAddr)->Aggregate(field, type); Py_RETURN_NONE; } } // namespace -static PyObject* AggregateDistinct(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggDistinct); } +static PyObject* AggregateDistinct(PyObject* self, PyObject* args) { + return aggregate(self, args, AggType::AggDistinct); +} static PyObject* AggregateSum(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggSum); } static PyObject* AggregateAvg(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggAvg); } static PyObject* AggregateMin(PyObject* self, PyObject* args) { return aggregate(self, args, AggType::AggMin); } @@ -848,15 +822,17 @@ static PyObject* addValue(PyObject* self, PyObject* args, QueryItemType type) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->AddValue(type, value); + getWrapper(queryWrapperAddr)->AddValue(type, value); Py_RETURN_NONE; } } // namespace -static PyObject* AggregationLimit(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryAggregationLimit); } -static PyObject* AggregationOffset(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryAggregationOffset); } +static PyObject* AggregationLimit(PyObject* self, PyObject* args) { + return addValue(self, args, QueryItemType::QueryAggregationLimit); +} +static PyObject* AggregationOffset(PyObject* self, PyObject* args) { + return addValue(self, args, QueryItemType::QueryAggregationOffset); +} static PyObject* AggregationSort(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -866,16 +842,14 @@ static PyObject* AggregationSort(PyObject* self, PyObject* args) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->AggregationSort(field, (desc != 0)); + getWrapper(queryWrapperAddr)->AggregationSort(field, (desc != 0)); return pyErr(errOK); } static PyObject* Aggregation(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &fieldsList)) { return nullptr; } @@ -895,9 +869,7 @@ static PyObject* Aggregation(PyObject* self, PyObject* args) { Py_XDECREF(fieldsList); - auto query = getWrapper(queryWrapperAddr); - - query->Aggregation(fields); + getWrapper(queryWrapperAddr)->Aggregation(fields); return pyErr(errOK); } @@ -906,7 +878,7 @@ static PyObject* Sort(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned desc = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &desc, &PyList_Type, &keysList)) { return nullptr; } @@ -926,9 +898,7 @@ static PyObject* Sort(PyObject* self, PyObject* args) { Py_XDECREF(keysList); - auto query = getWrapper(queryWrapperAddr); - - query->Sort(index, (desc != 0), keys); + getWrapper(queryWrapperAddr)->Sort(index, (desc != 0), keys); return pyErr(errOK); } @@ -940,9 +910,7 @@ PyObject* logOp(PyObject* self, PyObject* args, OpType opID) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->LogOp(opID); + getWrapper(queryWrapperAddr)->LogOp(opID); Py_RETURN_NONE; } @@ -958,15 +926,17 @@ static PyObject* total(PyObject* self, PyObject* args, CalcTotalMode mode) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->Total(mode); + getWrapper(queryWrapperAddr)->Total(mode); Py_RETURN_NONE; } } // namespace -static PyObject* ReqTotal(PyObject* self, PyObject* args) { return total(self, args, CalcTotalMode::ModeAccurateTotal); } -static PyObject* CachedTotal(PyObject* self, PyObject* args) { return total(self, args, CalcTotalMode::ModeCachedTotal); } +static PyObject* ReqTotal(PyObject* self, PyObject* args) { + return total(self, args, CalcTotalMode::ModeAccurateTotal); +} +static PyObject* CachedTotal(PyObject* self, PyObject* args) { + return total(self, args, CalcTotalMode::ModeCachedTotal); +} static PyObject* Limit(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryLimit); } static PyObject* Offset(PyObject* self, PyObject* args) { return addValue(self, args, QueryItemType::QueryOffset); } @@ -979,9 +949,7 @@ static PyObject* Strict(PyObject* self, PyObject* args) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - query->Strict(StrictMode(mode)); + getWrapper(queryWrapperAddr)->AddValue(QueryItemType::QueryStrictMode, mode); Py_RETURN_NONE; } @@ -993,8 +961,7 @@ PyObject* modifier(PyObject* self, PyObject* args, QueryItemType type) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - query->Modifier(type); + getWrapper(queryWrapperAddr)->Modifier(type); Py_RETURN_NONE; } @@ -1008,9 +975,8 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); size_t count = 0; - auto err = query->DeleteQuery(count); + auto err = getWrapper(queryWrapperAddr)->DeleteQuery(count); return Py_BuildValue("isI", err.code(), err.what().c_str(), count); } @@ -1066,9 +1032,7 @@ static PyObject* SetObject(PyObject* self, PyObject* args) { Py_DECREF(valuesList); - auto query = getWrapper(queryWrapperAddr); - - query->SetObject(field, values); + getWrapper(queryWrapperAddr)->SetObject(field, values); return pyErr(errOK); } @@ -1096,9 +1060,7 @@ static PyObject* Set(PyObject* self, PyObject* args) { Py_DECREF(valuesList); - auto query = getWrapper(queryWrapperAddr); - - query->Set(field, values, QueryWrapper::IsExpression::No); + getWrapper(queryWrapperAddr)->Set(field, values, QueryWrapper::IsExpression::No); return pyErr(errOK); } @@ -1170,15 +1132,13 @@ static PyObject* On(PyObject* self, PyObject* args) { return nullptr; } - auto query = getWrapper(queryWrapperAddr); - - auto err = query->On(index, CondType(condition), joinIndex); + auto err = getWrapper(queryWrapperAddr)->On(index, CondType(condition), joinIndex); return pyErr(err); } static PyObject* SelectFilter(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* fieldsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &fieldsList)) { return nullptr; } @@ -1198,16 +1158,14 @@ static PyObject* SelectFilter(PyObject* self, PyObject* args) { Py_XDECREF(fieldsList); - auto query = getWrapper(queryWrapperAddr); - - query->SelectFilter(fields); + getWrapper(queryWrapperAddr)->SelectFilter(fields); return pyErr(errOK); } static PyObject* AddFunctions(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - PyObject* functionsList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* functionsList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &functionsList)) { return nullptr; } @@ -1227,16 +1185,14 @@ static PyObject* AddFunctions(PyObject* self, PyObject* args) { Py_XDECREF(functionsList); - auto query = getWrapper(queryWrapperAddr); - - query->AddFunctions(functions); + getWrapper(queryWrapperAddr)->AddFunctions(functions); return pyErr(errOK); } static PyObject* AddEqualPosition(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; - PyObject* equalPosesList = nullptr; // borrowed ref after ParseTuple if passed + PyObject* equalPosesList = nullptr; // borrowed ref after ParseTuple if passed if (!PyArg_ParseTuple(args, "kO!", &queryWrapperAddr, &PyList_Type, &equalPosesList)) { return nullptr; } @@ -1256,9 +1212,7 @@ static PyObject* AddEqualPosition(PyObject* self, PyObject* args) { Py_XDECREF(equalPosesList); - auto query = getWrapper(queryWrapperAddr); - - query->AddEqualPosition(equalPoses); + getWrapper(queryWrapperAddr)->AddEqualPosition(equalPoses); return pyErr(errOK); } diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 87b6f61..8346fbd 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -46,10 +46,10 @@ static PyObject* GetAggregationResults(PyObject* self, PyObject* args); static PyObject* GetExplainResults(PyObject* self, PyObject* args); // transaction (sync) static PyObject* NewTransaction(PyObject* self, PyObject* args); -static PyObject* ItemInsertTransaction(PyObject* self, PyObject* args); -static PyObject* ItemUpdateTransaction(PyObject* self, PyObject* args); -static PyObject* ItemUpsertTransaction(PyObject* self, PyObject* args); -static PyObject* ItemDeleteTransaction(PyObject* self, PyObject* args); +static PyObject* InsertTransaction(PyObject* self, PyObject* args); +static PyObject* UpdateTransaction(PyObject* self, PyObject* args); +static PyObject* UpsertTransaction(PyObject* self, PyObject* args); +static PyObject* DeleteTransaction(PyObject* self, PyObject* args); static PyObject* CommitTransaction(PyObject* self, PyObject* args); static PyObject* RollbackTransaction(PyObject* self, PyObject* args); // query @@ -131,10 +131,10 @@ static PyMethodDef module_methods[] = { {"get_explain_results", GetExplainResults, METH_VARARGS, "get explain results"}, // transaction (sync) {"new_transaction", NewTransaction, METH_VARARGS, "start new transaction"}, - {"item_insert_transaction", ItemInsertTransaction, METH_VARARGS, "item insert transaction"}, - {"item_update_transaction", ItemUpdateTransaction, METH_VARARGS, "item update transaction"}, - {"item_upsert_transaction", ItemUpsertTransaction, METH_VARARGS, "item upsert transaction"}, - {"item_delete_transaction", ItemDeleteTransaction, METH_VARARGS, "item delete transaction"}, + {"item_insert_transaction", InsertTransaction, METH_VARARGS, "item insert transaction"}, + {"item_update_transaction", UpdateTransaction, METH_VARARGS, "item update transaction"}, + {"item_upsert_transaction", UpsertTransaction, METH_VARARGS, "item upsert transaction"}, + {"item_delete_transaction", DeleteTransaction, METH_VARARGS, "item delete transaction"}, {"commit_transaction", CommitTransaction, METH_VARARGS, "apply changes. Free transaction object memory"}, {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback changes. Free transaction object memory"}, // query diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 67fce7b..581010e 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -31,6 +31,17 @@ def __init__(self, api, qres_wrapper_ptr, qres_iter_count, qres_total_count): self.err_code = 0 self.err_msg = "" + def __raise_on_error(self): + """Checks if there is an error code and raises with an error message + + #### Raises: + Exception: Raises with an error message of API return on non-zero error code + + """ + + if self.err_code: + raise Exception(self.err_msg) + def __iter__(self): """Returns the current iteration result @@ -49,8 +60,7 @@ def __next__(self): if self.pos < self.qres_iter_count: self.pos += 1 self.err_code, self.err_msg, res = self.api.query_results_iterate(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.__raise_on_error() return res else: del self @@ -72,8 +82,7 @@ def status(self) -> None: """ self.err_code, self.err_msg = self.api.query_results_status(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.__raise_on_error() def count(self) -> int: """Returns a count of results for iterations @@ -115,8 +124,7 @@ def get_agg_results(self) -> dict: """ self.err_code, self.err_msg, res = self.api.get_agg_results(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.__raise_on_error() return res def get_explain_results(self) -> str: @@ -131,6 +139,5 @@ def get_explain_results(self) -> str: """ self.err_code, self.err_msg, res = self.api.get_explain_results(self.qres_wrapper_ptr) - if self.err_code: - raise Exception(self.err_msg) + self.__raise_on_error() return res From 1f726bee8a9f35a81b9962b19fe82e886ad2c637 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 2 Dec 2024 12:29:06 +0300 Subject: [PATCH 104/125] Fix. Add default value --- pyreindexer/query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index c89c6cb..ea6d7e3 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -173,7 +173,7 @@ def __where(self, index: str, condition: CondType, return self def where(self, index: str, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types],...]]) -> Query: + keys: Union[simple_types, tuple[list[simple_types],...]] = None) -> Query: """Adds where condition to DB query with args #### Arguments: @@ -194,7 +194,7 @@ def where(self, index: str, condition: CondType, return self.__where(index, condition, keys) def where_query(self, sub_query: Query, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types],...]]) -> Query: + keys: Union[simple_types, tuple[list[simple_types],...]] = None) -> Query: """Adds sub-query where condition to DB query with args #### Arguments: From d2c0a66c26cdb80bfb671475369ffaa63a8cb074 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 2 Dec 2024 14:08:14 +0300 Subject: [PATCH 105/125] Update minimal Reindexer version --- pyreindexer/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyreindexer/CMakeLists.txt b/pyreindexer/CMakeLists.txt index 6093c23..a410182 100644 --- a/pyreindexer/CMakeLists.txt +++ b/pyreindexer/CMakeLists.txt @@ -11,7 +11,7 @@ endif() enable_testing() set(PY_MIN_VERSION 3.6) -set(RX_MIN_VERSION 3.24.0) # ToDo +set(RX_MIN_VERSION 3.30.0) find_package(PythonInterp ${PY_MIN_VERSION}) find_package(PythonLibs ${PY_MIN_VERSION}) find_package(reindexer CONFIG ${RX_MIN_VERSION}) From d3fd3152dd19e2774d096ae657f3cd2f500c7dd7 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 3 Dec 2024 10:50:47 +0300 Subject: [PATCH 106/125] Client configuration support for Reindexer --- README.md | 13 +++-- pyreindexer/example/main.py | 4 +- pyreindexer/lib/src/rawpyreindexer.cc | 31 +++++++++--- pyreindexer/lib/src/rawpyreindexer.h | 2 +- pyreindexer/lib/src/reindexerinterface.cc | 40 ++++++++++++++- pyreindexer/lib/src/reindexerinterface.h | 61 +++++++++-------------- pyreindexer/rx_connector.py | 39 ++++++++++++--- 7 files changed, 130 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 712b7ee..a76fdbb 100644 --- a/README.md +++ b/README.md @@ -738,8 +738,11 @@ An object representing the context of a Reindexer query ### Query.where ```python -def where(index: str, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query +def where( + index: str, + condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], + ...]] = None) -> Query ``` Adds where condition to DB query with args @@ -763,8 +766,10 @@ Adds where condition to DB query with args ```python def where_query( - sub_query: Query, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query + sub_query: Query, + condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], + ...]] = None) -> Query ``` Adds sub-query where condition to DB query with args diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 5ab37bd..dee3241 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -117,8 +117,8 @@ def query_example(db, namespace): def rx_example(): - db = RxConnector('builtin:///tmp/pyrx') -# db = RxConnector('cproto://127.0.0.1:6534/pyrx') + db = RxConnector('builtin:///tmp/pyrx', max_replication_updates_size = 10 * 1024 * 1024) +# db = RxConnector('cproto://127.0.0.1:6534/pyrx', enable_compression = True, fetch_amount = 500) namespace = 'test_table' diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 248845f..18b6d91 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -1,8 +1,8 @@ #include "rawpyreindexer.h" -#include "pyobjtools.h" #include "queryresults_wrapper.h" #include "query_wrapper.h" +#include "pyobjtools.h" #include "transaction_wrapper.h" #include "tools/serializer.h" @@ -10,12 +10,11 @@ namespace pyreindexer { using reindexer::Error; using reindexer::IndexDef; -using reindexer::NamespaceDef; using reindexer::WrSerializer; namespace { -uintptr_t initReindexer() { - auto db = std::make_unique(); +uintptr_t initReindexer(const ReindexerConfig& cfg) { + auto db = std::make_unique(cfg); return reinterpret_cast(db.release()); } @@ -59,7 +58,23 @@ PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { // common -------------------------------------------------------------------------------------------------------------- static PyObject* Init(PyObject* self, PyObject* args) { - uintptr_t rx = initReindexer(); + ReindexerConfig cfg; + char* clientName = nullptr; + unsigned enableCompression = 0; + unsigned startSpecialThread = 0; + unsigned maxReplUpdatesSize = 0; + if (!PyArg_ParseTuple(args, "iiiIIsIif", &cfg.fetchAmount, &cfg.connectTimeout, &cfg.requestTimeout, + &enableCompression, &startSpecialThread, &clientName, &maxReplUpdatesSize, + &cfg.allocatorCacheLimit, &cfg.allocatorCachePart)) { + return nullptr; + } + + cfg.enableCompression = (enableCompression != 0); + cfg.requestDedicatedThread = (startSpecialThread != 0); + cfg.appName = clientName; + cfg.maxReplUpdatesSize = maxReplUpdatesSize; + + uintptr_t rx = initReindexer(cfg); if (rx == 0) { PyErr_SetString(PyExc_RuntimeError, "Initialization error"); @@ -154,7 +169,7 @@ static PyObject* EnumNamespaces(PyObject* self, PyObject* args) { return nullptr; } - std::vector nsDefs; + std::vector nsDefs; auto err = getWrapper(rx)->EnumNamespaces(nsDefs, reindexer::EnumNamespacesOpts().WithClosed(enumAll)); if (!err.ok()) { return Py_BuildValue("is[]", err.code(), err.what().c_str()); @@ -336,7 +351,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { err = getWrapper(rx)->Delete(ns, item); break; default: - err = reindexer::Error(ErrorCode::errLogic, "Unknown item modify mode"); + err = Error(ErrorCode::errLogic, "Unknown item modify mode"); } return pyErr(err); @@ -577,7 +592,7 @@ PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) err = transaction->Modify(std::move(item), mode); return pyErr(err); default: - return pyErr(reindexer::Error(ErrorCode::errLogic, "Unknown item modify transaction mode")); + return pyErr(Error(ErrorCode::errLogic, "Unknown item modify transaction mode")); } return nullptr; diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 8346fbd..36cd507 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -100,7 +100,7 @@ static PyObject* AddEqualPosition(PyObject* self, PyObject* args); // clang-format off static PyMethodDef module_methods[] = { - {"init", Init, METH_NOARGS, "init reindexer instance"}, + {"init", Init, METH_VARARGS, "init reindexer instance"}, {"destroy", Destroy, METH_VARARGS, "destroy reindexer instance"}, {"connect", Connect, METH_VARARGS, "connect to reindexer database"}, {"select", Select, METH_VARARGS, "select query"}, diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 3f5b1fa..a291f38 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -1,4 +1,5 @@ #include "reindexerinterface.h" +#include #include "client/cororeindexer.h" #include "core/reindexer.h" #include "core/type_consts.h" @@ -10,11 +11,46 @@ namespace { const int QRESULTS_FLAGS = kResultsJson | kResultsWithRank | kResultsWithJoined; } + +struct ICommand { + virtual Error Status() const = 0; + virtual void Execute() = 0; + virtual bool IsExecuted() const = 0; + + virtual ~ICommand() = default; +}; + +class GenericCommand : public ICommand { +public: + using CallableT = std::function; + + GenericCommand(CallableT command) : command_(std::move(command)) {} + + Error Status() const override final { return err_; } + void Execute() override final { + err_ = command_(); + executed_.store(true, std::memory_order_release); + } + bool IsExecuted() const override final { return executed_.load(std::memory_order_acquire); } + +private: + CallableT command_; + Error err_{errOK}; + std::atomic_bool executed_{false}; +}; + template <> -ReindexerInterface::ReindexerInterface() {} +ReindexerInterface::ReindexerInterface(const ReindexerConfig& cfg) + : db_(reindexer::ReindexerConfig() + .WithUpdatesSize(cfg.maxReplUpdatesSize) + .WithAllocatorCacheLimits(cfg.allocatorCacheLimit, cfg.allocatorCachePart)) +{ } template <> -ReindexerInterface::ReindexerInterface() { +ReindexerInterface::ReindexerInterface(const ReindexerConfig& cfg) + : db_(reindexer::client::ReindexerConfig(4, 1, cfg.fetchAmount, 0, std::chrono::seconds(cfg.connectTimeout), + std::chrono::seconds(cfg.requestTimeout), cfg.enableCompression, cfg.requestDedicatedThread, cfg.appName)) +{ std::atomic_bool running{false}; executionThr_ = std::thread([this, &running] { cmdAsync_.set(loop_); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index ed5afd2..a3108fe 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -11,46 +11,33 @@ namespace pyreindexer { +using reindexer::EnumNamespacesOpts; using reindexer::Error; -using reindexer::net::ev::dynamic_loop; +using reindexer::Query; using reindexer::IndexDef; using reindexer::NamespaceDef; -using reindexer::EnumNamespacesOpts; class QueryResultsWrapper; class TransactionWrapper; - -struct ICommand { - virtual Error Status() const = 0; - virtual void Execute() = 0; - virtual bool IsExecuted() const = 0; - - virtual ~ICommand() = default; -}; - -class GenericCommand : public ICommand { -public: - using CallableT = std::function; - - GenericCommand(CallableT command) : command_(std::move(command)) {} - - Error Status() const override final { return err_; } - void Execute() override final { - err_ = command_(); - executed_.store(true, std::memory_order_release); - } - bool IsExecuted() const override final { return executed_.load(std::memory_order_acquire); } - -private: - CallableT command_; - Error err_{errOK}; - std::atomic_bool executed_{false}; +class ICommand; + +struct ReindexerConfig { + int fetchAmount{1000}; + int connectTimeout{0}; + int requestTimeout{0}; + bool enableCompression{false}; + bool requestDedicatedThread{false}; + std::string appName; + + size_t maxReplUpdatesSize{1024 * 1024 * 1024}; + int64_t allocatorCacheLimit{-1}; + float allocatorCachePart{-1.0}; }; template class ReindexerInterface { public: - ReindexerInterface(); + ReindexerInterface(const ReindexerConfig& cfg); ~ReindexerInterface(); Error Connect(const std::string& dsn) { @@ -129,13 +116,13 @@ class ReindexerInterface { Error RollbackTransaction(typename DBT::TransactionT& tr) { return execute([this, &tr] { return rollbackTransaction(tr); }); } - Error SelectQuery(const reindexer::Query& query, QueryResultsWrapper& result) { + Error SelectQuery(const Query& query, QueryResultsWrapper& result) { return execute([this, &query, &result] { return selectQuery(query, result); }); } - Error DeleteQuery(const reindexer::Query& query, size_t& count) { + Error DeleteQuery(const Query& query, size_t& count) { return execute([this, &query, &count] { return deleteQuery(query, count); }); } - Error UpdateQuery(const reindexer::Query& query, QueryResultsWrapper& result) { + Error UpdateQuery(const Query& query, QueryResultsWrapper& result) { return execute([this, &query, &result] { return updateQuery(query, result); }); } @@ -165,15 +152,15 @@ class ReindexerInterface { Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); Error commitTransaction(typename DBT::TransactionT& transaction, size_t& count); Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } - Error selectQuery(const reindexer::Query& query, QueryResultsWrapper& result); - Error deleteQuery(const reindexer::Query& query, size_t& count); - Error updateQuery(const reindexer::Query& query, QueryResultsWrapper& result); + Error selectQuery(const Query& query, QueryResultsWrapper& result); + Error deleteQuery(const Query& query, size_t& count); + Error updateQuery(const Query& query, QueryResultsWrapper& result); Error stop(); DBT db_; std::thread executionThr_; - dynamic_loop loop_; - ICommand* curCmd_ = nullptr; + reindexer::net::ev::dynamic_loop loop_; + ICommand* curCmd_{nullptr}; reindexer::net::ev::async cmdAsync_; std::mutex mtx_; std::condition_variable condVar_; diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 19612fc..d473ade 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -20,21 +20,49 @@ class RxConnector(RaiserMixin): """ - def __init__(self, dsn): + def __init__(self, dsn: str, *, + # cproto options + fetch_amount: int = 1000, + connect_timeout: int = 0, + request_timeout: int = 0, + enable_compression: bool = False, + start_special_thread: bool = False, + client_name: str = 'pyreindexer', + # builtin options + max_replication_updates_size: int = 1024 * 1024 * 1024, + allocator_cache_limit: int = -1, + allocator_cache_part: float = -1.0): """Constructs a new connector object. Initializes an error code and a Reindexer instance descriptor to zero #### Arguments: dsn (string): The connection string which contains a protocol - Examples: 'builtin:///tmp/pyrx', 'cproto://127.0.0.1:6534/pyrx - + Examples: 'builtin:///tmp/pyrx', 'cproto://127.0.0.1:6534/pyrx + + cproto options: + fetch_amount (int): The number of items that will be fetched by one operation + connect_timeout (int): Connection and database login timeout value + request_timeout (int): Request execution timeout value + enable_compression (bool): Flag enable/disable traffic compression + start_special_thread (bool): Determines whether to request a special thread of execution + on the server for this connection + client_name (string): Proper name of the application (as a client for Reindexer-server) + + built-in options: + max_replication_updates_size (int): Max pended replication updates size in bytes + allocator_cache_limit (int): Recommended maximum free cache size of tcmalloc memory allocator in bytes + allocator_cache_part (float): Recommended maximum free cache size of tcmalloc memory allocator in + relation to total reindexer allocated memory size, in units """ self.err_code = 0 self.err_msg = '' self.rx = 0 self._api_import(dsn) - self._api_init(dsn) + self.rx = self.api.init(fetch_amount, connect_timeout, request_timeout, enable_compression, + start_special_thread, client_name, max_replication_updates_size, + allocator_cache_limit, allocator_cache_part) + self._api_connect(dsn) def __del__(self): """Closes an API instance on a connector object deletion if the API is initialized @@ -384,7 +412,7 @@ def _api_import(self, dsn): raise Exception( "Unknown Reindexer connection protocol for dsn: ", dsn) - def _api_init(self, dsn): + def _api_connect(self, dsn): """Initializes Reindexer instance and connects to a database specified in dsn Obtains a pointer to Reindexer instance @@ -397,7 +425,6 @@ def _api_init(self, dsn): """ - self.rx = self.api.init() self.raise_on_not_init() self.err_code, self.err_msg = self.api.connect(self.rx, dsn) self.raise_on_error() From 1bccaaef7c3587d8a5705ab75655bc1dffb24499 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Tue, 3 Dec 2024 11:13:13 +0300 Subject: [PATCH 107/125] add exceptions --- pyreindexer/example/main.py | 7 ++-- pyreindexer/exceptions.py | 14 ++++++++ pyreindexer/query.py | 37 ++++++++++++--------- pyreindexer/query_results.py | 5 ++- pyreindexer/raiser_mixin.py | 7 ++-- pyreindexer/rx_connector.py | 3 +- pyreindexer/tests/tests/test_index.py | 7 ++-- pyreindexer/tests/tests/test_namespace.py | 4 ++- pyreindexer/tests/tests/test_sql.py | 4 ++- pyreindexer/tests/tests/test_transaction.py | 21 ++++++------ pyreindexer/transaction.py | 7 ++-- 11 files changed, 76 insertions(+), 40 deletions(-) create mode 100644 pyreindexer/exceptions.py diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 5ab37bd..1ae7ae3 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -1,3 +1,4 @@ +from exceptions import ApiError from pyreindexer import RxConnector from pyreindexer.query import CondType @@ -20,7 +21,7 @@ def create_index_example(db, namespace): try: db.index_add(namespace, index_definition) - except (Exception,): + except ApiError: db.index_drop(namespace, 'id') db.index_add(namespace, index_definition) @@ -63,7 +64,7 @@ def transaction_example(db, namespace, items_in_base): items_count = len(items_in_base) # delete first few items - for i in range(int(items_count/2)): + for i in range(int(items_count / 2)): transaction.delete(items_in_base[i]) # update last one item, overwrite field 'value' @@ -118,7 +119,7 @@ def query_example(db, namespace): def rx_example(): db = RxConnector('builtin:///tmp/pyrx') -# db = RxConnector('cproto://127.0.0.1:6534/pyrx') + # db = RxConnector('cproto://127.0.0.1:6534/pyrx') namespace = 'test_table' diff --git a/pyreindexer/exceptions.py b/pyreindexer/exceptions.py new file mode 100644 index 0000000..0d602f7 --- /dev/null +++ b/pyreindexer/exceptions.py @@ -0,0 +1,14 @@ +class ApiError(Exception): + pass + + +class QueryError(ApiError): + pass + + +class QueryResultsError(QueryError): + pass + + +class TransactionError(ApiError): + pass diff --git a/pyreindexer/query.py b/pyreindexer/query.py index ea6d7e3..8702a1b 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -4,11 +4,18 @@ from enum import Enum from typing import Optional, Union +from exceptions import QueryError from pyreindexer.point import Point from pyreindexer.query_results import QueryResults -class CondType(Enum): +class ExtendedEnum(Enum): + + def __eq__(self, other): + return self.value == other + + +class CondType(ExtendedEnum): CondAny = 0 CondEq = 1 CondLt = 2 @@ -23,21 +30,21 @@ class CondType(Enum): CondDWithin = 11 -class StrictMode(Enum): +class StrictMode(ExtendedEnum): NotSet = 0 Empty = 1 Names = 2 Indexes = 3 -class JoinType(Enum): +class JoinType(ExtendedEnum): LeftJoin = 0 InnerJoin = 1 OrInnerJoin = 2 Merge = 3 -class LogLevel(Enum): +class LogLevel(ExtendedEnum): Off = 0 Error = 1 Warning = 2 @@ -96,7 +103,7 @@ def __raise_on_error(self): """ if self.err_code: - raise Exception(self.err_msg) + raise QueryError(self.err_msg) @staticmethod def __convert_to_list(param: Union[simple_types, tuple[list[simple_types], ...]]) -> list: @@ -125,7 +132,7 @@ def __convert_to_list(param: Union[simple_types, tuple[list[simple_types], ...]] if not isinstance(res, list): res = list(res) if len(res) == 0 or (len(res) > 0 and not isinstance(res[0], list)): - wrap : list = [res] + wrap: list = [res] res = wrap return res @@ -164,7 +171,7 @@ def __where(self, index: str, condition: CondType, """ if condition == CondType.CondDWithin: - raise Exception("In this case, use a special method 'dwithin'") + raise QueryError("In this case, use a special method 'dwithin'") params = self.__convert_to_list(keys) @@ -173,7 +180,7 @@ def __where(self, index: str, condition: CondType, return self def where(self, index: str, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types],...]] = None) -> Query: + keys: Union[simple_types, tuple[list[simple_types], ...]] = None) -> Query: """Adds where condition to DB query with args #### Arguments: @@ -194,7 +201,7 @@ def where(self, index: str, condition: CondType, return self.__where(index, condition, keys) def where_query(self, sub_query: Query, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types],...]] = None) -> Query: + keys: Union[simple_types, tuple[list[simple_types], ...]] = None) -> Query: """Adds sub-query where condition to DB query with args #### Arguments: @@ -521,7 +528,7 @@ def aggregate_facet(self, *fields: str) -> Query._AggregateFacet: return self._AggregateFacet(self) def sort(self, index: str, desc: bool = False, - keys: Union[simple_types, tuple[list[simple_types],...]] = None) -> Query: + keys: Union[simple_types, tuple[list[simple_types], ...]] = None) -> Query: """Applies sort order to return from query items. If values argument specified, then items equal to values, if found will be placed in the top positions. Forced sort is support for the first sorting field only @@ -759,7 +766,7 @@ def delete(self) -> int: """ if (self.root is not None) or (len(self.join_queries) > 0): - raise Exception("Delete does not support joined queries") + raise QueryError("Delete does not support joined queries") self.err_code, self.err_msg, number = self.api.delete_query(self.query_wrapper_ptr) self.__raise_on_error() @@ -782,7 +789,7 @@ def set_object(self, field: str, values: list[simple_types]) -> Query: """ if values is None: - raise Exception("A required parameter is not specified. `values` can't be None") + raise QueryError("A required parameter is not specified. `values` can't be None") self.err_code, self.err_msg = self.api.set_object(self.query_wrapper_ptr, field, values) self.__raise_on_error() @@ -851,7 +858,7 @@ def update(self) -> QueryResults: """ if (self.root is not None) or (len(self.join_queries) > 0): - raise Exception("Update does not support joined queries") + raise QueryError("Update does not support joined queries") (self.err_code, self.err_msg, wrapper_ptr, iter_count, total_count) = self.api.update_query(self.query_wrapper_ptr) @@ -915,7 +922,7 @@ def __join(self, query: Query, field: str, join_type: JoinType) -> Query: return self.root.__join(query, field, join_type) if query.root is not None: - raise Exception("Query.join call on already joined query. You should create new Query") + raise QueryError("Query.join call on already joined query. You should create new Query") # index of join query self.api.join(self.query_wrapper_ptr, join_type.value, query.query_wrapper_ptr) @@ -1009,7 +1016,7 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: """ if self.root is None: - raise Exception("Can't join on root query") + raise QueryError("Can't join on root query") self.api.on(self.query_wrapper_ptr, index, condition.value, join_index) return self diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 581010e..b4d147f 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,3 +1,6 @@ +from exceptions import QueryResultsError + + class QueryResults: """QueryResults is a disposable iterator of Reindexer results for such queries as SELECT etc. When the results are fetched the iterator closes and frees a memory of results buffer of Reindexer @@ -40,7 +43,7 @@ def __raise_on_error(self): """ if self.err_code: - raise Exception(self.err_msg) + raise QueryResultsError(self.err_msg) def __iter__(self): """Returns the current iteration result diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index 045df65..f483a9e 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -1,3 +1,6 @@ +from exceptions import ApiError + + class RaiserMixin: """RaiserMixin contains methods for checking some typical API bad events and raise if there is a necessity @@ -15,7 +18,7 @@ def raise_on_error(self): """ if self.err_code: - raise Exception(self.err_msg) + raise ApiError(self.err_msg) def raise_on_not_init(self): """Checks if there is an error code and raises with an error message @@ -26,7 +29,7 @@ def raise_on_not_init(self): """ if self.rx <= 0: - raise Exception("Connection is not initialized") + raise ConnectionError("Connection is not initialized") def raise_if_error(func): diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 19612fc..9c3221d 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -381,8 +381,7 @@ def _api_import(self, dsn): elif dsn.startswith('cproto://'): self.api = __import__('rawpyreindexerc') else: - raise Exception( - "Unknown Reindexer connection protocol for dsn: ", dsn) + raise ConnectionError(f"Unknown Reindexer connection protocol for dsn: {dsn}") def _api_init(self, dsn): """Initializes Reindexer instance and connects to a database specified in dsn diff --git a/pyreindexer/tests/tests/test_index.py b/pyreindexer/tests/tests/test_index.py index 5887e75..c67fcf8 100644 --- a/pyreindexer/tests/tests/test_index.py +++ b/pyreindexer/tests/tests/test_index.py @@ -1,5 +1,6 @@ from hamcrest import * +from exceptions import ApiError from tests.helpers.base_helper import get_ns_description from tests.test_data.constants import index_definition, updated_index_definition @@ -46,7 +47,7 @@ def test_cannot_add_index_with_same_name(self, db, namespace): db.index.create(namespace, index_definition) # Then ("Check that we can't add index with the same name") assert_that(calling(db.index.create).with_args(namespace, updated_index_definition), - raises(Exception, pattern="Index '.*' already exists with different settings"), + raises(ApiError, pattern="Index '.*' already exists with different settings"), "Index with existing name was created") def test_cannot_update_not_existing_index_in_namespace(self, db, namespace): @@ -54,7 +55,7 @@ def test_cannot_update_not_existing_index_in_namespace(self, db, namespace): # When ("Update index") # Then ("Check that we can't update index that was not created") assert_that(calling(db.index.update).with_args(namespace, index_definition), - raises(Exception, pattern=f"Index 'id' not found in '{namespace}'"), + raises(ApiError, pattern=f"Index 'id' not found in '{namespace}'"), "Not existing index was updated") def test_cannot_delete_not_existing_index_in_namespace(self, db, namespace): @@ -63,5 +64,5 @@ def test_cannot_delete_not_existing_index_in_namespace(self, db, namespace): # Then ("Check that we can't delete index that was not created") index_name = 'id' assert_that(calling(db.index.drop).with_args(namespace, index_name), - raises(Exception, pattern=f"Cannot remove index {index_name}: doesn't exist"), + raises(ApiError, pattern=f"Cannot remove index {index_name}: doesn't exist"), "Not existing index was deleted") diff --git a/pyreindexer/tests/tests/test_namespace.py b/pyreindexer/tests/tests/test_namespace.py index feee497..26cda86 100644 --- a/pyreindexer/tests/tests/test_namespace.py +++ b/pyreindexer/tests/tests/test_namespace.py @@ -1,5 +1,7 @@ from hamcrest import * +from exceptions import ApiError + class TestCrudNamespace: @@ -30,4 +32,4 @@ def test_cannot_delete_ns_not_created(self, db): # Then ("Check that we cannot delete namespace that does not exist") namespace_name = 'test_ns' assert_that(calling(db.namespace.drop).with_args(namespace_name), - raises(Exception, pattern=f"Namespace '{namespace_name}' does not exist")) + raises(ApiError, pattern=f"Namespace '{namespace_name}' does not exist")) diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index b186a9d..acb55b2 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -1,5 +1,7 @@ from hamcrest import * +from exceptions import ApiError + class TestSqlQueries: def test_sql_select(self, db, namespace, index, item): @@ -55,7 +57,7 @@ def test_sql_select_with_syntax_error(self, db, namespace, index, item): query = "SELECT *" # Then ("Check that selected item is in result") assert_that(calling(db.query.sql).with_args(query), - raises(Exception, pattern="Expected .* but found"), + raises(ApiError, pattern="Expected .* but found"), "Error wasn't raised when syntax was incorrect") def test_sql_select_with_aggregations(self, db, namespace, index, items): diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index 1eea358..a00181a 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -1,5 +1,6 @@ from hamcrest import * +from exceptions import TransactionError from tests.helpers.base_helper import get_ns_items from tests.helpers.transaction import * from tests.test_data.constants import item_definition @@ -14,7 +15,7 @@ def test_negative_commit_after_rollback(self, db, namespace): transaction.rollback() # Then ("Commit transaction") assert_that(calling(transaction.commit).with_args(), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_rollback_after_commit(self, db, namespace): # Given("Create namespace") @@ -24,7 +25,7 @@ def test_negative_rollback_after_commit(self, db, namespace): transaction.commit() # Then ("Rollback transaction") assert_that(calling(transaction.rollback).with_args(), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_insert_after_rollback(self, db, namespace, index): # Given("Create namespace with index") @@ -34,7 +35,7 @@ def test_negative_insert_after_rollback(self, db, namespace, index): transaction.rollback() # Then ("Insert transaction") assert_that(calling(transaction.insert_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_update_after_rollback(self, db, namespace, index): # Given("Create namespace with index") @@ -44,7 +45,7 @@ def test_negative_update_after_rollback(self, db, namespace, index): transaction.rollback() # Then ("Update transaction") assert_that(calling(transaction.update_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_upsert_after_rollback(self, db, namespace, index): # Given("Create namespace with index") @@ -54,7 +55,7 @@ def test_negative_upsert_after_rollback(self, db, namespace, index): transaction.rollback() # Then ("Upsert transaction") assert_that(calling(transaction.upsert_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_delete_after_rollback(self, db, namespace, index): # Given("Create namespace with index") @@ -64,7 +65,7 @@ def test_negative_delete_after_rollback(self, db, namespace, index): transaction.rollback() # Then ("Delete transaction") assert_that(calling(transaction.delete_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_insert_after_commit(self, db, namespace, index): # Given("Create namespace with index") @@ -74,7 +75,7 @@ def test_negative_insert_after_commit(self, db, namespace, index): transaction.commit() # Then ("Insert transaction") assert_that(calling(transaction.insert_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_update_after_commit(self, db, namespace, index): # Given("Create namespace with index") @@ -84,7 +85,7 @@ def test_negative_update_after_commit(self, db, namespace, index): transaction.commit() # Then ("Update transaction") assert_that(calling(transaction.update_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_upsert_after_commit(self, db, namespace, index): # Given("Create namespace with index") @@ -94,7 +95,7 @@ def test_negative_upsert_after_commit(self, db, namespace, index): transaction.commit() # Then ("Upsert transaction") assert_that(calling(transaction.upsert_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_negative_delete_after_commit(self, db, namespace, index): # Given("Create namespace with index") @@ -104,7 +105,7 @@ def test_negative_delete_after_commit(self, db, namespace, index): transaction.commit() # Then ("Delete transaction") assert_that(calling(transaction.delete_item).with_args(item_definition), - raises(Exception, matching=has_string("Transaction is over"))) + raises(TransactionError, matching=has_string("Transaction is over"))) def test_create_item_insert(self, db, namespace, index): # Given("Create namespace with index") diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 618d764..1bd013a 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,3 +1,6 @@ +from exceptions import TransactionError + + def raise_if_error(func): def wrapper(self, *args, **kwargs): self._raise_on_is_over() @@ -50,7 +53,7 @@ def _raise_on_error(self): """ if self.err_code: - raise Exception(self.err_msg) + raise TransactionError(self.err_msg) def _raise_on_is_over(self): """Checks the state of a transaction and returns an error message when necessary @@ -61,7 +64,7 @@ def _raise_on_is_over(self): """ if self.transaction_wrapper_ptr <= 0: - raise Exception("Transaction is over") + raise TransactionError("Transaction is over") @raise_if_error def insert(self, item_def, precepts=None): From a2f80cd986a6d586da077dfc88abbffbf0e3dde8 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 3 Dec 2024 11:43:24 +0300 Subject: [PATCH 108/125] Client configuration support for Reindexer. Fix MacOS build --- pyreindexer/lib/src/reindexerinterface.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index a291f38..de44c7e 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -11,8 +11,9 @@ namespace { const int QRESULTS_FLAGS = kResultsJson | kResultsWithRank | kResultsWithJoined; } - -struct ICommand { +class ICommand { +public: + virtual ~ICommand() = default; virtual Error Status() const = 0; virtual void Execute() = 0; virtual bool IsExecuted() const = 0; From db70321b7c45dfb3e22cc4640414b0d0e9901ce8 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 3 Dec 2024 11:53:50 +0300 Subject: [PATCH 109/125] Client configuration support for Reindexer. ReFix MacOS build --- pyreindexer/lib/src/reindexerinterface.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index de44c7e..fc33a13 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -13,7 +13,6 @@ namespace { class ICommand { public: - virtual ~ICommand() = default; virtual Error Status() const = 0; virtual void Execute() = 0; virtual bool IsExecuted() const = 0; From fcba1b55959990254e8087dee2ec82bf38f1b4a9 Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Tue, 3 Dec 2024 15:23:02 +0300 Subject: [PATCH 110/125] fix raises in docstrings --- pyreindexer/exceptions.py | 4 -- pyreindexer/query.py | 62 +++++++++++++++-------------- pyreindexer/query_results.py | 12 +++--- pyreindexer/raiser_mixin.py | 4 +- pyreindexer/rx_connector.py | 76 ++++++++++++++++++------------------ pyreindexer/transaction.py | 36 ++++++++--------- 6 files changed, 97 insertions(+), 97 deletions(-) diff --git a/pyreindexer/exceptions.py b/pyreindexer/exceptions.py index 0d602f7..bf18d9d 100644 --- a/pyreindexer/exceptions.py +++ b/pyreindexer/exceptions.py @@ -6,9 +6,5 @@ class QueryError(ApiError): pass -class QueryResultsError(QueryError): - pass - - class TransactionError(ApiError): pass diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 8702a1b..146df7d 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Optional, Union -from exceptions import QueryError +from exceptions import ApiError, QueryError from pyreindexer.point import Point from pyreindexer.query_results import QueryResults @@ -98,12 +98,12 @@ def __raise_on_error(self): """Checks if there is an error code and raises with an error message #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ if self.err_code: - raise QueryError(self.err_msg) + raise ApiError(self.err_msg) @staticmethod def __convert_to_list(param: Union[simple_types, tuple[list[simple_types], ...]]) -> list: @@ -166,7 +166,8 @@ def __where(self, index: str, condition: CondType, (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + QueryError: Raises with an error message if inappropriate condition is used + ApiError: Raises with an error message of API return on non-zero error code """ @@ -194,7 +195,7 @@ def where(self, index: str, condition: CondType, (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -215,7 +216,7 @@ def where_query(self, sub_query: Query, condition: CondType, (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -260,7 +261,7 @@ def where_composite(self, index: str, condition: CondType, keys: tuple[list[simp (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -281,7 +282,7 @@ def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -314,7 +315,7 @@ def open_bracket(self) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -329,7 +330,7 @@ def close_bracket(self) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -349,7 +350,7 @@ def match(self, index: str, *keys: str) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -542,7 +543,7 @@ def sort(self, index: str, desc: bool = False, (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -582,7 +583,7 @@ def sort_stfield_distance(self, first_field: str, second_field: str, desc: bool) (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -740,8 +741,8 @@ def execute(self) -> QueryResults: (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code """ @@ -760,8 +761,8 @@ def delete(self) -> int: (int): Number of deleted elements #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + QueryError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code """ @@ -783,8 +784,8 @@ def set_object(self, field: str, values: list[simple_types]) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code - Exception: Raises with an error message if no values are specified + QueryError: Raises with an error message if no values are specified + ApiError: Raises with an error message of API return on non-zero error code """ @@ -806,7 +807,7 @@ def set(self, field: str, values: list[simple_types]) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -852,8 +853,8 @@ def update(self) -> QueryResults: (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + QueryError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code """ @@ -872,8 +873,8 @@ def must_execute(self) -> QueryResults: (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code """ @@ -888,8 +889,8 @@ def get(self) -> (str, bool): (:tuple:string,bool): 1st string item and found flag #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code """ @@ -1013,6 +1014,9 @@ def on(self, index: str, condition: CondType, join_index: str) -> Query: #### Returns: (:obj:`Query`): Query object for further customizations + #### Raises: + QueryError: Raises with an error message when query is in an invalid state + """ if self.root is None: @@ -1034,7 +1038,7 @@ def select(self, *fields: str) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -1054,7 +1058,7 @@ def functions(self, *functions: str) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -1074,7 +1078,7 @@ def equal_position(self, *equal_position: str) -> Query: (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index b4d147f..68a808e 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,4 +1,4 @@ -from exceptions import QueryResultsError +from exceptions import ApiError class QueryResults: @@ -38,12 +38,12 @@ def __raise_on_error(self): """Checks if there is an error code and raises with an error message #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ if self.err_code: - raise QueryResultsError(self.err_msg) + raise ApiError(self.err_msg) def __iter__(self): """Returns the current iteration result @@ -80,7 +80,7 @@ def status(self) -> None: """Check status #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -122,7 +122,7 @@ def get_agg_results(self) -> dict: (:obj:`dict`): Dictionary with all results for the current query #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -137,7 +137,7 @@ def get_explain_results(self) -> str: (string): Formatted string with explain of results for the current query #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index f483a9e..17e68d8 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -13,7 +13,7 @@ def raise_on_error(self): """Checks if there is an error code and raises with an error message #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ @@ -24,7 +24,7 @@ def raise_on_not_init(self): """Checks if there is an error code and raises with an error message #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet """ diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 9c3221d..0474112 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -48,7 +48,7 @@ def close(self) -> None: """Closes an API instance with Reindexer resources freeing #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet """ @@ -62,8 +62,8 @@ def namespace_open(self, namespace) -> None: namespace (string): A name of a namespace #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -77,8 +77,8 @@ def namespace_close(self, namespace) -> None: namespace (string): A name of a namespace #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -111,8 +111,8 @@ def namespaces_enum(self, enum_not_opened=False) -> List[Dict[str, str]]: (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -128,8 +128,8 @@ def index_add(self, namespace, index_def) -> None: index_def (dict): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -144,8 +144,8 @@ def index_update(self, namespace, index_def) -> None: index_def (dict): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -160,8 +160,8 @@ def index_drop(self, namespace, index_name) -> None: index_name (string): A name of an index #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -177,8 +177,8 @@ def item_insert(self, namespace, item_def, precepts=None) -> None: precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -195,8 +195,8 @@ def item_update(self, namespace, item_def, precepts=None) -> None: precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -213,8 +213,8 @@ def item_upsert(self, namespace, item_def, precepts=None) -> None: precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -230,8 +230,8 @@ def item_delete(self, namespace, item_def) -> None: item_def (dict): A dictionary of item definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -247,8 +247,8 @@ def meta_put(self, namespace, key, value) -> None: value (string): A metadata for storage #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -266,8 +266,8 @@ def meta_get(self, namespace, key) -> str: string: A metadata value #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -283,8 +283,8 @@ def meta_delete(self, namespace, key) -> None: key (string): A key in a storage of Reindexer where metadata is kept #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -301,8 +301,8 @@ def meta_enum(self, namespace) -> List[str]: (:obj:`list` of :obj:`str`): A list of all metadata keys #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -320,8 +320,8 @@ def select(self, query: str) -> QueryResults: (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -339,8 +339,8 @@ def new_transaction(self, namespace) -> Transaction: (:obj:`Transaction`): A new transaction #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -358,7 +358,7 @@ def new_query(self, namespace: str) -> Query: (:obj:`Query`): A new query #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet """ @@ -372,7 +372,7 @@ def _api_import(self, dsn): dsn (string): The connection string which contains a protocol #### Raises: - Exception: Raises an exception if a connection protocol is unrecognized + ConnectionError: Raises an exception if a connection protocol is unrecognized """ @@ -391,8 +391,8 @@ def _api_init(self, dsn): dsn (string): The connection string which contains a protocol #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code """ @@ -405,7 +405,7 @@ def _api_close(self): """Destructs Reindexer instance correctly and resets memory pointer #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet """ diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index 1bd013a..e38264a 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,4 +1,4 @@ -from exceptions import TransactionError +from exceptions import ApiError, TransactionError def raise_if_error(func): @@ -48,18 +48,18 @@ def _raise_on_error(self): """Checks if there is an error code and raises with an error message #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code """ if self.err_code: - raise TransactionError(self.err_msg) + raise ApiError(self.err_msg) def _raise_on_is_over(self): """Checks the state of a transaction and returns an error message when necessary #### Raises: - Exception: Raises with an error message of API return if Transaction is over + TransactionError: Raises with an error message of API return if Transaction is over """ @@ -75,8 +75,8 @@ def insert(self, item_def, precepts=None): precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ @@ -92,8 +92,8 @@ def update(self, item_def, precepts=None): precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ @@ -109,8 +109,8 @@ def upsert(self, item_def, precepts=None): precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ @@ -125,8 +125,8 @@ def delete(self, item_def): item_def (dict): A dictionary of item definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ @@ -137,8 +137,8 @@ def commit(self): """Applies changes #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ @@ -150,8 +150,8 @@ def commit_with_count(self) -> int: """Applies changes and return the number of count of changed items #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ @@ -164,8 +164,8 @@ def rollback(self): """Rollbacks changes #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code """ From aaeb1fa5db4c202d12db64b9ee4a990a684fe43f Mon Sep 17 00:00:00 2001 From: Nikita Bushmakin Date: Wed, 4 Dec 2024 11:40:36 +0300 Subject: [PATCH 111/125] small ref, fix expected exception --- pyreindexer/query.py | 19 +++++++------------ pyreindexer/tests/tests/test_query.py | 14 ++++++-------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 146df7d..fa95f77 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -119,22 +119,18 @@ def __convert_to_list(param: Union[simple_types, tuple[list[simple_types], ...]] """ if param is None: - return list() + return [] if isinstance(param, str) or not isinstance(param, Iterable): - result: list = [param] - return result + return [param] if isinstance(param, list): return param - res = param - if not isinstance(res, list): - res = list(res) - if len(res) == 0 or (len(res) > 0 and not isinstance(res[0], list)): - wrap: list = [res] - res = wrap - return res + result = list(param) + if len(result) == 0 or not isinstance(result[0], list): + result: list = [result] + return result @staticmethod def __convert_strs_to_list(param: tuple[str, ...]) -> list[str]: @@ -148,8 +144,7 @@ def __convert_strs_to_list(param: tuple[str, ...]) -> list[str]: """ - result = list() if param is None else list(param) - return result + return [] if param is None else list(param) def __where(self, index: str, condition: CondType, keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query: diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 8d76010..a774b2e 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -5,6 +5,7 @@ import pytest from hamcrest import * +from exceptions import ApiError from point import Point from query import CondType, LogLevel, StrictMode from tests.helpers.base_helper import calculate_distance, get_ns_items @@ -342,8 +343,7 @@ def test_query_select_strict_mode_names_default(self, db, namespace, index, item query.strict(strict_mode).where("rand", CondType.CondEq, 1) err_msg = "Current query strict mode allows filtering by existing fields only. " \ f"There are no fields with name 'rand' in namespace '{namespace}'" - assert_that(calling(query.execute).with_args(), - raises(Exception, pattern=err_msg)) + assert_that(calling(query.execute).with_args(), raises(ApiError, pattern=err_msg)) def test_query_select_strict_mode_indexes(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -353,8 +353,7 @@ def test_query_select_strict_mode_indexes(self, db, namespace, index, items): query.strict(StrictMode.Indexes).where("rand", CondType.CondEq, 1) err_msg = "Current query strict mode allows filtering by indexes only. " \ f"There are no indexes with name 'rand' in namespace '{namespace}'" - assert_that(calling(query.execute).with_args(), - raises(Exception, pattern=err_msg)) + assert_that(calling(query.execute).with_args(), raises(ApiError, pattern=err_msg)) def test_query_select_wal_any(self, db, namespace, index, items): # Given("Create namespace with index and items") @@ -363,7 +362,7 @@ def test_query_select_wal_any(self, db, namespace, index, items): # When ("Make select query with lsn any") err_msg = "WAL queries are not supported" query.where("#lsn", CondType.CondAny, None) - assert_that(calling(query.execute).with_args(), raises(Exception, pattern=err_msg)) + assert_that(calling(query.execute).with_args(), raises(ApiError, pattern=err_msg)) class TestQuerySelectAggregations: @@ -530,8 +529,7 @@ def test_query_select_sort_strict_mode_indexes(self, db, namespace, index, items query.strict(StrictMode.Indexes).sort("rand") err_msg = "Current query strict mode allows sort by index fields only. " \ f"There are no indexes with name 'rand' in namespace '{namespace}'" - assert_that(calling(query.execute).with_args(), - raises(Exception, pattern=err_msg)) + assert_that(calling(query.execute).with_args(), raises(ApiError, pattern=err_msg)) class TestQuerySelectJoin: @@ -583,7 +581,7 @@ def test_cannot_query_select_join_without_on(self, db, namespace, index, items, query2 = db.query.new(second_namespace) # When ("Try to ake select query join without on") assert_that(calling(query1.inner_join(query2, "joined").execute).with_args(), - raises(Exception, pattern="Join without ON conditions")) + raises(ApiError, pattern="Join without ON conditions")) def test_query_select_merge_with_joins(self, db, namespace, index, items, second_namespace, second_item): # Given("Create two namespaces with index and items") From b4360b301b2efc57c0a3279f848d3a0b07e52da4 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 4 Dec 2024 14:59:46 +0300 Subject: [PATCH 112/125] Update readme file --- README.md | 168 +++++++++++++++++++----------------- pyreindexer/rx_connector.py | 2 +- 2 files changed, 89 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 712b7ee..7c3d561 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ def close() -> None Closes an API instance with Reindexer resources freeing #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet @@ -141,8 +141,8 @@ Opens a namespace specified or creates a namespace if it does not exist namespace (string): A name of a namespace #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -158,8 +158,8 @@ Closes a namespace specified namespace (string): A name of a namespace #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -196,8 +196,8 @@ Gets a list of namespaces available (:obj:`list` of :obj:`dict`): A list of dictionaries which describe each namespace #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -214,8 +214,8 @@ Adds an index to the namespace specified index_def (dict): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -232,8 +232,8 @@ Updates an index in the namespace specified index_def (dict): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -250,8 +250,8 @@ Drops an index from the namespace specified index_name (string): A name of an index #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -269,8 +269,8 @@ Inserts an item with its precepts to the namespace specified precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -288,8 +288,8 @@ Updates an item with its precepts in the namespace specified precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -307,8 +307,8 @@ Updates an item with its precepts in the namespace specified. Creates the item i precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -325,8 +325,8 @@ Deletes an item from the namespace specified item_def (dict): A dictionary of item definition #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -344,8 +344,8 @@ Puts metadata to a storage of Reindexer by key value (string): A metadata for storage #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -365,8 +365,8 @@ Gets metadata from a storage of Reindexer by key specified string: A metadata value #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -383,8 +383,8 @@ Deletes metadata from a storage of Reindexer by key specified key (string): A key in a storage of Reindexer where metadata is kept #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -403,8 +403,8 @@ Gets a list of metadata keys from a storage of Reindexer (:obj:`list` of :obj:`str`): A list of all metadata keys #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -423,8 +423,8 @@ Executes an SQL query and returns query results (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -443,8 +443,8 @@ Starts a new transaction and return the transaction object to processing (:obj:`Transaction`): A new transaction #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet - Exception: Raises with an error message of API return on non-zero error code + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet + ApiError: Raises with an error message of API return on non-zero error code @@ -463,7 +463,7 @@ Creates a new query and return the query object to processing (:obj:`Query`): A new query #### Raises: - Exception: Raises with an error message when Reindexer instance is not initialized yet + ConnectionError: Raises with an error message when Reindexer instance is not initialized yet @@ -499,7 +499,7 @@ def status() -> None Check status #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -541,7 +541,7 @@ Returns aggregation results for the current query (:obj:`dict`): Dictionary with all results for the current query #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -557,7 +557,7 @@ Returns explain results for the current query (string): Formatted string with explain of results for the current query #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -594,8 +594,8 @@ Inserts an item with its precepts to the transaction precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -612,8 +612,8 @@ Updates an item with its precepts to the transaction precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -630,8 +630,8 @@ Updates an item with its precepts to the transaction. Creates the item if it not precepts (:obj:`list` of :obj:`str`): A dictionary of index definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -647,8 +647,8 @@ Deletes an item from the transaction item_def (dict): A dictionary of item definition #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -661,8 +661,8 @@ def commit() Applies changes #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -675,8 +675,8 @@ def commit_with_count() -> int Applies changes and return the number of count of changed items #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -689,8 +689,8 @@ def rollback() Rollbacks changes #### Raises: - Exception: Raises with an error message of API return if Transaction is over - Exception: Raises with an error message of API return on non-zero error code + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code @@ -738,8 +738,11 @@ An object representing the context of a Reindexer query ### Query.where ```python -def where(index: str, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query +def where( + index: str, + condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], + ...]] = None) -> Query ``` Adds where condition to DB query with args @@ -755,7 +758,7 @@ Adds where condition to DB query with args (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -763,8 +766,10 @@ Adds where condition to DB query with args ```python def where_query( - sub_query: Query, condition: CondType, - keys: Union[simple_types, tuple[list[simple_types], ...]]) -> Query + sub_query: Query, + condition: CondType, + keys: Union[simple_types, tuple[list[simple_types], + ...]] = None) -> Query ``` Adds sub-query where condition to DB query with args @@ -780,7 +785,7 @@ Adds sub-query where condition to DB query with args (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -826,7 +831,7 @@ Adds where condition to DB query with interface args for composite indexes (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -850,7 +855,7 @@ Adds where condition to DB query with UUID as string args. (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -885,7 +890,7 @@ Opens bracket for where condition to DB query (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -901,7 +906,7 @@ Closes bracket for where condition to DB query (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -922,7 +927,7 @@ Adds string EQ-condition to DB query with string args (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -1065,7 +1070,7 @@ Applies sort order to return from query items. If values argument specified, the (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -1107,7 +1112,7 @@ Applies geometry sort order to return from query items. Wrapper for geometry sor (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -1289,8 +1294,8 @@ Executes a select query (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code @@ -1306,8 +1311,8 @@ Executes a query, and delete items, matches query (int): Number of deleted elements #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + QueryError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code @@ -1327,8 +1332,8 @@ Adds an update query to an object field for an update query (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code - Exception: Raises with an error message if no values are specified + QueryError: Raises with an error message if no values are specified + ApiError: Raises with an error message of API return on non-zero error code @@ -1348,7 +1353,7 @@ Adds a field update request to the update request (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -1397,8 +1402,8 @@ Executes update query, and update fields in items, which matches query (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + QueryError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code @@ -1414,8 +1419,8 @@ Executes a query, and update fields in items, which matches query, with status c (:obj:`QueryResults`): A QueryResults iterator #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code @@ -1431,8 +1436,8 @@ Executes a query, and return 1 JSON item (:tuple:string,bool): 1st string item and found flag #### Raises: - Exception: Raises with an error message when query is in an invalid state - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message when query is in an invalid state + ApiError: Raises with an error message of API return on non-zero error code @@ -1526,6 +1531,9 @@ On specifies join condition #### Returns: (:obj:`Query`): Query object for further customizations +#### Raises: + QueryError: Raises with an error message when query is in an invalid state + ### Query.select @@ -1546,7 +1554,7 @@ Sets list of columns in this namespace to be finally selected. (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -1565,7 +1573,7 @@ Adds sql-functions to query (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code @@ -1584,7 +1592,7 @@ Adds equal position fields to arrays queries (:obj:`Query`): Query object for further customizations #### Raises: - Exception: Raises with an error message of API return on non-zero error code + ApiError: Raises with an error message of API return on non-zero error code diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index 0474112..73023b6 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -26,7 +26,7 @@ def __init__(self, dsn): #### Arguments: dsn (string): The connection string which contains a protocol - Examples: 'builtin:///tmp/pyrx', 'cproto://127.0.0.1:6534/pyrx + Examples: 'builtin:///tmp/pyrx', 'cproto://127.0.0.1:6534/pyrx' """ From 28ff2dc25336af554fe1307ad301799650d1faff Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 4 Dec 2024 15:13:17 +0300 Subject: [PATCH 113/125] Fix build --- pyreindexer/example/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 1ae7ae3..07d3681 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -1,5 +1,5 @@ -from exceptions import ApiError from pyreindexer import RxConnector +from pyreindexer.exceptions import ApiError from pyreindexer.query import CondType From 4748384fa6f3e5e369c78222640cdc088c169dcb Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Wed, 4 Dec 2024 16:06:38 +0300 Subject: [PATCH 114/125] Fix build --- pyreindexer/query.py | 4 ++-- pyreindexer/query_results.py | 2 +- pyreindexer/raiser_mixin.py | 2 +- pyreindexer/transaction.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyreindexer/query.py b/pyreindexer/query.py index fa95f77..298121e 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -4,9 +4,9 @@ from enum import Enum from typing import Optional, Union -from exceptions import ApiError, QueryError -from pyreindexer.point import Point +from pyreindexer.exceptions import ApiError, QueryError from pyreindexer.query_results import QueryResults +from pyreindexer.point import Point class ExtendedEnum(Enum): diff --git a/pyreindexer/query_results.py b/pyreindexer/query_results.py index 68a808e..0c350e8 100644 --- a/pyreindexer/query_results.py +++ b/pyreindexer/query_results.py @@ -1,4 +1,4 @@ -from exceptions import ApiError +from pyreindexer.exceptions import ApiError class QueryResults: diff --git a/pyreindexer/raiser_mixin.py b/pyreindexer/raiser_mixin.py index 17e68d8..c57a6f4 100644 --- a/pyreindexer/raiser_mixin.py +++ b/pyreindexer/raiser_mixin.py @@ -1,4 +1,4 @@ -from exceptions import ApiError +from pyreindexer.exceptions import ApiError class RaiserMixin: diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index e38264a..ad1bc48 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,4 +1,4 @@ -from exceptions import ApiError, TransactionError +from pyreindexer.exceptions import ApiError, TransactionError def raise_if_error(func): From 794eb5f6a92711757b4d784ae30275f72fb4b437 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 5 Dec 2024 19:18:45 +0300 Subject: [PATCH 115/125] Fix build --- .github/workflows/test.sh | 1 + pyreindexer/tests/tests/test_index.py | 2 +- pyreindexer/tests/tests/test_namespace.py | 2 +- pyreindexer/tests/tests/test_query.py | 2 +- pyreindexer/tests/tests/test_sql.py | 2 +- pyreindexer/tests/tests/test_transaction.py | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.sh b/.github/workflows/test.sh index 6af6d2e..1262d9e 100755 --- a/.github/workflows/test.sh +++ b/.github/workflows/test.sh @@ -1,3 +1,4 @@ +export PYTHONPATH=$(pwd) python3 example/main.py || exit 1 python3 -m pytest tests/tests || exit 1 reindexer_server --db /tmp/reindex_test -l0 --serverlog="" --corelog="" --httplog="" --rpclog="" & diff --git a/pyreindexer/tests/tests/test_index.py b/pyreindexer/tests/tests/test_index.py index c67fcf8..b3537da 100644 --- a/pyreindexer/tests/tests/test_index.py +++ b/pyreindexer/tests/tests/test_index.py @@ -1,6 +1,6 @@ from hamcrest import * -from exceptions import ApiError +from pyreindexer.exceptions import ApiError from tests.helpers.base_helper import get_ns_description from tests.test_data.constants import index_definition, updated_index_definition diff --git a/pyreindexer/tests/tests/test_namespace.py b/pyreindexer/tests/tests/test_namespace.py index 26cda86..e5927e8 100644 --- a/pyreindexer/tests/tests/test_namespace.py +++ b/pyreindexer/tests/tests/test_namespace.py @@ -1,6 +1,6 @@ from hamcrest import * -from exceptions import ApiError +from pyreindexer.exceptions import ApiError class TestCrudNamespace: diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index a774b2e..986557e 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -5,7 +5,7 @@ import pytest from hamcrest import * -from exceptions import ApiError +from pyreindexer.exceptions import ApiError from point import Point from query import CondType, LogLevel, StrictMode from tests.helpers.base_helper import calculate_distance, get_ns_items diff --git a/pyreindexer/tests/tests/test_sql.py b/pyreindexer/tests/tests/test_sql.py index acb55b2..2107ca4 100644 --- a/pyreindexer/tests/tests/test_sql.py +++ b/pyreindexer/tests/tests/test_sql.py @@ -1,6 +1,6 @@ from hamcrest import * -from exceptions import ApiError +from pyreindexer.exceptions import ApiError class TestSqlQueries: diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index a00181a..f5e7545 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -1,6 +1,6 @@ from hamcrest import * -from exceptions import TransactionError +from pyreindexer.exceptions import TransactionError from tests.helpers.base_helper import get_ns_items from tests.helpers.transaction import * from tests.test_data.constants import item_definition From ab6601582a82eaf38125c72325eb631fa13e2ff9 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 5 Dec 2024 20:43:42 +0300 Subject: [PATCH 116/125] Review changes --- pyreindexer/lib/src/rawpyreindexer.cc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 248845f..4c07237 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -298,8 +298,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { Py_DECREF(itemDefDict); - char* json = const_cast(wrSer.c_str()); - err = item.Unsafe().FromJSON(json, 0, mode == ModeDelete); + err = item.Unsafe().FromJSON(wrSer.c_str(), 0, mode == ModeDelete); if (!err.ok()) { Py_XDECREF(preceptsList); @@ -317,7 +316,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { return pyErr(err); } - item.SetPrecepts(itemPrecepts); + item.SetPrecepts(itemPrecepts); // ToDo after migrate on v.4, do std::move } Py_XDECREF(preceptsList); @@ -545,8 +544,7 @@ PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) Py_DECREF(defDict); - char* json = const_cast(wrSer.c_str()); - err = item.Unsafe().FromJSON(json, 0, mode == ModeDelete); + err = item.Unsafe().FromJSON(wrSer.c_str(), 0, mode == ModeDelete); if (!err.ok()) { Py_XDECREF(precepts); @@ -564,7 +562,7 @@ PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) return pyErr(err); } - item.SetPrecepts(itemPrecepts); + item.SetPrecepts(itemPrecepts); // ToDo after migrate on v.4, do std::move } Py_XDECREF(precepts); @@ -594,7 +592,6 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args) { return nullptr; } - assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); size_t count = 0; auto err = getWrapper(transactionWrapperAddr)->Commit(count); @@ -609,7 +606,6 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { return nullptr; } - assert((StopTransactionMode::Commit == stopMode) || (StopTransactionMode::Rollback == stopMode)); auto err = getWrapper(transactionWrapperAddr)->Rollback(); deleteWrapper(transactionWrapperAddr); From a39b9661c999f0c44d914821ebb445235c097249 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 5 Dec 2024 20:51:04 +0300 Subject: [PATCH 117/125] Review changes --- pyreindexer/lib/include/queryresults_wrapper.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyreindexer/lib/include/queryresults_wrapper.h b/pyreindexer/lib/include/queryresults_wrapper.h index bd0a653..c008dfd 100644 --- a/pyreindexer/lib/include/queryresults_wrapper.h +++ b/pyreindexer/lib/include/queryresults_wrapper.h @@ -61,10 +61,8 @@ class QueryResultsWrapper { void FetchResults() { assert(qres_.has_value()); + // when results are fetched iterator closes and frees memory of results buffer of Reindexer ++it_; - if (it_ == qres_->end()) { - it_ = qres_->begin(); - } } const std::string& GetExplainResults() & noexcept { From f280c663e4bdb636559c7a94a0a5dc64efd7da47 Mon Sep 17 00:00:00 2001 From: "Alexander.A.Utkin" Date: Mon, 9 Dec 2024 19:14:01 +0300 Subject: [PATCH 118/125] Review changes --- README.md | 20 ++--- pyreindexer/lib/include/pyobjtools.cc | 12 +-- pyreindexer/lib/include/pyobjtools.h | 7 +- pyreindexer/lib/include/query_wrapper.cc | 49 ++++++------ pyreindexer/lib/include/query_wrapper.h | 33 ++++---- pyreindexer/lib/src/rawpyreindexer.cc | 98 ++++++++++++----------- pyreindexer/lib/src/reindexerinterface.cc | 4 +- pyreindexer/lib/src/reindexerinterface.h | 2 +- pyreindexer/query.py | 13 +-- 9 files changed, 125 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 7c3d561..7761757 100644 --- a/README.md +++ b/README.md @@ -838,7 +838,7 @@ Adds where condition to DB query with interface args for composite indexes ### Query.where\_uuid ```python -def where_uuid(index: str, condition: CondType, *keys: str) -> Query +def where_uuid(index: str, condition: CondType, *uuids: UUID) -> Query ``` Adds where condition to DB query with UUID as string args. @@ -848,7 +848,7 @@ Adds where condition to DB query with UUID as string args. #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (*string): Value of index to be compared with. For composite indexes keys must be list, + uuids (*:obj:`UUID`): Value of index to be compared with. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -1051,19 +1051,21 @@ Gets fields facet value. Applicable to multiple data fields and the result of th ```python def sort( - index: str, - desc: bool = False, - keys: Union[simple_types, tuple[list[simple_types], - ...]] = None) -> Query + index: str, + desc: bool = False, + forced_sort_values: Union[simple_types, tuple[list[simple_types], + ...]] = None +) -> Query ``` -Applies sort order to return from query items. If values argument specified, then items equal to values, - if found will be placed in the top positions. Forced sort is support for the first sorting field only +Applies sort order to return from query items. If forced_sort_values argument specified, then items equal to + values, if found will be placed in the top positions. Forced sort is support for the first sorting field + only #### Arguments: index (string): The index name desc (bool): Sort in descending order - keys (union[simple_types, (list[simple_types], ...)]): + forced_sort_values (union[simple_types, (list[simple_types], ...)]): Value of index to match. For composite indexes keys must be list, with value of each sub-index #### Returns: diff --git a/pyreindexer/lib/include/pyobjtools.cc b/pyreindexer/lib/include/pyobjtools.cc index dde4ab1..2a13c59 100644 --- a/pyreindexer/lib/include/pyobjtools.cc +++ b/pyreindexer/lib/include/pyobjtools.cc @@ -99,8 +99,8 @@ void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer) { } } -std::vector PyObjectToJson(PyObject** obj) { - std::vector values; +reindexer::h_vector PyObjectToJson(PyObject** obj) { + reindexer::h_vector values; reindexer::WrSerializer wrSer; if (PyDict_Check(*obj)) { @@ -132,8 +132,8 @@ std::vector PyObjectToJson(PyObject** obj) { return values; } -std::vector ParseStrListToStrVec(PyObject** list) { - std::vector result; +reindexer::h_vector ParseStrListToStrVec(PyObject** list) { + reindexer::h_vector result; Py_ssize_t sz = PyList_Size(*list); result.reserve(sz); @@ -191,8 +191,8 @@ reindexer::Variant convert(PyObject** value) { return {}; } -std::vector ParseListToVec(PyObject** list) { - std::vector result; +reindexer::VariantArray ParseListToVec(PyObject** list) { + reindexer::VariantArray result; Py_ssize_t sz = PyList_Size(*list); result.reserve(sz); diff --git a/pyreindexer/lib/include/pyobjtools.h b/pyreindexer/lib/include/pyobjtools.h index 5b691e2..4dc28ee 100644 --- a/pyreindexer/lib/include/pyobjtools.h +++ b/pyreindexer/lib/include/pyobjtools.h @@ -3,14 +3,15 @@ #include #include #include "core/keyvalue/variant.h" +#include "estl/h_vector.h" namespace pyreindexer { -std::vector ParseStrListToStrVec(PyObject** list); -std::vector ParseListToVec(PyObject** list); +reindexer::h_vector ParseStrListToStrVec(PyObject** list); +reindexer::VariantArray ParseListToVec(PyObject** list); void PyObjectToJson(PyObject** obj, reindexer::WrSerializer& wrSer); -std::vector PyObjectToJson(PyObject** obj); +reindexer::h_vector PyObjectToJson(PyObject** obj); PyObject* PyObjectFromJson(reindexer::span json); } // namespace pyreindexer diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 4fd929f..5d178b3 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -11,7 +11,7 @@ QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { ser_.PutVString(ns); } -void QueryWrapper::Where(std::string_view index, CondType condition, const std::vector& keys) { +void QueryWrapper::Where(std::string_view index, CondType condition, const reindexer::VariantArray& keys) { ser_.PutVarUint(QueryItemType::QueryCondition); ser_.PutVString(index); ser_.PutVarUint(nextOperation_); @@ -22,7 +22,7 @@ void QueryWrapper::Where(std::string_view index, CondType condition, const std:: ++queriesCount_; } -void QueryWrapper::WhereSubQuery(QueryWrapper& query, CondType condition, const std::vector& keys) { +void QueryWrapper::WhereSubQuery(QueryWrapper& query, CondType condition, const reindexer::VariantArray& keys) { ser_.PutVarUint(QueryItemType::QuerySubQueryCondition); ser_.PutVarUint(nextOperation_); ser_.PutVString(query.ser_.Slice()); @@ -44,7 +44,8 @@ void QueryWrapper::WhereFieldSubQuery(std::string_view index, CondType condition ++queriesCount_; } -void QueryWrapper::WhereUUID(std::string_view index, CondType condition, const std::vector& keys) { +void QueryWrapper::WhereUUID(std::string_view index, CondType condition, + const reindexer::h_vector& keys) { ser_.PutVarUint(QueryItemType::QueryCondition); ser_.PutVString(index); ser_.PutVarUint(nextOperation_); @@ -129,7 +130,7 @@ void QueryWrapper::Aggregate(std::string_view index, AggType type) { ser_.PutVString(index); } -void QueryWrapper::Aggregation(const std::vector& fields) { +void QueryWrapper::Aggregation(const reindexer::h_vector& fields) { ser_.PutVarUint(QueryItemType::QueryAggregation); ser_.PutVarUint(AggType::AggFacet); ser_.PutVarUint(fields.size()); @@ -138,11 +139,11 @@ void QueryWrapper::Aggregation(const std::vector& fields) { } } -void QueryWrapper::Sort(std::string_view index, bool desc, const std::vector& keys) { +void QueryWrapper::Sort(std::string_view index, bool desc, const reindexer::VariantArray& sortValues) { ser_.PutVarUint(QueryItemType::QuerySortIndex); ser_.PutVString(index); ser_.PutVarUint(desc? 1 : 0); - putKeys(keys); + putKeys(sortValues); } void QueryWrapper::LogOp(OpType op) { @@ -183,7 +184,8 @@ void serializeJoinQuery(JoinType type, reindexer::WrSerializer& data, reindexer: } } // namespace -void QueryWrapper::addJoinQueries(const std::vector& queries, reindexer::WrSerializer& buffer) const { +void QueryWrapper::addJoinQueries(const reindexer::h_vector& queries, reindexer::WrSerializer& buffer) + const { for (auto query : queries) { serializeJoinQuery(query->joinType_, query->ser_, buffer); } @@ -217,7 +219,7 @@ reindexer::Error QueryWrapper::prepareQuery(reindexer::Query& query) { return error; } -reindexer::Error QueryWrapper::ExecuteQuery(ExecuteType type, QueryResultsWrapper& qr) { +reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { reindexer::Query query; auto err = prepareQuery(query); if (!err.ok()) { @@ -228,17 +230,16 @@ reindexer::Error QueryWrapper::ExecuteQuery(ExecuteType type, QueryResultsWrappe return reindexer::Error(ErrorCode::errQueryExec, "WAL queries are not supported"); } - switch (type) { - case ExecuteType::Select: - err = db_->SelectQuery(query, qr); - break; - case ExecuteType::Update: - err = db_->UpdateQuery(query, qr); - break; - default: - assert(false); + return db_->SelectQuery(query, qr); +} + +reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { + reindexer::Query query; + auto err = prepareQuery(query); + if (!err.ok()) { + return err; } - return err; + return db_->UpdateQuery(query, qr); } reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { @@ -250,7 +251,7 @@ reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { return db_->DeleteQuery(query, count); } -void QueryWrapper::Set(std::string_view field, const std::vector& values, +void QueryWrapper::Set(std::string_view field, const reindexer::VariantArray& values, IsExpression isExpression) { ser_.PutVarUint(QueryItemType::QueryUpdateFieldV2); ser_.PutVString(field); @@ -262,7 +263,7 @@ void QueryWrapper::Set(std::string_view field, const std::vector& values) { +void QueryWrapper::SetObject(std::string_view field, const reindexer::h_vector& values) { ser_.PutVarUint(QueryItemType::QueryUpdateObject); ser_.PutVString(field); ser_.PutVarUint(values.size()); // values count @@ -315,21 +316,21 @@ reindexer::Error QueryWrapper::On(std::string_view joinField, CondType condition return errOK; } -void QueryWrapper::SelectFilter(const std::vector& fields) { +void QueryWrapper::SelectFilter(const reindexer::h_vector& fields) { for (const auto& field : fields) { ser_.PutVarUint(QueryItemType::QuerySelectFilter); ser_.PutVString(field); } } -void QueryWrapper::AddFunctions(const std::vector& functions) { +void QueryWrapper::AddFunctions(const reindexer::h_vector& functions) { for (const auto& function : functions) { ser_.PutVarUint(QueryItemType::QuerySelectFunction); ser_.PutVString(function); } } -void QueryWrapper::AddEqualPosition(const std::vector& equalPositions) { +void QueryWrapper::AddEqualPosition(const reindexer::h_vector& equalPositions) { ser_.PutVarUint(QueryItemType::QueryEqualPosition); ser_.PutVarUint(openedBrackets_.empty()? 0 : int(openedBrackets_.back() + 1)); ser_.PutVarUint(equalPositions.size()); @@ -338,7 +339,7 @@ void QueryWrapper::AddEqualPosition(const std::vector& equalPositio } } -void QueryWrapper::putKeys(const std::vector& keys) { +void QueryWrapper::putKeys(const reindexer::VariantArray& keys) { ser_.PutVarUint(keys.size()); for (const auto& key : keys) { ser_.PutVariant(key); diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index b694b50..920757f 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -3,6 +3,7 @@ #include #include "core/type_consts.h" +#include "estl/h_vector.h" #include "tools/serializer.h" #include "tools/errors.h" #ifdef PYREINDEXER_CPROTO @@ -26,10 +27,10 @@ class QueryWrapper { public: QueryWrapper(DBInterface* db, std::string_view ns); - void Where(std::string_view index, CondType condition, const std::vector& keys); - void WhereSubQuery(QueryWrapper& query, CondType condition, const std::vector& keys); + void Where(std::string_view index, CondType condition, const reindexer::VariantArray& keys); + void WhereSubQuery(QueryWrapper& query, CondType condition, const reindexer::VariantArray& keys); void WhereFieldSubQuery(std::string_view index, CondType condition, QueryWrapper& query); - void WhereUUID(std::string_view index, CondType condition, const std::vector& keys); + void WhereUUID(std::string_view index, CondType condition, const reindexer::h_vector& keys); void WhereBetweenFields(std::string_view firstField, CondType condition, std::string_view secondField); @@ -39,10 +40,10 @@ class QueryWrapper { void DWithin(std::string_view index, double x, double y, double distance); void Aggregate(std::string_view index, AggType type); - void Aggregation(const std::vector& fields); + void Aggregation(const reindexer::h_vector& fields); void AggregationSort(std::string_view field, bool desc); - void Sort(std::string_view index, bool desc, const std::vector& keys); + void Sort(std::string_view index, bool desc, const reindexer::VariantArray& sortValues); void LogOp(OpType op); @@ -52,13 +53,13 @@ class QueryWrapper { void Modifier(QueryItemType type); - enum class ExecuteType { Select, Update }; - reindexer::Error ExecuteQuery(ExecuteType type, QueryResultsWrapper& qr); + reindexer::Error SelectQuery(QueryResultsWrapper& qr); + reindexer::Error UpdateQuery(QueryResultsWrapper& qr); reindexer::Error DeleteQuery(size_t& count); enum class IsExpression { Yes, No }; - void Set(std::string_view field, const std::vector& values, IsExpression isExpression); - void SetObject(std::string_view field, const std::vector& values); + void Set(std::string_view field, const reindexer::VariantArray& values, IsExpression isExpression); + void SetObject(std::string_view field, const reindexer::h_vector& values); void Drop(std::string_view field); @@ -67,17 +68,17 @@ class QueryWrapper { reindexer::Error On(std::string_view joinField, CondType condition, std::string_view joinIndex); - void SelectFilter(const std::vector& fields); + void SelectFilter(const reindexer::h_vector& fields); - void AddFunctions(const std::vector& functions); - void AddEqualPosition(const std::vector& equalPositions); + void AddFunctions(const reindexer::h_vector& functions); + void AddEqualPosition(const reindexer::h_vector& equalPositions); DBInterface* GetDB() const { return db_; } private: - void addJoinQueries(const std::vector& queries, reindexer::WrSerializer& buffer) const; + void addJoinQueries(const reindexer::h_vector& queries, reindexer::WrSerializer& buffer) const; reindexer::Error prepareQuery(reindexer::Query& query); - void putKeys(const std::vector& keys); + void putKeys(const reindexer::VariantArray& keys); DBInterface* db_{nullptr}; reindexer::WrSerializer ser_; @@ -86,8 +87,8 @@ class QueryWrapper { unsigned queriesCount_{0}; std::deque openedBrackets_; JoinType joinType_{JoinType::LeftJoin}; - std::vector joinQueries_; - std::vector mergedQueries_; + reindexer::h_vector joinQueries_; + reindexer::h_vector mergedQueries_; }; } // namespace pyreindexer diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 4c07237..5cde19e 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -99,17 +99,16 @@ static PyObject* Select(PyObject* self, PyObject* args) { } auto db = getWrapper(rx); - auto qresWrapper = std::make_unique(db); - auto err = qresWrapper->Select(query); - + auto qresult = std::make_unique(db); + auto err = qresult->Select(query); if (!err.ok()) { return Py_BuildValue("iskII", err.code(), err.what().c_str(), 0, 0); } - auto count = qresWrapper->Count(); - auto totalCount = qresWrapper->TotalCount(); + const auto count = qresult->Count(); + const auto totalCount = qresult->TotalCount(); return Py_BuildValue("iskII", err.code(), err.what().c_str(), - reinterpret_cast(qresWrapper.release()), count, totalCount); + reinterpret_cast(qresult.release()), count, totalCount); } // namespace ---------------------------------------------------------------------------------------------------------- @@ -298,7 +297,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { Py_DECREF(itemDefDict); - err = item.Unsafe().FromJSON(wrSer.c_str(), 0, mode == ModeDelete); + err = item.Unsafe().FromJSON(wrSer.Slice(), 0, mode == ModeDelete); if (!err.ok()) { Py_XDECREF(preceptsList); @@ -306,8 +305,7 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { } if (preceptsList != nullptr && mode != ModeDelete) { - std::vector itemPrecepts; - + reindexer::h_vector itemPrecepts; try { itemPrecepts = ParseStrListToStrVec(&preceptsList); } catch (const Error& err) { @@ -316,7 +314,8 @@ PyObject* itemModify(PyObject* self, PyObject* args, ItemModifyMode mode) { return pyErr(err); } - item.SetPrecepts(itemPrecepts); // ToDo after migrate on v.4, do std::move + std::vector prets(itemPrecepts.begin(), itemPrecepts.end()); + item.SetPrecepts(prets); // ToDo after migrate on v.4, do std::move } Py_XDECREF(preceptsList); @@ -544,7 +543,7 @@ PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) Py_DECREF(defDict); - err = item.Unsafe().FromJSON(wrSer.c_str(), 0, mode == ModeDelete); + err = item.Unsafe().FromJSON(wrSer.Slice(), 0, mode == ModeDelete); if (!err.ok()) { Py_XDECREF(precepts); @@ -552,7 +551,7 @@ PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) } if (precepts != nullptr && mode != ModeDelete) { - std::vector itemPrecepts; + reindexer::h_vector itemPrecepts; try { itemPrecepts = ParseStrListToStrVec(&precepts); @@ -562,7 +561,8 @@ PyObject* modifyTransaction(PyObject* self, PyObject* args, ItemModifyMode mode) return pyErr(err); } - item.SetPrecepts(itemPrecepts); // ToDo after migrate on v.4, do std::move + std::vector prets(itemPrecepts.begin(), itemPrecepts.end()); + item.SetPrecepts(prets); // ToDo after migrate on v.4, do std::move } Py_XDECREF(precepts); @@ -595,7 +595,7 @@ static PyObject* CommitTransaction(PyObject* self, PyObject* args) { size_t count = 0; auto err = getWrapper(transactionWrapperAddr)->Commit(count); - deleteWrapper(transactionWrapperAddr); + deleteWrapper(transactionWrapperAddr); // free memory return Py_BuildValue("isI", err.code(), err.what().c_str(), count); } @@ -608,7 +608,7 @@ static PyObject* RollbackTransaction(PyObject* self, PyObject* args) { auto err = getWrapper(transactionWrapperAddr)->Rollback(); - deleteWrapper(transactionWrapperAddr); + deleteWrapper(transactionWrapperAddr); // free memory return pyErr(err); } @@ -649,7 +649,7 @@ static PyObject* Where(PyObject* self, PyObject* args) { Py_XINCREF(keysList); - std::vector keys; + reindexer::VariantArray keys; if (keysList != nullptr) { try { keys = ParseListToVec(&keysList); @@ -678,7 +678,7 @@ static PyObject* WhereSubQuery(PyObject* self, PyObject* args) { Py_XINCREF(keysList); - std::vector keys; + reindexer::VariantArray keys; if (keysList != nullptr) { try { keys = ParseListToVec(&keysList); @@ -693,7 +693,6 @@ static PyObject* WhereSubQuery(PyObject* self, PyObject* args) { auto query = getWrapper(queryWrapperAddr); auto subQuery = getWrapper(subQueryWrapperAddr); - query->WhereSubQuery(*subQuery, CondType(cond), keys); return pyErr(errOK); @@ -727,7 +726,7 @@ static PyObject* WhereUUID(PyObject* self, PyObject* args) { Py_XINCREF(keysList); - std::vector keys; + reindexer::h_vector keys; if (keysList != nullptr) { try { keys = ParseStrListToStrVec(&keysList); @@ -852,7 +851,7 @@ static PyObject* Aggregation(PyObject* self, PyObject* args) { Py_XINCREF(fieldsList); - std::vector fields; + reindexer::h_vector fields; if (fieldsList != nullptr) { try { fields = ParseStrListToStrVec(&fieldsList); @@ -874,27 +873,27 @@ static PyObject* Sort(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; char* index = nullptr; unsigned desc = 0; - PyObject* keysList = nullptr; // borrowed ref after ParseTuple if passed - if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &desc, &PyList_Type, &keysList)) { + PyObject* sortValuesList = nullptr; // borrowed ref after ParseTuple if passed + if (!PyArg_ParseTuple(args, "ksIO!", &queryWrapperAddr, &index, &desc, &PyList_Type, &sortValuesList)) { return nullptr; } - Py_XINCREF(keysList); + Py_XINCREF(sortValuesList); - std::vector keys; - if (keysList != nullptr) { + reindexer::VariantArray sortValues; + if (sortValuesList != nullptr) { try { - keys = ParseListToVec(&keysList); + sortValues = ParseListToVec(&sortValuesList); } catch (const Error& err) { - Py_DECREF(keysList); + Py_DECREF(sortValuesList); return pyErr(err); } } - Py_XDECREF(keysList); + Py_XDECREF(sortValuesList); - getWrapper(queryWrapperAddr)->Sort(index, (desc != 0), keys); + getWrapper(queryWrapperAddr)->Sort(index, (desc != 0), sortValues); return pyErr(errOK); } @@ -978,32 +977,39 @@ static PyObject* DeleteQuery(PyObject* self, PyObject* args) { } namespace { -static PyObject* executeQuery(PyObject* self, PyObject* args, QueryWrapper::ExecuteType type) { +enum class ExecuteType { Select, Update }; +static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) { uintptr_t queryWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &queryWrapperAddr)) { return nullptr; } auto query = getWrapper(queryWrapperAddr); + auto qresult = std::make_unique(query->GetDB()); + Error err; + switch (type) { + case ExecuteType::Select: + err = query->SelectQuery(*qresult); + break; + case ExecuteType::Update: + err = query->UpdateQuery(*qresult); + break; + default: + return pyErr(reindexer::Error(ErrorCode::errLogic, "Unknown query execute mode")); + } - auto qresWrapper = std::make_unique(query->GetDB()); - auto err = query->ExecuteQuery(type, *qresWrapper); if (!err.ok()) { return Py_BuildValue("iskII", err.code(), err.what().c_str(), 0, 0, 0); } - auto count = qresWrapper->Count(); - auto totalCount = qresWrapper->TotalCount(); + const auto count = qresult->Count(); + const auto totalCount = qresult->TotalCount(); return Py_BuildValue("iskII", err.code(), err.what().c_str(), - reinterpret_cast(qresWrapper.release()), count, totalCount); + reinterpret_cast(qresult.release()), count, totalCount); } } // namespace -static PyObject* SelectQuery(PyObject* self, PyObject* args) { - return executeQuery(self, args, QueryWrapper::ExecuteType::Select); -} -static PyObject* UpdateQuery(PyObject* self, PyObject* args) { - return executeQuery(self, args, QueryWrapper::ExecuteType::Update); -} +static PyObject* SelectQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Select); } +static PyObject* UpdateQuery(PyObject* self, PyObject* args) { return executeQuery(self, args, ExecuteType::Update); } static PyObject* SetObject(PyObject* self, PyObject* args) { uintptr_t queryWrapperAddr = 0; @@ -1015,7 +1021,7 @@ static PyObject* SetObject(PyObject* self, PyObject* args) { Py_INCREF(valuesList); - std::vector values; + reindexer::h_vector values; if (valuesList != nullptr) { try { values = PyObjectToJson(&valuesList); @@ -1043,7 +1049,7 @@ static PyObject* Set(PyObject* self, PyObject* args) { Py_INCREF(valuesList); - std::vector values; + reindexer::VariantArray values; if (valuesList != nullptr) { try { values = ParseListToVec(&valuesList); @@ -1141,7 +1147,7 @@ static PyObject* SelectFilter(PyObject* self, PyObject* args) { Py_XINCREF(fieldsList); - std::vector fields; + reindexer::h_vector fields; if (fieldsList != nullptr) { try { fields = ParseStrListToStrVec(&fieldsList); @@ -1168,7 +1174,7 @@ static PyObject* AddFunctions(PyObject* self, PyObject* args) { Py_XINCREF(functionsList); - std::vector functions; + reindexer::h_vector functions; if (functionsList != nullptr) { try { functions = ParseStrListToStrVec(&functionsList); @@ -1195,7 +1201,7 @@ static PyObject* AddEqualPosition(PyObject* self, PyObject* args) { Py_XINCREF(equalPosesList); - std::vector equalPoses; + reindexer::h_vector equalPoses; if (equalPosesList != nullptr) { try { equalPoses = ParseStrListToStrVec(&equalPosesList); diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 3f5b1fa..04bd50b 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -7,7 +7,7 @@ namespace pyreindexer { namespace { - const int QRESULTS_FLAGS = kResultsJson | kResultsWithRank | kResultsWithJoined; + const int QRESULTS_FLAGS = kResultsJson | kResultsWithJoined; } template <> @@ -58,7 +58,7 @@ ReindexerInterface::~ReindexerInterface() { template Error ReindexerInterface::Select(const std::string& query, QueryResultsWrapper& result) { - return execute([this, query, &result] { + return execute([this, &query, &result] { typename DBT::QueryResultsT qres(QRESULTS_FLAGS); auto res = select(query, qres); result.Wrap(std::move(qres)); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index ed5afd2..96d7d4d 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -43,7 +43,7 @@ class GenericCommand : public ICommand { private: CallableT command_; - Error err_{errOK}; + Error err_; std::atomic_bool executed_{false}; }; diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 298121e..80f2646 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -524,14 +524,15 @@ def aggregate_facet(self, *fields: str) -> Query._AggregateFacet: return self._AggregateFacet(self) def sort(self, index: str, desc: bool = False, - keys: Union[simple_types, tuple[list[simple_types], ...]] = None) -> Query: - """Applies sort order to return from query items. If values argument specified, then items equal to values, - if found will be placed in the top positions. Forced sort is support for the first sorting field only + forced_sort_values: Union[simple_types, tuple[list[simple_types], ...]] = None) -> Query: + """Applies sort order to return from query items. If forced_sort_values argument specified, then items equal to + values, if found will be placed in the top positions. Forced sort is support for the first sorting field + only #### Arguments: index (string): The index name desc (bool): Sort in descending order - keys (union[simple_types, (list[simple_types], ...)]): + forced_sort_values (union[simple_types, (list[simple_types], ...)]): Value of index to match. For composite indexes keys must be list, with value of each sub-index #### Returns: @@ -542,9 +543,9 @@ def sort(self, index: str, desc: bool = False, """ - params = self.__convert_to_list(keys) + values = self.__convert_to_list(forced_sort_values) - self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, params) + self.err_code, self.err_msg = self.api.sort(self.query_wrapper_ptr, index, desc, values) self.__raise_on_error() return self From 1e4a49febfd0f8c13c3550a319e1e8759d26783a Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 9 Dec 2024 20:47:59 +0300 Subject: [PATCH 119/125] Review changes. Fix build --- pyreindexer/lib/include/query_wrapper.cc | 10 +++++----- pyreindexer/lib/include/query_wrapper.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 5d178b3..7c1161a 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -184,14 +184,14 @@ void serializeJoinQuery(JoinType type, reindexer::WrSerializer& data, reindexer: } } // namespace -void QueryWrapper::addJoinQueries(const reindexer::h_vector& queries, reindexer::WrSerializer& buffer) +void QueryWrapper::addJoinQueries(const reindexer::h_vector& queries, reindexer::WrSerializer& buffer) const { for (auto query : queries) { serializeJoinQuery(query->joinType_, query->ser_, buffer); } } -reindexer::Error QueryWrapper::prepareQuery(reindexer::Query& query) { +reindexer::Error QueryWrapper::CreateQuery(reindexer::Query& query) { reindexer::Error error = errOK; try { // current query (root) @@ -221,7 +221,7 @@ reindexer::Error QueryWrapper::prepareQuery(reindexer::Query& query) { reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { reindexer::Query query; - auto err = prepareQuery(query); + auto err = CreateQuery(query); if (!err.ok()) { return err; } @@ -235,7 +235,7 @@ reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { reindexer::Query query; - auto err = prepareQuery(query); + auto err = CreateQuery(query); if (!err.ok()) { return err; } @@ -244,7 +244,7 @@ reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { reindexer::Query query; - auto err = prepareQuery(query); + auto err = CreateQuery(query); if (!err.ok()) { return err; } diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 920757f..1e0163f 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -74,10 +74,10 @@ class QueryWrapper { void AddEqualPosition(const reindexer::h_vector& equalPositions); DBInterface* GetDB() const { return db_; } + reindexer::Error CreateQuery(reindexer::Query& query); private: - void addJoinQueries(const reindexer::h_vector& queries, reindexer::WrSerializer& buffer) const; - reindexer::Error prepareQuery(reindexer::Query& query); + void addJoinQueries(const reindexer::h_vector& queries, reindexer::WrSerializer& buffer) const; void putKeys(const reindexer::VariantArray& keys); DBInterface* db_{nullptr}; From efd1436c72d266f3eb8954ff556de1105c80c4b4 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 10 Dec 2024 18:14:58 +0300 Subject: [PATCH 120/125] Add query in transaction --- README.md | 42 ++++++++++++++++- pyreindexer/example/main.py | 45 ++++++++++++++----- pyreindexer/lib/include/query_wrapper.cc | 11 +++-- pyreindexer/lib/include/query_wrapper.h | 8 ++-- pyreindexer/lib/include/transaction_wrapper.h | 6 +++ pyreindexer/lib/src/rawpyreindexer.cc | 34 ++++++++++++-- pyreindexer/lib/src/rawpyreindexer.h | 4 ++ pyreindexer/lib/src/reindexerinterface.cc | 6 +++ pyreindexer/lib/src/reindexerinterface.h | 4 ++ pyreindexer/query.py | 9 ++-- pyreindexer/tests/helpers/api.py | 9 ++++ pyreindexer/tests/tests/test_query.py | 4 +- pyreindexer/tests/tests/test_transaction.py | 28 ++++++++++++ pyreindexer/transaction.py | 35 +++++++++++++++ 14 files changed, 219 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 7761757..8143b40 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,10 @@ * [Transaction](#pyreindexer.transaction.Transaction) * [insert](#pyreindexer.transaction.Transaction.insert) * [update](#pyreindexer.transaction.Transaction.update) + * [update\_query](#pyreindexer.transaction.Transaction.update_query) * [upsert](#pyreindexer.transaction.Transaction.upsert) * [delete](#pyreindexer.transaction.Transaction.delete) + * [delete\_query](#pyreindexer.transaction.Transaction.delete_query) * [commit](#pyreindexer.transaction.Transaction.commit) * [commit\_with\_count](#pyreindexer.transaction.Transaction.commit_with_count) * [rollback](#pyreindexer.transaction.Transaction.rollback) @@ -615,6 +617,25 @@ Updates an item with its precepts to the transaction TransactionError: Raises with an error message of API return if Transaction is over ApiError: Raises with an error message of API return on non-zero error code + + +### Transaction.update\_query + +```python +def update_query(query: Query) +``` + +Updates items with the transaction + Read-committed isolation is available for read operations. + Changes made in active transaction is invisible to current and another transactions. + +#### Arguments: + query (:obj:`Query`): A query object to modify + +#### Raises: + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code + ### Transaction.upsert @@ -650,6 +671,25 @@ Deletes an item from the transaction TransactionError: Raises with an error message of API return if Transaction is over ApiError: Raises with an error message of API return on non-zero error code + + +### Transaction.delete\_query + +```python +def delete_query(query: Query) +``` + +Deletes items with the transaction + Read-committed isolation is available for read operations. + Changes made in active transaction is invisible to current and another transactions. + +#### Arguments: + query (:obj:`Query`): A query object to modify + +#### Raises: + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code + ### Transaction.commit @@ -848,7 +888,7 @@ Adds where condition to DB query with UUID as string args. #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - uuids (*:obj:`UUID`): Value of index to be compared with. For composite indexes keys must be list, + uuids (*:obj:`UUID`): Value of index to be compared with. For composite indexes uuids must be list, with value of each sub-index #### Returns: diff --git a/pyreindexer/example/main.py b/pyreindexer/example/main.py index 07d3681..462de5b 100644 --- a/pyreindexer/example/main.py +++ b/pyreindexer/example/main.py @@ -57,8 +57,20 @@ def select_item_query_example(db, namespace): return db.select("SELECT * FROM " + namespace + " WHERE name='" + item_name_for_lookup + "'") +def select_all_item_query_example(db, namespace): + return db.select("SELECT * FROM " + namespace) + +def print_all_records_from_namespace(db, namespace, message): + selected_items_tr = select_all_item_query_example(db, namespace) + + res_count = selected_items_tr.count() + print(message, res_count) + + for item in selected_items_tr: + print('Item: ', item) def transaction_example(db, namespace, items_in_base): + # start transaction transaction = db.new_transaction(namespace) items_count = len(items_in_base) @@ -75,18 +87,10 @@ def transaction_example(db, namespace, items_in_base): # stop transaction and commit changes to namespace transaction.commit() - # print records from namespace - selected_items_tr = select_item_query_example(db, namespace) - - res_count = selected_items_tr.count() - print('Transaction results count: ', res_count) - - # disposable QueryResults iterator - for item in selected_items_tr: - print('Item: ', item) - + print_all_records_from_namespace(db, namespace, 'Transaction results count: ') def query_example(db, namespace): + # query all items any_items = (db.new_query(namespace) .where('value', CondType.CondAny) .sort('id') @@ -95,6 +99,7 @@ def query_example(db, namespace): for item in any_items: print('Item: ', item) + # query some items selected_items = (db.new_query(namespace) .where('value', CondType.CondEq, 'check') .sort('id') @@ -104,11 +109,13 @@ def query_example(db, namespace): for item in selected_items: print('Item: ', item) + # delete some items del_count = (db.new_query(namespace) .where('name', CondType.CondEq, 'item_1') .delete()) print(f'Deleted count: {del_count}') + # query all actual items any_items = (db.new_query(namespace) .where('value', CondType.CondAny) .must_execute()) @@ -116,6 +123,22 @@ def query_example(db, namespace): for item in any_items: print('Item: ', item) +def modify_query_transaction(db, namespace): + # start transaction + transaction = db.new_transaction(namespace) + + # create an update query and set it for the transaction + query_upd = db.new_query(namespace).where("id", CondType.CondGe, 4).set("name", ["update_with_query_tx"]) + transaction.update_query(query_upd) + + # create a delete query and set it for the transaction + query_del = db.new_query(namespace).where("id", CondType.CondLt, 3) + transaction.delete_query(query_del) + + # stop transaction and commit changes to namespace + count = transaction.commit() + + print_all_records_from_namespace(db, namespace, 'Transaction with Query results count: ') def rx_example(): db = RxConnector('builtin:///tmp/pyrx') @@ -149,6 +172,8 @@ def rx_example(): query_example(db, namespace) + modify_query_transaction(db, namespace) + db.close() diff --git a/pyreindexer/lib/include/query_wrapper.cc b/pyreindexer/lib/include/query_wrapper.cc index 7c1161a..58f7445 100644 --- a/pyreindexer/lib/include/query_wrapper.cc +++ b/pyreindexer/lib/include/query_wrapper.cc @@ -4,6 +4,7 @@ #include "core/query/query.h" #include "core/keyvalue/uuid.h" +#include "queryresults_wrapper.h" namespace pyreindexer { QueryWrapper::QueryWrapper(DBInterface* db, std::string_view ns) : db_{db} { @@ -219,7 +220,7 @@ reindexer::Error QueryWrapper::CreateQuery(reindexer::Query& query) { return error; } -reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { +reindexer::Error QueryWrapper::SelectQuery(std::unique_ptr& qr) { reindexer::Query query; auto err = CreateQuery(query); if (!err.ok()) { @@ -230,16 +231,18 @@ reindexer::Error QueryWrapper::SelectQuery(QueryResultsWrapper& qr) { return reindexer::Error(ErrorCode::errQueryExec, "WAL queries are not supported"); } - return db_->SelectQuery(query, qr); + qr = std::make_unique(db_); + return db_->SelectQuery(query, *qr); } -reindexer::Error QueryWrapper::UpdateQuery(QueryResultsWrapper& qr) { +reindexer::Error QueryWrapper::UpdateQuery(std::unique_ptr& qr) { reindexer::Query query; auto err = CreateQuery(query); if (!err.ok()) { return err; } - return db_->UpdateQuery(query, qr); + qr = std::make_unique(db_); + return db_->UpdateQuery(query, *qr); } reindexer::Error QueryWrapper::DeleteQuery(size_t& count) { diff --git a/pyreindexer/lib/include/query_wrapper.h b/pyreindexer/lib/include/query_wrapper.h index 1e0163f..b3a5efd 100644 --- a/pyreindexer/lib/include/query_wrapper.h +++ b/pyreindexer/lib/include/query_wrapper.h @@ -13,7 +13,6 @@ #endif #include "reindexerinterface.h" -#include "queryresults_wrapper.h" namespace pyreindexer { @@ -23,6 +22,8 @@ using DBInterface = ReindexerInterface; using DBInterface = ReindexerInterface; #endif +class QueryResultsWrapper; + class QueryWrapper { public: QueryWrapper(DBInterface* db, std::string_view ns); @@ -53,8 +54,8 @@ class QueryWrapper { void Modifier(QueryItemType type); - reindexer::Error SelectQuery(QueryResultsWrapper& qr); - reindexer::Error UpdateQuery(QueryResultsWrapper& qr); + reindexer::Error SelectQuery(std::unique_ptr& qr); + reindexer::Error UpdateQuery(std::unique_ptr& qr); reindexer::Error DeleteQuery(size_t& count); enum class IsExpression { Yes, No }; @@ -73,7 +74,6 @@ class QueryWrapper { void AddFunctions(const reindexer::h_vector& functions); void AddEqualPosition(const reindexer::h_vector& equalPositions); - DBInterface* GetDB() const { return db_; } reindexer::Error CreateQuery(reindexer::Query& query); private: diff --git a/pyreindexer/lib/include/transaction_wrapper.h b/pyreindexer/lib/include/transaction_wrapper.h index 0356a82..9ad5754 100644 --- a/pyreindexer/lib/include/transaction_wrapper.h +++ b/pyreindexer/lib/include/transaction_wrapper.h @@ -1,6 +1,7 @@ #pragma once #include "reindexerinterface.h" +#include "core/query/query.h" #ifdef PYREINDEXER_CPROTO #include "client/cororeindexer.h" @@ -46,6 +47,11 @@ class TransactionWrapper { return db_->Modify(*transaction_, std::move(item), mode); } + Error Modify(reindexer::Query&& query) { + assert(transaction_.has_value()); + return db_->Modify(*transaction_, std::move(query)); + } + Error Commit(size_t& count) { assert(transaction_.has_value()); return db_->CommitTransaction(*transaction_, count); diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index 5cde19e..e2e6dd7 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -586,6 +586,34 @@ static PyObject* UpdateTransaction(PyObject* self, PyObject* args) { return modi static PyObject* UpsertTransaction(PyObject* self, PyObject* args) { return modifyTransaction(self, args, ModeUpsert); } static PyObject* DeleteTransaction(PyObject* self, PyObject* args) { return modifyTransaction(self, args, ModeDelete); } +namespace { +PyObject* modifyQueryTransaction(PyObject* self, PyObject* args, QueryType type) { + uintptr_t transactionWrapperAddr = 0; + uintptr_t queryWrapperAddr = 0; + if (!PyArg_ParseTuple(args, "kk", &transactionWrapperAddr, &queryWrapperAddr)) { + return nullptr; + } + + auto query = getWrapper(queryWrapperAddr); + + reindexer::Query rxQuery; + auto err = query->CreateQuery(rxQuery); + if (!err.ok()) { + return pyErr(err); + } + rxQuery.type_ = type; + + err = getWrapper(transactionWrapperAddr)->Modify(std::move(rxQuery)); + return pyErr(err); +} +}; +static PyObject* UpdateQueryTransaction(PyObject* self, PyObject* args) { + return modifyQueryTransaction(self, args, QueryType::QueryUpdate); +} +static PyObject* DeleteQueryTransaction(PyObject* self, PyObject* args) { + return modifyQueryTransaction(self, args, QueryType::QueryDelete); +} + static PyObject* CommitTransaction(PyObject* self, PyObject* args) { uintptr_t transactionWrapperAddr = 0; if (!PyArg_ParseTuple(args, "k", &transactionWrapperAddr)) { @@ -985,14 +1013,14 @@ static PyObject* executeQuery(PyObject* self, PyObject* args, ExecuteType type) } auto query = getWrapper(queryWrapperAddr); - auto qresult = std::make_unique(query->GetDB()); + std::unique_ptr qresult; Error err; switch (type) { case ExecuteType::Select: - err = query->SelectQuery(*qresult); + err = query->SelectQuery(qresult); break; case ExecuteType::Update: - err = query->UpdateQuery(*qresult); + err = query->UpdateQuery(qresult); break; default: return pyErr(reindexer::Error(ErrorCode::errLogic, "Unknown query execute mode")); diff --git a/pyreindexer/lib/src/rawpyreindexer.h b/pyreindexer/lib/src/rawpyreindexer.h index 8346fbd..1ea5162 100644 --- a/pyreindexer/lib/src/rawpyreindexer.h +++ b/pyreindexer/lib/src/rawpyreindexer.h @@ -50,6 +50,8 @@ static PyObject* InsertTransaction(PyObject* self, PyObject* args); static PyObject* UpdateTransaction(PyObject* self, PyObject* args); static PyObject* UpsertTransaction(PyObject* self, PyObject* args); static PyObject* DeleteTransaction(PyObject* self, PyObject* args); +static PyObject* UpdateQueryTransaction(PyObject* self, PyObject* args); +static PyObject* DeleteQueryTransaction(PyObject* self, PyObject* args); static PyObject* CommitTransaction(PyObject* self, PyObject* args); static PyObject* RollbackTransaction(PyObject* self, PyObject* args); // query @@ -135,6 +137,8 @@ static PyMethodDef module_methods[] = { {"item_update_transaction", UpdateTransaction, METH_VARARGS, "item update transaction"}, {"item_upsert_transaction", UpsertTransaction, METH_VARARGS, "item upsert transaction"}, {"item_delete_transaction", DeleteTransaction, METH_VARARGS, "item delete transaction"}, + {"modify_transaction", UpdateQueryTransaction, METH_VARARGS, "update query transaction"}, + {"delete_transaction", DeleteQueryTransaction, METH_VARARGS, "delete query transaction"}, {"commit_transaction", CommitTransaction, METH_VARARGS, "apply changes. Free transaction object memory"}, {"rollback_transaction", RollbackTransaction, METH_VARARGS, "rollback changes. Free transaction object memory"}, // query diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index 04bd50b..43a2e6d 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -96,6 +96,12 @@ Error ReindexerInterface::modify(reindexer::cl return transaction.Modify(std::move(item), mode); } +template +Error ReindexerInterface::modify(typename DBT::TransactionT& transaction, reindexer::Query&& query) { + transaction.Modify(std::move(query)); + return errOK; +} + template Error ReindexerInterface::commitTransaction(typename DBT::TransactionT& transaction, size_t& count) { typename DBT::QueryResultsT qres(QRESULTS_FLAGS); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 96d7d4d..ea5d731 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -123,6 +123,9 @@ class ReindexerInterface { Error Modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode) { return execute([this, &tr, &item, mode] { return modify(tr, std::move(item), mode); }); } + Error Modify(typename DBT::TransactionT& tr, reindexer::Query&& query) { + return execute([this, &tr, &query] { return modify(tr, std::move(query)); }); + } Error CommitTransaction(typename DBT::TransactionT& tr, size_t& count) { return execute([this, &tr, &count] { return commitTransaction(tr, count); }); } @@ -163,6 +166,7 @@ class ReindexerInterface { typename DBT::TransactionT startTransaction(std::string_view ns) { return db_.NewTransaction({ns.data(), ns.size()}); } typename DBT::ItemT newItem(typename DBT::TransactionT& tr) { return tr.NewItem(); } Error modify(typename DBT::TransactionT& tr, typename DBT::ItemT&& item, ItemModifyMode mode); + Error modify(typename DBT::TransactionT& tr, reindexer::Query&& query); Error commitTransaction(typename DBT::TransactionT& transaction, size_t& count); Error rollbackTransaction(typename DBT::TransactionT& tr) { return db_.RollBackTransaction(tr); } Error selectQuery(const reindexer::Query& query, QueryResultsWrapper& result); diff --git a/pyreindexer/query.py b/pyreindexer/query.py index 80f2646..b3b041e 100644 --- a/pyreindexer/query.py +++ b/pyreindexer/query.py @@ -3,6 +3,7 @@ from collections.abc import Iterable from enum import Enum from typing import Optional, Union +from uuid import UUID from pyreindexer.exceptions import ApiError, QueryError from pyreindexer.query_results import QueryResults @@ -262,7 +263,7 @@ def where_composite(self, index: str, condition: CondType, keys: tuple[list[simp return self.__where(index, condition, keys) - def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: + def where_uuid(self, index: str, condition: CondType, *uuids: UUID) -> Query: """Adds where condition to DB query with UUID as string args. This function applies binary encoding to the UUID value. `index` MUST be declared as uuid index in this case @@ -270,7 +271,7 @@ def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: #### Arguments: index (string): Field name used in condition clause condition (:enum:`CondType`): Type of condition - keys (*string): Value of index to be compared with. For composite indexes keys must be list, + uuids (*:obj:`UUID`): Value of index to be compared with. For composite indexes uuids must be list, with value of each sub-index #### Returns: @@ -281,7 +282,9 @@ def where_uuid(self, index: str, condition: CondType, *keys: str) -> Query: """ - params: list = self.__convert_strs_to_list(keys) + params: list[str] = [] + for item in uuids: + params.append(str(item)) self.err_code, self.err_msg = self.api.where_uuid(self.query_wrapper_ptr, index, condition.value, params) self.__raise_on_error() diff --git a/pyreindexer/tests/helpers/api.py b/pyreindexer/tests/helpers/api.py index 26d0c77..8ba94ba 100644 --- a/pyreindexer/tests/helpers/api.py +++ b/pyreindexer/tests/helpers/api.py @@ -191,6 +191,15 @@ def delete_item(self, item): """ Delete item from transaction """ return self.tx.delete(item) + @api_method + def update_query(self, query): + """ Call update query in transaction """ + return self.tx.update_query(query) + @api_method + def delete_query(self, query): + """ Call delete query in transaction """ + return self.tx.delete_query(query) + @api_method def begin(self, ns_name) -> "_TransactionApi": """ Begin new transaction """ diff --git a/pyreindexer/tests/tests/test_query.py b/pyreindexer/tests/tests/test_query.py index 986557e..f4b44da 100644 --- a/pyreindexer/tests/tests/test_query.py +++ b/pyreindexer/tests/tests/test_query.py @@ -1,6 +1,7 @@ import copy import json import random +import uuid import pytest from hamcrest import * @@ -168,7 +169,8 @@ def test_query_select_where_uuid(self, db, namespace, index): query = db.query.new(namespace) # When ("Make select query with where_uuid") item = items[1] - query_result = list(query.where_uuid("uuid", CondType.CondEq, item["uuid"]).must_execute()) + some_uuid = uuid.UUID(item["uuid"]) + query_result = list(query.where_uuid("uuid", CondType.CondEq, some_uuid).must_execute()) # Then ("Check that selected item is in result") assert_that(query_result, equal_to([item]), "Wrong query results") diff --git a/pyreindexer/tests/tests/test_transaction.py b/pyreindexer/tests/tests/test_transaction.py index f5e7545..0adbf8b 100644 --- a/pyreindexer/tests/tests/test_transaction.py +++ b/pyreindexer/tests/tests/test_transaction.py @@ -5,6 +5,8 @@ from tests.helpers.transaction import * from tests.test_data.constants import item_definition +from pyreindexer.query import CondType + class TestCrudTransaction: def test_negative_commit_after_rollback(self, db, namespace): @@ -183,3 +185,29 @@ def test_rollback_transaction(self, db, namespace, index): select_result = get_ns_items(db, namespace) # Then ("Check that list of items in namespace is empty") assert_that(select_result, empty(), "Transaction: item list is not empty") + + def test_transaction_query_delete(self, db, namespace, index, items): + # Given("Create namespace with items") + # When ("Delete items with transaction") + transaction = db.tx.begin(namespace) + query = db.query.new(namespace) + query.where('id', CondType.CondGe, 0) + transaction.delete_query(query) + transaction.commit() + # Then ("Check that items is deleted") + select_result = get_ns_items(db, namespace) + assert_that(select_result, empty(), "Transaction: item wasn't deleted") + + def test_transaction_query_update(self, db, namespace, index, item): + # Given("Create namespace with item") + # When ("Update item") + item_definition_updated = {'id': 100, 'val': "new_value"} + transaction = db.tx.begin(namespace) + query = db.query.new(namespace) + query.where('id', CondType.CondEq, 100).set('val', ['new_value']) + transaction.update_query(query) + transaction.commit() + # Then ("Check that item is updated") + select_result = get_ns_items(db, namespace) + assert_that(select_result, has_length(1), "Transaction: item wasn't updated") + assert_that(select_result, has_item(item_definition_updated), "Transaction: item wasn't updated") diff --git a/pyreindexer/transaction.py b/pyreindexer/transaction.py index ad1bc48..f41c0ac 100644 --- a/pyreindexer/transaction.py +++ b/pyreindexer/transaction.py @@ -1,4 +1,5 @@ from pyreindexer.exceptions import ApiError, TransactionError +from pyreindexer.query import Query def raise_if_error(func): @@ -100,6 +101,23 @@ def update(self, item_def, precepts=None): precepts = [] if precepts is None else precepts self.err_code, self.err_msg = self.api.item_update_transaction(self.transaction_wrapper_ptr, item_def, precepts) + @raise_if_error + def update_query(self, query: Query): + """Updates items with the transaction + Read-committed isolation is available for read operations. + Changes made in active transaction is invisible to current and another transactions. + + #### Arguments: + query (:obj:`Query`): A query object to modify + + #### Raises: + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.modify_transaction(self.transaction_wrapper_ptr, query.query_wrapper_ptr) + @raise_if_error def upsert(self, item_def, precepts=None): """Updates an item with its precepts to the transaction. Creates the item if it not exists @@ -132,6 +150,23 @@ def delete(self, item_def): self.err_code, self.err_msg = self.api.item_delete_transaction(self.transaction_wrapper_ptr, item_def) + @raise_if_error + def delete_query(self, query: Query): + """Deletes items with the transaction + Read-committed isolation is available for read operations. + Changes made in active transaction is invisible to current and another transactions. + + #### Arguments: + query (:obj:`Query`): A query object to modify + + #### Raises: + TransactionError: Raises with an error message of API return if Transaction is over + ApiError: Raises with an error message of API return on non-zero error code + + """ + + self.err_code, self.err_msg = self.api.delete_transaction(self.transaction_wrapper_ptr, query.query_wrapper_ptr) + @raise_if_error def commit(self): """Applies changes From b15395b6b0827cbb52de2f181b61e62d374d882e Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Tue, 10 Dec 2024 18:16:14 +0300 Subject: [PATCH 121/125] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 402b923..faf8b60 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.2.45', + version='0.3', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', maintainer='Reindexer Team', From d6479e599beccb1a2d625a9b110d315d4f488962 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 12 Dec 2024 17:53:30 +0300 Subject: [PATCH 122/125] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index faf8b60..fff9416 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.3', + version='0.3.1', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', maintainer='Reindexer Team', From 985b5304dc2a8044c6cc4d5e48f04b90c0f949a2 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Thu, 12 Dec 2024 18:02:50 +0300 Subject: [PATCH 123/125] Correct README --- README.md | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/README.md b/README.md index 8143b40..7e7488d 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,8 @@ * [Transaction](#pyreindexer.transaction.Transaction) * [insert](#pyreindexer.transaction.Transaction.insert) * [update](#pyreindexer.transaction.Transaction.update) - * [update\_query](#pyreindexer.transaction.Transaction.update_query) * [upsert](#pyreindexer.transaction.Transaction.upsert) * [delete](#pyreindexer.transaction.Transaction.delete) - * [delete\_query](#pyreindexer.transaction.Transaction.delete_query) * [commit](#pyreindexer.transaction.Transaction.commit) * [commit\_with\_count](#pyreindexer.transaction.Transaction.commit_with_count) * [rollback](#pyreindexer.transaction.Transaction.rollback) @@ -617,25 +615,6 @@ Updates an item with its precepts to the transaction TransactionError: Raises with an error message of API return if Transaction is over ApiError: Raises with an error message of API return on non-zero error code - - -### Transaction.update\_query - -```python -def update_query(query: Query) -``` - -Updates items with the transaction - Read-committed isolation is available for read operations. - Changes made in active transaction is invisible to current and another transactions. - -#### Arguments: - query (:obj:`Query`): A query object to modify - -#### Raises: - TransactionError: Raises with an error message of API return if Transaction is over - ApiError: Raises with an error message of API return on non-zero error code - ### Transaction.upsert @@ -671,25 +650,6 @@ Deletes an item from the transaction TransactionError: Raises with an error message of API return if Transaction is over ApiError: Raises with an error message of API return on non-zero error code - - -### Transaction.delete\_query - -```python -def delete_query(query: Query) -``` - -Deletes items with the transaction - Read-committed isolation is available for read operations. - Changes made in active transaction is invisible to current and another transactions. - -#### Arguments: - query (:obj:`Query`): A query object to modify - -#### Raises: - TransactionError: Raises with an error message of API return if Transaction is over - ApiError: Raises with an error message of API return on non-zero error code - ### Transaction.commit From 70ef6a75b7d538b274338307e9439c3ef9cc3899 Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 16 Dec 2024 12:08:04 +0300 Subject: [PATCH 124/125] Review changes --- pyreindexer/lib/src/rawpyreindexer.cc | 14 +++++++------- pyreindexer/lib/src/reindexerinterface.cc | 17 +++++++++++++++-- pyreindexer/lib/src/reindexerinterface.h | 5 +++-- pyreindexer/rx_connector.py | 16 +++++++++------- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/pyreindexer/lib/src/rawpyreindexer.cc b/pyreindexer/lib/src/rawpyreindexer.cc index ca73712..f8d60e4 100644 --- a/pyreindexer/lib/src/rawpyreindexer.cc +++ b/pyreindexer/lib/src/rawpyreindexer.cc @@ -60,22 +60,22 @@ PyObject* queryResultsWrapperIterate(uintptr_t qresWrapperAddr) { static PyObject* Init(PyObject* self, PyObject* args) { ReindexerConfig cfg; char* clientName = nullptr; - int connectTimeout = 0; - int requestTimeout = 0; + int netTimeout = 0; unsigned enableCompression = 0; unsigned startSpecialThread = 0; + unsigned syncRxCoroCount = 0; unsigned maxReplUpdatesSize = 0; - if (!PyArg_ParseTuple(args, "iiiIIsIif", &cfg.fetchAmount, &connectTimeout, &requestTimeout, - &enableCompression, &startSpecialThread, &clientName, &maxReplUpdatesSize, - &cfg.allocatorCacheLimit, &cfg.allocatorCachePart)) { + if (!PyArg_ParseTuple(args, "iiiIIsIIif", &cfg.fetchAmount, &cfg.reconnectAttempts, &netTimeout, + &enableCompression, &startSpecialThread, &clientName, &syncRxCoroCount, + &maxReplUpdatesSize, &cfg.allocatorCacheLimit, &cfg.allocatorCachePart)) { return nullptr; } - cfg.connectTimeout = std::chrono::seconds(connectTimeout); - cfg.requestTimeout = std::chrono::seconds(requestTimeout); + cfg.netTimeout = std::chrono::milliseconds(netTimeout); cfg.enableCompression = (enableCompression != 0); cfg.requestDedicatedThread = (startSpecialThread != 0); cfg.appName = clientName; + cfg.syncRxCoroCount = syncRxCoroCount; cfg.maxReplUpdatesSize = maxReplUpdatesSize; uintptr_t rx = initReindexer(cfg); diff --git a/pyreindexer/lib/src/reindexerinterface.cc b/pyreindexer/lib/src/reindexerinterface.cc index c522c68..674d140 100644 --- a/pyreindexer/lib/src/reindexerinterface.cc +++ b/pyreindexer/lib/src/reindexerinterface.cc @@ -38,6 +38,20 @@ class GenericCommand : public ICommand { std::atomic_bool executed_{false}; }; +namespace { +reindexer::client::ReindexerConfig makeClientConfig(const ReindexerConfig& cfg) { + reindexer::client::ReindexerConfig config; + config.FetchAmount = cfg.fetchAmount; + config.ReconnectAttempts = cfg.reconnectAttempts; + // config.NetTimeout = cfg.netTimeout; // ToDo after migrate on v.4 + config.EnableCompression = cfg.enableCompression; + config.RequestDedicatedThread = cfg.requestDedicatedThread; + config.AppName = cfg.appName; + //config.SyncRxCoroCount = cfg.syncRxCoroCount; // ToDo after migrate on v.4 + return config; +} +} // namespace + template <> ReindexerInterface::ReindexerInterface(const ReindexerConfig& cfg) : db_(reindexer::ReindexerConfig().WithUpdatesSize(cfg.maxReplUpdatesSize) @@ -46,8 +60,7 @@ ReindexerInterface::ReindexerInterface(const ReindexerConf template <> ReindexerInterface::ReindexerInterface(const ReindexerConfig& cfg) - : db_(reindexer::client::ReindexerConfig(4, 1, cfg.fetchAmount, 0, cfg.connectTimeout, cfg.requestTimeout, - cfg.enableCompression, cfg.requestDedicatedThread, cfg.appName)) { + : db_(makeClientConfig(cfg)) { std::atomic_bool running{false}; executionThr_ = std::thread([this, &running] { cmdAsync_.set(loop_); diff --git a/pyreindexer/lib/src/reindexerinterface.h b/pyreindexer/lib/src/reindexerinterface.h index 1c0272d..7ac08d5 100644 --- a/pyreindexer/lib/src/reindexerinterface.h +++ b/pyreindexer/lib/src/reindexerinterface.h @@ -24,11 +24,12 @@ class ICommand; struct ReindexerConfig { int fetchAmount{1000}; - std::chrono::seconds connectTimeout{0}; - std::chrono::seconds requestTimeout{0}; + int reconnectAttempts{0}; + std::chrono::milliseconds netTimeout{0}; bool enableCompression{false}; bool requestDedicatedThread{false}; std::string appName; + unsigned int syncRxCoroCount{10}; size_t maxReplUpdatesSize{1024 * 1024 * 1024}; int64_t allocatorCacheLimit{-1}; diff --git a/pyreindexer/rx_connector.py b/pyreindexer/rx_connector.py index e9bce74..48b9c8c 100644 --- a/pyreindexer/rx_connector.py +++ b/pyreindexer/rx_connector.py @@ -23,11 +23,12 @@ class RxConnector(RaiserMixin): def __init__(self, dsn: str, *, # cproto options fetch_amount: int = 1000, - connect_timeout: int = 0, - request_timeout: int = 0, + reconnect_attempts: int = 0, + net_timeout: int = 0, enable_compression: bool = False, start_special_thread: bool = False, client_name: str = 'pyreindexer', + sync_rxcoro_count: int = 10, # builtin options max_replication_updates_size: int = 1024 * 1024 * 1024, allocator_cache_limit: int = -1, @@ -41,12 +42,13 @@ def __init__(self, dsn: str, *, cproto options: fetch_amount (int): The number of items that will be fetched by one operation - connect_timeout (int): Connection and database login timeout value [seconds] - request_timeout (int): Request execution timeout value [seconds] + reconnect_attempts (int): Number of reconnection attempts when connection is lost + net_timeout (int): Connection and database login timeout value [milliseconds] enable_compression (bool): Flag enable/disable traffic compression start_special_thread (bool): Determines whether to request a special thread of execution on the server for this connection client_name (string): Proper name of the application (as a client for Reindexer-server) + sync_rxcoro_count (int): Client concurrency per connection built-in options: max_replication_updates_size (int): Max pended replication updates size in bytes @@ -59,9 +61,9 @@ def __init__(self, dsn: str, *, self.err_msg = '' self.rx = 0 self._api_import(dsn) - self.rx = self.api.init(fetch_amount, connect_timeout, request_timeout, enable_compression, - start_special_thread, client_name, max_replication_updates_size, - allocator_cache_limit, allocator_cache_part) + self.rx = self.api.init(fetch_amount, reconnect_attempts, net_timeout, enable_compression, + start_special_thread, client_name, sync_rxcoro_count, + max_replication_updates_size, allocator_cache_limit, allocator_cache_part) self._api_connect(dsn) def __del__(self): From cc3783378e74572ed7efd86f93fadfda73069d3e Mon Sep 17 00:00:00 2001 From: "Alexander.A,Utkin" Date: Mon, 16 Dec 2024 16:51:49 +0300 Subject: [PATCH 125/125] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fff9416..ad91610 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def build_cmake(self, ext): setup(name=PACKAGE_NAME, - version='0.3.1', + version='0.3.2', description='A connector that allows to interact with Reindexer', author='Igor Tulmentyev', maintainer='Reindexer Team',