[feat][sdk] Support sdk documnet search all

include/dingosdk/document.h

@@ -95,7 +95,7 @@ class DocValue {
   double DoubleValue() const;
   std::string StringValue() const;
   std::string BytesValue() const;
-  bool BoolValue()const;
+  bool BoolValue() const;
   std::string DatetimeValue() const;
 
   std::string ToString() const;
@@ -157,6 +157,8 @@ struct DocSearchParam {
   std::vector<std::string> column_names;
   bool with_scalar_data{false};
   std::vector<std::string> selected_keys;
+  //search all need ,The maximum number of results to return.
+  int32_t query_limited{40960};
 };
 
 struct DocWithStore {
@@ -229,6 +231,10 @@ class DocumentClient {
   Status SearchByIndexName(int64_t schema_id, const std::string& index_name, const DocSearchParam& search_param,
                            DocSearchResult& out_result);
 
+  Status SearchAllByIndexId(int64_t index_id, const DocSearchParam& search_param, DocSearchResult& out_result);
+  Status SearchAllByIndexName(int64_t schema_id, const std::string& index_name, const DocSearchParam& search_param,
+                           DocSearchResult& out_result);
+
   Status DeleteByIndexId(int64_t index_id, const std::vector<int64_t>& doc_ids,
                          std::vector<DocDeleteResult>& out_result);
   Status DeleteByIndexName(int64_t schema_id, const std::string& index_name, const std::vector<int64_t>& doc_ids,

python/src/document_bindings.cc

@@ -57,14 +57,14 @@ void DefineDocumentBindings(pybind11::module& m) {
       .def_static("FromDouble", &DocValue::FromDouble)
       .def_static("FromString", &DocValue::FromString)
       .def_static("FromBytes", &DocValue::FromBytes)
-      .def_static("FromBool",&DocValue::FromBool)
+      .def_static("FromBool", &DocValue::FromBool)
       .def_static("FromDatetime", &DocValue::FromDatetime)
       .def("GetType", &DocValue::GetType)
       .def("IntValue", &DocValue::IntValue)
       .def("DoubleValue", &DocValue::DoubleValue)
       .def("StringValue", &DocValue::StringValue)
       .def("BytesValue", &DocValue::BytesValue)
-      .def("BoolValue",&DocValue::BoolValue)
+      .def("BoolValue", &DocValue::BoolValue)
       .def("DatetimeValue", &DocValue::DatetimeValue)
       .def("ToString", &DocValue::ToString);
 
@@ -101,7 +101,8 @@ void DefineDocumentBindings(pybind11::module& m) {
       .def_readwrite("doc_ids", &DocSearchParam::doc_ids)
       .def_readwrite("column_names", &DocSearchParam::column_names)
       .def_readwrite("with_scalar_data", &DocSearchParam::with_scalar_data)
-      .def_readwrite("selected_keys", &DocSearchParam::selected_keys);
+      .def_readwrite("selected_keys", &DocSearchParam::selected_keys)
+      .def_readwrite("query_limited", &DocSearchParam::query_limited);
 
   py::class_<DocWithStore>(m, "DocWithStore")
       .def(py::init<>())
@@ -180,6 +181,19 @@ void DefineDocumentBindings(pybind11::module& m) {
              Status status = documentclient.SearchByIndexName(schema_id, index_name, search_param, out_result);
              return std::make_tuple(status, out_result);
            })
+      .def("SearchAllByIndexId",
+           [](DocumentClient& documentclient, int64_t index_id, const DocSearchParam& search_param) {
+             DocSearchResult out_result;
+             Status status = documentclient.SearchAllByIndexId(index_id, search_param, out_result);
+             return std::make_tuple(status, out_result);
+           })
+      .def("SearchAllByIndexName",
+           [](DocumentClient& documentclient, int64_t schema_id, const std::string& index_name,
+              const DocSearchParam& search_param) {
+             DocSearchResult out_result;
+             Status status = documentclient.SearchAllByIndexName(schema_id, index_name, search_param, out_result);
+             return std::make_tuple(status, out_result);
+           })
       .def("DeleteByIndexId",
            [](DocumentClient& documentclient, int64_t index_id, const std::vector<int64_t>& doc_ids) {
              std::vector<DocDeleteResult> out_result;

setup.py

@@ -139,7 +139,7 @@ def build_extension(self, ext: CMakeExtension) -> None:
 """
 setup(
     name="dingosdk",
-    version="0.2rc1.post3",
+    version="0.2rc1.post4",
     author="DingoDB",
     author_email="[email protected]",
     description="dingo-store python sdk",

src/example/sdk_document_example.cc

@@ -340,6 +340,149 @@ static void DocumentSearch(bool use_index_name = false) {
   }
 }
 
+static void DocumentSearchAll(bool use_index_name = false) {
+  {
+    dingodb::sdk::DocSearchParam param;
+    param.with_scalar_data = true;
+    param.query_string = "discover";
+    param.use_id_filter = false;
+
+    Status tmp;
+    DocSearchResult result;
+    if (use_index_name) {
+      tmp = g_doc_client->SearchAllByIndexName(g_schema_id, g_index_name, param, result);
+    } else {
+      tmp = g_doc_client->SearchAllByIndexId(g_index_id, param, result);
+    }
+
+    DINGO_LOG(INFO) << "vector search discover status: " << tmp.ToString();
+    DINGO_LOG(INFO) << "vector search discover result:" << result.ToString();
+    if (tmp.ok()) {
+      CHECK_EQ(result.doc_sores.size(), 1);
+    }
+  }
+
+  {
+    dingodb::sdk::DocSearchParam param;
+    param.with_scalar_data = true;
+    param.query_string = "of";
+    param.use_id_filter = false;
+
+    Status tmp;
+    DocSearchResult result;
+    if (use_index_name) {
+      tmp = g_doc_client->SearchAllByIndexName(g_schema_id, g_index_name, param, result);
+    } else {
+      tmp = g_doc_client->SearchAllByIndexId(g_index_id, param, result);
+    }
+
+    DINGO_LOG(INFO) << "vector search of with limit 10 status: " << tmp.ToString();
+    DINGO_LOG(INFO) << "vector search of with limit 10 result:" << result.ToString();
+    if (tmp.ok()) {
+      CHECK_EQ(result.doc_sores.size(), 5);
+    }
+  }
+
+  {
+    dingodb::sdk::DocSearchParam param;
+    param.with_scalar_data = true;
+    param.query_string = "of";
+    param.use_id_filter = true;
+    param.query_limited = 4096;
+    param.doc_ids = {
+        13,
+        15,
+    };
+
+    Status tmp;
+    DocSearchResult result;
+    if (use_index_name) {
+      tmp = g_doc_client->SearchAllByIndexName(g_schema_id, g_index_name, param, result);
+    } else {
+      tmp = g_doc_client->SearchAllByIndexId(g_index_id, param, result);
+    }
+
+    DINGO_LOG(INFO) << "vector search of with id filter status: " << tmp.ToString();
+    DINGO_LOG(INFO) << "vector search of with id filter result:" << result.ToString();
+    if (tmp.ok()) {
+      CHECK_EQ(result.doc_sores.size(), 2);
+    }
+  }
+
+  {
+    dingodb::sdk::DocSearchParam param;
+    param.with_scalar_data = true;
+    param.query_string = R"(text:"of" AND i64: >= 1013)";
+    param.use_id_filter = true;
+    param.query_limited = 4096;
+    param.doc_ids = {
+        9,
+        11,
+        13,
+        15,
+    };
+
+    Status tmp;
+    DocSearchResult result;
+    if (use_index_name) {
+      tmp = g_doc_client->SearchAllByIndexName(g_schema_id, g_index_name, param, result);
+    } else {
+      tmp = g_doc_client->SearchAllByIndexId(g_index_id, param, result);
+    }
+
+    DINGO_LOG(INFO) << "vector search of with id filter status: " << tmp.ToString();
+    DINGO_LOG(INFO) << "vector search of with id filter result:" << result.ToString();
+    if (tmp.ok()) {
+      CHECK_EQ(result.doc_sores.size(), 2);
+    }
+  }
+
+  {
+    dingodb::sdk::DocSearchParam param;
+    param.with_scalar_data = true;
+    param.query_string = R"( bool:true)";
+    param.use_id_filter = true;
+    param.doc_ids = {15, 17, 19, 21};
+    param.query_limited = 4096;
+
+    Status tmp;
+    DocSearchResult result;
+    if (use_index_name) {
+      tmp = g_doc_client->SearchAllByIndexName(g_schema_id, g_index_name, param, result);
+    } else {
+      tmp = g_doc_client->SearchAllByIndexId(g_index_id, param, result);
+    }
+
+    DINGO_LOG(INFO) << "vector search of with id filter status: " << tmp.ToString();
+    DINGO_LOG(INFO) << "vector search of with id filter result:" << result.ToString();
+    if (tmp.ok()) {
+      CHECK_EQ(result.doc_sores.size(), 2);
+    }
+  }
+
+  {
+    dingodb::sdk::DocSearchParam param;
+    param.with_scalar_data = true;
+    param.query_string = fmt::format("{}:\"{}\"", "datetime", time_1);
+    param.use_id_filter = true;
+    param.doc_ids = {9, 11, 13, 15};
+
+    Status tmp;
+    DocSearchResult result;
+    if (use_index_name) {
+      tmp = g_doc_client->SearchAllByIndexName(g_schema_id, g_index_name, param, result);
+    } else {
+      tmp = g_doc_client->SearchAllByIndexId(g_index_id, param, result);
+    }
+
+    DINGO_LOG(INFO) << "vector search of with id filter status: " << tmp.ToString();
+    DINGO_LOG(INFO) << "vector search of with id filter result:" << result.ToString();
+    if (tmp.ok()) {
+      CHECK_EQ(result.doc_sores.size(), 4);
+    }
+  }
+}
+
 static void DocumentQuey(bool use_index_name = false) {
   dingodb::sdk::DocQueryParam param;
   param.doc_ids = g_doc_ids;
@@ -578,6 +721,7 @@ int main(int argc, char* argv[]) {
 
     DocumentAdd();
     DocumentSearch();
+    DocumentSearchAll();
     DocumentQuey();
     DocumentGetBorder();
     DocumentScanQuery();
@@ -594,6 +738,7 @@ int main(int argc, char* argv[]) {
 
     DocumentAdd(true);
     DocumentSearch(true);
+    DocumentSearchAll(true);
     DocumentQuey(true);
     DocumentGetBorder(true);
     DocumentScanQuery(true);

src/sdk/CMakeLists.txt

@@ -84,6 +84,7 @@ set(SDK_SRCS
   document/document_get_index_metrics_task.cc
   document/document_scan_query_task.cc
   document/document_search_task.cc
+  document/document_search_all_task.cc
   document/document_update_task.cc
   document/document_get_auto_increment_id_task.cc
   document/document_update_auto_increment_task.cc

src/sdk/document/document_client.cc

@@ -25,6 +25,7 @@
 #include "sdk/document/document_get_index_metrics_task.h"
 #include "sdk/document/document_index_cache.h"
 #include "sdk/document/document_scan_query_task.h"
+#include "sdk/document/document_search_all_task.h"
 #include "sdk/document/document_search_task.h"
 #include "sdk/document/document_update_auto_increment_task.h"
 #include "sdk/document/document_update_task.h"
@@ -82,6 +83,22 @@ Status DocumentClient::SearchByIndexName(int64_t schema_id, const std::string& i
   return task.Run();
 }
 
+Status DocumentClient::SearchAllByIndexId(int64_t index_id, const DocSearchParam& search_param,
+                                       DocSearchResult& out_result) {
+  DocumentSearchAllTask task(stub_, index_id, search_param, out_result);
+  return task.Run();
+}
+
+Status DocumentClient::SearchAllByIndexName(int64_t schema_id, const std::string& index_name,
+                                         const DocSearchParam& search_param, DocSearchResult& out_result) {
+  int64_t index_id{0};
+  DINGO_RETURN_NOT_OK(
+      stub_.GetDocumentIndexCache()->GetIndexIdByKey(EncodeDocumentIndexCacheKey(schema_id, index_name), index_id));
+  CHECK_GT(index_id, 0);
+  DocumentSearchAllTask task(stub_, index_id, search_param, out_result);
+  return task.Run();
+}
+
 Status DocumentClient::DeleteByIndexId(int64_t index_id, const std::vector<int64_t>& doc_ids,
                                        std::vector<DocDeleteResult>& out_result) {
   DocumentDeleteTask task(stub_, index_id, doc_ids, out_result);