Merge pull request #80 from zilliztech/support-milvus-2.4

feat: support muti vectors, new vector types, index types

README.md

@@ -30,8 +30,8 @@ Python >= 3.8.5
 
 #### Install from PyPI (Recommended)
 
-Run `pip install pymilvus==2.3.4`
-Run `pip install milvus-cli==0.4.2`
+Run `pip install pymilvus==2.4.3`
+Run `pip install milvus-cli==0.4.3`
 
 #### Install from a tarball
 

milvus_cli/Collection.py

@@ -38,7 +38,12 @@ def create_collection(
         for field in fields:
             [fieldName, fieldType, *restData] = field.split(":")
             upperFieldType = fieldType.upper()
-            if upperFieldType in ["BINARY_VECTOR", "FLOAT_VECTOR"]:
+            if upperFieldType in [
+                "BINARY_VECTOR",
+                "FLOAT_VECTOR",
+                "BFLOAT16_VECTOR",
+                "FLOAT16_VECTOR",
+            ]:
                 fieldList.append(
                     FieldSchema(
                         name=fieldName,

milvus_cli/Connection.py

@@ -20,7 +20,6 @@ def showConnection(self, showAll=False):
         tempAlias = self.alias
         try:
             allConnections = connections.list_connections()
-
             if showAll:
                 return allConnections
 

milvus_cli/Index.py

@@ -86,14 +86,14 @@ def get_index_build_progress(
             indexName,
         )
 
-    def list_indexes(
-        self,
-        collectionName,
-    ):
+    def list_indexes(self, collectionName, onlyData=False):
         target = getTargetCollection(
             collectionName,
         )
         result = target.indexes
+
+        if onlyData:
+            return result
         rows = list(
             map(
                 lambda x: [

milvus_cli/Types.py

@@ -32,6 +32,9 @@ def __str__(self):
     "VARCHAR",
     "BINARY_VECTOR",
     "FLOAT_VECTOR",
+    "FLOAT16_VECTOR",
+    "BFLOAT16_VECTOR",
+    "SPARSE_FLOAT_VECTOR",
     "JSON",
     "ARRAY",
 ]
@@ -48,9 +51,12 @@ def __str__(self):
     "DISKANN",
     "GPU_IVF_FLAT",
     "GPU_IVF_PQ",
+    "SPARSE_INVERTED_INDEX",
+    "SPARSE_WAND",
     "SCANN",
     "STL_SORT",
     "Trie",
+    "INVERTED",
     "",
 ]
 
@@ -118,12 +124,21 @@ def __str__(self):
         "index_building_parameters": ["nlist", "with_raw_data"],
         "search_parameters": ["nprobe", "reorder_k"],
     },
+    "SPARSE_INVERTED_INDEX": {
+        "index_building_parameters": ["drop_ratio_build"],
+        "search_parameters": ["drop_ratio_search"],
+    },
+    "SPARSE_WAND": {
+        "index_building_parameters": ["drop_ratio_build"],
+        "search_parameters": ["drop_ratio_search"],
+    },
 }
 
 DupSearchParams = reduce(
     lambda x, y: x + IndexTypesMap[y]["search_parameters"], IndexTypesMap.keys(), []
 )
 SearchParams = list(dict.fromkeys(DupSearchParams))
+SearchParams.append("group_by_field")
 
 MetricTypes = [
     "L2",
@@ -149,6 +164,9 @@ def __str__(self):
     23: "JSON",
     100: "BINARY_VECTOR",
     101: "FLOAT_VECTOR",
+    102: "FLOAT16_VECTOR",
+    103: "BFLOAT16_VECTOR",
+    104: "SPARSE_FLOAT_VECTOR",
     999: "UNKNOWN",
 }
 

milvus_cli/Validation.py

@@ -40,11 +40,18 @@ def validateCollectionParameter(collectionName, primaryField, fields):
                     str(FiledDataTypes)
                 )
             )
-        if upperFieldType in ["BINARY_VECTOR", "FLOAT_VECTOR"]:
+        if upperFieldType in [
+            "BINARY_VECTOR",
+            "FLOAT_VECTOR",
+            "FLOAT16_VECTOR",
+            "BFLOAT16_VECTOR",
+        ]:
             try:
                 int(restData[0])
             except ValueError as e:
-                raise ParameterException("""Vector's dim should be int.""")
+                raise ParameterException(
+                    """{} Vector's dim should be int.""".format(upperFieldType)
+                )
     # Dedup field name.
     newNames = list(set(fieldNames))
     if not (len(newNames) == len(fieldNames)):
@@ -152,7 +159,8 @@ def validateSearchParams(
                     )
                 )
             try:
-                paramDict[paramName] = int(paramValue)
+                if paramName != "group_by_field":
+                    paramDict[paramName] = int(paramValue)
             except ValueError as e:
                 raise ParameterException("""Search parameter's value should be int.""")
         result["param"] = {"metric_type": metricType}

milvus_cli/scripts/connection_cli.py

@@ -36,10 +36,10 @@ def connect(obj, uri, token):
         click.echo(message=e, err=True)
     else:
         click.echo("Connect Milvus successfully.")
-        address, username = obj.connection.showConnection()
+        connectionInfo = obj.connection.showConnection()
         click.echo(
             tabulate(
-                [["Address", address], ["User", username], ["Alias", "default"]],
+                [["Address", list(connectionInfo)[0]], ["Alias", "default"]],
                 tablefmt="pretty",
             )
         )

milvus_cli/scripts/data_cli.py

@@ -210,8 +210,6 @@ def search(obj):
 
         The vector field used to search of collection (vector): vector
 
-        Metric type: L2
-
         Search parameter nprobe's value: 10
 
         The max number of returned record, also known as topk: 2
@@ -221,7 +219,6 @@ def search(obj):
     Example-2(collection has index):
 
         Collection name (car, test_collection): car
-
         \b
         The vectors of search data (the length of data is number of query (nq),
         the dim of every vector in data must be equal to vector field’s of
@@ -241,10 +238,10 @@ def search(obj):
 
         The vector field used to search of collection (vector): vector
 
-        Metric type: L2
-
         Search parameter nprobe's value: 10
 
+        Groups search results by a specified field to ensure diversity and avoid returning multiple results from the same group.: color
+
         The specified number of decimal places of returned distance [-1]: 5
 
         The max number of returned record, also known as topk: 2
@@ -264,12 +261,16 @@ def search(obj):
         "The vector field used to search of collection",
         type=click.Choice(obj.collection.list_field_names(collectionName)),
     )
-    indexDetails = obj.index.get_vector_index(collectionName)
+    indexes = obj.index.list_indexes(collectionName, onlyData=True)
+    for index in indexes:
+        if index.field_name == annsField:
+            indexDetails = index
+            break
     hasIndex = not not indexDetails
     if indexDetails:
-        index_type = indexDetails["index_type"]
+        index_type = indexDetails._index_params["index_type"]
         search_parameters = IndexTypesMap[index_type]["search_parameters"]
-        metric_type = indexDetails["metric_type"]
+        metric_type = indexDetails._index_params["metric_type"]
         click.echo(f"Metric type: {metric_type}")
         metricType = metric_type
         params = []
@@ -279,6 +280,14 @@ def search(obj):
     else:
         metricType = ""
         params = []
+
+    groupByField = click.prompt(
+        "Groups search results by a specified field to ensure diversity and avoid returning multiple results from the same group.",
+        default=None,
+        type=str,
+    )
+    if groupByField != None:
+        params += [f"group_by_field:{groupByField}"]
     roundDecimal = click.prompt(
         "The specified number of decimal places of returned distance",
         default=-1,
@@ -319,6 +328,7 @@ def search(obj):
             hasIndex=hasIndex,
             guarantee_timestamp=guarantee_timestamp,
         )
+
     except ParameterException as pe:
         click.echo("Error!\n{}".format(str(pe)))
 

milvus_cli/scripts/index_cli.py

@@ -51,7 +51,7 @@ def createIndex(obj):
         )
         params = []
 
-        ignoreIndexType = ["", "AUTOINDEX", "Trie", "STL_SORT"]
+        ignoreIndexType = ["", "AUTOINDEX", "Trie", "STL_SORT", "INVERTED"]
         if indexType not in ignoreIndexType:
             index_building_parameters = IndexTypesMap[indexType][
                 "index_building_parameters"

milvus_cli/utils.py

@@ -7,14 +7,12 @@
 
 
 def getPackageVersion():
-    import pkg_resources  # part of setuptools
+    import pkg_resources
 
     try:
         version = pkg_resources.require("milvus_cli")[0].version
     except Exception as e:
-        raise ParameterException(
-            "Could not get version under single executable file mode."
-        )
+        raise ParameterException(e)
     return version
 
 

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="milvus_cli",
-    version="v0.4.2",
+    version="v0.4.3",
     author="Milvus Team",
     author_email="[email protected]",
     url="https://github.com/zilliztech/milvus_cli",
@@ -17,7 +17,7 @@
     include_package_data=True,
     install_requires=[
         "Click==8.0.1",
-        "pymilvus==2.3.4",
+        "pymilvus==2.4.3",
         "tabulate==0.8.9",
         "requests==2.31.0",
     ],