Index.upsert() now accepts List[dict]

Upsert will now accept vectors represented as dicts.
This provides a nice way to pass vectors with sparse values, rather than isntantiating `Vector()` objects

Also, this will allow us to represent entire datasets in the future as dataframes, where each vector is a dictionary (one row of the dataframe).

Author: igiloh-pinecone

pinecone/core/grpc/index_grpc.py

@@ -5,6 +5,7 @@
 from abc import ABC, abstractmethod
 from functools import wraps
 from typing import NamedTuple, Optional, Dict, Iterable, Union, List, Tuple, Any
+from collections.abc import Mapping
 
 import certifi
 import grpc
@@ -263,7 +264,7 @@ def stub_class(self):
         return VectorServiceStub
 
     def upsert(self,
-               vectors: Union[List[GRPCVector], List[Tuple]],
+               vectors: Union[List[GRPCVector], List[tuple], List[dict]],
                async_req: bool = False,
                namespace: Optional[str] = None,
                batch_size: Optional[int] = None,
@@ -274,18 +275,25 @@ def upsert(self,
         If a new value is upserted for an existing vector id, it will overwrite the previous value.
 
         Examples:
-            >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])],
-            >>>               namespace='ns1', async_req=True)
+            >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}),
+                              ('id2', [1.0, 2.0, 3.0])
+                              ],
+                              namespace='ns1', async_req=True)
+            >>> index.upsert([{'id': 'id1', 'values': [1.0, 2.0, 3.0], 'metadata': {'key': 'value'}},
+                              {'id': 'id2',
+                                        'values': [1.0, 2.0, 3.0],
+                                        'sprase_values': {'indices': [1, 8], 'values': [0.2, 0.4]},
+                              ])
             >>> index.upsert([GRPCVector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}),
-            >>>               GRPCVector(id='id2', values=[1.0, 2.0, 3.0]),
-            >>>               GRPCVector(id='id3',
-            >>>                          values=[1.0, 2.0, 3.0],
-            >>>                          sparse_values=GRPCSparseValues(indices=[1, 2], values=[0.2, 0.4]))])
+                              GRPCVector(id='id2', values=[1.0, 2.0, 3.0]),
+                              GRPCVector(id='id3',
+                                         values=[1.0, 2.0, 3.0],
+                                         sparse_values=GRPCSparseValues(indices=[1, 2], values=[0.2, 0.4]))])
 
         Args:
             vectors (Union[List[Vector], List[Tuple]]): A list of vectors to upsert.
 
-                     A vector can be represented by a 1) GRPCVector object or a 2) tuple.
+                     A vector can be represented by a 1) GRPCVector object, a 2) tuple or 3) a dictionary
                      1) if a tuple is used, it must be of the form (id, values, metadata) or (id, values).
                         where id is a string, vector is a list of floats, and metadata is a dict.
                         Examples: ('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])
@@ -299,6 +307,10 @@ def upsert(self,
                                             values=[1.0, 2.0, 3.0],
                                             sparse_values=GRPCSparseValues(indices=[1, 2], values=[0.2, 0.4]))
 
+                    3) if a dictionary is used, it must be in the form
+                       {'id': str, 'values': List[float], 'sparse_values': {'indices': List[int], 'values': List[float]},
+                        'metadata': dict}
+
                     Note: the dimension of each vector must match the dimension of the index.
             async_req (bool): If True, the upsert operation will be performed asynchronously.
                               Cannot be used with batch_size.
@@ -320,9 +332,20 @@ def upsert(self,
         def _vector_transform(item):
             if isinstance(item, GRPCVector):
                 return item
-            if isinstance(item, tuple):
+            elif isinstance(item, tuple):
+                if len(item) > 3:
+                    raise ValueError(f"Found a tuple of length {len(item)} which is not supported. " 
+                                     f"Vectors can be represented as tuples either the form (id, values, metadata) or (id, values). "
+                                     f"To pass sparse values please use either dicts or a GRPCVector objects as inputs.")
                 id, values, metadata = fix_tuple_length(item, 3)
                 return GRPCVector(id=id, values=values, metadata=dict_to_proto_struct(metadata) or {})
+            elif isinstance(item, Mapping):
+                sparse_values = None
+                if 'sparse_values' in item:
+                    indices = item['sparse_values'].get('indices', None)
+                    values = item['sparse_values'].get('values', None)
+                    sparse_values = GRPCSparseValues(indices=indices, values=values)
+                return GRPCVector(id=item['id'], values=item['values'], sparse_values=sparse_values, metadata=dict_to_proto_struct(item.get('metadata', None)))
             raise ValueError(f"Invalid vector value passed: cannot interpret type {type(item)}")
 
         timeout = kwargs.pop('timeout', None)

pinecone/index.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2020-2021 Pinecone Systems Inc. All right reserved.
 #
 from tqdm import tqdm
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
 from typing import Union, List, Tuple, Optional, Dict, Any
 
 from .core.client.model.sparse_values import SparseValues
@@ -63,7 +63,7 @@ def __init__(self, index_name: str, pool_threads=1):
 
     @validate_and_convert_errors
     def upsert(self,
-               vectors: Union[List[Vector], List[Tuple]],
+               vectors: Union[List[Vector], List[tuple], List[dict]],
                namespace: Optional[str] = None,
                batch_size: Optional[int] = None,
                show_progress: bool = True,
@@ -77,18 +77,25 @@ def upsert(self,
         To upsert in parallel follow: https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel
 
         Examples:
-            >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])])
+            >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}),
+                              ('id2', [1.0, 2.0, 3.0]),
+                              ])
+            >>> index.upsert([{'id': 'id1', 'values': [1.0, 2.0, 3.0], 'metadata': {'key': 'value'}},
+                              {'id': 'id2',
+                                        'values': [1.0, 2.0, 3.0],
+                                        'sprase_values': {'indices': [1, 8], 'values': [0.2, 0.4]},
+                              ])
             >>> index.upsert([Vector(id='id1',
-            >>>                      values=[1.0, 2.0, 3.0],
-            >>>                      metadata={'key': 'value'}),
-            >>>               Vector(id='id2',
-            >>>                      values=[1.0, 2.0, 3.0],
-            >>>                      sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))])
+                                     values=[1.0, 2.0, 3.0],
+                                     metadata={'key': 'value'}),
+                              Vector(id='id2',
+                                     values=[1.0, 2.0, 3.0],
+                                     sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))])
 
         Args:
             vectors (Union[List[Vector], List[Tuple]]): A list of vectors to upsert.
 
-                     A vector can be represented by a 1) Vector object or a 2) tuple.
+                     A vector can be represented by a 1) Vector object, a 2) tuple or 3) a dictionary
                      1) if a tuple is used, it must be of the form (id, values, meatadaa) or (id, values).
                         where id is a string, vector is a list of floats, metadata is a dict,
                         and sparse_values is a dict of the form {'indices': List[int], 'values': List[float]}.
@@ -110,6 +117,10 @@ def upsert(self,
 
                     Note: the dimension of each vector must match the dimension of the index.
 
+                3) if a dictionary is used, it must be in the form
+                   {'id': str, 'values': List[float], 'sparse_values': {'indices': List[int], 'values': List[float]},
+                    'metadata': dict}
+
             namespace (str): The namespace to write to. If not specified, the default namespace is used. [optional]
             batch_size (int): The number of vectors to upsert in each batch.
                                If not specified, all vectors will be upserted in a single batch. [optional]
@@ -154,9 +165,21 @@ def _upsert_batch(self,
         def _vector_transform(item: Union[Vector, Tuple]):
             if isinstance(item, Vector):
                 return item
-            if isinstance(item, tuple):
+            elif isinstance(item, tuple):
+                if len(item) > 3:
+                    raise ValueError(f"Found a tuple of length {len(item)} which is not supported. " 
+                                     f"Vectors can be represented as tuples either the form (id, values, metadata) or (id, values). "
+                                     f"To pass sparse values please use either dicts or a Vector objects as inputs.")
                 id, values, metadata = fix_tuple_length(item, 3)
                 return Vector(id=id, values=values, metadata=metadata or {}, _check_type=_check_type)
+            elif isinstance(item, Mapping):
+                sparse_values = None
+                if 'sparse_values' in item:
+                    indices = item['sparse_values'].get('indices', [])
+                    values = item['sparse_values'].get('values', [])
+                    sparse_values = SparseValues(indices=indices, values=values)
+                return Vector(id=item['id'], values=item['values'], sparse_values=sparse_values, metadata=item.get('metadata', {}), _check_type=_check_type)
+
             raise ValueError(f"Invalid vector value passed: cannot interpret type {type(item)}")
 
         return self._vector_api.upsert(