From 427aa601ba88c38fca8e58f1539026fd7d6002a8 Mon Sep 17 00:00:00 2001 From: Anush Date: Thu, 12 Sep 2024 12:17:32 +0530 Subject: [PATCH] chore: Move tools/ and proto/ to internal/ (#55) --- CONTRIBUTING.md | 12 +- .../tools}/generate_proto_go.sh | 4 +- {tools => internal/tools}/sync_proto.sh | 6 +- proto/collections.proto | 654 ------------ proto/collections_service.proto | 60 -- proto/json_with_int.proto | 61 -- proto/points.proto | 965 ------------------ proto/points_service.proto | 123 --- proto/qdrant.proto | 19 - proto/snapshots_service.proto | 74 -- 10 files changed, 11 insertions(+), 1967 deletions(-) rename {tools => internal/tools}/generate_proto_go.sh (94%) rename {tools => internal/tools}/sync_proto.sh (90%) delete mode 100644 proto/collections.proto delete mode 100644 proto/collections_service.proto delete mode 100644 proto/json_with_int.proto delete mode 100644 proto/points.proto delete mode 100644 proto/points_service.proto delete mode 100644 proto/qdrant.proto delete mode 100644 proto/snapshots_service.proto diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0dbf61c..377ad1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,22 +63,22 @@ The client uses generated stubs from upstream Qdrant proto definitions, which ar 1. Download and generate the latest client stubs by running the following command from the project root: ```bash -BRANCH=dev sh tools/sync_proto.sh +BRANCH=dev sh internal/tools/sync_proto.sh ``` -2. Update the test image value in [`qdrant_test/image_test.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant_test/image_test.go) to `qdrant/qdrant:dev`. +2. Update the test image value in [`qdrant_test/image_test.go`](https://github.com/qdrant/go-client/blob/master/qdrant_test/image_test.go) to `qdrant/qdrant:dev`. 3. Remove the gRPC server definitions from the auto-generated code. There is currently [no way](https://github.com/golang/protobuf/issues/373) to skip generating Go server definitions. -You’ll need to manually delete them from [`snapshots_service_grpc.pb.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/snapshots_service_grpc.pb.go), [`points_service_grpc.pb.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/points_service.pb.go), and [`collections_service_grpc.pb.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/collections_service_grpc.pb.go). +You’ll need to manually delete them from [`snapshots_service_grpc.pb.go`](https://github.com/qdrant/go-client/blob/master/qdrant/snapshots_service_grpc.pb.go), [`points_service_grpc.pb.go`](https://github.com/qdrant/go-client/blob/master/qdrant/points_service.pb.go), and [`collections_service_grpc.pb.go`](https://github.com/qdrant/go-client/blob/master/qdrant/collections_service_grpc.pb.go). Remove lines starting from comments like `// CollectionsServer is the server API for Collections service.` until the end of the file. [Here’s an example commit](https://github.com/qdrant/go-client/commit/6d04e31bb2acccf54f964a634df8930533642892). -4. Implement new Qdrant methods in [`points.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/points.go), [`collections.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/collections.go), or [`qdrant.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/qdrant.go) as needed. +4. Implement new Qdrant methods in [`points.go`](https://github.com/qdrant/go-client/blob/master/qdrant/points.go), [`collections.go`](https://github.com/qdrant/go-client/blob/master/qdrant/collections.go), or [`qdrant.go`](https://github.com/qdrant/go-client/blob/master/qdrant/qdrant.go) as needed. -5. If there are any new `oneOf` properties in the proto definitions, add helper constructors in [`oneof_factory.go`](https://github.com/qdrant/go-client/blob/new-client/qdrant/oneof_factory.go) following the existing patterns. +5. If there are any new `oneOf` properties in the proto definitions, add helper constructors in [`oneof_factory.go`](https://github.com/qdrant/go-client/blob/master/qdrant/oneof_factory.go) following the existing patterns. 6. Submit your pull request and get those approvals. @@ -89,7 +89,7 @@ Once the new Qdrant version is live: 1. Run the following command: ```bash -BRANCH=master sh tools/sync_proto.sh +BRANCH=master sh internal/tools/sync_proto.sh ``` 2. Update the test image value in `qdrant_test/image_test.go` to `qdrant/qdrant:NEW_VERSION`. diff --git a/tools/generate_proto_go.sh b/internal/tools/generate_proto_go.sh similarity index 94% rename from tools/generate_proto_go.sh rename to internal/tools/generate_proto_go.sh index 0d6ae89..bf618d2 100755 --- a/tools/generate_proto_go.sh +++ b/internal/tools/generate_proto_go.sh @@ -4,12 +4,12 @@ PROJECT_ROOT="$(pwd)/$(dirname "$0")/../" QDRANT_PROTO_DIR='proto' -go install google.golang.org/protobuf/cmd/protoc-gen-go@1.34.2 +go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 export PATH="$PATH:$(go env GOPATH)/bin" -PROTO_DIR=./proto +PROTO_DIR=./internal/proto OUT_DIR=./qdrant PACKAGE_NAME="github.com/qdrant/go-client;qdrant" diff --git a/tools/sync_proto.sh b/internal/tools/sync_proto.sh similarity index 90% rename from tools/sync_proto.sh rename to internal/tools/sync_proto.sh index 18a545f..5e2a832 100755 --- a/tools/sync_proto.sh +++ b/internal/tools/sync_proto.sh @@ -3,7 +3,7 @@ set -e BRANCH=${BRANCH:-"master"} -PROJECT_ROOT="$(pwd)/$(dirname "$0")/../" +PROJECT_ROOT="$(pwd)/$(dirname "$0")/../../" cd $(mktemp -d) @@ -16,7 +16,7 @@ PROTO_DIR="$(pwd)/lib/api/src/grpc/proto" # Ensure current path is project root cd $PROJECT_ROOT -CLIENT_DIR="proto" +CLIENT_DIR="internal/proto" cp $PROTO_DIR/*.proto $CLIENT_DIR/ @@ -39,4 +39,4 @@ sed -i ' # Remove csharp option from proto files sed -i '/option csharp_namespace = .*/d' $CLIENT_DIR/*.proto -sh tools/generate_proto_go.sh +./internal/tools/generate_proto_go.sh diff --git a/proto/collections.proto b/proto/collections.proto deleted file mode 100644 index 864ca6a..0000000 --- a/proto/collections.proto +++ /dev/null @@ -1,654 +0,0 @@ -syntax = "proto3"; -package qdrant; - -enum Datatype { - Default = 0; - Float32 = 1; - Uint8 = 2; - Float16 = 3; -} - -message VectorParams { - uint64 size = 1; // Size of the vectors - Distance distance = 2; // Distance function used for comparing vectors - optional HnswConfigDiff hnsw_config = 3; // Configuration of vector HNSW graph. If omitted - the collection configuration will be used - optional QuantizationConfig quantization_config = 4; // Configuration of vector quantization config. If omitted - the collection configuration will be used - optional bool on_disk = 5; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM. - optional Datatype datatype = 6; // Data type of the vectors - optional MultiVectorConfig multivector_config = 7; // Configuration for multi-vector search -} - -message VectorParamsDiff { - optional HnswConfigDiff hnsw_config = 1; // Update params for HNSW index. If empty object - it will be unset - optional QuantizationConfigDiff quantization_config = 2; // Update quantization params. If none - it is left unchanged. - optional bool on_disk = 3; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM. -} - -message VectorParamsMap { - map map = 1; -} - -message VectorParamsDiffMap { - map map = 1; -} - -message VectorsConfig { - oneof config { - VectorParams params = 1; - VectorParamsMap params_map = 2; - } -} - -message VectorsConfigDiff { - oneof config { - VectorParamsDiff params = 1; - VectorParamsDiffMap params_map = 2; - } -} - -enum Modifier { - None = 0; - Idf = 1; // Apply Inverse Document Frequency -} - -message SparseVectorParams { - optional SparseIndexConfig index = 1; // Configuration of sparse index - optional Modifier modifier = 2; // If set - apply modifier to the vector values -} - -message SparseVectorConfig { - map map = 1; -} - -enum MultiVectorComparator { - MaxSim = 0; -} - -message MultiVectorConfig { - MultiVectorComparator comparator = 1; // Comparator for multi-vector search -} - - -message GetCollectionInfoRequest { - string collection_name = 1; // Name of the collection -} - -message CollectionExistsRequest { - string collection_name = 1; -} - -message CollectionExists { - bool exists = 1; -} - -message CollectionExistsResponse { - CollectionExists result = 1; - double time = 2; // Time spent to process -} - -message ListCollectionsRequest { -} - -message CollectionDescription { - string name = 1; // Name of the collection -} - -message GetCollectionInfoResponse { - CollectionInfo result = 1; - double time = 2; // Time spent to process -} - -message ListCollectionsResponse { - repeated CollectionDescription collections = 1; - double time = 2; // Time spent to process -} - -enum Distance { - UnknownDistance = 0; - Cosine = 1; - Euclid = 2; - Dot = 3; - Manhattan = 4; -} - -enum CollectionStatus { - UnknownCollectionStatus = 0; - Green = 1; // All segments are ready - Yellow = 2; // Optimization in process - Red = 3; // Something went wrong - Grey = 4; // Optimization is pending -} - -enum PayloadSchemaType { - UnknownType = 0; - Keyword = 1; - Integer = 2; - Float = 3; - Geo = 4; - Text = 5; - Bool = 6; - Datetime = 7; - Uuid = 8; -} - -enum QuantizationType { - UnknownQuantization = 0; - Int8 = 1; -} - -enum CompressionRatio { - x4 = 0; - x8 = 1; - x16 = 2; - x32 = 3; - x64 = 4; -} - -message OptimizerStatus { - bool ok = 1; - string error = 2; -} - -message HnswConfigDiff { - /* - Number of edges per node in the index graph. Larger the value - more accurate the search, more space required. - */ - optional uint64 m = 1; - /* - Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build the index. - */ - optional uint64 ef_construct = 2; - /* - Minimal size (in KiloBytes) of vectors for additional payload-based indexing. - If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used - - in this case full-scan search should be preferred by query planner and additional indexing is not required. - Note: 1 Kb = 1 vector of size 256 - */ - optional uint64 full_scan_threshold = 3; - /* - Number of parallel threads used for background index building. - If 0 - automatically select from 8 to 16. - Best to keep between 8 and 16 to prevent likelihood of building broken/inefficient HNSW graphs. - On small CPUs, less threads are used. - */ - optional uint64 max_indexing_threads = 4; - /* - Store HNSW index on disk. If set to false, the index will be stored in RAM. - */ - optional bool on_disk = 5; - /* - Number of additional payload-aware links per node in the index graph. If not set - regular M parameter will be used. - */ - optional uint64 payload_m = 6; -} - -message SparseIndexConfig { - /* - Prefer a full scan search upto (excluding) this number of vectors. - Note: this is number of vectors, not KiloBytes. - */ - optional uint64 full_scan_threshold = 1; - /* - Store inverted index on disk. If set to false, the index will be stored in RAM. - */ - optional bool on_disk = 2; - /* - Datatype used to store weights in the index. - */ - optional Datatype datatype = 3; -} - -message WalConfigDiff { - optional uint64 wal_capacity_mb = 1; // Size of a single WAL block file - optional uint64 wal_segments_ahead = 2; // Number of segments to create in advance -} - -message OptimizersConfigDiff { - /* - The minimal fraction of deleted vectors in a segment, required to perform segment optimization - */ - optional double deleted_threshold = 1; - /* - The minimal number of vectors in a segment, required to perform segment optimization - */ - optional uint64 vacuum_min_vector_number = 2; - /* - Target amount of segments the optimizer will try to keep. - Real amount of segments may vary depending on multiple parameters: - - - Amount of stored points. - - Current write RPS. - - It is recommended to select the default number of segments as a factor of the number of search threads, - so that each segment would be handled evenly by one of the threads. - */ - optional uint64 default_segment_number = 3; - /* - Do not create segments larger this size (in kilobytes). - Large segments might require disproportionately long indexation times, - therefore it makes sense to limit the size of segments. - - If indexing speed is more important - make this parameter lower. - If search speed is more important - make this parameter higher. - Note: 1Kb = 1 vector of size 256 - If not set, will be automatically selected considering the number of available CPUs. - */ - optional uint64 max_segment_size = 4; - /* - Maximum size (in kilobytes) of vectors to store in-memory per segment. - Segments larger than this threshold will be stored as read-only memmaped file. - - Memmap storage is disabled by default, to enable it, set this threshold to a reasonable value. - - To disable memmap storage, set this to `0`. - - Note: 1Kb = 1 vector of size 256 - */ - optional uint64 memmap_threshold = 5; - /* - Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing - - Default value is 20,000, based on . - - To disable vector indexing, set to `0`. - - Note: 1kB = 1 vector of size 256. - */ - optional uint64 indexing_threshold = 6; - /* - Interval between forced flushes. - */ - optional uint64 flush_interval_sec = 7; - /* - Max number of threads (jobs) for running optimizations per shard. - Note: each optimization job will also use `max_indexing_threads` threads by itself for index building. - If null - have no limit and choose dynamically to saturate CPU. - If 0 - no optimization threads, optimizations will be disabled. - */ - optional uint64 max_optimization_threads = 8; -} - -message ScalarQuantization { - QuantizationType type = 1; // Type of quantization - optional float quantile = 2; // Number of bits to use for quantization - optional bool always_ram = 3; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage -} - -message ProductQuantization { - CompressionRatio compression = 1; // Compression ratio - optional bool always_ram = 2; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage -} - -message BinaryQuantization { - optional bool always_ram = 1; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage -} - -message QuantizationConfig { - oneof quantization { - ScalarQuantization scalar = 1; - ProductQuantization product = 2; - BinaryQuantization binary = 3; - } -} - -message Disabled { - -} - -message QuantizationConfigDiff { - oneof quantization { - ScalarQuantization scalar = 1; - ProductQuantization product = 2; - Disabled disabled = 3; - BinaryQuantization binary = 4; - } -} - -enum ShardingMethod { - Auto = 0; // Auto-sharding based on record ids - Custom = 1; // Shard by user-defined key -} - -message CreateCollection { - string collection_name = 1; // Name of the collection - reserved 2; // Deprecated - reserved 3; // Deprecated - optional HnswConfigDiff hnsw_config = 4; // Configuration of vector index - optional WalConfigDiff wal_config = 5; // Configuration of the Write-Ahead-Log - optional OptimizersConfigDiff optimizers_config = 6; // Configuration of the optimizers - optional uint32 shard_number = 7; // Number of shards in the collection, default is 1 for standalone, otherwise equal to the number of nodes. Minimum is 1 - optional bool on_disk_payload = 8; // If true - point's payload will not be stored in memory - optional uint64 timeout = 9; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied - optional VectorsConfig vectors_config = 10; // Configuration for vectors - optional uint32 replication_factor = 11; // Number of replicas of each shard that network tries to maintain, default = 1 - optional uint32 write_consistency_factor = 12; // How many replicas should apply the operation for us to consider it successful, default = 1 - optional string init_from_collection = 13; // Specify name of the other collection to copy data from - optional QuantizationConfig quantization_config = 14; // Quantization configuration of vector - optional ShardingMethod sharding_method = 15; // Sharding method - optional SparseVectorConfig sparse_vectors_config = 16; // Configuration for sparse vectors -} - -message UpdateCollection { - string collection_name = 1; // Name of the collection - optional OptimizersConfigDiff optimizers_config = 2; // New configuration parameters for the collection. This operation is blocking, it will only proceed once all current optimizations are complete - optional uint64 timeout = 3; // Wait timeout for operation commit in seconds if blocking, if not specified - default value will be supplied - optional CollectionParamsDiff params = 4; // New configuration parameters for the collection - optional HnswConfigDiff hnsw_config = 5; // New HNSW parameters for the collection index - optional VectorsConfigDiff vectors_config = 6; // New vector parameters - optional QuantizationConfigDiff quantization_config = 7; // Quantization configuration of vector - optional SparseVectorConfig sparse_vectors_config = 8; // New sparse vector parameters -} - -message DeleteCollection { - string collection_name = 1; // Name of the collection - optional uint64 timeout = 2; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied -} - -message CollectionOperationResponse { - bool result = 1; // if operation made changes - double time = 2; // Time spent to process -} - -message CollectionParams { - reserved 1; // Deprecated - reserved 2; // Deprecated - uint32 shard_number = 3; // Number of shards in collection - bool on_disk_payload = 4; // If true - point's payload will not be stored in memory - optional VectorsConfig vectors_config = 5; // Configuration for vectors - optional uint32 replication_factor = 6; // Number of replicas of each shard that network tries to maintain - optional uint32 write_consistency_factor = 7; // How many replicas should apply the operation for us to consider it successful - optional uint32 read_fan_out_factor = 8; // Fan-out every read request to these many additional remote nodes (and return first available response) - optional ShardingMethod sharding_method = 9; // Sharding method - optional SparseVectorConfig sparse_vectors_config = 10; // Configuration for sparse vectors -} - -message CollectionParamsDiff { - optional uint32 replication_factor = 1; // Number of replicas of each shard that network tries to maintain - optional uint32 write_consistency_factor = 2; // How many replicas should apply the operation for us to consider it successful - optional bool on_disk_payload = 3; // If true - point's payload will not be stored in memory - optional uint32 read_fan_out_factor = 4; // Fan-out every read request to these many additional remote nodes (and return first available response) -} - -message CollectionConfig { - CollectionParams params = 1; // Collection parameters - HnswConfigDiff hnsw_config = 2; // Configuration of vector index - OptimizersConfigDiff optimizer_config = 3; // Configuration of the optimizers - WalConfigDiff wal_config = 4; // Configuration of the Write-Ahead-Log - optional QuantizationConfig quantization_config = 5; // Configuration of the vector quantization -} - -enum TokenizerType { - Unknown = 0; - Prefix = 1; - Whitespace = 2; - Word = 3; - Multilingual = 4; -} - -message KeywordIndexParams { - optional bool is_tenant = 1; // If true - used for tenant optimization. - optional bool on_disk = 2; // If true - store index on disk. -} - -message IntegerIndexParams { - bool lookup = 1; // If true - support direct lookups. - bool range = 2; // If true - support ranges filters. - optional bool is_principal = 3; // If true - use this key to organize storage of the collection data. This option assumes that this key will be used in majority of filtered requests. - optional bool on_disk = 4; // If true - store index on disk. -} - -message FloatIndexParams { - optional bool on_disk = 1; // If true - store index on disk. - optional bool is_principal = 2; // If true - use this key to organize storage of the collection data. This option assumes that this key will be used in majority of filtered requests. -} - -message GeoIndexParams { -} - -message TextIndexParams { - TokenizerType tokenizer = 1; // Tokenizer type - optional bool lowercase = 2; // If true - all tokens will be lowercase - optional uint64 min_token_len = 3; // Minimal token length - optional uint64 max_token_len = 4; // Maximal token length -} - -message BoolIndexParams { -} - -message DatetimeIndexParams { - optional bool on_disk = 1; // If true - store index on disk. - optional bool is_principal = 2; // If true - use this key to organize storage of the collection data. This option assumes that this key will be used in majority of filtered requests. -} - -message UuidIndexParams { - optional bool is_tenant = 1; // If true - used for tenant optimization. - optional bool on_disk = 2; // If true - store index on disk. -} - -message PayloadIndexParams { - oneof index_params { - KeywordIndexParams keyword_index_params = 3; // Parameters for keyword index - IntegerIndexParams integer_index_params = 2; // Parameters for integer index - FloatIndexParams float_index_params = 4; // Parameters for float index - GeoIndexParams geo_index_params = 5; // Parameters for geo index - TextIndexParams text_index_params = 1; // Parameters for text index - BoolIndexParams bool_index_params = 6; // Parameters for bool index - DatetimeIndexParams datetime_index_params = 7; // Parameters for datetime index - UuidIndexParams uuid_index_params = 8; // Parameters for uuid index - } -} - -message PayloadSchemaInfo { - PayloadSchemaType data_type = 1; // Field data type - optional PayloadIndexParams params = 2; // Field index parameters - optional uint64 points = 3; // Number of points indexed within this field indexed -} - -message CollectionInfo { - CollectionStatus status = 1; // operating condition of the collection - OptimizerStatus optimizer_status = 2; // status of collection optimizers - optional uint64 vectors_count = 3; // Approximate number of vectors in the collection - uint64 segments_count = 4; // Number of independent segments - reserved 5; // Deprecated - reserved 6; // Deprecated - CollectionConfig config = 7; // Configuration - map payload_schema = 8; // Collection data types - optional uint64 points_count = 9; // Approximate number of points in the collection - optional uint64 indexed_vectors_count = 10; // Approximate number of indexed vectors in the collection. -} - -message ChangeAliases { - repeated AliasOperations actions = 1; // List of actions - optional uint64 timeout = 2; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied -} - -message AliasOperations { - oneof action { - CreateAlias create_alias = 1; - RenameAlias rename_alias = 2; - DeleteAlias delete_alias = 3; - } -} - -message CreateAlias { - string collection_name = 1; // Name of the collection - string alias_name = 2; // New name of the alias -} - -message RenameAlias { - string old_alias_name = 1; // Name of the alias to rename - string new_alias_name = 2; // Name of the alias -} - -message DeleteAlias { - string alias_name = 1; // Name of the alias -} - -message ListAliasesRequest { -} - -message ListCollectionAliasesRequest { - string collection_name = 1; // Name of the collection -} - -message AliasDescription { - string alias_name = 1; // Name of the alias - string collection_name = 2; // Name of the collection -} - -message ListAliasesResponse { - repeated AliasDescription aliases = 1; - double time = 2; // Time spent to process -} - -message CollectionClusterInfoRequest { - string collection_name = 1; // Name of the collection -} - -enum ReplicaState { - Active = 0; // Active and sound - Dead = 1; // Failed for some reason - Partial = 2; // The shard is partially loaded and is currently receiving data from other shards - Initializing = 3; // Collection is being created - Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards - PartialSnapshot = 5; // Deprecated: snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard - Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true - Resharding = 7; // Points are being migrated to this shard as part of resharding -} - -message ShardKey { - oneof key { - string keyword = 1; // String key - uint64 number = 2; // Number key - } -} - -message LocalShardInfo { - uint32 shard_id = 1; // Local shard id - uint64 points_count = 2; // Number of points in the shard - ReplicaState state = 3; // Is replica active - optional ShardKey shard_key = 4; // User-defined shard key -} - -message RemoteShardInfo { - uint32 shard_id = 1; // Local shard id - uint64 peer_id = 2; // Remote peer id - ReplicaState state = 3; // Is replica active - optional ShardKey shard_key = 4; // User-defined shard key -} - -message ShardTransferInfo { - uint32 shard_id = 1; // Local shard id - optional uint32 to_shard_id = 5; - uint64 from = 2; - uint64 to = 3; - bool sync = 4; // If `true` transfer is a synchronization of a replicas; If `false` transfer is a moving of a shard from one peer to another -} - -message ReshardingInfo { - uint32 shard_id = 1; - uint64 peer_id = 2; - optional ShardKey shard_key = 3; -} - -message CollectionClusterInfoResponse { - uint64 peer_id = 1; // ID of this peer - uint64 shard_count = 2; // Total number of shards - repeated LocalShardInfo local_shards = 3; // Local shards - repeated RemoteShardInfo remote_shards = 4; // Remote shards - repeated ShardTransferInfo shard_transfers = 5; // Shard transfers - // TODO(resharding): enable on release: - // repeated ReshardingInfo resharding_operations = 6; // Resharding operations -} - -message MoveShard { - uint32 shard_id = 1; // Local shard id - optional uint32 to_shard_id = 5; - uint64 from_peer_id = 2; - uint64 to_peer_id = 3; - optional ShardTransferMethod method = 4; -} - -message ReplicateShard { - uint32 shard_id = 1; // Local shard id - optional uint32 to_shard_id = 5; - uint64 from_peer_id = 2; - uint64 to_peer_id = 3; - optional ShardTransferMethod method = 4; -} - -message AbortShardTransfer { - uint32 shard_id = 1; // Local shard id - optional uint32 to_shard_id = 4; - uint64 from_peer_id = 2; - uint64 to_peer_id = 3; -} - -message RestartTransfer { - uint32 shard_id = 1; // Local shard id - optional uint32 to_shard_id = 5; - uint64 from_peer_id = 2; - uint64 to_peer_id = 3; - ShardTransferMethod method = 4; -} - -enum ShardTransferMethod { - StreamRecords = 0; // Stream shard records in batches - Snapshot = 1; // Snapshot the shard and recover it on the target peer - WalDelta = 2; // Resolve WAL delta between peers and transfer the difference - ReshardingStreamRecords = 3; // Stream shard records in batches for resharding -} - -message Replica { - uint32 shard_id = 1; - uint64 peer_id = 2; -} - -message CreateShardKey { - ShardKey shard_key = 1; // User-defined shard key - optional uint32 shards_number = 2; // Number of shards to create per shard key - optional uint32 replication_factor = 3; // Number of replicas of each shard to create - repeated uint64 placement = 4; // List of peer ids, allowed to create shards. If empty - all peers are allowed -} - -message DeleteShardKey { - ShardKey shard_key = 1; // Shard key to delete -} - -message UpdateCollectionClusterSetupRequest { - string collection_name = 1; // Name of the collection - oneof operation { - MoveShard move_shard = 2; - ReplicateShard replicate_shard = 3; - AbortShardTransfer abort_transfer = 4; - Replica drop_replica = 5; - CreateShardKey create_shard_key = 7; - DeleteShardKey delete_shard_key = 8; - RestartTransfer restart_transfer = 9; - } - optional uint64 timeout = 6; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied -} - -message UpdateCollectionClusterSetupResponse { - bool result = 1; -} - -message CreateShardKeyRequest { - string collection_name = 1; // Name of the collection - CreateShardKey request = 2; // Request to create shard key - optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied -} - -message DeleteShardKeyRequest { - string collection_name = 1; // Name of the collection - DeleteShardKey request = 2; // Request to delete shard key - optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied -} - -message CreateShardKeyResponse { - bool result = 1; -} - -message DeleteShardKeyResponse { - bool result = 1; -} diff --git a/proto/collections_service.proto b/proto/collections_service.proto deleted file mode 100644 index 0a77844..0000000 --- a/proto/collections_service.proto +++ /dev/null @@ -1,60 +0,0 @@ -syntax = "proto3"; - -import "collections.proto"; - -package qdrant; - -service Collections { - /* - Get detailed information about specified existing collection - */ - rpc Get (GetCollectionInfoRequest) returns (GetCollectionInfoResponse) {} - /* - Get list name of all existing collections - */ - rpc List (ListCollectionsRequest) returns (ListCollectionsResponse) {} - /* - Create new collection with given parameters - */ - rpc Create (CreateCollection) returns (CollectionOperationResponse) {} - /* - Update parameters of the existing collection - */ - rpc Update (UpdateCollection) returns (CollectionOperationResponse) {} - /* - Drop collection and all associated data - */ - rpc Delete (DeleteCollection) returns (CollectionOperationResponse) {} - /* - Update Aliases of the existing collection - */ - rpc UpdateAliases (ChangeAliases) returns (CollectionOperationResponse) {} - /* - Get list of all aliases for a collection - */ - rpc ListCollectionAliases (ListCollectionAliasesRequest) returns (ListAliasesResponse) {} - /* - Get list of all aliases for all existing collections - */ - rpc ListAliases (ListAliasesRequest) returns (ListAliasesResponse) {} - /* - Get cluster information for a collection - */ - rpc CollectionClusterInfo (CollectionClusterInfoRequest) returns (CollectionClusterInfoResponse) {} - /* - Check the existence of a collection - */ - rpc CollectionExists (CollectionExistsRequest) returns (CollectionExistsResponse) {} - /* - Update cluster setup for a collection - */ - rpc UpdateCollectionClusterSetup (UpdateCollectionClusterSetupRequest) returns (UpdateCollectionClusterSetupResponse) {} - /* - Create shard key - */ - rpc CreateShardKey (CreateShardKeyRequest) returns (CreateShardKeyResponse) {} - /* - Delete shard key - */ - rpc DeleteShardKey (DeleteShardKeyRequest) returns (DeleteShardKeyResponse) {} -} diff --git a/proto/json_with_int.proto b/proto/json_with_int.proto deleted file mode 100644 index 3fc496e..0000000 --- a/proto/json_with_int.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Fork of the google.protobuf.Value with explicit support for integer values - -syntax = "proto3"; - -package qdrant; - -// `Struct` represents a structured data value, consisting of fields -// which map to dynamically typed values. In some languages, `Struct` -// might be supported by a native representation. For example, in -// scripting languages like JS a struct is represented as an -// object. The details of that representation are described together -// with the proto support for the language. -// -// The JSON representation for `Struct` is a JSON object. -message Struct { - // Unordered map of dynamically typed values. - map fields = 1; -} - -// `Value` represents a dynamically typed value which can be either -// null, a number, a string, a boolean, a recursive struct value, or a -// list of values. A producer of value is expected to set one of those -// variants, absence of any variant indicates an error. -// -// The JSON representation for `Value` is a JSON value. -message Value { - // The kind of value. - oneof kind { - // Represents a null value. - NullValue null_value = 1; - // Represents a double value. - double double_value = 2; - // Represents an integer value - int64 integer_value = 3; - // Represents a string value. - string string_value = 4; - // Represents a boolean value. - bool bool_value = 5; - // Represents a structured value. - Struct struct_value = 6; - // Represents a repeated `Value`. - ListValue list_value = 7; - } -} - -// `NullValue` is a singleton enumeration to represent the null value for the -// `Value` type union. -// -// The JSON representation for `NullValue` is JSON `null`. -enum NullValue { - // Null value. - NULL_VALUE = 0; -} - -// `ListValue` is a wrapper around a repeated field of values. -// -// The JSON representation for `ListValue` is a JSON array. -message ListValue { - // Repeated field of dynamically typed values. - repeated Value values = 1; -} diff --git a/proto/points.proto b/proto/points.proto deleted file mode 100644 index a34047e..0000000 --- a/proto/points.proto +++ /dev/null @@ -1,965 +0,0 @@ -syntax = "proto3"; - -package qdrant; - -import "collections.proto"; -import "google/protobuf/timestamp.proto"; -import "json_with_int.proto"; - - -enum WriteOrderingType { - Weak = 0; // Write operations may be reordered, works faster, default - Medium = 1; // Write operations go through dynamically selected leader, may be inconsistent for a short period of time in case of leader change - Strong = 2; // Write operations go through the permanent leader, consistent, but may be unavailable if leader is down -} - -message WriteOrdering { - WriteOrderingType type = 1; // Write ordering guarantees -} - -enum ReadConsistencyType { - All = 0; // Send request to all nodes and return points which are present on all of them - Majority = 1; // Send requests to all nodes and return points which are present on majority of them - Quorum = 2; // Send requests to half + 1 nodes, return points which are present on all of them -} - -message ReadConsistency { - oneof value { - ReadConsistencyType type = 1; // Common read consistency configurations - uint64 factor = 2; // Send request to a specified number of nodes, and return points which are present on all of them - } -} - -// --------------------------------------------- -// ------------- Point Id Requests ------------- -// --------------------------------------------- - -message PointId { - oneof point_id_options { - uint64 num = 1; // Numerical ID of the point - string uuid = 2; // UUID - } -} - -message SparseIndices { - repeated uint32 data = 1; -} - -// Legacy vector format, which determines the vector type by the configuration of its fields. -message Vector { - repeated float data = 1; // Vector data (flatten for multi vectors) - optional SparseIndices indices = 2; // Sparse indices for sparse vectors - optional uint32 vectors_count = 3; // Number of vectors per multi vector -} - -message DenseVector { - repeated float data = 1; -} - -message SparseVector { - repeated float values = 1; - repeated uint32 indices = 2; -} - -message MultiDenseVector { - repeated DenseVector vectors = 1; -} - -// Vector type to be used in queries. Ids will be substituted with their corresponding vectors from the collection. -message VectorInput { - oneof variant { - PointId id = 1; - DenseVector dense = 2; - SparseVector sparse = 3; - MultiDenseVector multi_dense = 4; - } -} - -// --------------------------------------------- -// ----------------- ShardKeySelector ---------- -// --------------------------------------------- - -message ShardKeySelector { - repeated ShardKey shard_keys = 1; // List of shard keys which should be used in the request -} - - -// --------------------------------------------- -// ---------------- RPC Requests --------------- -// --------------------------------------------- - -message UpsertPoints { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - repeated PointStruct points = 3; - optional WriteOrdering ordering = 4; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 5; // Option for custom sharding to specify used shard keys -} - -message DeletePoints { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - PointsSelector points = 3; // Affected points - optional WriteOrdering ordering = 4; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 5; // Option for custom sharding to specify used shard keys -} - -message GetPoints { - string collection_name = 1; // name of the collection - repeated PointId ids = 2; // List of points to retrieve - reserved 3; // deprecated "with_vector" field - WithPayloadSelector with_payload = 4; // Options for specifying which payload to include or not - optional WithVectorsSelector with_vectors = 5; // Options for specifying which vectors to include into response - optional ReadConsistency read_consistency = 6; // Options for specifying read consistency guarantees - optional ShardKeySelector shard_key_selector = 7; // Specify in which shards to look for the points, if not specified - look in all shards - optional uint64 timeout = 8; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message UpdatePointVectors { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - repeated PointVectors points = 3; // List of points and vectors to update - optional WriteOrdering ordering = 4; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 5; // Option for custom sharding to specify used shard keys -} - -message PointVectors { - PointId id = 1; // ID to update vectors for - Vectors vectors = 2; // Named vectors to update, leave others intact -} - -message DeletePointVectors { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - PointsSelector points_selector = 3; // Affected points - VectorsSelector vectors = 4; // List of vector names to delete - optional WriteOrdering ordering = 5; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 6; // Option for custom sharding to specify used shard keys -} - -message SetPayloadPoints { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - map payload = 3; // New payload values - reserved 4; // List of point to modify, deprecated - optional PointsSelector points_selector = 5; // Affected points - optional WriteOrdering ordering = 6; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 7; // Option for custom sharding to specify used shard keys - optional string key = 8; // Option for indicate property of payload -} - -message DeletePayloadPoints { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - repeated string keys = 3; // List of keys to delete - reserved 4; // Affected points, deprecated - optional PointsSelector points_selector = 5; // Affected points - optional WriteOrdering ordering = 6; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 7; // Option for custom sharding to specify used shard keys -} - -message ClearPayloadPoints { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - PointsSelector points = 3; // Affected points - optional WriteOrdering ordering = 4; // Write ordering guarantees - optional ShardKeySelector shard_key_selector = 5; // Option for custom sharding to specify used shard keys -} - -enum FieldType { - FieldTypeKeyword = 0; - FieldTypeInteger = 1; - FieldTypeFloat = 2; - FieldTypeGeo = 3; - FieldTypeText = 4; - FieldTypeBool = 5; - FieldTypeDatetime = 6; - FieldTypeUuid = 7; -} - -message CreateFieldIndexCollection { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - string field_name = 3; // Field name to index - optional FieldType field_type = 4; // Field type. - optional PayloadIndexParams field_index_params = 5; // Payload index params. - optional WriteOrdering ordering = 6; // Write ordering guarantees -} - -message DeleteFieldIndexCollection { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - string field_name = 3; // Field name to delete - optional WriteOrdering ordering = 4; // Write ordering guarantees -} - -message PayloadIncludeSelector { - repeated string fields = 1; // List of payload keys to include into result -} - -message PayloadExcludeSelector { - repeated string fields = 1; // List of payload keys to exclude from the result -} - -message WithPayloadSelector { - oneof selector_options { - bool enable = 1; // If `true` - return all payload, if `false` - none - PayloadIncludeSelector include = 2; - PayloadExcludeSelector exclude = 3; - } -} - -message NamedVectors { - map vectors = 1; -} - -message Vectors { - oneof vectors_options { - Vector vector = 1; - NamedVectors vectors = 2; - } -} - -message VectorsSelector { - repeated string names = 1; // List of vectors to include into result -} - -message WithVectorsSelector { - oneof selector_options { - bool enable = 1; // If `true` - return all vectors, if `false` - none - VectorsSelector include = 2; // List of payload keys to include into result - } -} - -message QuantizationSearchParams { - /* - If set to true, search will ignore quantized vector data - */ - optional bool ignore = 1; - - /* - If true, use original vectors to re-score top-k results. If ignored, qdrant decides automatically does rescore enabled or not. - */ - optional bool rescore = 2; - - /* - Oversampling factor for quantization. - - Defines how many extra vectors should be pre-selected using quantized index, - and then re-scored using original vectors. - - For example, if `oversampling` is 2.4 and `limit` is 100, then 240 vectors will be pre-selected using quantized index, - and then top-100 will be returned after re-scoring. - */ - optional double oversampling = 3; -} - -message SearchParams { - /* - Params relevant to HNSW index. Size of the beam in a beam-search. - Larger the value - more accurate the result, more time required for search. - */ - optional uint64 hnsw_ef = 1; - - /* - Search without approximation. If set to true, search may run long but with exact results. - */ - optional bool exact = 2; - - /* - If set to true, search will ignore quantized vector data - */ - optional QuantizationSearchParams quantization = 3; - /* - If enabled, the engine will only perform search among indexed or small segments. - Using this option prevents slow searches in case of delayed index, but does not - guarantee that all uploaded vectors will be included in search results - */ - optional bool indexed_only = 4; -} - -message SearchPoints { - string collection_name = 1; // name of the collection - repeated float vector = 2; // vector - Filter filter = 3; // Filter conditions - return only those points that satisfy the specified conditions - uint64 limit = 4; // Max number of result - reserved 5; // deprecated "with_vector" field - WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not - SearchParams params = 7; // Search config - optional float score_threshold = 8; // If provided - cut off results with worse scores - optional uint64 offset = 9; // Offset of the result - optional string vector_name = 10; // Which vector to use for search, if not specified - use default vector - optional WithVectorsSelector with_vectors = 11; // Options for specifying which vectors to include into response - optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees - optional uint64 timeout = 13; // If set, overrides global timeout setting for this request. Unit is seconds. - optional ShardKeySelector shard_key_selector = 14; // Specify in which shards to look for the points, if not specified - look in all shards - optional SparseIndices sparse_indices = 15; -} - -message SearchBatchPoints { - string collection_name = 1; // Name of the collection - repeated SearchPoints search_points = 2; - optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees - optional uint64 timeout = 4; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message WithLookup { - string collection = 1; // Name of the collection to use for points lookup - optional WithPayloadSelector with_payload = 2; // Options for specifying which payload to include (or not) - optional WithVectorsSelector with_vectors = 3; // Options for specifying which vectors to include (or not) -} - - -message SearchPointGroups { - string collection_name = 1; // Name of the collection - repeated float vector = 2; // Vector to compare against - Filter filter = 3; // Filter conditions - return only those points that satisfy the specified conditions - uint32 limit = 4; // Max number of result - WithPayloadSelector with_payload = 5; // Options for specifying which payload to include or not - SearchParams params = 6; // Search config - optional float score_threshold = 7; // If provided - cut off results with worse scores - optional string vector_name = 8; // Which vector to use for search, if not specified - use default vector - optional WithVectorsSelector with_vectors = 9; // Options for specifying which vectors to include into response - string group_by = 10; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups. - uint32 group_size = 11; // Maximum amount of points to return per group - optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees - optional WithLookup with_lookup = 13; // Options for specifying how to use the group id to lookup points in another collection - optional uint64 timeout = 14; // If set, overrides global timeout setting for this request. Unit is seconds. - optional ShardKeySelector shard_key_selector = 15; // Specify in which shards to look for the points, if not specified - look in all shards - optional SparseIndices sparse_indices = 16; -} - -enum Direction { - Asc = 0; - Desc = 1; -} - -message StartFrom { - oneof value { - double float = 1; - int64 integer = 2; - google.protobuf.Timestamp timestamp = 3; - string datetime = 4; - } -} - -message OrderBy { - string key = 1; // Payload key to order by - optional Direction direction = 2; // Ascending or descending order - optional StartFrom start_from = 3; // Start from this value -} - -message ScrollPoints { - string collection_name = 1; - Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions - optional PointId offset = 3; // Start with this ID - optional uint32 limit = 4; // Max number of result - reserved 5; // deprecated "with_vector" field - WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not - optional WithVectorsSelector with_vectors = 7; // Options for specifying which vectors to include into response - optional ReadConsistency read_consistency = 8; // Options for specifying read consistency guarantees - optional ShardKeySelector shard_key_selector = 9; // Specify in which shards to look for the points, if not specified - look in all shards - optional OrderBy order_by = 10; // Order the records by a payload field - optional uint64 timeout = 11; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -// How to use positive and negative vectors to find the results, default is `AverageVector`. -enum RecommendStrategy { - // Average positive and negative vectors and create a single query with the formula - // `query = avg_pos + avg_pos - avg_neg`. Then performs normal search. - AverageVector = 0; - - // Uses custom search objective. Each candidate is compared against all - // examples, its score is then chosen from the `max(max_pos_score, max_neg_score)`. - // If the `max_neg_score` is chosen then it is squared and negated. - BestScore = 1; -} - -message LookupLocation { - string collection_name = 1; - optional string vector_name = 2; // Which vector to use for search, if not specified - use default vector - optional ShardKeySelector shard_key_selector = 3; // Specify in which shards to look for the points, if not specified - look in all shards -} - -message RecommendPoints { - string collection_name = 1; // name of the collection - repeated PointId positive = 2; // Look for vectors closest to the vectors from these points - repeated PointId negative = 3; // Try to avoid vectors like the vector from these points - Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions - uint64 limit = 5; // Max number of result - reserved 6; // deprecated "with_vector" field - WithPayloadSelector with_payload = 7; // Options for specifying which payload to include or not - SearchParams params = 8; // Search config - optional float score_threshold = 9; // If provided - cut off results with worse scores - optional uint64 offset = 10; // Offset of the result - optional string using = 11; // Define which vector to use for recommendation, if not specified - default vector - optional WithVectorsSelector with_vectors = 12; // Options for specifying which vectors to include into response - optional LookupLocation lookup_from = 13; // Name of the collection to use for points lookup, if not specified - use current collection - optional ReadConsistency read_consistency = 14; // Options for specifying read consistency guarantees - optional RecommendStrategy strategy = 16; // How to use the example vectors to find the results - repeated Vector positive_vectors = 17; // Look for vectors closest to those - repeated Vector negative_vectors = 18; // Try to avoid vectors like this - optional uint64 timeout = 19; // If set, overrides global timeout setting for this request. Unit is seconds. - optional ShardKeySelector shard_key_selector = 20; // Specify in which shards to look for the points, if not specified - look in all shards -} - -message RecommendBatchPoints { - string collection_name = 1; // Name of the collection - repeated RecommendPoints recommend_points = 2; - optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees - optional uint64 timeout = 4; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message RecommendPointGroups { - string collection_name = 1; // Name of the collection - repeated PointId positive = 2; // Look for vectors closest to the vectors from these points - repeated PointId negative = 3; // Try to avoid vectors like the vector from these points - Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions - uint32 limit = 5; // Max number of groups in result - WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not - SearchParams params = 7; // Search config - optional float score_threshold = 8; // If provided - cut off results with worse scores - optional string using = 9; // Define which vector to use for recommendation, if not specified - default vector - optional WithVectorsSelector with_vectors = 10; // Options for specifying which vectors to include into response - optional LookupLocation lookup_from = 11; // Name of the collection to use for points lookup, if not specified - use current collection - string group_by = 12; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups. - uint32 group_size = 13; // Maximum amount of points to return per group - optional ReadConsistency read_consistency = 14; // Options for specifying read consistency guarantees - optional WithLookup with_lookup = 15; // Options for specifying how to use the group id to lookup points in another collection - optional RecommendStrategy strategy = 17; // How to use the example vectors to find the results - repeated Vector positive_vectors = 18; // Look for vectors closest to those - repeated Vector negative_vectors = 19; // Try to avoid vectors like this - optional uint64 timeout = 20; // If set, overrides global timeout setting for this request. Unit is seconds. - optional ShardKeySelector shard_key_selector = 21; // Specify in which shards to look for the points, if not specified - look in all shards -} - -message TargetVector { - oneof target { - VectorExample single = 1; - - // leaving extensibility for possibly adding multi-target - } -} - -message VectorExample { - oneof example { - PointId id = 1; - Vector vector = 2; - } -} - -message ContextExamplePair { - VectorExample positive = 1; - VectorExample negative = 2; -} - -message DiscoverPoints { - string collection_name = 1; // name of the collection - TargetVector target = 2; // Use this as the primary search objective - repeated ContextExamplePair context = 3; // Search will be constrained by these pairs of examples - Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions - uint64 limit = 5; // Max number of result - WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not - SearchParams params = 7; // Search config - optional uint64 offset = 8; // Offset of the result - optional string using = 9; // Define which vector to use for recommendation, if not specified - default vector - optional WithVectorsSelector with_vectors = 10; // Options for specifying which vectors to include into response - optional LookupLocation lookup_from = 11; // Name of the collection to use for points lookup, if not specified - use current collection - optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees - optional uint64 timeout = 13; // If set, overrides global timeout setting for this request. Unit is seconds. - optional ShardKeySelector shard_key_selector = 14; // Specify in which shards to look for the points, if not specified - look in all shards -} - -message DiscoverBatchPoints { - string collection_name = 1; // Name of the collection - repeated DiscoverPoints discover_points = 2; - optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees - optional uint64 timeout = 4; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message CountPoints { - string collection_name = 1; // Name of the collection - Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions - optional bool exact = 3; // If `true` - return exact count, if `false` - return approximate count - optional ReadConsistency read_consistency = 4; // Options for specifying read consistency guarantees - optional ShardKeySelector shard_key_selector = 5; // Specify in which shards to look for the points, if not specified - look in all shards - optional uint64 timeout = 6; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message RecommendInput { - repeated VectorInput positive = 1; // Look for vectors closest to the vectors from these points - repeated VectorInput negative = 2; // Try to avoid vectors like the vector from these points - optional RecommendStrategy strategy = 3; // How to use the provided vectors to find the results -} - -message ContextInputPair { - VectorInput positive = 1; // A positive vector - VectorInput negative = 2; // Repel from this vector -} - -message DiscoverInput { - VectorInput target = 1; // Use this as the primary search objective - ContextInput context = 2; // Search space will be constrained by these pairs of vectors -} - -message ContextInput { - repeated ContextInputPair pairs = 1; // Search space will be constrained by these pairs of vectors -} - -enum Fusion { - RRF = 0; // Reciprocal Rank Fusion - DBSF = 1; // Distribution-Based Score Fusion -} - -// Sample points from the collection -// -// Available sampling methods: -// -// * `random` - Random sampling -enum Sample { - Random = 0; -} - -message Query { - oneof variant { - VectorInput nearest = 1; // Find the nearest neighbors to this vector. - RecommendInput recommend = 2; // Use multiple positive and negative vectors to find the results. - DiscoverInput discover = 3; // Search for nearest points, but constrain the search space with context - ContextInput context = 4; // Return points that live in positive areas. - OrderBy order_by = 5; // Order the points by a payload field. - Fusion fusion = 6; // Fuse the results of multiple prefetches. - Sample sample = 7; // Sample points from the collection. - } -} - -message PrefetchQuery { - repeated PrefetchQuery prefetch = 1; // Sub-requests to perform first. If present, the query will be performed on the results of the prefetches. - optional Query query = 2; // Query to perform. If missing, returns points ordered by their IDs. - optional string using = 3; // Define which vector to use for querying. If missing, the default vector is is used. - optional Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions. - optional SearchParams params = 5; // Search params for when there is no prefetch. - optional float score_threshold = 6; // Return points with scores better than this threshold. - optional uint64 limit = 7; // Max number of points. Default is 10 - optional LookupLocation lookup_from = 8; // The location to use for IDs lookup, if not specified - use the current collection and the 'using' vector -} - -message QueryPoints { - string collection_name = 1; // Name of the collection - repeated PrefetchQuery prefetch = 2; // Sub-requests to perform first. If present, the query will be performed on the results of the prefetches. - optional Query query = 3; // Query to perform. If missing, returns points ordered by their IDs. - optional string using = 4; // Define which vector to use for querying. If missing, the default vector is used. - optional Filter filter = 5; // Filter conditions - return only those points that satisfy the specified conditions. - optional SearchParams params = 6; // Search params for when there is no prefetch. - optional float score_threshold = 7; // Return points with scores better than this threshold. - optional uint64 limit = 8; // Max number of points. Default is 10. - optional uint64 offset = 9; // Offset of the result. Skip this many points. Default is 0. - optional WithVectorsSelector with_vectors = 10; // Options for specifying which vectors to include into the response. - optional WithPayloadSelector with_payload = 11; // Options for specifying which payload to include or not. - optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees. - optional ShardKeySelector shard_key_selector = 13; // Specify in which shards to look for the points, if not specified - look in all shards. - optional LookupLocation lookup_from = 14; // The location to use for IDs lookup, if not specified - use the current collection and the 'using' vector - optional uint64 timeout = 15; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message QueryBatchPoints { - string collection_name = 1; - repeated QueryPoints query_points = 2; - optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees - optional uint64 timeout = 4; // If set, overrides global timeout setting for this request. Unit is seconds. -} - -message QueryPointGroups { - string collection_name = 1; // Name of the collection - repeated PrefetchQuery prefetch = 2; // Sub-requests to perform first. If present, the query will be performed on the results of the prefetches. - optional Query query = 3; // Query to perform. If missing, returns points ordered by their IDs. - optional string using = 4; // Define which vector to use for querying. If missing, the default vector is used. - optional Filter filter = 5; // Filter conditions - return only those points that satisfy the specified conditions. - optional SearchParams params = 6; // Search params for when there is no prefetch. - optional float score_threshold = 7; // Return points with scores better than this threshold. - WithPayloadSelector with_payload = 8; // Options for specifying which payload to include or not - optional WithVectorsSelector with_vectors = 9; // Options for specifying which vectors to include into response - optional LookupLocation lookup_from = 10; // The location to use for IDs lookup, if not specified - use the current collection and the 'using' vector - optional uint64 limit = 11; // Max number of points. Default is 3. - optional uint64 group_size = 12; // Maximum amount of points to return per group. Default to 10. - string group_by = 13; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups. - optional ReadConsistency read_consistency = 14; // Options for specifying read consistency guarantees - optional WithLookup with_lookup = 15; // Options for specifying how to use the group id to lookup points in another collection - optional uint64 timeout = 16; // If set, overrides global timeout setting for this request. Unit is seconds. - optional ShardKeySelector shard_key_selector = 17; // Specify in which shards to look for the points, if not specified - look in all shards -} - -message FacetValue { - oneof variant { - string string_value = 1; // String value from the facet - } -} - -message FacetValueHit { - FacetValue value = 1; // Value from the facet - uint64 count = 2; // Number of points with this value -} - -message PointsUpdateOperation { - message PointStructList { - repeated PointStruct points = 1; - optional ShardKeySelector shard_key_selector = 2; // Option for custom sharding to specify used shard keys - } - message SetPayload { - map payload = 1; - optional PointsSelector points_selector = 2; // Affected points - optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys - optional string key = 4; // Option for indicate property of payload - } - message OverwritePayload { - map payload = 1; - optional PointsSelector points_selector = 2; // Affected points - optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys - optional string key = 4; // Option for indicate property of payload - } - message DeletePayload { - repeated string keys = 1; - optional PointsSelector points_selector = 2; // Affected points - optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys - } - message UpdateVectors { - repeated PointVectors points = 1; // List of points and vectors to update - optional ShardKeySelector shard_key_selector = 2; // Option for custom sharding to specify used shard keys - } - message DeleteVectors { - PointsSelector points_selector = 1; // Affected points - VectorsSelector vectors = 2; // List of vector names to delete - optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys - } - message DeletePoints { - PointsSelector points = 1; // Affected points - optional ShardKeySelector shard_key_selector = 2; // Option for custom sharding to specify used shard keys - } - message ClearPayload { - PointsSelector points = 1; // Affected points - optional ShardKeySelector shard_key_selector = 2; // Option for custom sharding to specify used shard keys - } - - oneof operation { - PointStructList upsert = 1; - PointsSelector delete_deprecated = 2 [deprecated=true]; - SetPayload set_payload = 3; - OverwritePayload overwrite_payload = 4; - DeletePayload delete_payload = 5; - PointsSelector clear_payload_deprecated = 6 [deprecated=true]; - UpdateVectors update_vectors = 7; - DeleteVectors delete_vectors = 8; - DeletePoints delete_points = 9; - ClearPayload clear_payload = 10; - } -} - -message UpdateBatchPoints { - string collection_name = 1; // name of the collection - optional bool wait = 2; // Wait until the changes have been applied? - repeated PointsUpdateOperation operations = 3; - optional WriteOrdering ordering = 4; // Write ordering guarantees -} - -// --------------------------------------------- -// ---------------- RPC Response --------------- -// --------------------------------------------- - -message PointsOperationResponse { - UpdateResult result = 1; - double time = 2; // Time spent to process -} - -message UpdateResult { - optional uint64 operation_id = 1; // Number of operation - UpdateStatus status = 2; // Operation status -} - -enum UpdateStatus { - UnknownUpdateStatus = 0; - Acknowledged = 1; // Update is received, but not processed yet - Completed = 2; // Update is applied and ready for search - ClockRejected = 3; // Internal: update is rejected due to an outdated clock -} - -message OrderValue { - oneof variant { - int64 int = 1; - double float = 2; - } -} - -message ScoredPoint { - PointId id = 1; // Point id - map payload = 2; // Payload - float score = 3; // Similarity score - reserved 4; // deprecated "vector" field - uint64 version = 5; // Last update operation applied to this point - optional Vectors vectors = 6; // Vectors to search - optional ShardKey shard_key = 7; // Shard key - optional OrderValue order_value = 8; // Order by value -} - -message GroupId { - oneof kind { - // Represents a double value. - uint64 unsigned_value = 1; - // Represents an integer value - int64 integer_value = 2; - // Represents a string value. - string string_value = 3; - } -} - -message PointGroup { - GroupId id = 1; // Group id - repeated ScoredPoint hits = 2; // Points in the group - RetrievedPoint lookup = 3; // Point(s) from the lookup collection that matches the group id -} - -message GroupsResult { - repeated PointGroup groups = 1; // Groups -} - -message SearchResponse { - repeated ScoredPoint result = 1; - double time = 2; // Time spent to process -} - -message QueryResponse { - repeated ScoredPoint result = 1; - double time = 2; // Time spent to process -} - -message QueryBatchResponse { - repeated BatchResult result = 1; - double time = 2; // Time spent to process -} - -message QueryGroupsResponse { - GroupsResult result = 1; - double time = 2; // Time spent to process -} - -message BatchResult { - repeated ScoredPoint result = 1; -} - -message SearchBatchResponse { - repeated BatchResult result = 1; - double time = 2; // Time spent to process -} - -message SearchGroupsResponse { - GroupsResult result = 1; - double time = 2; // Time spent to process -} - -message CountResponse { - CountResult result = 1; - double time = 2; // Time spent to process -} - -message ScrollResponse { - optional PointId next_page_offset = 1; // Use this offset for the next query - repeated RetrievedPoint result = 2; - double time = 3; // Time spent to process -} - -message CountResult { - uint64 count = 1; -} - -message RetrievedPoint { - PointId id = 1; - map payload = 2; - reserved 3; // deprecated "vector" field - optional Vectors vectors = 4; - optional ShardKey shard_key = 5; // Shard key - optional OrderValue order_value = 6; // Order-by value -} - -message GetResponse { - repeated RetrievedPoint result = 1; - double time = 2; // Time spent to process -} - -message RecommendResponse { - repeated ScoredPoint result = 1; - double time = 2; // Time spent to process -} - -message RecommendBatchResponse { - repeated BatchResult result = 1; - double time = 2; // Time spent to process -} - -message DiscoverResponse { - repeated ScoredPoint result = 1; - double time = 2; // Time spent to process -} - -message DiscoverBatchResponse { - repeated BatchResult result = 1; - double time = 2; // Time spent to process -} - -message RecommendGroupsResponse { - GroupsResult result = 1; - double time = 2; // Time spent to process -} - -message UpdateBatchResponse { - repeated UpdateResult result = 1; - double time = 2; // Time spent to process -} - -// --------------------------------------------- -// ------------- Filter Conditions ------------- -// --------------------------------------------- - -message Filter { - repeated Condition should = 1; // At least one of those conditions should match - repeated Condition must = 2; // All conditions must match - repeated Condition must_not = 3; // All conditions must NOT match - optional MinShould min_should = 4; // At least minimum amount of given conditions should match -} - -message MinShould { - repeated Condition conditions = 1; - uint64 min_count = 2; -} - -message Condition { - oneof condition_one_of { - FieldCondition field = 1; - IsEmptyCondition is_empty = 2; - HasIdCondition has_id = 3; - Filter filter = 4; - IsNullCondition is_null = 5; - NestedCondition nested = 6; - } -} - -message IsEmptyCondition { - string key = 1; -} - -message IsNullCondition { - string key = 1; -} - -message HasIdCondition { - repeated PointId has_id = 1; -} - -message NestedCondition { - string key = 1; // Path to nested object - Filter filter = 2; // Filter condition -} - -message FieldCondition { - string key = 1; - Match match = 2; // Check if point has field with a given value - Range range = 3; // Check if points value lies in a given range - GeoBoundingBox geo_bounding_box = 4; // Check if points geolocation lies in a given area - GeoRadius geo_radius = 5; // Check if geo point is within a given radius - ValuesCount values_count = 6; // Check number of values for a specific field - GeoPolygon geo_polygon = 7; // Check if geo point is within a given polygon - DatetimeRange datetime_range = 8; // Check if datetime is within a given range -} - -message Match { - oneof match_value { - string keyword = 1; // Match string keyword - int64 integer = 2; // Match integer - bool boolean = 3; // Match boolean - string text = 4; // Match text - RepeatedStrings keywords = 5; // Match multiple keywords - RepeatedIntegers integers = 6; // Match multiple integers - RepeatedIntegers except_integers = 7; // Match any other value except those integers - RepeatedStrings except_keywords = 8; // Match any other value except those keywords - } -} - -message RepeatedStrings { - repeated string strings = 1; -} - -message RepeatedIntegers { - repeated int64 integers = 1; -} - -message Range { - optional double lt = 1; - optional double gt = 2; - optional double gte = 3; - optional double lte = 4; -} - -message DatetimeRange { - optional google.protobuf.Timestamp lt = 1; - optional google.protobuf.Timestamp gt = 2; - optional google.protobuf.Timestamp gte = 3; - optional google.protobuf.Timestamp lte = 4; -} - -message GeoBoundingBox { - GeoPoint top_left = 1; // north-west corner - GeoPoint bottom_right = 2; // south-east corner -} - -message GeoRadius { - GeoPoint center = 1; // Center of the circle - float radius = 2; // In meters -} - -message GeoLineString { - repeated GeoPoint points = 1; // Ordered sequence of GeoPoints representing the line -} - -// For a valid GeoPolygon, both the exterior and interior GeoLineStrings must consist of a minimum of 4 points. -// Additionally, the first and last points of each GeoLineString must be the same. -message GeoPolygon { - GeoLineString exterior = 1; // The exterior line bounds the surface - repeated GeoLineString interiors = 2; // Interior lines (if present) bound holes within the surface -} - -message ValuesCount { - optional uint64 lt = 1; - optional uint64 gt = 2; - optional uint64 gte = 3; - optional uint64 lte = 4; -} - -// --------------------------------------------- -// -------------- Points Selector -------------- -// --------------------------------------------- - -message PointsSelector { - oneof points_selector_one_of { - PointsIdsList points = 1; - Filter filter = 2; - } -} - -message PointsIdsList { - repeated PointId ids = 1; -} - -// --------------------------------------------- -// ------------------- Point ------------------- -// --------------------------------------------- - - -message PointStruct { - PointId id = 1; - reserved 2; // deprecated "vector" field - map payload = 3; - optional Vectors vectors = 4; -} - - -message GeoPoint { - double lon = 1; - double lat = 2; -} diff --git a/proto/points_service.proto b/proto/points_service.proto deleted file mode 100644 index e88f0fd..0000000 --- a/proto/points_service.proto +++ /dev/null @@ -1,123 +0,0 @@ -syntax = "proto3"; - -import "points.proto"; - -package qdrant; - -service Points { - /* - Perform insert + updates on points. If a point with a given ID already exists - it will be overwritten. - */ - rpc Upsert (UpsertPoints) returns (PointsOperationResponse) {} - /* - Delete points - */ - rpc Delete (DeletePoints) returns (PointsOperationResponse) {} - /* - Retrieve points - */ - rpc Get (GetPoints) returns (GetResponse) {} - /* - Update named vectors for point - */ - rpc UpdateVectors (UpdatePointVectors) returns (PointsOperationResponse) {} - /* - Delete named vectors for points - */ - rpc DeleteVectors (DeletePointVectors) returns (PointsOperationResponse) {} - /* - Set payload for points - */ - rpc SetPayload (SetPayloadPoints) returns (PointsOperationResponse) {} - /* - Overwrite payload for points - */ - rpc OverwritePayload (SetPayloadPoints) returns (PointsOperationResponse) {} - /* - Delete specified key payload for points - */ - rpc DeletePayload (DeletePayloadPoints) returns (PointsOperationResponse) {} - /* - Remove all payload for specified points - */ - rpc ClearPayload (ClearPayloadPoints) returns (PointsOperationResponse) {} - /* - Create index for field in collection - */ - rpc CreateFieldIndex (CreateFieldIndexCollection) returns (PointsOperationResponse) {} - /* - Delete field index for collection - */ - rpc DeleteFieldIndex (DeleteFieldIndexCollection) returns (PointsOperationResponse) {} - /* - Retrieve closest points based on vector similarity and given filtering conditions - */ - rpc Search (SearchPoints) returns (SearchResponse) {} - /* - Retrieve closest points based on vector similarity and given filtering conditions - */ - rpc SearchBatch (SearchBatchPoints) returns (SearchBatchResponse) {} - /* - Retrieve closest points based on vector similarity and given filtering conditions, grouped by a given field - */ - rpc SearchGroups (SearchPointGroups) returns (SearchGroupsResponse) {} - /* - Iterate over all or filtered points - */ - rpc Scroll (ScrollPoints) returns (ScrollResponse) {} - /* - Look for the points which are closer to stored positive examples and at the same time further to negative examples. - */ - rpc Recommend (RecommendPoints) returns (RecommendResponse) {} - /* - Look for the points which are closer to stored positive examples and at the same time further to negative examples. - */ - rpc RecommendBatch (RecommendBatchPoints) returns (RecommendBatchResponse) {} - /* - Look for the points which are closer to stored positive examples and at the same time further to negative examples, grouped by a given field - */ - rpc RecommendGroups (RecommendPointGroups) returns (RecommendGroupsResponse) {} - /* - Use context and a target to find the most similar points to the target, constrained by the context. - - When using only the context (without a target), a special search - called context search - is performed where - pairs of points are used to generate a loss that guides the search towards the zone where - most positive examples overlap. This means that the score minimizes the scenario of - finding a point closer to a negative than to a positive part of a pair. - - Since the score of a context relates to loss, the maximum score a point can get is 0.0, - and it becomes normal that many points can have a score of 0.0. - - When using target (with or without context), the score behaves a little different: The - integer part of the score represents the rank with respect to the context, while the - decimal part of the score relates to the distance to the target. The context part of the score for - each pair is calculated +1 if the point is closer to a positive than to a negative part of a pair, - and -1 otherwise. - */ - rpc Discover (DiscoverPoints) returns (DiscoverResponse) {} - /* - Batch request points based on { positive, negative } pairs of examples, and/or a target - */ - rpc DiscoverBatch (DiscoverBatchPoints) returns (DiscoverBatchResponse) {} - /* - Count points in collection with given filtering conditions - */ - rpc Count (CountPoints) returns (CountResponse) {} - - /* - Perform multiple update operations in one request - */ - rpc UpdateBatch (UpdateBatchPoints) returns (UpdateBatchResponse) {} - /* - Universally query points. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries. - */ - rpc Query (QueryPoints) returns (QueryResponse) {} - /* - Universally query points in a batch fashion. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries. - */ - rpc QueryBatch (QueryBatchPoints) returns (QueryBatchResponse) {} - /* - Universally query points in a group fashion. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries. - */ - rpc QueryGroups (QueryPointGroups) returns (QueryGroupsResponse) {} -} diff --git a/proto/qdrant.proto b/proto/qdrant.proto deleted file mode 100644 index f6990d6..0000000 --- a/proto/qdrant.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; - -import "collections_service.proto"; -import "points_service.proto"; -import "snapshots_service.proto"; - -package qdrant; - -service Qdrant { - rpc HealthCheck (HealthCheckRequest) returns (HealthCheckReply) {} -} - -message HealthCheckRequest {} - -message HealthCheckReply { - string title = 1; - string version = 2; - optional string commit = 3; -} diff --git a/proto/snapshots_service.proto b/proto/snapshots_service.proto deleted file mode 100644 index e5a3c1a..0000000 --- a/proto/snapshots_service.proto +++ /dev/null @@ -1,74 +0,0 @@ -syntax = "proto3"; - -package qdrant; - -import "google/protobuf/timestamp.proto"; - -service Snapshots { - /* - Create collection snapshot - */ - rpc Create (CreateSnapshotRequest) returns (CreateSnapshotResponse) {} - /* - List collection snapshots - */ - rpc List (ListSnapshotsRequest) returns (ListSnapshotsResponse) {} - /* - Delete collection snapshot - */ - rpc Delete (DeleteSnapshotRequest) returns (DeleteSnapshotResponse) {} - /* - Create full storage snapshot - */ - rpc CreateFull (CreateFullSnapshotRequest) returns (CreateSnapshotResponse) {} - /* - List full storage snapshots - */ - rpc ListFull (ListFullSnapshotsRequest) returns (ListSnapshotsResponse) {} - /* - Delete full storage snapshot - */ - rpc DeleteFull (DeleteFullSnapshotRequest) returns (DeleteSnapshotResponse) {} -} - -message CreateFullSnapshotRequest {} - -message ListFullSnapshotsRequest {} - -message DeleteFullSnapshotRequest { - string snapshot_name = 1; // Name of the full snapshot -} - -message CreateSnapshotRequest { - string collection_name = 1; // Name of the collection -} - -message ListSnapshotsRequest { - string collection_name = 1; // Name of the collection -} - -message DeleteSnapshotRequest { - string collection_name = 1; // Name of the collection - string snapshot_name = 2; // Name of the collection snapshot -} - -message SnapshotDescription { - string name = 1; // Name of the snapshot - google.protobuf.Timestamp creation_time = 2; // Creation time of the snapshot - int64 size = 3; // Size of the snapshot in bytes - optional string checksum = 4; // SHA256 digest of the snapshot file -} - -message CreateSnapshotResponse { - SnapshotDescription snapshot_description = 1; - double time = 2; // Time spent to process -} - -message ListSnapshotsResponse { - repeated SnapshotDescription snapshot_descriptions = 1; - double time = 2; // Time spent to process -} - -message DeleteSnapshotResponse { - double time = 1; // Time spent to process -}