From 42d0fb5085124906812b03bea887fd6fdac6fcd9 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Fri, 4 Oct 2024 23:12:25 -0400 Subject: [PATCH 01/37] Made the SchemaObject store and return List of VectorConfig. This is needed because tables need multiple vector config. --- .../jsonapi/api/v1/CollectionResource.java | 4 +- .../executor/DatabaseSchemaObject.java | 6 +- .../executor/KeyspaceSchemaObject.java | 5 +- .../cqldriver/executor/SchemaObject.java | 9 ++- .../cqldriver/executor/TableSchemaObject.java | 5 +- .../cqldriver/executor/VectorConfig.java | 41 +++++++---- .../service/embedding/DataVectorizer.java | 24 ++++--- .../processor/MeteredCommandProcessor.java | 2 +- .../collections/CollectionSchemaObject.java | 69 +++++++++++-------- .../CollectionSettingsV0Reader.java | 6 +- .../CollectionSettingsV1Reader.java | 8 ++- .../stargate/sgv2/jsonapi/TestConstants.java | 5 +- .../CollectionSchemaObjectTest.java | 3 +- .../executor/NamespaceCacheTest.java | 4 +- .../operation/DataVectorizerTest.java | 12 ++-- .../operation/TestEmbeddingProvider.java | 12 ++-- .../FindCollectionOperationTest.java | 2 +- .../InsertCollectionOperationTest.java | 2 +- .../collections/OperationTestBase.java | 2 +- .../ReadAndUpdateCollectionOperationTest.java | 2 +- .../CommandResolverWithVectorizerTest.java | 3 +- 21 files changed, 139 insertions(+), 87 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java index 95000bb42..08c8c09c4 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java @@ -198,7 +198,7 @@ public Uni> postCommand( // TODO: refactor this code to be cleaner so it assigns on one line EmbeddingProvider embeddingProvider = null; final VectorConfig.VectorizeConfig vectorizeConfig = - schemaObject.vectorConfig().vectorizeConfig(); + schemaObject.vectorConfigs().get(0).vectorizeConfig(); if (vectorizeConfig != null) { embeddingProvider = embeddingProviderFactory.getConfiguration( @@ -206,7 +206,7 @@ public Uni> postCommand( dataApiRequestInfo.getCassandraToken(), vectorizeConfig.provider(), vectorizeConfig.modelName(), - schemaObject.vectorConfig().vectorSize(), + schemaObject.vectorConfigs().get(0).vectorSize(), vectorizeConfig.parameters(), vectorizeConfig.authentication(), command.getClass().getSimpleName()); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java index 600f6106a..071e90375 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; +import java.util.List; + public class DatabaseSchemaObject extends SchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.DATABASE; @@ -9,8 +11,8 @@ public DatabaseSchemaObject() { } @Override - public VectorConfig vectorConfig() { - return VectorConfig.notEnabledVectorConfig(); + public List vectorConfigs() { + return List.of(VectorConfig.notEnabledVectorConfig()); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java index b0ea4f89f..08176a76e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; +import java.util.List; public class KeyspaceSchemaObject extends SchemaObject { @@ -39,8 +40,8 @@ public static KeyspaceSchemaObject fromSchemaObject(TableSchemaObject table) { } @Override - public VectorConfig vectorConfig() { - return VectorConfig.notEnabledVectorConfig(); + public List vectorConfigs() { + return List.of(VectorConfig.notEnabledVectorConfig()); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java index bd147a509..8d90b3e50 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; +import java.util.List; + /** A Collection or Table the command works on */ public abstract class SchemaObject { @@ -29,12 +31,13 @@ public SchemaObjectName name() { } /** - * Subclasses must always return an instance of VectorConfig, if there is no vector config they - * should return VectorConfig.notEnabledVectorConfig() + * Subclasses must always return List of VectorConfig, if there is no vector config they should + * return VectorConfig.notEnabledVectorConfig(). This needs to be a list because a table can have + * multiple vector columns. * * @return */ - public abstract VectorConfig vectorConfig(); + public abstract List vectorConfigs(); /** * Call to get an instance of the appropriate {@link IndexUsage} for this schema object diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 3e4ef7d62..b78f722c4 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import java.util.List; public class TableSchemaObject extends TableBasedSchemaObject { @@ -11,8 +12,8 @@ public TableSchemaObject(TableMetadata tableMetadata) { } @Override - public VectorConfig vectorConfig() { - return VectorConfig.notEnabledVectorConfig(); + public List vectorConfigs() { + return List.of(VectorConfig.notEnabledVectorConfig()); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java index dfb52b543..103b087e0 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java @@ -15,6 +15,7 @@ */ public record VectorConfig( boolean vectorEnabled, + String fieldName, int vectorSize, SimilarityFunction similarityFunction, VectorizeConfig vectorizeConfig) { @@ -22,20 +23,43 @@ public record VectorConfig( // TODO: this is an immutable record, this can be singleton // TODO: Remove the use of NULL for the objects like vectorizeConfig public static VectorConfig notEnabledVectorConfig() { - return new VectorConfig(false, -1, null, null); + return new VectorConfig(false, "", -1, null, null); } - // convert a vector jsonNode from table comment to vectorConfig + // convert a vector jsonNode from table comment to vectorConfig, used for collection public static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { // dimension, similarityFunction, must exist int dimension = jsonNode.get("dimension").asInt(); SimilarityFunction similarityFunction = SimilarityFunction.fromString(jsonNode.get("metric").asText()); + return fromJson("$vectorize", dimension, similarityFunction, jsonNode, objectMapper); + } + + // convert a vector jsonNode from table extension to vectorConfig, used for tables + public static VectorConfig fromJson( + String fieldName, + int dimension, + SimilarityFunction similarityFunction, + JsonNode jsonNode, + ObjectMapper objectMapper) { VectorizeConfig vectorizeConfig = null; // construct vectorizeConfig JsonNode vectorizeServiceNode = jsonNode.get("service"); if (vectorizeServiceNode != null) { + vectorizeConfig = VectorizeConfig.fromJson(vectorizeServiceNode, objectMapper); + } + return new VectorConfig(true, fieldName, dimension, similarityFunction, vectorizeConfig); + } + + public record VectorizeConfig( + String provider, + String modelName, + Map authentication, + Map parameters) { + + protected static VectorizeConfig fromJson( + JsonNode vectorizeServiceNode, ObjectMapper objectMapper) { // provider, modelName, must exist String provider = vectorizeServiceNode.get("provider").asText(); String modelName = vectorizeServiceNode.get("modelName").asText(); @@ -51,17 +75,8 @@ public static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper vectorizeServiceParameterNode == null ? null : objectMapper.convertValue(vectorizeServiceParameterNode, Map.class); - vectorizeConfig = - new VectorizeConfig( - provider, modelName, vectorizeServiceAuthentication, vectorizeServiceParameter); + return new VectorizeConfig( + provider, modelName, vectorizeServiceAuthentication, vectorizeServiceParameter); } - - return new VectorConfig(true, dimension, similarityFunction, vectorizeConfig); } - - public record VectorizeConfig( - String provider, - String modelName, - Map authentication, - Map parameters) {} } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java index 0557ae329..744e78ccc 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java @@ -14,6 +14,7 @@ import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.SchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.embedding.operation.EmbeddingProvider; import java.util.ArrayList; import java.util.HashMap; @@ -30,6 +31,7 @@ public class DataVectorizer { private final JsonNodeFactory nodeFactory; private final EmbeddingCredentials embeddingCredentials; private final SchemaObject schemaObject; + private final VectorConfig vectorConfig; /** * Constructor @@ -49,6 +51,8 @@ public DataVectorizer( this.nodeFactory = nodeFactory; this.embeddingCredentials = embeddingCredentials; this.schemaObject = schemaObject; + vectorConfig = + schemaObject.vectorConfigs().isEmpty() ? null : schemaObject.vectorConfigs().get(0); } /** @@ -114,7 +118,7 @@ public Uni vectorize(List documents) { if (vectorData.size() != vectorizeTexts.size()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' didn't return the expected number of embeddings. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), + vectorConfig.vectorizeConfig().provider(), vectorizeTexts.size(), vectorData.size()); } @@ -125,11 +129,11 @@ public Uni vectorize(List documents) { JsonNode document = documents.get(position); float[] vector = vectorData.get(vectorPosition); // check if all vectors have the expected size - if (vector.length != schemaObject.vectorConfig().vectorSize()) { + if (vector.length != vectorConfig.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), - schemaObject.vectorConfig().vectorSize(), + vectorConfig.vectorizeConfig().provider(), + vectorConfig.vectorSize(), vector.length); } final ArrayNode arrayNode = nodeFactory.arrayNode(vector.length); @@ -170,11 +174,11 @@ public Uni vectorize(String vectorizeContent) { vectorData -> { float[] vector = vectorData.get(0); // check if vector have the expected size - if (vector.length != schemaObject.vectorConfig().vectorSize()) { + if (vector.length != vectorConfig.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), - schemaObject.vectorConfig().vectorSize(), + vectorConfig.vectorizeConfig().provider(), + vectorConfig.vectorSize(), vector.length); } return vector; @@ -212,11 +216,11 @@ public Uni vectorize(SortClause sortClause) { vectorData -> { float[] vector = vectorData.get(0); // check if vector have the expected size - if (vector.length != schemaObject.vectorConfig().vectorSize()) { + if (vector.length != vectorConfig.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), - schemaObject.vectorConfig().vectorSize(), + vectorConfig.vectorizeConfig().provider(), + vectorConfig.vectorSize(), vector.length); } sortExpressions.clear(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index a2d2a581f..18e5a559e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -244,7 +244,7 @@ private Tags getCustomTags( errorCodeTag = Tag.of(jsonApiMetricsConfig.errorCode(), errorCode); } Tag vectorEnabled = - commandContext.schemaObject().vectorConfig().vectorEnabled() + commandContext.schemaObject().vectorConfigs().get(0).vectorEnabled() ? Tag.of(jsonApiMetricsConfig.vectorEnabled(), "true") : Tag.of(jsonApiMetricsConfig.vectorEnabled(), "false"); JsonApiMetricsConfig.SortType sortType = getVectorTypeTag(command); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java index 3c054d51c..01d001a1a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java @@ -17,6 +17,7 @@ import io.stargate.sgv2.jsonapi.service.cqldriver.executor.*; import io.stargate.sgv2.jsonapi.service.projection.IndexingProjector; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -35,16 +36,16 @@ public final class CollectionSchemaObject extends TableBasedSchemaObject { SchemaObjectName.MISSING, null, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), null); private final IdConfig idConfig; - private final VectorConfig vectorConfig; + private final List vectorConfigs; private final CollectionIndexingConfig indexingConfig; private final TableMetadata tableMetadata; /** - * @param vectorConfig + * @param vectorConfigs * @param indexingConfig */ public CollectionSchemaObject( @@ -52,13 +53,13 @@ public CollectionSchemaObject( String name, TableMetadata tableMetadata, IdConfig idConfig, - VectorConfig vectorConfig, + List vectorConfigs, CollectionIndexingConfig indexingConfig) { this( new SchemaObjectName(keypaceName, name), tableMetadata, idConfig, - vectorConfig, + vectorConfigs, indexingConfig); } @@ -66,12 +67,12 @@ public CollectionSchemaObject( SchemaObjectName name, TableMetadata tableMetadata, IdConfig idConfig, - VectorConfig vectorConfig, + List vectorConfigs, CollectionIndexingConfig indexingConfig) { super(TYPE, name, tableMetadata); this.idConfig = idConfig; - this.vectorConfig = vectorConfig; + this.vectorConfigs = vectorConfigs; this.indexingConfig = indexingConfig; this.tableMetadata = tableMetadata; } @@ -80,12 +81,12 @@ public CollectionSchemaObject( // effectively public CollectionSchemaObject withIdType(CollectionIdType idType) { return new CollectionSchemaObject( - name(), tableMetadata, new IdConfig(idType), vectorConfig, indexingConfig); + name(), tableMetadata, new IdConfig(idType), vectorConfigs, indexingConfig); } @Override - public VectorConfig vectorConfig() { - return vectorConfig; + public List vectorConfigs() { + return vectorConfigs; } @Override @@ -220,7 +221,7 @@ private static CollectionSchemaObject createCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - new VectorConfig(true, vectorSize, function, null), + List.of(new VectorConfig(true, "$vector", vectorSize, function, null)), null); } else { return new CollectionSchemaObject( @@ -228,7 +229,7 @@ private static CollectionSchemaObject createCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), null); } } else { @@ -272,6 +273,16 @@ private static CollectionSchemaObject createCollectionSettings( } } + // convert a vector jsonNode from table comment to vectorConfig, used for collection + private static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { + // dimension, similarityFunction, must exist + int dimension = jsonNode.get("dimension").asInt(); + SimilarityFunction similarityFunction = + SimilarityFunction.fromString(jsonNode.get("metric").asText()); + + return VectorConfig.fromJson("$vector", dimension, similarityFunction, jsonNode, objectMapper); + } + public static CreateCollectionCommand collectionSettingToCreateCollectionCommand( CollectionSchemaObject collectionSetting) { @@ -279,25 +290,25 @@ public static CreateCollectionCommand collectionSettingToCreateCollectionCommand CreateCollectionCommand.Options options = null; CreateCollectionCommand.Options.VectorSearchConfig vectorSearchConfig = null; CreateCollectionCommand.Options.IndexingConfig indexingConfig = null; - // populate the vectorSearchConfig - if (collectionSetting.vectorConfig().vectorEnabled()) { + // populate the vectorSearchConfig, Default will be the index 0 since there is only one vector + // column supported for collection + final VectorConfig vectorConfig = collectionSetting.vectorConfigs().get(0); + if (vectorConfig.vectorEnabled()) { VectorizeConfig vectorizeConfig = null; - if (collectionSetting.vectorConfig().vectorizeConfig() != null) { - Map authentication = - collectionSetting.vectorConfig().vectorizeConfig().authentication(); - Map parameters = - collectionSetting.vectorConfig().vectorizeConfig().parameters(); + if (vectorConfig.vectorizeConfig() != null) { + Map authentication = vectorConfig.vectorizeConfig().authentication(); + Map parameters = vectorConfig.vectorizeConfig().parameters(); vectorizeConfig = new VectorizeConfig( - collectionSetting.vectorConfig().vectorizeConfig().provider(), - collectionSetting.vectorConfig().vectorizeConfig().modelName(), + vectorConfig.vectorizeConfig().provider(), + vectorConfig.vectorizeConfig().modelName(), authentication == null ? null : Map.copyOf(authentication), parameters == null ? null : Map.copyOf(parameters)); } vectorSearchConfig = new CreateCollectionCommand.Options.VectorSearchConfig( - collectionSetting.vectorConfig().vectorSize(), - collectionSetting.vectorConfig().similarityFunction().name().toLowerCase(), + vectorConfig.vectorSize(), + vectorConfig.similarityFunction().name().toLowerCase(), vectorizeConfig); } // populate the indexingConfig @@ -331,11 +342,11 @@ public CollectionIndexingConfig indexingConfig() { // TODO: these helper functions break encapsulation for very little benefit public SimilarityFunction similarityFunction() { - return vectorConfig().similarityFunction(); + return vectorConfigs().get(0).similarityFunction(); } public boolean isVectorEnabled() { - return vectorConfig() != null && vectorConfig().vectorEnabled(); + return !(vectorConfigs().isEmpty()) && vectorConfigs().get(0).vectorEnabled(); } // TODO: the overrides below were auto added when migrating from a record to a class, not sure @@ -347,13 +358,13 @@ public boolean equals(Object obj) { var that = (CollectionSchemaObject) obj; return Objects.equals(this.name, that.name) && Objects.equals(this.idConfig, that.idConfig) - && Objects.equals(this.vectorConfig, that.vectorConfig) + && Objects.equals(this.vectorConfigs, that.vectorConfigs) && Objects.equals(this.indexingConfig, that.indexingConfig); } @Override public int hashCode() { - return Objects.hash(name, idConfig, vectorConfig, indexingConfig); + return Objects.hash(name, idConfig, vectorConfigs, indexingConfig); } @Override @@ -365,8 +376,8 @@ public String toString() { + "idConfig=" + idConfig + ", " - + "vectorConfig=" - + vectorConfig + + "vectorConfigs=" + + vectorConfigs + ", " + "indexingConfig=" + indexingConfig diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java index 0cf67b0be..59c6da015 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java @@ -6,6 +6,7 @@ import io.stargate.sgv2.jsonapi.config.constants.TableCommentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.List; /** * schema_version 0 is before we introduce schema_version into the C* table comment of data api @@ -26,7 +27,8 @@ public CollectionSchemaObject readCollectionSettings( int vectorSize, SimilarityFunction function) { - VectorConfig vectorConfig = new VectorConfig(vectorEnabled, vectorSize, function, null); + VectorConfig vectorConfig = + new VectorConfig(vectorEnabled, "$vector", vectorSize, function, null); CollectionIndexingConfig indexingConfig = null; JsonNode indexing = commentConfigNode.path(TableCommentConstants.COLLECTION_INDEXING_KEY); if (!indexing.isMissingNode()) { @@ -37,7 +39,7 @@ public CollectionSchemaObject readCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - vectorConfig, + List.of(vectorConfig), indexingConfig); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java index de0c3a7e6..1c6bd7bb5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.config.constants.TableCommentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; +import java.util.List; /** * schema_version 1 sample: @@ -44,6 +45,11 @@ public CollectionSchemaObject readCollectionSettings( } return new CollectionSchemaObject( - keyspaceName, collectionName, tableMetadata, idConfig, vectorConfig, indexingConfig); + keyspaceName, + collectionName, + tableMetadata, + idConfig, + List.of(vectorConfig), + indexingConfig); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java index f13ab7a04..a0bb02ff3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java @@ -6,6 +6,7 @@ import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; import io.stargate.sgv2.jsonapi.service.schema.collections.IdConfig; +import java.util.List; import org.apache.commons.lang3.RandomStringUtils; /** @@ -26,7 +27,7 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), null); public static final CollectionSchemaObject VECTOR_COLLECTION_SCHEMA_OBJECT = @@ -34,7 +35,7 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), null); public static final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java index 2e8402d72..9a968c28d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java @@ -13,6 +13,7 @@ import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import org.junit.jupiter.api.Test; @QuarkusTest @@ -30,7 +31,7 @@ public void ensureSingleProjectorCreation() { "collectionName", null, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), indexingConfig); IndexingProjector indexingProj = settings.indexingProjector(); assertThat(indexingProj) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java index 3bd97a2b4..fb6166965 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java @@ -164,7 +164,7 @@ public void checkValidJsonApiTable() { assertThat(schemaObject) .satisfies( s -> { - assertThat(s.vectorConfig().vectorEnabled()).isFalse(); + assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); }); } @@ -296,7 +296,7 @@ public void checkValidJsonApiTableWithIndexing() { assertThat(collectionSchemaObject) .satisfies( s -> { - assertThat(s.vectorConfig().vectorEnabled()).isFalse(); + assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); assertThat(s.indexingConfig().denied()).containsExactly("comment"); }); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java index 1ac981fc7..299945381 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java @@ -230,11 +230,13 @@ public void testWithUnmatchedVectorSize() { "collections", null, IdConfig.defaultIdConfig(), - new VectorConfig( - true, - 4, - SimilarityFunction.COSINE, - new VectorConfig.VectorizeConfig("custom", "custom", null, null)), + List.of( + new VectorConfig( + true, + "$vectorize", + 4, + SimilarityFunction.COSINE, + new VectorConfig.VectorizeConfig("custom", "custom", null, null))), null); List documents = new ArrayList<>(); for (int i = 0; i < 2; i++) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java index ac27f0f20..5e78ff3e4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java @@ -19,11 +19,13 @@ public class TestEmbeddingProvider extends EmbeddingProvider { TestConstants.SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig( - true, - 3, - SimilarityFunction.COSINE, - new VectorConfig.VectorizeConfig("custom", "custom", null, null)), + List.of( + new VectorConfig( + true, + "Svectorize", + 3, + SimilarityFunction.COSINE, + new VectorConfig.VectorizeConfig("custom", "custom", null, null))), null), new TestEmbeddingProvider(), "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java index 41e45f309..ba192f9d9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java @@ -82,7 +82,7 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java index 1ad31e4b2..b71d90ecf 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java @@ -98,7 +98,7 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java index ccb32d8e2..ff6f05157 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java @@ -46,7 +46,7 @@ public class OperationTestBase { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), null); protected final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = KeyspaceSchemaObject.fromSchemaObject(COLLECTION_SCHEMA_OBJECT); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java index f8a8e6d02..7e5b7648d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java @@ -117,7 +117,7 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java index 45cefb809..25a0a98ae 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java @@ -41,6 +41,7 @@ import io.stargate.sgv2.jsonapi.service.updater.DocumentUpdater; import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import jakarta.inject.Inject; +import java.util.List; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -83,7 +84,7 @@ class Resolve { new SchemaObjectName(KEYSPACE_NAME, COLLECTION_NAME), null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), null), null, null, From c016640069f71be89a592a55453584efe796cf11 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Sun, 6 Oct 2024 22:36:12 -0400 Subject: [PATCH 02/37] Changes for TableSchemaObject to deserialize vector and vectorize information --- .../cqldriver/executor/NamespaceCache.java | 2 +- .../cqldriver/executor/TableSchemaObject.java | 71 ++++++++++++++++++- .../resolver/CreateTableCommandResolver.java | 15 ++-- .../sgv2/jsonapi/fixtures/CqlFixture.java | 3 +- .../testdata/SchemaObjectTestData.java | 3 +- ...DefaultDriverExceptionHandlerTestData.java | 3 +- 6 files changed, 87 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java index 3e0439575..181aa0b31 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java @@ -104,7 +104,7 @@ private Uni loadSchemaObject( } // 04-Sep-2024, tatu: Used to check that API Tables enabled; no longer checked here - return new TableSchemaObject(table); + return TableSchemaObject.getTableSettings(table, objectMapper); }); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index b78f722c4..e4660ab0f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -1,23 +1,90 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.data.ByteUtils; +import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.IndexMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.api.core.type.VectorType; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; public class TableSchemaObject extends TableBasedSchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.TABLE; - public TableSchemaObject(TableMetadata tableMetadata) { + private final List vectorConfigs; + + private TableSchemaObject(TableMetadata tableMetadata) { super(TYPE, tableMetadata); + vectorConfigs = new ArrayList<>(); } @Override public List vectorConfigs() { - return List.of(VectorConfig.notEnabledVectorConfig()); + return vectorConfigs; } @Override public IndexUsage newIndexUsage() { return IndexUsage.NO_OP; } + + public static TableSchemaObject getTableSettings( + TableMetadata tableMetadata, ObjectMapper objectMapper) { + Map extensions = + (Map) + tableMetadata.getOptions().get(CqlIdentifier.fromInternal("extensions")); + String vectorize = extensions != null ? extensions.get("vectorize") : null; + Map resultMap = new HashMap<>(); + if (vectorize != null) { + String vectorizeJson = new String(ByteUtils.fromHexString(vectorize).array()); + // Define the TypeReference for Map + TypeReference> typeRef = + new TypeReference>() {}; + + // Convert JSON string to Map + try { + resultMap = objectMapper.readValue(vectorizeJson, typeRef); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + TableSchemaObject tableSchemaObject = new TableSchemaObject(tableMetadata); + + for (Map.Entry column : tableMetadata.getColumns().entrySet()) { + if (column.getValue().getType() instanceof VectorType vectorType) { + final Optional index = tableMetadata.getIndex(column.getKey()); + SimilarityFunction similarityFunction = SimilarityFunction.COSINE; + if (index.isPresent()) { + final IndexMetadata indexMetadata = index.get(); + final Map indexOptions = indexMetadata.getOptions(); + final String similarityFunctionValue = indexOptions.get("similarity_function"); + if (similarityFunctionValue != null) { + similarityFunction = SimilarityFunction.fromString(similarityFunctionValue); + } + } + int dimension = vectorType.getDimensions(); + VectorConfig vectorConfig = + new VectorConfig( + true, + column.getKey().asInternal(), + dimension, + similarityFunction, + resultMap.get(column.getKey().asInternal())); + tableSchemaObject.vectorConfigs.add(vectorConfig); + } + } + if (tableSchemaObject.vectorConfigs.isEmpty()) { + tableSchemaObject.vectorConfigs().add(VectorConfig.notEnabledVectorConfig()); + } + return tableSchemaObject; + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java index 7f25c46ed..b95e8100a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java @@ -11,6 +11,7 @@ import io.stargate.sgv2.jsonapi.config.OperationsConfig; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.operation.*; import io.stargate.sgv2.jsonapi.service.operation.Operation; import io.stargate.sgv2.jsonapi.service.operation.tables.CreateTableAttemptBuilder; @@ -39,7 +40,7 @@ public Operation resolveKeyspaceCommand( .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getApiDataType())); List partitionKeys = Arrays.stream(command.definition().primaryKey().keys()).toList(); - Map vectorizeConfigMap = + Map vectorizeConfigMap = command.definition().columns().entrySet().stream() .filter( e -> @@ -50,9 +51,15 @@ public Operation resolveKeyspaceCommand( Map.Entry::getKey, e -> { ComplexTypes.VectorType vectorType = ((ComplexTypes.VectorType) e.getValue()); - final VectorizeConfig vectorConfig = vectorType.getVectorConfig(); - validateVectorize.validateService(vectorConfig, vectorType.getDimension()); - return vectorConfig; + final VectorizeConfig vectorizeConfig = vectorType.getVectorConfig(); + validateVectorize.validateService(vectorizeConfig, vectorType.getDimension()); + VectorConfig.VectorizeConfig dbVectorConfig = + new VectorConfig.VectorizeConfig( + vectorizeConfig.provider(), + vectorizeConfig.modelName(), + vectorizeConfig.authentication(), + vectorizeConfig.parameters()); + return dbVectorConfig; })); if (partitionKeys.isEmpty()) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java index 28d0e3b66..0c3be13d6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.fixtures; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.fixtures.data.DefaultData; import io.stargate.sgv2.jsonapi.fixtures.data.FixtureData; import io.stargate.sgv2.jsonapi.fixtures.identifiers.BaseFixtureIdentifiers; @@ -62,7 +63,7 @@ public CqlFixture( this.cqlData = cqlData; this.tableFixture = tableFixture; this.tableMetadata = tableFixture.tableMetadata(identifiers); - this.tableSchemaObject = new TableSchemaObject(tableMetadata); + this.tableSchemaObject = TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); } public FixtureIdentifiers identifiers() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java index b13fde1ef..c25a1214f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.fixtures.testdata; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; public class SchemaObjectTestData extends TestDataSuplier { @@ -9,6 +10,6 @@ public SchemaObjectTestData(TestData testData) { } public TableSchemaObject emptyTableSchemaObject() { - return new TableSchemaObject(testData.tableMetadata().empty()); + return TableSchemaObject.getTableSettings(testData.tableMetadata().empty(), new ObjectMapper()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java index d80645866..5d7ec9331 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java @@ -2,6 +2,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.internal.core.metadata.schema.DefaultTableMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import java.util.UUID; @@ -32,6 +33,6 @@ public DefaultDriverExceptionHandlerTestData() { Map.of(), Map.of(), Map.of()); - TABLE_SCHEMA_OBJECT = new TableSchemaObject(tableMetadata); + TABLE_SCHEMA_OBJECT = TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); } } From d590679d92c4e03c41063d08dc7bd2eabc6dbd55 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:44:53 -0400 Subject: [PATCH 03/37] Changed the vector config to unmodifiable list --- .../cqldriver/executor/TableSchemaObject.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index e4660ab0f..4b0dcd289 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,9 +23,9 @@ public class TableSchemaObject extends TableBasedSchemaObject { private final List vectorConfigs; - private TableSchemaObject(TableMetadata tableMetadata) { + private TableSchemaObject(TableMetadata tableMetadata, List vectorConfigs) { super(TYPE, tableMetadata); - vectorConfigs = new ArrayList<>(); + this.vectorConfigs = vectorConfigs; } @Override @@ -37,6 +38,13 @@ public IndexUsage newIndexUsage() { return IndexUsage.NO_OP; } + /** + * Get table schema object from table metadata + * + * @param tableMetadata + * @param objectMapper + * @return + */ public static TableSchemaObject getTableSettings( TableMetadata tableMetadata, ObjectMapper objectMapper) { Map extensions = @@ -57,8 +65,8 @@ public static TableSchemaObject getTableSettings( throw new RuntimeException(e); } } - TableSchemaObject tableSchemaObject = new TableSchemaObject(tableMetadata); + List vectorConfigs = new ArrayList<>(); for (Map.Entry column : tableMetadata.getColumns().entrySet()) { if (column.getValue().getType() instanceof VectorType vectorType) { final Optional index = tableMetadata.getIndex(column.getKey()); @@ -79,12 +87,12 @@ public static TableSchemaObject getTableSettings( dimension, similarityFunction, resultMap.get(column.getKey().asInternal())); - tableSchemaObject.vectorConfigs.add(vectorConfig); + vectorConfigs.add(vectorConfig); } } - if (tableSchemaObject.vectorConfigs.isEmpty()) { - tableSchemaObject.vectorConfigs().add(VectorConfig.notEnabledVectorConfig()); + if (vectorConfigs.isEmpty()) { + vectorConfigs.add(VectorConfig.notEnabledVectorConfig()); } - return tableSchemaObject; + return new TableSchemaObject(tableMetadata, Collections.unmodifiableList(vectorConfigs)); } } From aa656efac9a9e526399d199f833350a348659776 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:46:55 -0400 Subject: [PATCH 04/37] Fixed the comments --- .../sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java index 8d90b3e50..13cd961b8 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java @@ -32,7 +32,7 @@ public SchemaObjectName name() { /** * Subclasses must always return List of VectorConfig, if there is no vector config they should - * return VectorConfig.notEnabledVectorConfig(). This needs to be a list because a table can have + * return VectorConfig.notEnabledVectorConfig(). This needs to be a list to support tables with * multiple vector columns. * * @return From 02bc38e6f5b51db752f90c74a97e4709bc0d71d1 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:50:21 -0400 Subject: [PATCH 05/37] Fixed the comments --- .../sgv2/jsonapi/service/embedding/DataVectorizer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java index 744e78ccc..3035b9843 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java @@ -51,8 +51,9 @@ public DataVectorizer( this.nodeFactory = nodeFactory; this.embeddingCredentials = embeddingCredentials; this.schemaObject = schemaObject; - vectorConfig = - schemaObject.vectorConfigs().isEmpty() ? null : schemaObject.vectorConfigs().get(0); + // This is getting element at 0 since only one vector is stored in the schema. + // This logic needs to be changed to handle multiple vectors columns for tables, + vectorConfig = schemaObject.vectorConfigs().get(0); } /** From 24da97553041fc318027c2b4439d7d1153b8245b Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:05:00 -0400 Subject: [PATCH 06/37] Use $vectorize field name from constant `DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD` --- .../cqldriver/executor/VectorConfig.java | 10 ++++++++-- .../processor/MeteredCommandProcessor.java | 8 +++++++- .../collections/CollectionSchemaObject.java | 17 ++++++++++++++--- .../collections/CollectionSettingsV0Reader.java | 8 +++++++- .../io/stargate/sgv2/jsonapi/TestConstants.java | 3 ++- .../embedding/operation/DataVectorizerTest.java | 3 ++- .../operation/TestEmbeddingProvider.java | 3 ++- .../FindCollectionOperationTest.java | 3 ++- .../InsertCollectionOperationTest.java | 3 ++- .../CommandResolverWithVectorizerTest.java | 3 ++- 10 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java index 103b087e0..1d6bd7ee9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import java.util.Map; @@ -23,7 +24,7 @@ public record VectorConfig( // TODO: this is an immutable record, this can be singleton // TODO: Remove the use of NULL for the objects like vectorizeConfig public static VectorConfig notEnabledVectorConfig() { - return new VectorConfig(false, "", -1, null, null); + return new VectorConfig(false, null, -1, null, null); } // convert a vector jsonNode from table comment to vectorConfig, used for collection @@ -33,7 +34,12 @@ public static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper SimilarityFunction similarityFunction = SimilarityFunction.fromString(jsonNode.get("metric").asText()); - return fromJson("$vectorize", dimension, similarityFunction, jsonNode, objectMapper); + return fromJson( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + dimension, + similarityFunction, + jsonNode, + objectMapper); } // convert a vector jsonNode from table extension to vectorConfig, used for tables diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 18e5a559e..4b7e1ca85 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -22,12 +22,14 @@ import io.stargate.sgv2.jsonapi.config.CommandLevelLoggingConfig; import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.SchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Produces; import jakarta.inject.Inject; import jakarta.inject.Singleton; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -243,8 +245,12 @@ private Tags getCustomTags( result.errors().get(0).fieldsForMetricsTag().getOrDefault("errorCode", UNKNOWN_VALUE); errorCodeTag = Tag.of(jsonApiMetricsConfig.errorCode(), errorCode); } + final Optional first = + commandContext.schemaObject().vectorConfigs().stream() + .filter(a -> a.vectorEnabled()) + .findFirst(); Tag vectorEnabled = - commandContext.schemaObject().vectorConfigs().get(0).vectorEnabled() + first.isPresent() ? Tag.of(jsonApiMetricsConfig.vectorEnabled(), "true") : Tag.of(jsonApiMetricsConfig.vectorEnabled(), "false"); JsonApiMetricsConfig.SortType sortType = getVectorTypeTag(command); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java index 01d001a1a..b4a66f573 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java @@ -221,7 +221,13 @@ private static CollectionSchemaObject createCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, "$vector", vectorSize, function, null)), + List.of( + new VectorConfig( + true, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + vectorSize, + function, + null)), null); } else { return new CollectionSchemaObject( @@ -273,14 +279,19 @@ private static CollectionSchemaObject createCollectionSettings( } } - // convert a vector jsonNode from table comment to vectorConfig, used for collection + // convert a vector jsonNode from cql table comment to vectorConfig, used for collection private static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { // dimension, similarityFunction, must exist int dimension = jsonNode.get("dimension").asInt(); SimilarityFunction similarityFunction = SimilarityFunction.fromString(jsonNode.get("metric").asText()); - return VectorConfig.fromJson("$vector", dimension, similarityFunction, jsonNode, objectMapper); + return VectorConfig.fromJson( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + dimension, + similarityFunction, + jsonNode, + objectMapper); } public static CreateCollectionCommand collectionSettingToCreateCollectionCommand( diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java index 59c6da015..eb4c76791 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java @@ -3,6 +3,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.config.constants.TableCommentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; @@ -28,7 +29,12 @@ public CollectionSchemaObject readCollectionSettings( SimilarityFunction function) { VectorConfig vectorConfig = - new VectorConfig(vectorEnabled, "$vector", vectorSize, function, null); + new VectorConfig( + vectorEnabled, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + vectorSize, + function, + null); CollectionIndexingConfig indexingConfig = null; JsonNode indexing = commentConfigNode.path(TableCommentConstants.COLLECTION_INDEXING_KEY); if (!indexing.isMissingNode()) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java index a0bb02ff3..aec1064f9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.config.feature.ApiFeatures; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.*; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; @@ -35,7 +36,7 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), + List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), null); public static final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java index 299945381..32f91221e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java @@ -13,6 +13,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortClause; import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortExpression; import io.stargate.sgv2.jsonapi.api.request.EmbeddingCredentials; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; @@ -233,7 +234,7 @@ public void testWithUnmatchedVectorSize() { List.of( new VectorConfig( true, - "$vectorize", + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, 4, SimilarityFunction.COSINE, new VectorConfig.VectorizeConfig("custom", "custom", null, null))), diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java index 5e78ff3e4..05c43c681 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java @@ -4,6 +4,7 @@ import io.stargate.sgv2.jsonapi.TestConstants; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.request.EmbeddingCredentials; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; @@ -22,7 +23,7 @@ public class TestEmbeddingProvider extends EmbeddingProvider { List.of( new VectorConfig( true, - "Svectorize", + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, 3, SimilarityFunction.COSINE, new VectorConfig.VectorizeConfig("custom", "custom", null, null))), diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java index ba192f9d9..5cab53449 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java @@ -25,6 +25,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.mappers.ThrowableToErrorMapper; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; @@ -82,7 +83,7 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), + List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java index b71d90ecf..1a7b8e981 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java @@ -20,6 +20,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; @@ -98,7 +99,7 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), + List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java index 25a0a98ae..d0a984f81 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java @@ -19,6 +19,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.impl.UpdateOneCommand; import io.stargate.sgv2.jsonapi.api.request.DataApiRequestInfo; import io.stargate.sgv2.jsonapi.config.OperationsConfig; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.config.feature.ApiFeatures; import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; @@ -84,7 +85,7 @@ class Resolve { new SchemaObjectName(KEYSPACE_NAME, COLLECTION_NAME), null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), + List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), null), null, null, From ecabca3b43b6e2c0d938e842b009d0cd079d7a39 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:05:58 -0400 Subject: [PATCH 07/37] Use $vectorize field name from constant `DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD` --- .../ReadAndUpdateCollectionOperationTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java index 7e5b7648d..ef9f326fc 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java @@ -23,6 +23,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateClause; import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateOperator; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.cqldriver.serializer.CQLBindValues; @@ -117,7 +118,13 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, "$vectorize", -1, SimilarityFunction.COSINE, null)), + List.of( + new VectorConfig( + true, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null)), null), null, "testCommand", From bc30acbfe03eee5b9759b050b69eca48ec8e181b Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:06:02 -0400 Subject: [PATCH 08/37] Use $vectorize field name from constant `DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD` --- src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java | 8 +++++++- .../service/embedding/operation/DataVectorizerTest.java | 2 +- .../embedding/operation/TestEmbeddingProvider.java | 2 +- .../collections/FindCollectionOperationTest.java | 8 +++++++- .../collections/InsertCollectionOperationTest.java | 8 +++++++- .../resolver/CommandResolverWithVectorizerTest.java | 8 +++++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java index aec1064f9..f24fe600d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java @@ -36,7 +36,13 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), + List.of( + new VectorConfig( + true, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null)), null); public static final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java index 32f91221e..9bdf8863d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java @@ -234,7 +234,7 @@ public void testWithUnmatchedVectorSize() { List.of( new VectorConfig( true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, 4, SimilarityFunction.COSINE, new VectorConfig.VectorizeConfig("custom", "custom", null, null))), diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java index 05c43c681..fa5353107 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java @@ -23,7 +23,7 @@ public class TestEmbeddingProvider extends EmbeddingProvider { List.of( new VectorConfig( true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, 3, SimilarityFunction.COSINE, new VectorConfig.VectorizeConfig("custom", "custom", null, null))), diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java index 5cab53449..e350d81bb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java @@ -83,7 +83,13 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), + List.of( + new VectorConfig( + true, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java index 1a7b8e981..9f4beb013 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java @@ -99,7 +99,13 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), + List.of( + new VectorConfig( + true, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null)), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java index d0a984f81..80c2c37dc 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java @@ -85,7 +85,13 @@ class Resolve { new SchemaObjectName(KEYSPACE_NAME, COLLECTION_NAME), null, IdConfig.defaultIdConfig(), - List.of(new VectorConfig(true, DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, -1, SimilarityFunction.COSINE, null)), + List.of( + new VectorConfig( + true, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null)), null), null, null, From f0cb7bdc61271e478f5db216d7e2337a95c8ad3a Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:48:16 -0400 Subject: [PATCH 09/37] Backup commit for list tables --- .../jsonapi/api/model/command/Command.java | 1 + .../ColumnDefinitionSerializer.java | 36 ++++ .../model/command/impl/ListTablesCommand.java | 23 +++ .../command/table/definition/PrimaryKey.java | 10 +- .../table/definition/datatype/ColumnType.java | 45 +---- .../definition/datatype/ComplexTypes.java | 36 ++++ .../definition/datatype/PrimitiveTypes.java | 167 ++++-------------- .../cqldriver/executor/TableSchemaObject.java | 94 ++++++++++ .../operation/tables/ListTablesOperation.java | 113 ++++++++++++ .../service/schema/tables/ApiDataTypeDef.java | 13 -- .../schema/tables/ComplexApiDataType.java | 19 +- 11 files changed, 363 insertions(+), 194 deletions(-) create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java index 7640337c8..bc18ad581 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java @@ -64,6 +64,7 @@ enum CommandName { FIND_ONE("findOne"), INSERT_MANY("insertMany"), INSERT_ONE("insertOne"), + LIST_TABLES("listTables"), UPDATE_MANY("updateMany"), UPDATE_ONE("updateOne"), BEGIN_OFFLINE_SESSION("beginOfflineSession"), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java new file mode 100644 index 000000000..3d7c9994f --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java @@ -0,0 +1,36 @@ +package io.stargate.sgv2.jsonapi.api.model.command.deserializers; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ColumnType; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ComplexTypes; +import java.io.IOException; + +/** + * Custom serializer to encode the column type to the JSON payload This is required because + * composite and custom column types may need additional properties to be serialized + */ +public class ColumnDefinitionSerializer extends JsonSerializer { + + @Override + public void serialize( + ColumnType columnType, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeStringField("type", columnType.name()); + if (columnType instanceof ComplexTypes.MapType mt) { + jsonGenerator.writeStringField("keyType", mt.keyType()); + jsonGenerator.writeStringField("valueType", mt.valueType()); + } else if (columnType instanceof ComplexTypes.ListType lt) { + jsonGenerator.writeStringField("valueType", lt.valueType()); + } else if (columnType instanceof ComplexTypes.SetType st) { + jsonGenerator.writeStringField("st", lt.valueType()); + } else if (columnType instanceof ComplexTypes.VectorType vt) { + jsonGenerator.writeNumberField("dimension", vectorType.dimension()); + if (vectorType.getVectorConfig() != null) + jsonGenerator.writeObjectField("service", vt.getVectorConfig()); + } + jsonGenerator.writeEndObject(); + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java new file mode 100644 index 000000000..b1ae2f8d1 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java @@ -0,0 +1,23 @@ +package io.stargate.sgv2.jsonapi.api.model.command.impl; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.stargate.sgv2.jsonapi.api.model.command.CollectionOnlyCommand; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.media.Schema; + +@Schema(description = "Command that lists all available tables in a namespace.") +@JsonTypeName("listTables") +public record ListTablesCommand(Options options) implements CollectionOnlyCommand { + public record Options( + @Schema( + description = "include table properties.", + type = SchemaType.BOOLEAN, + implementation = Boolean.class) + boolean explain) {} + + /** {@inheritDoc} */ + @Override + public CommandName commandName() { + return CommandName.LIST_TABLES; + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java index abfd0d14b..a96881e2d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.api.model.command.table.definition; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.PrimaryKeyDeserializer; @@ -16,9 +17,14 @@ // implementation = Object.class, // description = "Represents the table primary key") public record PrimaryKey( - @NotNull @Schema(description = "Columns that make the partition keys", type = SchemaType.ARRAY) + @NotNull + @Schema(description = "Columns that make the partition keys", type = SchemaType.ARRAY) + @JsonProperty("partitionBy") + @JsonInclude(JsonInclude.Include.NON_NULL) String[] keys, - @Nullable @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) + @Nullable + @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) + @JsonProperty("partitionSort") OrderingKey[] orderingKeys) { public record OrderingKey(String column, Order order) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index d2da91411..4c0e7c318 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -1,7 +1,9 @@ package io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionDeserializer; +import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionSerializer; import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; @@ -10,10 +12,13 @@ /** Interface for column types. This is used to define the type of a column in a table. */ @JsonDeserialize(using = ColumnDefinitionDeserializer.class) +@JsonSerialize(using = ColumnDefinitionSerializer.class) public interface ColumnType { // Returns api data type. ApiDataType getApiDataType(); + public String name(); + static List getSupportedTypes() { return List.of( "ascii", @@ -46,42 +51,6 @@ static ColumnType fromString( // TODO: the name of the type should be a part of the ColumnType interface, and use a map for // the lookup switch (type) { - case "ascii": - return PrimitiveTypes.ASCII; - case "bigint": - return PrimitiveTypes.BIGINT; - case "blob": - return PrimitiveTypes.BINARY; - case "boolean": - return PrimitiveTypes.BOOLEAN; - case "date": - return PrimitiveTypes.DATE; - case "decimal": - return PrimitiveTypes.DECIMAL; - case "double": - return PrimitiveTypes.DOUBLE; - case "duration": - return PrimitiveTypes.DURATION; - case "float": - return PrimitiveTypes.FLOAT; - case "inet": - return PrimitiveTypes.INET; - case "int": - return PrimitiveTypes.INT; - case "smallint": - return PrimitiveTypes.SMALLINT; - case "text": - return PrimitiveTypes.TEXT; - case "time": - return PrimitiveTypes.TIME; - case "timestamp": - return PrimitiveTypes.TIMESTAMP; - case "tinyint": - return PrimitiveTypes.TINYINT; - case "uuid": - return PrimitiveTypes.UUID; - case "varint": - return PrimitiveTypes.VARINT; case "map": { if (keyType == null || valueType == null) { @@ -134,6 +103,10 @@ static ColumnType fromString( } default: { + ColumnType columnType = PrimitiveTypes.fromString(type); + if (columnType != null) { + return columnType; + } Map errorMessageFormattingValues = Map.of( "type", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index 4bb640961..b0db01137 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -24,6 +24,19 @@ public ApiDataType getApiDataType() { (PrimitiveApiDataType) keyType.getApiDataType(), (PrimitiveApiDataType) valueType.getApiDataType()); } + + @Override + public String name() { + return getApiDataType().getApiName(); + } + + public String keyType() { + return keyType.getApiDataType().getApiName(); + } + + public String valueType() { + return valueType.getApiDataType().getApiName(); + } } /** A list type implementation */ @@ -38,6 +51,15 @@ public ListType(ColumnType valueType) { public ApiDataType getApiDataType() { return new ComplexApiDataType.ListType((PrimitiveApiDataType) valueType.getApiDataType()); } + + @Override + public String name() { + return "list"; + } + + public String valueType() { + return valueType.getApiDataType().getApiName(); + } } /** A set type implementation */ @@ -52,6 +74,15 @@ public SetType(ColumnType valueType) { public ApiDataType getApiDataType() { return new ComplexApiDataType.SetType((PrimitiveApiDataType) valueType.getApiDataType()); } + + @Override + public String name() { + return "set"; + } + + public String valueType() { + return valueType.getApiDataType().getApiName(); + } } /* Vector type */ @@ -77,6 +108,11 @@ public VectorizeConfig getVectorConfig() { return vectorConfig; } + @Override + public String name() { + return "vector"; + } + public int getDimension() { return vectorSize; } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java index 630bb798c..e839de4a9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java @@ -2,154 +2,57 @@ import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; import io.stargate.sgv2.jsonapi.service.schema.tables.PrimitiveApiDataType; +import java.util.HashMap; +import java.util.Map; /** Interface for primitive column types similar to what is defined in cassandra java driver. */ -public class PrimitiveTypes { +public enum PrimitiveTypes implements ColumnType { // TODO: add a private ctor to stop this class from being instantiated or make abstract - public static final ColumnType ASCII = new Ascii(); - public static final ColumnType BIGINT = new BigInt(); - public static final ColumnType BINARY = new Binary(); - public static final ColumnType BOOLEAN = new Boolean(); - public static final ColumnType DATE = new Date(); - public static final ColumnType DECIMAL = new Decimal(); - public static final ColumnType DOUBLE = new Double(); - public static final ColumnType DURATION = new Duration(); - public static final ColumnType FLOAT = new Float(); - public static final ColumnType INET = new Inet(); - public static final ColumnType INT = new Int(); - public static final ColumnType SMALLINT = new SmallInt(); - public static final ColumnType TEXT = new Text(); - public static final ColumnType TIME = new Time(); - public static final ColumnType TIMESTAMP = new Timestamp(); - public static final ColumnType TINYINT = new TinyInt(); - public static final ColumnType UUID = new Uuid(); - public static final ColumnType VARINT = new VarInt(); + ASCII(PrimitiveApiDataType.ASCII), + BIGINT(PrimitiveApiDataType.BIGINT), + BINARY(PrimitiveApiDataType.BINARY), + BOOLEAN(PrimitiveApiDataType.BOOLEAN), + DATE(PrimitiveApiDataType.DATE), + DECIMAL(PrimitiveApiDataType.DECIMAL), + DOUBLE(PrimitiveApiDataType.DOUBLE), + DURATION(PrimitiveApiDataType.DURATION), + FLOAT(PrimitiveApiDataType.FLOAT), + INET(PrimitiveApiDataType.INET), + INT(PrimitiveApiDataType.INT), + SMALLINT(PrimitiveApiDataType.SMALLINT), + TEXT(PrimitiveApiDataType.TEXT), + TIME(PrimitiveApiDataType.TIME), + TIMESTAMP(PrimitiveApiDataType.TIMESTAMP), + TINYINT(PrimitiveApiDataType.TINYINT), + UUID(PrimitiveApiDataType.UUID), + VARINT(PrimitiveApiDataType.VARINT); - private static class Text implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TEXT; - } + @Override + public ApiDataType getApiDataType() { + return getApiDataType; } - private static class Int implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.INT; - } + PrimitiveTypes(ApiDataType getApiDataType) { + this.getApiDataType = getApiDataType; } - private static class Boolean implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BOOLEAN; - } - } + private ApiDataType getApiDataType; - private static class BigInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BIGINT; - } - } + private static Map columnTypeMap = new HashMap<>(); - private static class Decimal implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DECIMAL; + static { + for (PrimitiveTypes type : PrimitiveTypes.values()) { + columnTypeMap.put(type.getApiDataType().getApiName(), type); } } - private static class Double implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DOUBLE; - } + public static ColumnType fromString(String type) { + return columnTypeMap.get(type); } - private static class Float implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.FLOAT; - } - } - - private static class SmallInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.SMALLINT; - } - } - - private static class TinyInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TINYINT; - } - } - - private static class VarInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.VARINT; - } - } - - private static class Ascii implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.ASCII; - } - } - - private static class Binary implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BINARY; - } - } - - private static class Date implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DATE; - } - } - - private static class Duration implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DURATION; - } - } - - private static class Time implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TIME; - } - } - - private static class Timestamp implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TIMESTAMP; - } - } - - private static class Inet implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.INET; - } - } - - private static class Uuid implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.UUID; - } + public static String typeAsString(ColumnType type) { + return type.getApiDataType().getApiName(); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 4b0dcd289..7768bd8a2 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -2,20 +2,32 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.data.ByteUtils; +import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; import com.datastax.oss.driver.api.core.metadata.schema.IndexMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.api.core.type.MapType; import com.datastax.oss.driver.api.core.type.VectorType; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.PrimaryKey; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ColumnType; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ComplexTypes; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.PrimitiveTypes; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDef; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDefs; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; public class TableSchemaObject extends TableBasedSchemaObject { @@ -95,4 +107,86 @@ public static TableSchemaObject getTableSettings( } return new TableSchemaObject(tableMetadata, Collections.unmodifiableList(vectorConfigs)); } + + public TableResponse toTableResponse() { + String tableName = name().table(); + HashMap columnsDefinition = new HashMap<>(); + for (Map.Entry column : + tableMetadata().getColumns().entrySet()) { + ColumnType type = getColumnType(column.getKey().asInternal(), column.getValue()); + columnsDefinition.put(column.getKey().asInternal(), type); + } + + final List partitionBy = + tableMetadata().getPartitionKey().stream() + .map(column -> column.getName().asInternal()) + .collect(Collectors.toList()); + final List partitionSort = + tableMetadata().getClusteringColumns().entrySet().stream() + .map( + entry -> + new PrimaryKey.OrderingKey( + entry.getKey().getName().asInternal(), + entry.getValue() == ClusteringOrder.ASC + ? PrimaryKey.OrderingKey.Order.ASC + : PrimaryKey.OrderingKey.Order.DESC)) + .collect(Collectors.toList()); + PrimaryKey primaryKey = + new PrimaryKey( + partitionBy.toArray(new String[0]), + partitionSort.toArray(new PrimaryKey.OrderingKey[0])); + return new TableResponse( + tableName, + new TableResponse.TableDefinition( + new TableResponse.TableDefinition.ColumnsDefinition(columnsDefinition), primaryKey)); + } + + // Need to handle frozen types + private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadata) { + if (columnMetadata.getType() instanceof VectorType vt) { + // Schema will always have VectorConfig for vector type + VectorConfig vectorConfig = + vectorConfigs.stream().filter(vc -> vc.fieldName().equals(columnName)).findFirst().get(); + VectorizeConfig vectorizeConfig = + vectorConfig.vectorizeConfig() == null + ? null + : new VectorizeConfig( + vectorConfig.vectorizeConfig().provider(), + vectorConfig.vectorizeConfig().modelName(), + vectorConfig.vectorizeConfig().authentication(), + vectorConfig.vectorizeConfig().parameters()); + return new ComplexTypes.VectorType(PrimitiveTypes.FLOAT, vt.getDimensions(), vectorizeConfig); + } else if (columnMetadata.getType() instanceof MapType mt) { + return new ComplexTypes.MapType( + PrimitiveTypes.fromString(mt.getKeyType().toString()), + PrimitiveTypes.fromString(mt.getValueType().toString())); + } else if (columnMetadata.getType() + instanceof com.datastax.oss.driver.api.core.type.ListType lt) { + return new ComplexTypes.ListType(PrimitiveTypes.fromString(lt.getElementType().toString())); + } else if (columnMetadata.getType() + instanceof com.datastax.oss.driver.api.core.type.SetType st) { + return new ComplexTypes.SetType(PrimitiveTypes.fromString(st.getElementType().toString())); + } else { + final Optional apiDataTypeDef = + ApiDataTypeDefs.from(columnMetadata.getType()); + if (apiDataTypeDef.isPresent()) + return PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName()); + else { + // Need to return unsupported type + throw new RuntimeException("Unknown data type: " + columnMetadata.getType()); + } + } + } + + @JsonPropertyOrder({"_id", "definition"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record TableResponse(String name, TableDefinition tableDefinition) { + + @JsonPropertyOrder({"columns", "primaryKey"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + record TableDefinition(ColumnsDefinition columns, PrimaryKey primaryKey) { + + record ColumnsDefinition(Map columns) {} + } + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java new file mode 100644 index 000000000..f6e918bc6 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -0,0 +1,113 @@ +package io.stargate.sgv2.jsonapi.service.operation.tables; + +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; +import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.api.request.DataApiRequestInfo; +import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; +import io.stargate.sgv2.jsonapi.service.operation.Operation; +import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * List tables operation. Uses {@link CQLSessionCache} to fetch all valid tables for a namespace. + * The schema check against the table is done in the {@link CollectionTableMatcher} and ignores + * them. + * + * @param explain - returns collection options if `true`; returns only collection names if `false` + * @param objectMapper {@link ObjectMapper} + * @param cqlSessionCache {@link CQLSessionCache} + * @param tableMatcher {@link CollectionTableMatcher} + * @param commandContext {@link CommandContext} + */ +public record ListTablesOperation( + boolean explain, + ObjectMapper objectMapper, + CQLSessionCache cqlSessionCache, + CollectionTableMatcher tableMatcher, + CommandContext commandContext) + implements Operation { + + // shared table matcher instance + // TODO: if this is static why does the record that have an instance variable passed by the ctor + // below ? + private static final CollectionTableMatcher TABLE_MATCHER = new CollectionTableMatcher(); + + public ListTablesOperation( + boolean explain, + ObjectMapper objectMapper, + CQLSessionCache cqlSessionCache, + CommandContext commandContext) { + this(explain, objectMapper, cqlSessionCache, TABLE_MATCHER, commandContext); + } + + /** {@inheritDoc} */ + @Override + public Uni> execute( + DataApiRequestInfo dataApiRequestInfo, QueryExecutor queryExecutor) { + KeyspaceMetadata keyspaceMetadata = + cqlSessionCache + .getSession(dataApiRequestInfo) + .getMetadata() + .getKeyspaces() + .get(CqlIdentifier.fromInternal(commandContext.schemaObject().name().keyspace())); + if (keyspaceMetadata == null) { + return Uni.createFrom() + .failure( + ErrorCodeV1.KEYSPACE_DOES_NOT_EXIST.toApiException( + "Unknown keyspace '%s', you must create it first", + commandContext.schemaObject().name().keyspace())); + } + return Uni.createFrom() + .item( + () -> { + List properties = + keyspaceMetadata + // get all tables + .getTables() + .values() + .stream() + // filter for valid collections + .filter(tableMatcher.negate()) + // map to name + .map(table -> TableSchemaObject.getTableSettings(table, objectMapper)) + // get as list + .toList(); + // Wrap the properties list into a command result + return new Result(explain, properties); + }); + } + + // simple result wrapper + private record Result(boolean explain, List tables) + implements Supplier { + + @Override + public CommandResult get() { + if (explain) { + final List createCollectionCommands = + tables().stream() + .map(tableSchemaObject -> tableSchemaObject.toTableResponse()) + .toList(); + Map statuses = + Map.of(CommandStatus.EXISTING_COLLECTIONS, createCollectionCommands); + return new CommandResult(statuses); + } else { + List tables = + tables().stream().map(schemaObject -> schemaObject.name().table()).toList(); + Map statuses = Map.of(CommandStatus.EXISTING_COLLECTIONS, tables); + return new CommandResult(statuses); + } + } + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java index 4bbd6f0d3..c3d02b5a5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java @@ -3,7 +3,6 @@ import com.datastax.oss.driver.api.core.type.*; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.Objects; -import java.util.Optional; /** * The definition of a type the API supports for a table column. @@ -42,18 +41,6 @@ public boolean isContainer() { return cqlType instanceof ContainerType; } - public Optional cqlTypeAsList() { - return cqlType instanceof ListType listType ? Optional.of(listType) : Optional.empty(); - } - - public Optional cqlTypeAsSet() { - return cqlType instanceof SetType setType ? Optional.of(setType) : Optional.empty(); - } - - public Optional cqlTypeAsMap() { - return cqlType instanceof MapType mapType ? Optional.of(mapType) : Optional.empty(); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java index a27eb0418..28afa0397 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java @@ -9,17 +9,14 @@ public abstract class ComplexApiDataType implements ApiDataType { private final String apiName; private final PrimitiveApiDataType keyType; private final PrimitiveApiDataType valueType; - private final int vectorSize; + private final int dimension; public ComplexApiDataType( - String apiName, - PrimitiveApiDataType keyType, - PrimitiveApiDataType valueType, - int vectorSize) { + String apiName, PrimitiveApiDataType keyType, PrimitiveApiDataType valueType, int dimension) { this.apiName = apiName; this.keyType = keyType; this.valueType = valueType; - this.vectorSize = vectorSize; + this.dimension = dimension; } public PrimitiveApiDataType getKeyType() { @@ -30,8 +27,8 @@ public PrimitiveApiDataType getValueType() { return valueType; } - public int getVectorSize() { - return vectorSize; + public int getDimension() { + return dimension; } public abstract DataType getCqlType(); @@ -77,14 +74,14 @@ public DataType getCqlType() { } public static class VectorType extends ComplexApiDataType { - public VectorType(PrimitiveApiDataType valueType, int vectorSize) { - super("vector", null, valueType, vectorSize); + public VectorType(PrimitiveApiDataType valueType, int dimension) { + super("vector", null, valueType, dimension); } @Override public DataType getCqlType() { return new ExtendedVectorType( - ApiDataTypeDefs.from(getValueType()).get().getCqlType(), getVectorSize()); + ApiDataTypeDefs.from(getValueType()).get().getCqlType(), getDimension()); } } From 550ce05eabea8922fb721b646d6e8233d708fedd Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:22:48 -0400 Subject: [PATCH 10/37] Updated the code based on review. --- .../jsonapi/api/v1/CollectionResource.java | 13 +- .../jsonapi/exception/SchemaException.java | 1 + .../executor/DatabaseSchemaObject.java | 6 +- .../executor/KeyspaceSchemaObject.java | 5 +- .../cqldriver/executor/SchemaObject.java | 9 +- .../cqldriver/executor/TableSchemaObject.java | 59 ++++---- .../cqldriver/executor/VectorConfig.java | 141 ++++++++++-------- .../service/embedding/DataVectorizer.java | 37 +++-- .../processor/MeteredCommandProcessor.java | 9 +- .../resolver/CreateTableCommandResolver.java | 6 +- .../collections/CollectionSchemaObject.java | 73 ++++----- .../CollectionSettingsV0Reader.java | 12 +- .../CollectionSettingsV1Reader.java | 11 +- src/main/resources/errors.yaml | 6 + .../stargate/sgv2/jsonapi/TestConstants.java | 17 ++- .../CollectionSchemaObjectTest.java | 3 +- .../executor/NamespaceCacheTest.java | 4 +- .../operation/DataVectorizerTest.java | 16 +- .../operation/TestEmbeddingProvider.java | 16 +- .../FindCollectionOperationTest.java | 15 +- .../InsertCollectionOperationTest.java | 15 +- .../collections/OperationTestBase.java | 2 +- .../ReadAndUpdateCollectionOperationTest.java | 15 +- .../CommandResolverWithVectorizerTest.java | 15 +- 24 files changed, 279 insertions(+), 227 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java index 08c8c09c4..a378cbc94 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java @@ -197,8 +197,15 @@ public Uni> postCommand( } // TODO: refactor this code to be cleaner so it assigns on one line EmbeddingProvider embeddingProvider = null; - final VectorConfig.VectorizeConfig vectorizeConfig = - schemaObject.vectorConfigs().get(0).vectorizeConfig(); + VectorConfig vectorConfig = schemaObject.vectorConfig(); + final VectorConfig.ColumnVectorDefinition columnVectorDefinition = + vectorConfig.columnVectorDefinitions().isEmpty() + ? null + : vectorConfig.columnVectorDefinitions().get(0); + final VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = + columnVectorDefinition != null + ? columnVectorDefinition.vectorizeConfig() + : null; if (vectorizeConfig != null) { embeddingProvider = embeddingProviderFactory.getConfiguration( @@ -206,7 +213,7 @@ public Uni> postCommand( dataApiRequestInfo.getCassandraToken(), vectorizeConfig.provider(), vectorizeConfig.modelName(), - schemaObject.vectorConfigs().get(0).vectorSize(), + columnVectorDefinition.vectorSize(), vectorizeConfig.parameters(), vectorizeConfig.authentication(), command.getClass().getSimpleName()); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java b/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java index b2a5355c3..1f459321b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java @@ -17,6 +17,7 @@ public enum Code implements ErrorCode { COLUMN_DEFINITION_MISSING, COLUMN_TYPE_INCORRECT, COLUMN_TYPE_UNSUPPORTED, + INVALID_VECTORIZE_CONFIGURATION, LIST_TYPE_INCORRECT_DEFINITION, MAP_TYPE_INCORRECT_DEFINITION, MISSING_PRIMARY_KEYS, diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java index 071e90375..600f6106a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; -import java.util.List; - public class DatabaseSchemaObject extends SchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.DATABASE; @@ -11,8 +9,8 @@ public DatabaseSchemaObject() { } @Override - public List vectorConfigs() { - return List.of(VectorConfig.notEnabledVectorConfig()); + public VectorConfig vectorConfig() { + return VectorConfig.notEnabledVectorConfig(); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java index 08176a76e..b0ea4f89f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java @@ -1,7 +1,6 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; -import java.util.List; public class KeyspaceSchemaObject extends SchemaObject { @@ -40,8 +39,8 @@ public static KeyspaceSchemaObject fromSchemaObject(TableSchemaObject table) { } @Override - public List vectorConfigs() { - return List.of(VectorConfig.notEnabledVectorConfig()); + public VectorConfig vectorConfig() { + return VectorConfig.notEnabledVectorConfig(); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java index 13cd961b8..239a67d73 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; -import java.util.List; - /** A Collection or Table the command works on */ public abstract class SchemaObject { @@ -31,13 +29,12 @@ public SchemaObjectName name() { } /** - * Subclasses must always return List of VectorConfig, if there is no vector config they should - * return VectorConfig.notEnabledVectorConfig(). This needs to be a list to support tables with - * multiple vector columns. + * Subclasses must always return VectorConfig, if there is no vector config they should return + * VectorConfig.notEnabledVectorConfig(). * * @return */ - public abstract List vectorConfigs(); + public abstract VectorConfig vectorConfig(); /** * Call to get an instance of the appropriate {@link IndexUsage} for this schema object diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 4b0dcd289..43604d87a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -7,9 +7,11 @@ import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.core.type.VectorType; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -21,16 +23,16 @@ public class TableSchemaObject extends TableBasedSchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.TABLE; - private final List vectorConfigs; + private final VectorConfig vectorConfig; - private TableSchemaObject(TableMetadata tableMetadata, List vectorConfigs) { + private TableSchemaObject(TableMetadata tableMetadata, VectorConfig vectorConfig) { super(TYPE, tableMetadata); - this.vectorConfigs = vectorConfigs; + this.vectorConfig = vectorConfig; } @Override - public List vectorConfigs() { - return vectorConfigs; + public VectorConfig vectorConfig() { + return vectorConfig; } @Override @@ -51,22 +53,28 @@ public static TableSchemaObject getTableSettings( (Map) tableMetadata.getOptions().get(CqlIdentifier.fromInternal("extensions")); String vectorize = extensions != null ? extensions.get("vectorize") : null; - Map resultMap = new HashMap<>(); + Map resultMap = new HashMap<>(); if (vectorize != null) { - String vectorizeJson = new String(ByteUtils.fromHexString(vectorize).array()); - // Define the TypeReference for Map - TypeReference> typeRef = - new TypeReference>() {}; - - // Convert JSON string to Map try { - resultMap = objectMapper.readValue(vectorizeJson, typeRef); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + String vectorizeJson = + new String(ByteUtils.fromHexString(vectorize).array(), StandardCharsets.UTF_8); + // Convert JSON string to Map + JsonNode vectorizeByColumns = objectMapper.readTree(vectorizeJson); + Map vectorizeConfigMap = + new HashMap<>(); + while (vectorizeByColumns.fields().hasNext()) { + Map.Entry entry = vectorizeByColumns.fields().next(); + VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = + objectMapper.treeToValue( + entry.getValue(), VectorConfig.ColumnVectorDefinition.VectorizeConfig.class); + vectorizeConfigMap.put(entry.getKey(), vectorizeConfig); + } + } catch (JsonProcessingException | IllegalArgumentException e) { + throw SchemaException.Code.INVALID_VECTORIZE_CONFIGURATION.get(); } } - - List vectorConfigs = new ArrayList<>(); + VectorConfig vectorConfig; + List columnVectorDefinitions = new ArrayList<>(); for (Map.Entry column : tableMetadata.getColumns().entrySet()) { if (column.getValue().getType() instanceof VectorType vectorType) { final Optional index = tableMetadata.getIndex(column.getKey()); @@ -80,19 +88,20 @@ public static TableSchemaObject getTableSettings( } } int dimension = vectorType.getDimensions(); - VectorConfig vectorConfig = - new VectorConfig( - true, + VectorConfig.ColumnVectorDefinition columnVectorDefinition = + new VectorConfig.ColumnVectorDefinition( column.getKey().asInternal(), dimension, similarityFunction, resultMap.get(column.getKey().asInternal())); - vectorConfigs.add(vectorConfig); + columnVectorDefinitions.add(columnVectorDefinition); } } - if (vectorConfigs.isEmpty()) { - vectorConfigs.add(VectorConfig.notEnabledVectorConfig()); + if (columnVectorDefinitions.isEmpty()) { + vectorConfig = VectorConfig.notEnabledVectorConfig(); + } else { + vectorConfig = new VectorConfig(true, Collections.unmodifiableList(columnVectorDefinitions)); } - return new TableSchemaObject(tableMetadata, Collections.unmodifiableList(vectorConfigs)); + return new TableSchemaObject(tableMetadata, vectorConfig); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java index 1d6bd7ee9..4ffd447de 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java @@ -4,85 +4,104 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.List; import java.util.Map; /** * incorporates vectorizeConfig into vectorConfig * - * @param vectorEnabled - * @param vectorSize - * @param similarityFunction - * @param vectorizeConfig + * @param vectorEnabled - IF vector field is available for the table + * @param columnVectorDefinitions - List of columnVectorDefinitions each with respect to a + * column/field */ public record VectorConfig( - boolean vectorEnabled, - String fieldName, - int vectorSize, - SimilarityFunction similarityFunction, - VectorizeConfig vectorizeConfig) { + boolean vectorEnabled, List columnVectorDefinitions) { // TODO: this is an immutable record, this can be singleton // TODO: Remove the use of NULL for the objects like vectorizeConfig public static VectorConfig notEnabledVectorConfig() { - return new VectorConfig(false, null, -1, null, null); + return new VectorConfig(false, null); } - // convert a vector jsonNode from table comment to vectorConfig, used for collection - public static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { - // dimension, similarityFunction, must exist - int dimension = jsonNode.get("dimension").asInt(); - SimilarityFunction similarityFunction = - SimilarityFunction.fromString(jsonNode.get("metric").asText()); - - return fromJson( - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - dimension, - similarityFunction, - jsonNode, - objectMapper); - } - - // convert a vector jsonNode from table extension to vectorConfig, used for tables - public static VectorConfig fromJson( + /** + * Configuration for a column, In case of collection this will be of size one + * + * @param fieldName + * @param vectorSize + * @param similarityFunction + * @param vectorizeConfig + */ + public record ColumnVectorDefinition( String fieldName, - int dimension, + int vectorSize, SimilarityFunction similarityFunction, - JsonNode jsonNode, - ObjectMapper objectMapper) { - VectorizeConfig vectorizeConfig = null; - // construct vectorizeConfig - JsonNode vectorizeServiceNode = jsonNode.get("service"); - if (vectorizeServiceNode != null) { - vectorizeConfig = VectorizeConfig.fromJson(vectorizeServiceNode, objectMapper); + VectorizeConfig vectorizeConfig) { + + // convert a vector jsonNode from comment option to vectorConfig, used for collection + public static ColumnVectorDefinition fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { + // dimension, similarityFunction, must exist + int dimension = jsonNode.get("dimension").asInt(); + SimilarityFunction similarityFunction = + SimilarityFunction.fromString(jsonNode.get("metric").asText()); + + return fromJson( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + dimension, + similarityFunction, + jsonNode, + objectMapper); + } + + // convert a vector jsonNode from table extension to vectorConfig, used for tables + public static ColumnVectorDefinition fromJson( + String fieldName, + int dimension, + SimilarityFunction similarityFunction, + JsonNode jsonNode, + ObjectMapper objectMapper) { + VectorizeConfig vectorizeConfig = null; + // construct vectorizeConfig + JsonNode vectorizeServiceNode = jsonNode.get("service"); + if (vectorizeServiceNode != null) { + vectorizeConfig = VectorizeConfig.fromJson(vectorizeServiceNode, objectMapper); + } + return new ColumnVectorDefinition(fieldName, dimension, similarityFunction, vectorizeConfig); } - return new VectorConfig(true, fieldName, dimension, similarityFunction, vectorizeConfig); - } - public record VectorizeConfig( - String provider, - String modelName, - Map authentication, - Map parameters) { + /** + * Represent the vectorize configuration defined for a column + * + * @param provider + * @param modelName + * @param authentication + * @param parameters + */ + public record VectorizeConfig( + String provider, + String modelName, + Map authentication, + Map parameters) { - protected static VectorizeConfig fromJson( - JsonNode vectorizeServiceNode, ObjectMapper objectMapper) { - // provider, modelName, must exist - String provider = vectorizeServiceNode.get("provider").asText(); - String modelName = vectorizeServiceNode.get("modelName").asText(); - // construct VectorizeConfig.authentication, can be null - JsonNode vectorizeServiceAuthenticationNode = vectorizeServiceNode.get("authentication"); - Map vectorizeServiceAuthentication = - vectorizeServiceAuthenticationNode == null - ? null - : objectMapper.convertValue(vectorizeServiceAuthenticationNode, Map.class); - // construct VectorizeConfig.parameters, can be null - JsonNode vectorizeServiceParameterNode = vectorizeServiceNode.get("parameters"); - Map vectorizeServiceParameter = - vectorizeServiceParameterNode == null - ? null - : objectMapper.convertValue(vectorizeServiceParameterNode, Map.class); - return new VectorizeConfig( - provider, modelName, vectorizeServiceAuthentication, vectorizeServiceParameter); + protected static VectorizeConfig fromJson( + JsonNode vectorizeServiceNode, ObjectMapper objectMapper) { + // provider, modelName, must exist + String provider = vectorizeServiceNode.get("provider").asText(); + String modelName = vectorizeServiceNode.get("modelName").asText(); + // construct VectorizeConfig.authentication, can be null + JsonNode vectorizeServiceAuthenticationNode = vectorizeServiceNode.get("authentication"); + Map vectorizeServiceAuthentication = + vectorizeServiceAuthenticationNode == null + ? null + : objectMapper.convertValue(vectorizeServiceAuthenticationNode, Map.class); + // construct VectorizeConfig.parameters, can be null + JsonNode vectorizeServiceParameterNode = vectorizeServiceNode.get("parameters"); + Map vectorizeServiceParameter = + vectorizeServiceParameterNode == null + ? null + : objectMapper.convertValue(vectorizeServiceParameterNode, Map.class); + return new VectorizeConfig( + provider, modelName, vectorizeServiceAuthentication, vectorizeServiceParameter); + } } } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java index 3035b9843..1eca64f39 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java @@ -31,7 +31,6 @@ public class DataVectorizer { private final JsonNodeFactory nodeFactory; private final EmbeddingCredentials embeddingCredentials; private final SchemaObject schemaObject; - private final VectorConfig vectorConfig; /** * Constructor @@ -51,9 +50,6 @@ public DataVectorizer( this.nodeFactory = nodeFactory; this.embeddingCredentials = embeddingCredentials; this.schemaObject = schemaObject; - // This is getting element at 0 since only one vector is stored in the schema. - // This logic needs to be changed to handle multiple vectors columns for tables, - vectorConfig = schemaObject.vectorConfigs().get(0); } /** @@ -115,11 +111,16 @@ public Uni vectorize(List documents) { .onItem() .transform( vectorData -> { + final VectorConfig vectorConfig = schemaObject.vectorConfig(); + // This will be the first element for collection + final VectorConfig.ColumnVectorDefinition collectionVectorDefinition = + vectorConfig.columnVectorDefinitions().get(0); + // check if we get back the same number of vectors that we asked for if (vectorData.size() != vectorizeTexts.size()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' didn't return the expected number of embeddings. Expect: '%d'. Actual: '%d'", - vectorConfig.vectorizeConfig().provider(), + collectionVectorDefinition.vectorizeConfig().provider(), vectorizeTexts.size(), vectorData.size()); } @@ -130,11 +131,11 @@ public Uni vectorize(List documents) { JsonNode document = documents.get(position); float[] vector = vectorData.get(vectorPosition); // check if all vectors have the expected size - if (vector.length != vectorConfig.vectorSize()) { + if (vector.length != collectionVectorDefinition.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - vectorConfig.vectorizeConfig().provider(), - vectorConfig.vectorSize(), + collectionVectorDefinition.vectorizeConfig().provider(), + collectionVectorDefinition.vectorSize(), vector.length); } final ArrayNode arrayNode = nodeFactory.arrayNode(vector.length); @@ -173,13 +174,17 @@ public Uni vectorize(String vectorizeContent) { .onItem() .transform( vectorData -> { + final VectorConfig vectorConfig = schemaObject.vectorConfig(); + // This will be the first element for collection + final VectorConfig.ColumnVectorDefinition collectionVectorDefinition = + vectorConfig.columnVectorDefinitions().get(0); float[] vector = vectorData.get(0); // check if vector have the expected size - if (vector.length != vectorConfig.vectorSize()) { + if (vector.length != collectionVectorDefinition.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - vectorConfig.vectorizeConfig().provider(), - vectorConfig.vectorSize(), + collectionVectorDefinition.vectorizeConfig().provider(), + collectionVectorDefinition.vectorSize(), vector.length); } return vector; @@ -216,12 +221,16 @@ public Uni vectorize(SortClause sortClause) { .transform( vectorData -> { float[] vector = vectorData.get(0); + final VectorConfig vectorConfig = schemaObject.vectorConfig(); + // This will be the first element for collection + final VectorConfig.ColumnVectorDefinition collectionVectorDefinition = + vectorConfig.columnVectorDefinitions().get(0); // check if vector have the expected size - if (vector.length != vectorConfig.vectorSize()) { + if (vector.length != collectionVectorDefinition.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - vectorConfig.vectorizeConfig().provider(), - vectorConfig.vectorSize(), + collectionVectorDefinition.vectorizeConfig().provider(), + collectionVectorDefinition.vectorSize(), vector.length); } sortExpressions.clear(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 4b7e1ca85..5cb882f59 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -22,14 +22,12 @@ import io.stargate.sgv2.jsonapi.config.CommandLevelLoggingConfig; import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.SchemaObject; -import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Produces; import jakarta.inject.Inject; import jakarta.inject.Singleton; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -245,12 +243,9 @@ private Tags getCustomTags( result.errors().get(0).fieldsForMetricsTag().getOrDefault("errorCode", UNKNOWN_VALUE); errorCodeTag = Tag.of(jsonApiMetricsConfig.errorCode(), errorCode); } - final Optional first = - commandContext.schemaObject().vectorConfigs().stream() - .filter(a -> a.vectorEnabled()) - .findFirst(); + Tag vectorEnabled = - first.isPresent() + commandContext.schemaObject().vectorConfig().vectorEnabled() ? Tag.of(jsonApiMetricsConfig.vectorEnabled(), "true") : Tag.of(jsonApiMetricsConfig.vectorEnabled(), "false"); JsonApiMetricsConfig.SortType sortType = getVectorTypeTag(command); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java index b95e8100a..b117cbcad 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java @@ -40,7 +40,7 @@ public Operation resolveKeyspaceCommand( .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getApiDataType())); List partitionKeys = Arrays.stream(command.definition().primaryKey().keys()).toList(); - Map vectorizeConfigMap = + Map vectorizeConfigMap = command.definition().columns().entrySet().stream() .filter( e -> @@ -53,8 +53,8 @@ public Operation resolveKeyspaceCommand( ComplexTypes.VectorType vectorType = ((ComplexTypes.VectorType) e.getValue()); final VectorizeConfig vectorizeConfig = vectorType.getVectorConfig(); validateVectorize.validateService(vectorizeConfig, vectorType.getDimension()); - VectorConfig.VectorizeConfig dbVectorConfig = - new VectorConfig.VectorizeConfig( + VectorConfig.ColumnVectorDefinition.VectorizeConfig dbVectorConfig = + new VectorConfig.ColumnVectorDefinition.VectorizeConfig( vectorizeConfig.provider(), vectorizeConfig.modelName(), vectorizeConfig.authentication(), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java index b4a66f573..2fe10c8e6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java @@ -36,16 +36,16 @@ public final class CollectionSchemaObject extends TableBasedSchemaObject { SchemaObjectName.MISSING, null, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), null); private final IdConfig idConfig; - private final List vectorConfigs; + private final VectorConfig vectorConfig; private final CollectionIndexingConfig indexingConfig; private final TableMetadata tableMetadata; /** - * @param vectorConfigs + * @param vectorConfig * @param indexingConfig */ public CollectionSchemaObject( @@ -53,13 +53,13 @@ public CollectionSchemaObject( String name, TableMetadata tableMetadata, IdConfig idConfig, - List vectorConfigs, + VectorConfig vectorConfig, CollectionIndexingConfig indexingConfig) { this( new SchemaObjectName(keypaceName, name), tableMetadata, idConfig, - vectorConfigs, + vectorConfig, indexingConfig); } @@ -67,12 +67,12 @@ public CollectionSchemaObject( SchemaObjectName name, TableMetadata tableMetadata, IdConfig idConfig, - List vectorConfigs, + VectorConfig vectorConfig, CollectionIndexingConfig indexingConfig) { super(TYPE, name, tableMetadata); this.idConfig = idConfig; - this.vectorConfigs = vectorConfigs; + this.vectorConfig = vectorConfig; this.indexingConfig = indexingConfig; this.tableMetadata = tableMetadata; } @@ -81,12 +81,12 @@ public CollectionSchemaObject( // effectively public CollectionSchemaObject withIdType(CollectionIdType idType) { return new CollectionSchemaObject( - name(), tableMetadata, new IdConfig(idType), vectorConfigs, indexingConfig); + name(), tableMetadata, new IdConfig(idType), vectorConfig, indexingConfig); } @Override - public List vectorConfigs() { - return vectorConfigs; + public VectorConfig vectorConfig() { + return vectorConfig; } @Override @@ -221,13 +221,14 @@ private static CollectionSchemaObject createCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - vectorSize, - function, - null)), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + vectorSize, + function, + null))), null); } else { return new CollectionSchemaObject( @@ -235,7 +236,7 @@ private static CollectionSchemaObject createCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), null); } } else { @@ -280,13 +281,14 @@ private static CollectionSchemaObject createCollectionSettings( } // convert a vector jsonNode from cql table comment to vectorConfig, used for collection - private static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { + private static VectorConfig.ColumnVectorDefinition fromJson( + JsonNode jsonNode, ObjectMapper objectMapper) { // dimension, similarityFunction, must exist int dimension = jsonNode.get("dimension").asInt(); SimilarityFunction similarityFunction = SimilarityFunction.fromString(jsonNode.get("metric").asText()); - return VectorConfig.fromJson( + return VectorConfig.ColumnVectorDefinition.fromJson( DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, dimension, similarityFunction, @@ -303,23 +305,26 @@ public static CreateCollectionCommand collectionSettingToCreateCollectionCommand CreateCollectionCommand.Options.IndexingConfig indexingConfig = null; // populate the vectorSearchConfig, Default will be the index 0 since there is only one vector // column supported for collection - final VectorConfig vectorConfig = collectionSetting.vectorConfigs().get(0); + final VectorConfig vectorConfig = collectionSetting.vectorConfig(); if (vectorConfig.vectorEnabled()) { + // This will be size 1 for collection + VectorConfig.ColumnVectorDefinition vectorConfigColumn = + vectorConfig.columnVectorDefinitions().get(0); VectorizeConfig vectorizeConfig = null; - if (vectorConfig.vectorizeConfig() != null) { - Map authentication = vectorConfig.vectorizeConfig().authentication(); - Map parameters = vectorConfig.vectorizeConfig().parameters(); + if (vectorConfigColumn.vectorizeConfig() != null) { + Map authentication = vectorConfigColumn.vectorizeConfig().authentication(); + Map parameters = vectorConfigColumn.vectorizeConfig().parameters(); vectorizeConfig = new VectorizeConfig( - vectorConfig.vectorizeConfig().provider(), - vectorConfig.vectorizeConfig().modelName(), + vectorConfigColumn.vectorizeConfig().provider(), + vectorConfigColumn.vectorizeConfig().modelName(), authentication == null ? null : Map.copyOf(authentication), parameters == null ? null : Map.copyOf(parameters)); } vectorSearchConfig = new CreateCollectionCommand.Options.VectorSearchConfig( - vectorConfig.vectorSize(), - vectorConfig.similarityFunction().name().toLowerCase(), + vectorConfigColumn.vectorSize(), + vectorConfigColumn.similarityFunction().name().toLowerCase(), vectorizeConfig); } // populate the indexingConfig @@ -353,11 +358,11 @@ public CollectionIndexingConfig indexingConfig() { // TODO: these helper functions break encapsulation for very little benefit public SimilarityFunction similarityFunction() { - return vectorConfigs().get(0).similarityFunction(); + return vectorConfig().columnVectorDefinitions().get(0).similarityFunction(); } public boolean isVectorEnabled() { - return !(vectorConfigs().isEmpty()) && vectorConfigs().get(0).vectorEnabled(); + return vectorConfig().vectorEnabled(); } // TODO: the overrides below were auto added when migrating from a record to a class, not sure @@ -369,13 +374,13 @@ public boolean equals(Object obj) { var that = (CollectionSchemaObject) obj; return Objects.equals(this.name, that.name) && Objects.equals(this.idConfig, that.idConfig) - && Objects.equals(this.vectorConfigs, that.vectorConfigs) + && Objects.equals(this.vectorConfig, that.vectorConfig) && Objects.equals(this.indexingConfig, that.indexingConfig); } @Override public int hashCode() { - return Objects.hash(name, idConfig, vectorConfigs, indexingConfig); + return Objects.hash(name, idConfig, vectorConfig, indexingConfig); } @Override @@ -387,8 +392,8 @@ public String toString() { + "idConfig=" + idConfig + ", " - + "vectorConfigs=" - + vectorConfigs + + "vectorConfig=" + + vectorConfig + ", " + "indexingConfig=" + indexingConfig diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java index eb4c76791..753827934 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java @@ -31,10 +31,12 @@ public CollectionSchemaObject readCollectionSettings( VectorConfig vectorConfig = new VectorConfig( vectorEnabled, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - vectorSize, - function, - null); + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + vectorSize, + function, + null))); CollectionIndexingConfig indexingConfig = null; JsonNode indexing = commentConfigNode.path(TableCommentConstants.COLLECTION_INDEXING_KEY); if (!indexing.isMissingNode()) { @@ -45,7 +47,7 @@ public CollectionSchemaObject readCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - List.of(vectorConfig), + vectorConfig, indexingConfig); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java index 1c6bd7bb5..9d577a9e9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java @@ -26,7 +26,9 @@ public CollectionSchemaObject readCollectionSettings( VectorConfig vectorConfig = VectorConfig.notEnabledVectorConfig(); JsonNode vector = collectionOptionsNode.path(TableCommentConstants.COLLECTION_VECTOR_KEY); if (!vector.isMissingNode()) { - vectorConfig = VectorConfig.fromJson(vector, objectMapper); + VectorConfig.ColumnVectorDefinition columnVectorDefinition = + VectorConfig.ColumnVectorDefinition.fromJson(vector, objectMapper); + vectorConfig = new VectorConfig(true, List.of(columnVectorDefinition)); } // construct collectionSettings IndexingConfig CollectionIndexingConfig indexingConfig = null; @@ -45,11 +47,6 @@ public CollectionSchemaObject readCollectionSettings( } return new CollectionSchemaObject( - keyspaceName, - collectionName, - tableMetadata, - idConfig, - List.of(vectorConfig), - indexingConfig); + keyspaceName, collectionName, tableMetadata, idConfig, vectorConfig, indexingConfig); } } diff --git a/src/main/resources/errors.yaml b/src/main/resources/errors.yaml index 1c220c847..0bac92c2d 100644 --- a/src/main/resources/errors.yaml +++ b/src/main/resources/errors.yaml @@ -291,6 +291,12 @@ request-errors: "modelName": "NV-Embed-QA" } } + - scope: SCHEMA + code: INVALID_VECTORIZE_CONFIGURATION + title: Unable to parse vectorize configuration, schema invalid. + body: |- + Unable to parse vectorize configuration, schema invalid. + # ================================================================================================================ # Server Errors diff --git a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java index f24fe600d..e990ea1f9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java @@ -28,7 +28,7 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), null); public static final CollectionSchemaObject VECTOR_COLLECTION_SCHEMA_OBJECT = @@ -36,13 +36,14 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - -1, - SimilarityFunction.COSINE, - null)), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null); public static final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java index 9a968c28d..2e8402d72 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java @@ -13,7 +13,6 @@ import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import java.util.Arrays; import java.util.HashSet; -import java.util.List; import org.junit.jupiter.api.Test; @QuarkusTest @@ -31,7 +30,7 @@ public void ensureSingleProjectorCreation() { "collectionName", null, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), indexingConfig); IndexingProjector indexingProj = settings.indexingProjector(); assertThat(indexingProj) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java index fb6166965..3bd97a2b4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java @@ -164,7 +164,7 @@ public void checkValidJsonApiTable() { assertThat(schemaObject) .satisfies( s -> { - assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); + assertThat(s.vectorConfig().vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); }); } @@ -296,7 +296,7 @@ public void checkValidJsonApiTableWithIndexing() { assertThat(collectionSchemaObject) .satisfies( s -> { - assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); + assertThat(s.vectorConfig().vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); assertThat(s.indexingConfig().denied()).containsExactly("comment"); }); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java index 9bdf8863d..e61fd4510 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java @@ -231,13 +231,15 @@ public void testWithUnmatchedVectorSize() { "collections", null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - 4, - SimilarityFunction.COSINE, - new VectorConfig.VectorizeConfig("custom", "custom", null, null))), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + 4, + SimilarityFunction.COSINE, + new VectorConfig.ColumnVectorDefinition.VectorizeConfig( + "custom", "custom", null, null)))), null); List documents = new ArrayList<>(); for (int i = 0; i < 2; i++) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java index fa5353107..df37dbd63 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java @@ -20,13 +20,15 @@ public class TestEmbeddingProvider extends EmbeddingProvider { TestConstants.SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - 3, - SimilarityFunction.COSINE, - new VectorConfig.VectorizeConfig("custom", "custom", null, null))), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + 3, + SimilarityFunction.COSINE, + new VectorConfig.ColumnVectorDefinition.VectorizeConfig( + "custom", "custom", null, null)))), null), new TestEmbeddingProvider(), "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java index e350d81bb..aadbda484 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java @@ -83,13 +83,14 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - -1, - SimilarityFunction.COSINE, - null)), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java index 9f4beb013..d9f88307b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java @@ -99,13 +99,14 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - -1, - SimilarityFunction.COSINE, - null)), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java index ff6f05157..ccb32d8e2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java @@ -46,7 +46,7 @@ public class OperationTestBase { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), null); protected final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = KeyspaceSchemaObject.fromSchemaObject(COLLECTION_SCHEMA_OBJECT); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java index ef9f326fc..22030b965 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java @@ -118,13 +118,14 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - -1, - SimilarityFunction.COSINE, - null)), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java index 80c2c37dc..92b3c4de5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java @@ -85,13 +85,14 @@ class Resolve { new SchemaObjectName(KEYSPACE_NAME, COLLECTION_NAME), null, IdConfig.defaultIdConfig(), - List.of( - new VectorConfig( - true, - DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, - -1, - SimilarityFunction.COSINE, - null)), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, null, From d47be3a79d88944d9a9e8ace1845c0fcaed06ab6 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:44:50 -0400 Subject: [PATCH 11/37] Working listTables --- .../api/model/command/TableOnlyCommand.java | 4 +- .../ColumnDefinitionSerializer.java | 6 +-- .../model/command/impl/ListTablesCommand.java | 4 +- .../cqldriver/executor/TableSchemaObject.java | 46 ++++++++++--------- .../resolver/ListTablesCommandResolver.java | 40 ++++++++++++++++ 5 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java index 23b48456d..5e9e61a0c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java @@ -4,11 +4,13 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.stargate.sgv2.jsonapi.api.model.command.impl.CreateTableCommand; import io.stargate.sgv2.jsonapi.api.model.command.impl.DropTableCommand; +import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; /** Interface for all commands executed against a keyspace. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonSubTypes({ @JsonSubTypes.Type(value = CreateTableCommand.class), - @JsonSubTypes.Type(value = DropTableCommand.class) + @JsonSubTypes.Type(value = DropTableCommand.class), + @JsonSubTypes.Type(value = ListTablesCommand.class), }) public interface TableOnlyCommand extends KeyspaceCommand {} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java index 3d7c9994f..2ba784a24 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java @@ -25,10 +25,10 @@ public void serialize( } else if (columnType instanceof ComplexTypes.ListType lt) { jsonGenerator.writeStringField("valueType", lt.valueType()); } else if (columnType instanceof ComplexTypes.SetType st) { - jsonGenerator.writeStringField("st", lt.valueType()); + jsonGenerator.writeStringField("st", st.valueType()); } else if (columnType instanceof ComplexTypes.VectorType vt) { - jsonGenerator.writeNumberField("dimension", vectorType.dimension()); - if (vectorType.getVectorConfig() != null) + jsonGenerator.writeNumberField("dimension", vt.getDimension()); + if (vt.getVectorConfig() != null) jsonGenerator.writeObjectField("service", vt.getVectorConfig()); } jsonGenerator.writeEndObject(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java index b1ae2f8d1..8f0759ba7 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java @@ -1,13 +1,13 @@ package io.stargate.sgv2.jsonapi.api.model.command.impl; import com.fasterxml.jackson.annotation.JsonTypeName; -import io.stargate.sgv2.jsonapi.api.model.command.CollectionOnlyCommand; +import io.stargate.sgv2.jsonapi.api.model.command.TableOnlyCommand; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; import org.eclipse.microprofile.openapi.annotations.media.Schema; @Schema(description = "Command that lists all available tables in a namespace.") @JsonTypeName("listTables") -public record ListTablesCommand(Options options) implements CollectionOnlyCommand { +public record ListTablesCommand(Options options) implements TableOnlyCommand { public record Options( @Schema( description = "include table properties.", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index f389f5e87..acaa9ffdf 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -22,10 +22,12 @@ import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDef; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDefs; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -61,21 +63,26 @@ public IndexUsage newIndexUsage() { */ public static TableSchemaObject getTableSettings( TableMetadata tableMetadata, ObjectMapper objectMapper) { - Map extensions = - (Map) + Map extensions = + (Map) tableMetadata.getOptions().get(CqlIdentifier.fromInternal("extensions")); - String vectorize = extensions != null ? extensions.get("vectorize") : null; - Map resultMap = new HashMap<>(); - if (vectorize != null) { + String vectorizeJson = null; + if (extensions != null) { + ByteBuffer vectorizeBuffer = + (ByteBuffer) extensions.get("com.datastax.data-api.vectorize-config"); + vectorizeJson = + vectorizeBuffer != null + ? new String(ByteUtils.getArray(vectorizeBuffer.duplicate()), StandardCharsets.UTF_8) + : null; + } + Map vectorizeConfigMap = + new HashMap<>(); + if (vectorizeJson != null) { try { - String vectorizeJson = - new String(ByteUtils.fromHexString(vectorize).array(), StandardCharsets.UTF_8); - // Convert JSON string to Map JsonNode vectorizeByColumns = objectMapper.readTree(vectorizeJson); - Map vectorizeConfigMap = - new HashMap<>(); - while (vectorizeByColumns.fields().hasNext()) { - Map.Entry entry = vectorizeByColumns.fields().next(); + Iterator> it = vectorizeByColumns.fields(); + while (it.hasNext()) { + Map.Entry entry = it.next(); VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = objectMapper.treeToValue( entry.getValue(), VectorConfig.ColumnVectorDefinition.VectorizeConfig.class); @@ -105,7 +112,7 @@ public static TableSchemaObject getTableSettings( column.getKey().asInternal(), dimension, similarityFunction, - resultMap.get(column.getKey().asInternal())); + vectorizeConfigMap.get(column.getKey().asInternal())); columnVectorDefinitions.add(columnVectorDefinition); } } @@ -145,9 +152,7 @@ public TableResponse toTableResponse() { partitionBy.toArray(new String[0]), partitionSort.toArray(new PrimaryKey.OrderingKey[0])); return new TableResponse( - tableName, - new TableResponse.TableDefinition( - new TableResponse.TableDefinition.ColumnsDefinition(columnsDefinition), primaryKey)); + tableName, new TableResponse.TableDefinition(columnsDefinition, primaryKey)); } // Need to handle frozen types @@ -190,15 +195,12 @@ private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadat } } - @JsonPropertyOrder({"_id", "definition"}) + @JsonPropertyOrder({"name", "definition"}) @JsonInclude(JsonInclude.Include.NON_NULL) - public record TableResponse(String name, TableDefinition tableDefinition) { + public record TableResponse(String name, TableDefinition definition) { @JsonPropertyOrder({"columns", "primaryKey"}) @JsonInclude(JsonInclude.Include.NON_NULL) - record TableDefinition(ColumnsDefinition columns, PrimaryKey primaryKey) { - - record ColumnsDefinition(Map columns) {} - } + record TableDefinition(Map columns, PrimaryKey primaryKey) {} } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java new file mode 100644 index 000000000..83b9d3dd6 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java @@ -0,0 +1,40 @@ +package io.stargate.sgv2.jsonapi.service.resolver; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.api.model.command.impl.FindCollectionsCommand; +import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.operation.Operation; +import io.stargate.sgv2.jsonapi.service.operation.tables.ListTablesOperation; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +/** Command resolver for the {@link FindCollectionsCommand}. */ +@ApplicationScoped +public class ListTablesCommandResolver implements CommandResolver { + private final ObjectMapper objectMapper; + private final CQLSessionCache cqlSessionCache; + + @Inject + public ListTablesCommandResolver(ObjectMapper objectMapper, CQLSessionCache cqlSessionCache) { + this.objectMapper = objectMapper; + this.cqlSessionCache = cqlSessionCache; + } + + /** {@inheritDoc} */ + @Override + public Class getCommandClass() { + return ListTablesCommand.class; + } + + /** {@inheritDoc} */ + @Override + public Operation resolveKeyspaceCommand( + CommandContext ctx, ListTablesCommand command) { + + boolean explain = command.options() != null ? command.options().explain() : false; + return new ListTablesOperation(explain, objectMapper, cqlSessionCache, ctx); + } +} From 2683c66073daad6c4e7f83ac0d17809adb9e55dd Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:46:57 -0400 Subject: [PATCH 12/37] Fixed the vectorize config deserializer --- .../cqldriver/executor/TableSchemaObject.java | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 43604d87a..9560cbd55 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -11,10 +11,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -49,21 +51,26 @@ public IndexUsage newIndexUsage() { */ public static TableSchemaObject getTableSettings( TableMetadata tableMetadata, ObjectMapper objectMapper) { - Map extensions = - (Map) + Map extensions = + (Map) tableMetadata.getOptions().get(CqlIdentifier.fromInternal("extensions")); - String vectorize = extensions != null ? extensions.get("vectorize") : null; - Map resultMap = new HashMap<>(); - if (vectorize != null) { + String vectorizeJson = null; + if (extensions != null) { + ByteBuffer vectorizeBuffer = + (ByteBuffer) extensions.get("com.datastax.data-api.vectorize-config"); + vectorizeJson = + vectorizeBuffer != null + ? new String(ByteUtils.getArray(vectorizeBuffer.duplicate()), StandardCharsets.UTF_8) + : null; + } + Map vectorizeConfigMap = + new HashMap<>(); + if (vectorizeJson != null) { try { - String vectorizeJson = - new String(ByteUtils.fromHexString(vectorize).array(), StandardCharsets.UTF_8); - // Convert JSON string to Map JsonNode vectorizeByColumns = objectMapper.readTree(vectorizeJson); - Map vectorizeConfigMap = - new HashMap<>(); - while (vectorizeByColumns.fields().hasNext()) { - Map.Entry entry = vectorizeByColumns.fields().next(); + Iterator> it = vectorizeByColumns.fields(); + while (it.hasNext()) { + Map.Entry entry = it.next(); VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = objectMapper.treeToValue( entry.getValue(), VectorConfig.ColumnVectorDefinition.VectorizeConfig.class); @@ -93,7 +100,7 @@ public static TableSchemaObject getTableSettings( column.getKey().asInternal(), dimension, similarityFunction, - resultMap.get(column.getKey().asInternal())); + vectorizeConfigMap.get(column.getKey().asInternal())); columnVectorDefinitions.add(columnVectorDefinition); } } From e7e92cb967eb0ef9fd9482410100404f6e1fb938 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:16:45 -0400 Subject: [PATCH 13/37] Changes based on review --- .../sgv2/jsonapi/exception/SchemaException.java | 1 + .../cqldriver/executor/TableSchemaObject.java | 17 +++++++++++------ .../cqldriver/executor/VectorConfig.java | 2 +- src/main/resources/errors.yaml | 8 +++++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java b/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java index 1f459321b..ff9ec460f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java @@ -17,6 +17,7 @@ public enum Code implements ErrorCode { COLUMN_DEFINITION_MISSING, COLUMN_TYPE_INCORRECT, COLUMN_TYPE_UNSUPPORTED, + INVALID_CONFIGURATION, INVALID_VECTORIZE_CONFIGURATION, LIST_TYPE_INCORRECT_DEFINITION, MAP_TYPE_INCORRECT_DEFINITION, diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 9560cbd55..3f07082ac 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -71,13 +71,18 @@ public static TableSchemaObject getTableSettings( Iterator> it = vectorizeByColumns.fields(); while (it.hasNext()) { Map.Entry entry = it.next(); - VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = - objectMapper.treeToValue( - entry.getValue(), VectorConfig.ColumnVectorDefinition.VectorizeConfig.class); - vectorizeConfigMap.put(entry.getKey(), vectorizeConfig); + try { + VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = + objectMapper.treeToValue( + entry.getValue(), VectorConfig.ColumnVectorDefinition.VectorizeConfig.class); + vectorizeConfigMap.put(entry.getKey(), vectorizeConfig); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw SchemaException.Code.INVALID_VECTORIZE_CONFIGURATION.get( + Map.of("field", entry.getKey())); + } } - } catch (JsonProcessingException | IllegalArgumentException e) { - throw SchemaException.Code.INVALID_VECTORIZE_CONFIGURATION.get(); + } catch (JsonProcessingException e) { + throw SchemaException.Code.INVALID_CONFIGURATION.get(); } } VectorConfig vectorConfig; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java index 4ffd447de..229d6d0de 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java @@ -10,7 +10,7 @@ /** * incorporates vectorizeConfig into vectorConfig * - * @param vectorEnabled - IF vector field is available for the table + * @param vectorEnabled - If vector field is available for the table * @param columnVectorDefinitions - List of columnVectorDefinitions each with respect to a * column/field */ diff --git a/src/main/resources/errors.yaml b/src/main/resources/errors.yaml index 0bac92c2d..524195753 100644 --- a/src/main/resources/errors.yaml +++ b/src/main/resources/errors.yaml @@ -291,11 +291,17 @@ request-errors: "modelName": "NV-Embed-QA" } } + - scope: SCHEMA + code: INVALID_CONFIGURATION + title: Unable to parse configuration, schema invalid. + body: |- + Unable to parse configuration, schema invalid. + - scope: SCHEMA code: INVALID_VECTORIZE_CONFIGURATION title: Unable to parse vectorize configuration, schema invalid. body: |- - Unable to parse vectorize configuration, schema invalid. + Unable to parse vectorize configuration, schema invalid for field ${field}. # ================================================================================================================ From 27f3bef2721d515e648604aae3c7560ccbdbfda9 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:25:14 -0400 Subject: [PATCH 14/37] Resolve the merge compile error --- .../jsonapi/fixtures/testdata/WhereAnalyzerTestData.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java index c734859c1..7f3702d04 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java @@ -10,6 +10,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.exception.FilterException; import io.stargate.sgv2.jsonapi.exception.WarningException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; @@ -58,8 +59,8 @@ public WhereAnalyzerFixture( this.message = message; this.tableMetadata = tableMetadata; - this.analyzer = new WhereCQLClauseAnalyzer(new TableSchemaObject(tableMetadata)); - this.tableSchemaObject = new TableSchemaObject(tableMetadata); + this.analyzer = new WhereCQLClauseAnalyzer(TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper())); + this.tableSchemaObject = TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); this.expression = new LogicalExpressionTestData.ExpressionBuilder<>(this, expression, tableMetadata); } From 11b589e11e099ca098ebe7debce7cc32ac05a390 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:27:09 -0400 Subject: [PATCH 15/37] Resolve the merge compile error --- .../jsonapi/fixtures/testdata/WhereAnalyzerTestData.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java index 7f3702d04..2a60f7505 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java @@ -59,8 +59,11 @@ public WhereAnalyzerFixture( this.message = message; this.tableMetadata = tableMetadata; - this.analyzer = new WhereCQLClauseAnalyzer(TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper())); - this.tableSchemaObject = TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); + this.analyzer = + new WhereCQLClauseAnalyzer( + TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper())); + this.tableSchemaObject = + TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); this.expression = new LogicalExpressionTestData.ExpressionBuilder<>(this, expression, tableMetadata); } From 6ad43dc9cc9a1d36c361eeabb05d11c53e48e23d Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:33:06 -0400 Subject: [PATCH 16/37] Resolve the merge compile error --- .../jsonapi/service/cqldriver/executor/NamespaceCache.java | 3 +-- .../jsonapi/service/cqldriver/executor/TableSchemaObject.java | 3 +-- .../java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java | 2 +- .../sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java | 2 +- .../sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java | 4 ++-- .../executor/DefaultDriverExceptionHandlerTestData.java | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java index 181aa0b31..f106b87ce 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java @@ -103,8 +103,7 @@ private Uni loadSchemaObject( optionalTable.get(), objectMapper); } - // 04-Sep-2024, tatu: Used to check that API Tables enabled; no longer checked here - return TableSchemaObject.getTableSettings(table, objectMapper); + return TableSchemaObject.from(table, objectMapper); }); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 3f07082ac..38c07e5f5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -49,8 +49,7 @@ public IndexUsage newIndexUsage() { * @param objectMapper * @return */ - public static TableSchemaObject getTableSettings( - TableMetadata tableMetadata, ObjectMapper objectMapper) { + public static TableSchemaObject from(TableMetadata tableMetadata, ObjectMapper objectMapper) { Map extensions = (Map) tableMetadata.getOptions().get(CqlIdentifier.fromInternal("extensions")); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java index 0c3be13d6..6a267769e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java @@ -63,7 +63,7 @@ public CqlFixture( this.cqlData = cqlData; this.tableFixture = tableFixture; this.tableMetadata = tableFixture.tableMetadata(identifiers); - this.tableSchemaObject = TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); + this.tableSchemaObject = TableSchemaObject.from(tableMetadata, new ObjectMapper()); } public FixtureIdentifiers identifiers() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java index c25a1214f..2dbfaebf3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java @@ -10,6 +10,6 @@ public SchemaObjectTestData(TestData testData) { } public TableSchemaObject emptyTableSchemaObject() { - return TableSchemaObject.getTableSettings(testData.tableMetadata().empty(), new ObjectMapper()); + return TableSchemaObject.from(testData.tableMetadata().empty(), new ObjectMapper()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java index 2a60f7505..7445d16a8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java @@ -61,9 +61,9 @@ public WhereAnalyzerFixture( this.tableMetadata = tableMetadata; this.analyzer = new WhereCQLClauseAnalyzer( - TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper())); + TableSchemaObject.from(tableMetadata, new ObjectMapper())); this.tableSchemaObject = - TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); + TableSchemaObject.from(tableMetadata, new ObjectMapper()); this.expression = new LogicalExpressionTestData.ExpressionBuilder<>(this, expression, tableMetadata); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java index 5d7ec9331..4db187633 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java @@ -33,6 +33,6 @@ public DefaultDriverExceptionHandlerTestData() { Map.of(), Map.of(), Map.of()); - TABLE_SCHEMA_OBJECT = TableSchemaObject.getTableSettings(tableMetadata, new ObjectMapper()); + TABLE_SCHEMA_OBJECT = TableSchemaObject.from(tableMetadata, new ObjectMapper()); } } From dd2f2cbe7365c4db864c5419a90f92d450722cd5 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:39:53 -0400 Subject: [PATCH 17/37] Formatted the file --- .../jsonapi/fixtures/testdata/WhereAnalyzerTestData.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java index 7445d16a8..531388e26 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java @@ -60,10 +60,8 @@ public WhereAnalyzerFixture( this.message = message; this.tableMetadata = tableMetadata; this.analyzer = - new WhereCQLClauseAnalyzer( - TableSchemaObject.from(tableMetadata, new ObjectMapper())); - this.tableSchemaObject = - TableSchemaObject.from(tableMetadata, new ObjectMapper()); + new WhereCQLClauseAnalyzer(TableSchemaObject.from(tableMetadata, new ObjectMapper())); + this.tableSchemaObject = TableSchemaObject.from(tableMetadata, new ObjectMapper()); this.expression = new LogicalExpressionTestData.ExpressionBuilder<>(this, expression, tableMetadata); } From 5e4566155dff35a57b335222216a84ff9ddce95e Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 17:31:11 -0400 Subject: [PATCH 18/37] Fix for IT --- .../io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java index a378cbc94..ec46b6aa2 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java @@ -199,7 +199,8 @@ public Uni> postCommand( EmbeddingProvider embeddingProvider = null; VectorConfig vectorConfig = schemaObject.vectorConfig(); final VectorConfig.ColumnVectorDefinition columnVectorDefinition = - vectorConfig.columnVectorDefinitions().isEmpty() + vectorConfig.columnVectorDefinitions() == null + || vectorConfig.columnVectorDefinitions().isEmpty() ? null : vectorConfig.columnVectorDefinitions().get(0); final VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = From da04424956fa25f8e6136f1da3b95e4e701520e2 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:17:02 -0400 Subject: [PATCH 19/37] Added UNSUPPORTED type support --- .../ColumnDefinitionSerializer.java | 10 ++++- .../table/definition/datatype/ColumnType.java | 4 ++ .../definition/datatype/ComplexTypes.java | 31 ++++++++++++++++ .../cqldriver/executor/TableSchemaObject.java | 37 ++++++++++++++++--- .../operation/tables/ListTablesOperation.java | 2 +- 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java index 2ba784a24..076b6ed73 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java @@ -18,19 +18,25 @@ public void serialize( ColumnType columnType, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { jsonGenerator.writeStartObject(); - jsonGenerator.writeStringField("type", columnType.name()); + jsonGenerator.writeStringField("type", columnType.getApiName()); if (columnType instanceof ComplexTypes.MapType mt) { jsonGenerator.writeStringField("keyType", mt.keyType()); jsonGenerator.writeStringField("valueType", mt.valueType()); } else if (columnType instanceof ComplexTypes.ListType lt) { jsonGenerator.writeStringField("valueType", lt.valueType()); } else if (columnType instanceof ComplexTypes.SetType st) { - jsonGenerator.writeStringField("st", st.valueType()); + jsonGenerator.writeStringField("valueType", st.valueType()); } else if (columnType instanceof ComplexTypes.VectorType vt) { jsonGenerator.writeNumberField("dimension", vt.getDimension()); if (vt.getVectorConfig() != null) jsonGenerator.writeObjectField("service", vt.getVectorConfig()); + } else if (columnType instanceof ComplexTypes.UnsupportedType ut) { + jsonGenerator.writeObjectField( + "apiSupport", new ApiSupport(false, false, false, ut.cqlFormat())); } jsonGenerator.writeEndObject(); } + + public record ApiSupport( + boolean createTable, boolean insert, boolean read, String cqlDefinition) {} } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index 4c0e7c318..110df39f6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -19,6 +19,10 @@ public interface ColumnType { public String name(); + default String getApiName() { + return getApiDataType().getApiName(); + } + static List getSupportedTypes() { return List.of( "ascii", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index b0db01137..e4ad0c1c8 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -117,4 +117,35 @@ public int getDimension() { return vectorSize; } } + + /** + * Unsupported type implementation, returned in response when cql table has unsupported format + * column + */ + public static class UnsupportedType implements ColumnType { + private final String cqlFormat; + + public UnsupportedType(String cqlFormat) { + this.cqlFormat = cqlFormat; + } + + @Override + public ApiDataType getApiDataType() { + throw new UnsupportedOperationException("Unsupported type"); + } + + @Override + public String name() { + return "UNSUPPORTED"; + } + + @Override + public String getApiName() { + return "UNSUPPORTED"; + } + + public String cqlFormat() { + return cqlFormat; + } + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index c3be72286..7f0182c3b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -178,15 +178,42 @@ private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadat columnVectorDefinition.vectorizeConfig().parameters()); return new ComplexTypes.VectorType(PrimitiveTypes.FLOAT, vt.getDimensions(), vectorizeConfig); } else if (columnMetadata.getType() instanceof MapType mt) { - return new ComplexTypes.MapType( - PrimitiveTypes.fromString(mt.getKeyType().toString()), - PrimitiveTypes.fromString(mt.getValueType().toString())); + if (!mt.isFrozen()) { + final Optional apiDataTypeDefKey = ApiDataTypeDefs.from(mt.getKeyType()); + final Optional apiDataTypeDefValue = + ApiDataTypeDefs.from(mt.getValueType()); + if (apiDataTypeDefKey.isPresent() && apiDataTypeDefValue.isPresent()) { + return new ComplexTypes.MapType( + PrimitiveTypes.fromString(apiDataTypeDefKey.get().getApiType().getApiName()), + PrimitiveTypes.fromString(apiDataTypeDefValue.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(mt.asCql(true, false)); + } else if (columnMetadata.getType() instanceof com.datastax.oss.driver.api.core.type.ListType lt) { - return new ComplexTypes.ListType(PrimitiveTypes.fromString(lt.getElementType().toString())); + if (!lt.isFrozen()) { + final Optional apiDataTypeDef = ApiDataTypeDefs.from(lt.getElementType()); + if (apiDataTypeDef.isPresent()) { + return new ComplexTypes.ListType( + PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(lt.asCql(true, false)); + } else if (columnMetadata.getType() instanceof com.datastax.oss.driver.api.core.type.SetType st) { - return new ComplexTypes.SetType(PrimitiveTypes.fromString(st.getElementType().toString())); + if (!st.isFrozen()) { + final Optional apiDataTypeDef = ApiDataTypeDefs.from(st.getElementType()); + if (apiDataTypeDef.isPresent()) { + return new ComplexTypes.SetType( + PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(st.asCql(true, false)); } else { final Optional apiDataTypeDef = ApiDataTypeDefs.from(columnMetadata.getType()); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java index f6e918bc6..7776ca288 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -80,7 +80,7 @@ public Uni> execute( // filter for valid collections .filter(tableMatcher.negate()) // map to name - .map(table -> TableSchemaObject.getTableSettings(table, objectMapper)) + .map(table -> TableSchemaObject.from(table, objectMapper)) // get as list .toList(); // Wrap the properties list into a command result From d6cd08ff589c950ccf157a99fe19558ed5599026 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Fri, 4 Oct 2024 23:12:25 -0400 Subject: [PATCH 20/37] Made the SchemaObject store and return List of VectorConfig. This is needed because tables need multiple vector config. --- .../service/cqldriver/executor/DatabaseSchemaObject.java | 6 ++++-- .../service/cqldriver/executor/KeyspaceSchemaObject.java | 5 +++-- .../service/cqldriver/executor/TableSchemaObject.java | 2 +- .../jsonapi/service/processor/MeteredCommandProcessor.java | 2 +- .../schema/collections/CollectionSettingsV1Reader.java | 7 ++++++- .../api/configuration/CollectionSchemaObjectTest.java | 3 ++- .../service/cqldriver/executor/NamespaceCacheTest.java | 4 ++-- .../service/operation/collections/OperationTestBase.java | 2 +- 8 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java index 600f6106a..071e90375 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; +import java.util.List; + public class DatabaseSchemaObject extends SchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.DATABASE; @@ -9,8 +11,8 @@ public DatabaseSchemaObject() { } @Override - public VectorConfig vectorConfig() { - return VectorConfig.notEnabledVectorConfig(); + public List vectorConfigs() { + return List.of(VectorConfig.notEnabledVectorConfig()); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java index b0ea4f89f..08176a76e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; +import java.util.List; public class KeyspaceSchemaObject extends SchemaObject { @@ -39,8 +40,8 @@ public static KeyspaceSchemaObject fromSchemaObject(TableSchemaObject table) { } @Override - public VectorConfig vectorConfig() { - return VectorConfig.notEnabledVectorConfig(); + public List vectorConfigs() { + return List.of(VectorConfig.notEnabledVectorConfig()); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 38c07e5f5..b372414d8 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -33,7 +33,7 @@ private TableSchemaObject(TableMetadata tableMetadata, VectorConfig vectorConfig } @Override - public VectorConfig vectorConfig() { + public VectorConfig vectorConfig() { return vectorConfig; } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 5cb882f59..fb9d86c60 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -245,7 +245,7 @@ private Tags getCustomTags( } Tag vectorEnabled = - commandContext.schemaObject().vectorConfig().vectorEnabled() + commandContext.schemaObject().vectorConfigs().get(0).vectorEnabled() ? Tag.of(jsonApiMetricsConfig.vectorEnabled(), "true") : Tag.of(jsonApiMetricsConfig.vectorEnabled(), "false"); JsonApiMetricsConfig.SortType sortType = getVectorTypeTag(command); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java index 9d577a9e9..003d950c1 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java @@ -47,6 +47,11 @@ public CollectionSchemaObject readCollectionSettings( } return new CollectionSchemaObject( - keyspaceName, collectionName, tableMetadata, idConfig, vectorConfig, indexingConfig); + keyspaceName, + collectionName, + tableMetadata, + idConfig, + List.of(vectorConfig), + indexingConfig); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java index 2e8402d72..9a968c28d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java @@ -13,6 +13,7 @@ import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import org.junit.jupiter.api.Test; @QuarkusTest @@ -30,7 +31,7 @@ public void ensureSingleProjectorCreation() { "collectionName", null, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), indexingConfig); IndexingProjector indexingProj = settings.indexingProjector(); assertThat(indexingProj) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java index 3bd97a2b4..fb6166965 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java @@ -164,7 +164,7 @@ public void checkValidJsonApiTable() { assertThat(schemaObject) .satisfies( s -> { - assertThat(s.vectorConfig().vectorEnabled()).isFalse(); + assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); }); } @@ -296,7 +296,7 @@ public void checkValidJsonApiTableWithIndexing() { assertThat(collectionSchemaObject) .satisfies( s -> { - assertThat(s.vectorConfig().vectorEnabled()).isFalse(); + assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); assertThat(s.indexingConfig().denied()).containsExactly("comment"); }); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java index ccb32d8e2..ff6f05157 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java @@ -46,7 +46,7 @@ public class OperationTestBase { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - VectorConfig.notEnabledVectorConfig(), + List.of(VectorConfig.notEnabledVectorConfig()), null); protected final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = KeyspaceSchemaObject.fromSchemaObject(COLLECTION_SCHEMA_OBJECT); From 9f386d2327eec7ea3ecac3500c17b81b2e5271ff Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:48:16 -0400 Subject: [PATCH 21/37] Backup commit for list tables --- .../jsonapi/api/model/command/Command.java | 1 + .../ColumnDefinitionSerializer.java | 36 ++++ .../model/command/impl/ListTablesCommand.java | 23 +++ .../command/table/definition/PrimaryKey.java | 10 +- .../table/definition/datatype/ColumnType.java | 45 +---- .../definition/datatype/ComplexTypes.java | 36 ++++ .../definition/datatype/PrimitiveTypes.java | 167 ++++-------------- .../operation/tables/ListTablesOperation.java | 113 ++++++++++++ .../service/schema/tables/ApiDataTypeDef.java | 13 -- .../schema/tables/ComplexApiDataType.java | 19 +- 10 files changed, 269 insertions(+), 194 deletions(-) create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java index 7640337c8..bc18ad581 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java @@ -64,6 +64,7 @@ enum CommandName { FIND_ONE("findOne"), INSERT_MANY("insertMany"), INSERT_ONE("insertOne"), + LIST_TABLES("listTables"), UPDATE_MANY("updateMany"), UPDATE_ONE("updateOne"), BEGIN_OFFLINE_SESSION("beginOfflineSession"), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java new file mode 100644 index 000000000..3d7c9994f --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java @@ -0,0 +1,36 @@ +package io.stargate.sgv2.jsonapi.api.model.command.deserializers; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ColumnType; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ComplexTypes; +import java.io.IOException; + +/** + * Custom serializer to encode the column type to the JSON payload This is required because + * composite and custom column types may need additional properties to be serialized + */ +public class ColumnDefinitionSerializer extends JsonSerializer { + + @Override + public void serialize( + ColumnType columnType, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeStringField("type", columnType.name()); + if (columnType instanceof ComplexTypes.MapType mt) { + jsonGenerator.writeStringField("keyType", mt.keyType()); + jsonGenerator.writeStringField("valueType", mt.valueType()); + } else if (columnType instanceof ComplexTypes.ListType lt) { + jsonGenerator.writeStringField("valueType", lt.valueType()); + } else if (columnType instanceof ComplexTypes.SetType st) { + jsonGenerator.writeStringField("st", lt.valueType()); + } else if (columnType instanceof ComplexTypes.VectorType vt) { + jsonGenerator.writeNumberField("dimension", vectorType.dimension()); + if (vectorType.getVectorConfig() != null) + jsonGenerator.writeObjectField("service", vt.getVectorConfig()); + } + jsonGenerator.writeEndObject(); + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java new file mode 100644 index 000000000..b1ae2f8d1 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java @@ -0,0 +1,23 @@ +package io.stargate.sgv2.jsonapi.api.model.command.impl; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.stargate.sgv2.jsonapi.api.model.command.CollectionOnlyCommand; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.media.Schema; + +@Schema(description = "Command that lists all available tables in a namespace.") +@JsonTypeName("listTables") +public record ListTablesCommand(Options options) implements CollectionOnlyCommand { + public record Options( + @Schema( + description = "include table properties.", + type = SchemaType.BOOLEAN, + implementation = Boolean.class) + boolean explain) {} + + /** {@inheritDoc} */ + @Override + public CommandName commandName() { + return CommandName.LIST_TABLES; + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java index abfd0d14b..a96881e2d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.api.model.command.table.definition; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.PrimaryKeyDeserializer; @@ -16,9 +17,14 @@ // implementation = Object.class, // description = "Represents the table primary key") public record PrimaryKey( - @NotNull @Schema(description = "Columns that make the partition keys", type = SchemaType.ARRAY) + @NotNull + @Schema(description = "Columns that make the partition keys", type = SchemaType.ARRAY) + @JsonProperty("partitionBy") + @JsonInclude(JsonInclude.Include.NON_NULL) String[] keys, - @Nullable @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) + @Nullable + @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) + @JsonProperty("partitionSort") OrderingKey[] orderingKeys) { public record OrderingKey(String column, Order order) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index d2da91411..4c0e7c318 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -1,7 +1,9 @@ package io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionDeserializer; +import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionSerializer; import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; @@ -10,10 +12,13 @@ /** Interface for column types. This is used to define the type of a column in a table. */ @JsonDeserialize(using = ColumnDefinitionDeserializer.class) +@JsonSerialize(using = ColumnDefinitionSerializer.class) public interface ColumnType { // Returns api data type. ApiDataType getApiDataType(); + public String name(); + static List getSupportedTypes() { return List.of( "ascii", @@ -46,42 +51,6 @@ static ColumnType fromString( // TODO: the name of the type should be a part of the ColumnType interface, and use a map for // the lookup switch (type) { - case "ascii": - return PrimitiveTypes.ASCII; - case "bigint": - return PrimitiveTypes.BIGINT; - case "blob": - return PrimitiveTypes.BINARY; - case "boolean": - return PrimitiveTypes.BOOLEAN; - case "date": - return PrimitiveTypes.DATE; - case "decimal": - return PrimitiveTypes.DECIMAL; - case "double": - return PrimitiveTypes.DOUBLE; - case "duration": - return PrimitiveTypes.DURATION; - case "float": - return PrimitiveTypes.FLOAT; - case "inet": - return PrimitiveTypes.INET; - case "int": - return PrimitiveTypes.INT; - case "smallint": - return PrimitiveTypes.SMALLINT; - case "text": - return PrimitiveTypes.TEXT; - case "time": - return PrimitiveTypes.TIME; - case "timestamp": - return PrimitiveTypes.TIMESTAMP; - case "tinyint": - return PrimitiveTypes.TINYINT; - case "uuid": - return PrimitiveTypes.UUID; - case "varint": - return PrimitiveTypes.VARINT; case "map": { if (keyType == null || valueType == null) { @@ -134,6 +103,10 @@ static ColumnType fromString( } default: { + ColumnType columnType = PrimitiveTypes.fromString(type); + if (columnType != null) { + return columnType; + } Map errorMessageFormattingValues = Map.of( "type", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index 4bb640961..b0db01137 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -24,6 +24,19 @@ public ApiDataType getApiDataType() { (PrimitiveApiDataType) keyType.getApiDataType(), (PrimitiveApiDataType) valueType.getApiDataType()); } + + @Override + public String name() { + return getApiDataType().getApiName(); + } + + public String keyType() { + return keyType.getApiDataType().getApiName(); + } + + public String valueType() { + return valueType.getApiDataType().getApiName(); + } } /** A list type implementation */ @@ -38,6 +51,15 @@ public ListType(ColumnType valueType) { public ApiDataType getApiDataType() { return new ComplexApiDataType.ListType((PrimitiveApiDataType) valueType.getApiDataType()); } + + @Override + public String name() { + return "list"; + } + + public String valueType() { + return valueType.getApiDataType().getApiName(); + } } /** A set type implementation */ @@ -52,6 +74,15 @@ public SetType(ColumnType valueType) { public ApiDataType getApiDataType() { return new ComplexApiDataType.SetType((PrimitiveApiDataType) valueType.getApiDataType()); } + + @Override + public String name() { + return "set"; + } + + public String valueType() { + return valueType.getApiDataType().getApiName(); + } } /* Vector type */ @@ -77,6 +108,11 @@ public VectorizeConfig getVectorConfig() { return vectorConfig; } + @Override + public String name() { + return "vector"; + } + public int getDimension() { return vectorSize; } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java index 630bb798c..e839de4a9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java @@ -2,154 +2,57 @@ import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; import io.stargate.sgv2.jsonapi.service.schema.tables.PrimitiveApiDataType; +import java.util.HashMap; +import java.util.Map; /** Interface for primitive column types similar to what is defined in cassandra java driver. */ -public class PrimitiveTypes { +public enum PrimitiveTypes implements ColumnType { // TODO: add a private ctor to stop this class from being instantiated or make abstract - public static final ColumnType ASCII = new Ascii(); - public static final ColumnType BIGINT = new BigInt(); - public static final ColumnType BINARY = new Binary(); - public static final ColumnType BOOLEAN = new Boolean(); - public static final ColumnType DATE = new Date(); - public static final ColumnType DECIMAL = new Decimal(); - public static final ColumnType DOUBLE = new Double(); - public static final ColumnType DURATION = new Duration(); - public static final ColumnType FLOAT = new Float(); - public static final ColumnType INET = new Inet(); - public static final ColumnType INT = new Int(); - public static final ColumnType SMALLINT = new SmallInt(); - public static final ColumnType TEXT = new Text(); - public static final ColumnType TIME = new Time(); - public static final ColumnType TIMESTAMP = new Timestamp(); - public static final ColumnType TINYINT = new TinyInt(); - public static final ColumnType UUID = new Uuid(); - public static final ColumnType VARINT = new VarInt(); + ASCII(PrimitiveApiDataType.ASCII), + BIGINT(PrimitiveApiDataType.BIGINT), + BINARY(PrimitiveApiDataType.BINARY), + BOOLEAN(PrimitiveApiDataType.BOOLEAN), + DATE(PrimitiveApiDataType.DATE), + DECIMAL(PrimitiveApiDataType.DECIMAL), + DOUBLE(PrimitiveApiDataType.DOUBLE), + DURATION(PrimitiveApiDataType.DURATION), + FLOAT(PrimitiveApiDataType.FLOAT), + INET(PrimitiveApiDataType.INET), + INT(PrimitiveApiDataType.INT), + SMALLINT(PrimitiveApiDataType.SMALLINT), + TEXT(PrimitiveApiDataType.TEXT), + TIME(PrimitiveApiDataType.TIME), + TIMESTAMP(PrimitiveApiDataType.TIMESTAMP), + TINYINT(PrimitiveApiDataType.TINYINT), + UUID(PrimitiveApiDataType.UUID), + VARINT(PrimitiveApiDataType.VARINT); - private static class Text implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TEXT; - } + @Override + public ApiDataType getApiDataType() { + return getApiDataType; } - private static class Int implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.INT; - } + PrimitiveTypes(ApiDataType getApiDataType) { + this.getApiDataType = getApiDataType; } - private static class Boolean implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BOOLEAN; - } - } + private ApiDataType getApiDataType; - private static class BigInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BIGINT; - } - } + private static Map columnTypeMap = new HashMap<>(); - private static class Decimal implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DECIMAL; + static { + for (PrimitiveTypes type : PrimitiveTypes.values()) { + columnTypeMap.put(type.getApiDataType().getApiName(), type); } } - private static class Double implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DOUBLE; - } + public static ColumnType fromString(String type) { + return columnTypeMap.get(type); } - private static class Float implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.FLOAT; - } - } - - private static class SmallInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.SMALLINT; - } - } - - private static class TinyInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TINYINT; - } - } - - private static class VarInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.VARINT; - } - } - - private static class Ascii implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.ASCII; - } - } - - private static class Binary implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BINARY; - } - } - - private static class Date implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DATE; - } - } - - private static class Duration implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DURATION; - } - } - - private static class Time implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TIME; - } - } - - private static class Timestamp implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TIMESTAMP; - } - } - - private static class Inet implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.INET; - } - } - - private static class Uuid implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.UUID; - } + public static String typeAsString(ColumnType type) { + return type.getApiDataType().getApiName(); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java new file mode 100644 index 000000000..f6e918bc6 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -0,0 +1,113 @@ +package io.stargate.sgv2.jsonapi.service.operation.tables; + +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; +import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.api.request.DataApiRequestInfo; +import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; +import io.stargate.sgv2.jsonapi.service.operation.Operation; +import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * List tables operation. Uses {@link CQLSessionCache} to fetch all valid tables for a namespace. + * The schema check against the table is done in the {@link CollectionTableMatcher} and ignores + * them. + * + * @param explain - returns collection options if `true`; returns only collection names if `false` + * @param objectMapper {@link ObjectMapper} + * @param cqlSessionCache {@link CQLSessionCache} + * @param tableMatcher {@link CollectionTableMatcher} + * @param commandContext {@link CommandContext} + */ +public record ListTablesOperation( + boolean explain, + ObjectMapper objectMapper, + CQLSessionCache cqlSessionCache, + CollectionTableMatcher tableMatcher, + CommandContext commandContext) + implements Operation { + + // shared table matcher instance + // TODO: if this is static why does the record that have an instance variable passed by the ctor + // below ? + private static final CollectionTableMatcher TABLE_MATCHER = new CollectionTableMatcher(); + + public ListTablesOperation( + boolean explain, + ObjectMapper objectMapper, + CQLSessionCache cqlSessionCache, + CommandContext commandContext) { + this(explain, objectMapper, cqlSessionCache, TABLE_MATCHER, commandContext); + } + + /** {@inheritDoc} */ + @Override + public Uni> execute( + DataApiRequestInfo dataApiRequestInfo, QueryExecutor queryExecutor) { + KeyspaceMetadata keyspaceMetadata = + cqlSessionCache + .getSession(dataApiRequestInfo) + .getMetadata() + .getKeyspaces() + .get(CqlIdentifier.fromInternal(commandContext.schemaObject().name().keyspace())); + if (keyspaceMetadata == null) { + return Uni.createFrom() + .failure( + ErrorCodeV1.KEYSPACE_DOES_NOT_EXIST.toApiException( + "Unknown keyspace '%s', you must create it first", + commandContext.schemaObject().name().keyspace())); + } + return Uni.createFrom() + .item( + () -> { + List properties = + keyspaceMetadata + // get all tables + .getTables() + .values() + .stream() + // filter for valid collections + .filter(tableMatcher.negate()) + // map to name + .map(table -> TableSchemaObject.getTableSettings(table, objectMapper)) + // get as list + .toList(); + // Wrap the properties list into a command result + return new Result(explain, properties); + }); + } + + // simple result wrapper + private record Result(boolean explain, List tables) + implements Supplier { + + @Override + public CommandResult get() { + if (explain) { + final List createCollectionCommands = + tables().stream() + .map(tableSchemaObject -> tableSchemaObject.toTableResponse()) + .toList(); + Map statuses = + Map.of(CommandStatus.EXISTING_COLLECTIONS, createCollectionCommands); + return new CommandResult(statuses); + } else { + List tables = + tables().stream().map(schemaObject -> schemaObject.name().table()).toList(); + Map statuses = Map.of(CommandStatus.EXISTING_COLLECTIONS, tables); + return new CommandResult(statuses); + } + } + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java index 4bbd6f0d3..c3d02b5a5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java @@ -3,7 +3,6 @@ import com.datastax.oss.driver.api.core.type.*; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.Objects; -import java.util.Optional; /** * The definition of a type the API supports for a table column. @@ -42,18 +41,6 @@ public boolean isContainer() { return cqlType instanceof ContainerType; } - public Optional cqlTypeAsList() { - return cqlType instanceof ListType listType ? Optional.of(listType) : Optional.empty(); - } - - public Optional cqlTypeAsSet() { - return cqlType instanceof SetType setType ? Optional.of(setType) : Optional.empty(); - } - - public Optional cqlTypeAsMap() { - return cqlType instanceof MapType mapType ? Optional.of(mapType) : Optional.empty(); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java index a27eb0418..28afa0397 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java @@ -9,17 +9,14 @@ public abstract class ComplexApiDataType implements ApiDataType { private final String apiName; private final PrimitiveApiDataType keyType; private final PrimitiveApiDataType valueType; - private final int vectorSize; + private final int dimension; public ComplexApiDataType( - String apiName, - PrimitiveApiDataType keyType, - PrimitiveApiDataType valueType, - int vectorSize) { + String apiName, PrimitiveApiDataType keyType, PrimitiveApiDataType valueType, int dimension) { this.apiName = apiName; this.keyType = keyType; this.valueType = valueType; - this.vectorSize = vectorSize; + this.dimension = dimension; } public PrimitiveApiDataType getKeyType() { @@ -30,8 +27,8 @@ public PrimitiveApiDataType getValueType() { return valueType; } - public int getVectorSize() { - return vectorSize; + public int getDimension() { + return dimension; } public abstract DataType getCqlType(); @@ -77,14 +74,14 @@ public DataType getCqlType() { } public static class VectorType extends ComplexApiDataType { - public VectorType(PrimitiveApiDataType valueType, int vectorSize) { - super("vector", null, valueType, vectorSize); + public VectorType(PrimitiveApiDataType valueType, int dimension) { + super("vector", null, valueType, dimension); } @Override public DataType getCqlType() { return new ExtendedVectorType( - ApiDataTypeDefs.from(getValueType()).get().getCqlType(), getVectorSize()); + ApiDataTypeDefs.from(getValueType()).get().getCqlType(), getDimension()); } } From fc48d0d2c860719c746e36718ea39afe298bc8fe Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:22:48 -0400 Subject: [PATCH 22/37] Updated the code based on review. --- .../service/cqldriver/executor/DatabaseSchemaObject.java | 6 ++---- .../service/cqldriver/executor/KeyspaceSchemaObject.java | 5 ++--- .../schema/collections/CollectionSettingsV1Reader.java | 7 +------ .../api/configuration/CollectionSchemaObjectTest.java | 3 +-- .../service/cqldriver/executor/NamespaceCacheTest.java | 4 ++-- .../service/operation/collections/OperationTestBase.java | 2 +- 6 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java index 071e90375..600f6106a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DatabaseSchemaObject.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; -import java.util.List; - public class DatabaseSchemaObject extends SchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.DATABASE; @@ -11,8 +9,8 @@ public DatabaseSchemaObject() { } @Override - public List vectorConfigs() { - return List.of(VectorConfig.notEnabledVectorConfig()); + public VectorConfig vectorConfig() { + return VectorConfig.notEnabledVectorConfig(); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java index 08176a76e..b0ea4f89f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/KeyspaceSchemaObject.java @@ -1,7 +1,6 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; -import java.util.List; public class KeyspaceSchemaObject extends SchemaObject { @@ -40,8 +39,8 @@ public static KeyspaceSchemaObject fromSchemaObject(TableSchemaObject table) { } @Override - public List vectorConfigs() { - return List.of(VectorConfig.notEnabledVectorConfig()); + public VectorConfig vectorConfig() { + return VectorConfig.notEnabledVectorConfig(); } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java index 003d950c1..9d577a9e9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java @@ -47,11 +47,6 @@ public CollectionSchemaObject readCollectionSettings( } return new CollectionSchemaObject( - keyspaceName, - collectionName, - tableMetadata, - idConfig, - List.of(vectorConfig), - indexingConfig); + keyspaceName, collectionName, tableMetadata, idConfig, vectorConfig, indexingConfig); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java index 9a968c28d..2e8402d72 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/configuration/CollectionSchemaObjectTest.java @@ -13,7 +13,6 @@ import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import java.util.Arrays; import java.util.HashSet; -import java.util.List; import org.junit.jupiter.api.Test; @QuarkusTest @@ -31,7 +30,7 @@ public void ensureSingleProjectorCreation() { "collectionName", null, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), indexingConfig); IndexingProjector indexingProj = settings.indexingProjector(); assertThat(indexingProj) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java index fb6166965..3bd97a2b4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCacheTest.java @@ -164,7 +164,7 @@ public void checkValidJsonApiTable() { assertThat(schemaObject) .satisfies( s -> { - assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); + assertThat(s.vectorConfig().vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); }); } @@ -296,7 +296,7 @@ public void checkValidJsonApiTableWithIndexing() { assertThat(collectionSchemaObject) .satisfies( s -> { - assertThat(s.vectorConfigs().get(0).vectorEnabled()).isFalse(); + assertThat(s.vectorConfig().vectorEnabled()).isFalse(); assertThat(s.name.table()).isEqualTo("table"); assertThat(s.indexingConfig().denied()).containsExactly("comment"); }); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java index ff6f05157..ccb32d8e2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/OperationTestBase.java @@ -46,7 +46,7 @@ public class OperationTestBase { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - List.of(VectorConfig.notEnabledVectorConfig()), + VectorConfig.notEnabledVectorConfig(), null); protected final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = KeyspaceSchemaObject.fromSchemaObject(COLLECTION_SCHEMA_OBJECT); From e01992ee542024a41f04d3ce1edd0699d6bd8d2c Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:44:50 -0400 Subject: [PATCH 23/37] Working listTables --- .../api/model/command/TableOnlyCommand.java | 4 +- .../ColumnDefinitionSerializer.java | 6 +-- .../model/command/impl/ListTablesCommand.java | 4 +- .../resolver/ListTablesCommandResolver.java | 40 +++++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java index 23b48456d..5e9e61a0c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java @@ -4,11 +4,13 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.stargate.sgv2.jsonapi.api.model.command.impl.CreateTableCommand; import io.stargate.sgv2.jsonapi.api.model.command.impl.DropTableCommand; +import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; /** Interface for all commands executed against a keyspace. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonSubTypes({ @JsonSubTypes.Type(value = CreateTableCommand.class), - @JsonSubTypes.Type(value = DropTableCommand.class) + @JsonSubTypes.Type(value = DropTableCommand.class), + @JsonSubTypes.Type(value = ListTablesCommand.class), }) public interface TableOnlyCommand extends KeyspaceCommand {} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java index 3d7c9994f..2ba784a24 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java @@ -25,10 +25,10 @@ public void serialize( } else if (columnType instanceof ComplexTypes.ListType lt) { jsonGenerator.writeStringField("valueType", lt.valueType()); } else if (columnType instanceof ComplexTypes.SetType st) { - jsonGenerator.writeStringField("st", lt.valueType()); + jsonGenerator.writeStringField("st", st.valueType()); } else if (columnType instanceof ComplexTypes.VectorType vt) { - jsonGenerator.writeNumberField("dimension", vectorType.dimension()); - if (vectorType.getVectorConfig() != null) + jsonGenerator.writeNumberField("dimension", vt.getDimension()); + if (vt.getVectorConfig() != null) jsonGenerator.writeObjectField("service", vt.getVectorConfig()); } jsonGenerator.writeEndObject(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java index b1ae2f8d1..8f0759ba7 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java @@ -1,13 +1,13 @@ package io.stargate.sgv2.jsonapi.api.model.command.impl; import com.fasterxml.jackson.annotation.JsonTypeName; -import io.stargate.sgv2.jsonapi.api.model.command.CollectionOnlyCommand; +import io.stargate.sgv2.jsonapi.api.model.command.TableOnlyCommand; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; import org.eclipse.microprofile.openapi.annotations.media.Schema; @Schema(description = "Command that lists all available tables in a namespace.") @JsonTypeName("listTables") -public record ListTablesCommand(Options options) implements CollectionOnlyCommand { +public record ListTablesCommand(Options options) implements TableOnlyCommand { public record Options( @Schema( description = "include table properties.", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java new file mode 100644 index 000000000..83b9d3dd6 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java @@ -0,0 +1,40 @@ +package io.stargate.sgv2.jsonapi.service.resolver; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.api.model.command.impl.FindCollectionsCommand; +import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.operation.Operation; +import io.stargate.sgv2.jsonapi.service.operation.tables.ListTablesOperation; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +/** Command resolver for the {@link FindCollectionsCommand}. */ +@ApplicationScoped +public class ListTablesCommandResolver implements CommandResolver { + private final ObjectMapper objectMapper; + private final CQLSessionCache cqlSessionCache; + + @Inject + public ListTablesCommandResolver(ObjectMapper objectMapper, CQLSessionCache cqlSessionCache) { + this.objectMapper = objectMapper; + this.cqlSessionCache = cqlSessionCache; + } + + /** {@inheritDoc} */ + @Override + public Class getCommandClass() { + return ListTablesCommand.class; + } + + /** {@inheritDoc} */ + @Override + public Operation resolveKeyspaceCommand( + CommandContext ctx, ListTablesCommand command) { + + boolean explain = command.options() != null ? command.options().explain() : false; + return new ListTablesOperation(explain, objectMapper, cqlSessionCache, ctx); + } +} From df3bfd5fad22213a259798d594a26a6b12694b6b Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:17:02 -0400 Subject: [PATCH 24/37] Added UNSUPPORTED type support --- .../ColumnDefinitionSerializer.java | 10 ++++-- .../table/definition/datatype/ColumnType.java | 4 +++ .../definition/datatype/ComplexTypes.java | 31 +++++++++++++++++++ .../operation/tables/ListTablesOperation.java | 2 +- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java index 2ba784a24..076b6ed73 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java @@ -18,19 +18,25 @@ public void serialize( ColumnType columnType, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { jsonGenerator.writeStartObject(); - jsonGenerator.writeStringField("type", columnType.name()); + jsonGenerator.writeStringField("type", columnType.getApiName()); if (columnType instanceof ComplexTypes.MapType mt) { jsonGenerator.writeStringField("keyType", mt.keyType()); jsonGenerator.writeStringField("valueType", mt.valueType()); } else if (columnType instanceof ComplexTypes.ListType lt) { jsonGenerator.writeStringField("valueType", lt.valueType()); } else if (columnType instanceof ComplexTypes.SetType st) { - jsonGenerator.writeStringField("st", st.valueType()); + jsonGenerator.writeStringField("valueType", st.valueType()); } else if (columnType instanceof ComplexTypes.VectorType vt) { jsonGenerator.writeNumberField("dimension", vt.getDimension()); if (vt.getVectorConfig() != null) jsonGenerator.writeObjectField("service", vt.getVectorConfig()); + } else if (columnType instanceof ComplexTypes.UnsupportedType ut) { + jsonGenerator.writeObjectField( + "apiSupport", new ApiSupport(false, false, false, ut.cqlFormat())); } jsonGenerator.writeEndObject(); } + + public record ApiSupport( + boolean createTable, boolean insert, boolean read, String cqlDefinition) {} } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index 4c0e7c318..110df39f6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -19,6 +19,10 @@ public interface ColumnType { public String name(); + default String getApiName() { + return getApiDataType().getApiName(); + } + static List getSupportedTypes() { return List.of( "ascii", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index b0db01137..e4ad0c1c8 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -117,4 +117,35 @@ public int getDimension() { return vectorSize; } } + + /** + * Unsupported type implementation, returned in response when cql table has unsupported format + * column + */ + public static class UnsupportedType implements ColumnType { + private final String cqlFormat; + + public UnsupportedType(String cqlFormat) { + this.cqlFormat = cqlFormat; + } + + @Override + public ApiDataType getApiDataType() { + throw new UnsupportedOperationException("Unsupported type"); + } + + @Override + public String name() { + return "UNSUPPORTED"; + } + + @Override + public String getApiName() { + return "UNSUPPORTED"; + } + + public String cqlFormat() { + return cqlFormat; + } + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java index f6e918bc6..7776ca288 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -80,7 +80,7 @@ public Uni> execute( // filter for valid collections .filter(tableMatcher.negate()) // map to name - .map(table -> TableSchemaObject.getTableSettings(table, objectMapper)) + .map(table -> TableSchemaObject.from(table, objectMapper)) // get as list .toList(); // Wrap the properties list into a command result From 9a2b13670f88368d6fb597444a0fa3529e1d3625 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:30:25 -0400 Subject: [PATCH 25/37] Rebased to main --- .../cqldriver/executor/TableSchemaObject.java | 121 +++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index b372414d8..7f0182c3b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -2,15 +2,26 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.data.ByteUtils; +import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; import com.datastax.oss.driver.api.core.metadata.schema.IndexMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.api.core.type.MapType; import com.datastax.oss.driver.api.core.type.VectorType; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.PrimaryKey; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ColumnType; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ComplexTypes; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.PrimitiveTypes; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDef; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDefs; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -20,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; public class TableSchemaObject extends TableBasedSchemaObject { @@ -33,7 +45,7 @@ private TableSchemaObject(TableMetadata tableMetadata, VectorConfig vectorConfig } @Override - public VectorConfig vectorConfig() { + public VectorConfig vectorConfig() { return vectorConfig; } @@ -115,4 +127,111 @@ public static TableSchemaObject from(TableMetadata tableMetadata, ObjectMapper o } return new TableSchemaObject(tableMetadata, vectorConfig); } + + public TableResponse toTableResponse() { + String tableName = name().table(); + HashMap columnsDefinition = new HashMap<>(); + for (Map.Entry column : + tableMetadata().getColumns().entrySet()) { + ColumnType type = getColumnType(column.getKey().asInternal(), column.getValue()); + columnsDefinition.put(column.getKey().asInternal(), type); + } + + final List partitionBy = + tableMetadata().getPartitionKey().stream() + .map(column -> column.getName().asInternal()) + .collect(Collectors.toList()); + final List partitionSort = + tableMetadata().getClusteringColumns().entrySet().stream() + .map( + entry -> + new PrimaryKey.OrderingKey( + entry.getKey().getName().asInternal(), + entry.getValue() == ClusteringOrder.ASC + ? PrimaryKey.OrderingKey.Order.ASC + : PrimaryKey.OrderingKey.Order.DESC)) + .collect(Collectors.toList()); + PrimaryKey primaryKey = + new PrimaryKey( + partitionBy.toArray(new String[0]), + partitionSort.toArray(new PrimaryKey.OrderingKey[0])); + return new TableResponse( + tableName, new TableResponse.TableDefinition(columnsDefinition, primaryKey)); + } + + // Need to handle frozen types + private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadata) { + if (columnMetadata.getType() instanceof VectorType vt) { + // Schema will always have VectorConfig for vector type + VectorConfig.ColumnVectorDefinition columnVectorDefinition = + vectorConfig.columnVectorDefinitions().stream() + .filter(vc -> vc.fieldName().equals(columnName)) + .findFirst() + .get(); + VectorizeConfig vectorizeConfig = + columnVectorDefinition.vectorizeConfig() == null + ? null + : new VectorizeConfig( + columnVectorDefinition.vectorizeConfig().provider(), + columnVectorDefinition.vectorizeConfig().modelName(), + columnVectorDefinition.vectorizeConfig().authentication(), + columnVectorDefinition.vectorizeConfig().parameters()); + return new ComplexTypes.VectorType(PrimitiveTypes.FLOAT, vt.getDimensions(), vectorizeConfig); + } else if (columnMetadata.getType() instanceof MapType mt) { + if (!mt.isFrozen()) { + final Optional apiDataTypeDefKey = ApiDataTypeDefs.from(mt.getKeyType()); + final Optional apiDataTypeDefValue = + ApiDataTypeDefs.from(mt.getValueType()); + if (apiDataTypeDefKey.isPresent() && apiDataTypeDefValue.isPresent()) { + return new ComplexTypes.MapType( + PrimitiveTypes.fromString(apiDataTypeDefKey.get().getApiType().getApiName()), + PrimitiveTypes.fromString(apiDataTypeDefValue.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(mt.asCql(true, false)); + + } else if (columnMetadata.getType() + instanceof com.datastax.oss.driver.api.core.type.ListType lt) { + if (!lt.isFrozen()) { + final Optional apiDataTypeDef = ApiDataTypeDefs.from(lt.getElementType()); + if (apiDataTypeDef.isPresent()) { + return new ComplexTypes.ListType( + PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(lt.asCql(true, false)); + + } else if (columnMetadata.getType() + instanceof com.datastax.oss.driver.api.core.type.SetType st) { + if (!st.isFrozen()) { + final Optional apiDataTypeDef = ApiDataTypeDefs.from(st.getElementType()); + if (apiDataTypeDef.isPresent()) { + return new ComplexTypes.SetType( + PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(st.asCql(true, false)); + } else { + final Optional apiDataTypeDef = + ApiDataTypeDefs.from(columnMetadata.getType()); + if (apiDataTypeDef.isPresent()) + return PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName()); + else { + // Need to return unsupported type + throw new RuntimeException("Unknown data type: " + columnMetadata.getType()); + } + } + } + + @JsonPropertyOrder({"name", "definition"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record TableResponse(String name, TableDefinition definition) { + + @JsonPropertyOrder({"columns", "primaryKey"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + record TableDefinition(Map columns, PrimaryKey primaryKey) {} + } } From 79861775f3a691fd6df7e1a928918c3924f459a6 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:33:56 -0400 Subject: [PATCH 26/37] Rebased to main --- .../sgv2/jsonapi/service/processor/MeteredCommandProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index fb9d86c60..5cb882f59 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -245,7 +245,7 @@ private Tags getCustomTags( } Tag vectorEnabled = - commandContext.schemaObject().vectorConfigs().get(0).vectorEnabled() + commandContext.schemaObject().vectorConfig().vectorEnabled() ? Tag.of(jsonApiMetricsConfig.vectorEnabled(), "true") : Tag.of(jsonApiMetricsConfig.vectorEnabled(), "false"); JsonApiMetricsConfig.SortType sortType = getVectorTypeTag(command); From b3e7cb1523d004388a2f4e443f4483d6d544be28 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:36:46 -0400 Subject: [PATCH 27/37] Removed unused method --- .../command/table/definition/datatype/PrimitiveTypes.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java index e839de4a9..ccdb62ebc 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java @@ -51,8 +51,4 @@ public ApiDataType getApiDataType() { public static ColumnType fromString(String type) { return columnTypeMap.get(type); } - - public static String typeAsString(ColumnType type) { - return type.getApiDataType().getApiName(); - } } From e42aaa282619349f759c89853fddffc55a87dc42 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:41:40 -0400 Subject: [PATCH 28/37] Added comments --- .../service/cqldriver/executor/TableSchemaObject.java | 11 +++++++++++ .../service/operation/tables/ListTablesOperation.java | 2 +- .../service/resolver/ListTablesCommandResolver.java | 3 +-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 7f0182c3b..1bc73e3f5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -128,6 +128,11 @@ public static TableSchemaObject from(TableMetadata tableMetadata, ObjectMapper o return new TableSchemaObject(tableMetadata, vectorConfig); } + /** + * Convert table schema object to table response which is returned as response for `listTables` + * + * @return + */ public TableResponse toTableResponse() { String tableName = name().table(); HashMap columnsDefinition = new HashMap<>(); @@ -226,6 +231,12 @@ private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadat } } + /** + * Object used to build the response for listTables command + * + * @param name + * @param definition + */ @JsonPropertyOrder({"name", "definition"}) @JsonInclude(JsonInclude.Include.NON_NULL) public record TableResponse(String name, TableDefinition definition) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java index 7776ca288..0696850ac 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -24,7 +24,7 @@ * The schema check against the table is done in the {@link CollectionTableMatcher} and ignores * them. * - * @param explain - returns collection options if `true`; returns only collection names if `false` + * @param explain - returns tables schema if `true`; returns only tables names if `false` * @param objectMapper {@link ObjectMapper} * @param cqlSessionCache {@link CQLSessionCache} * @param tableMatcher {@link CollectionTableMatcher} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java index 83b9d3dd6..6f62db096 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; -import io.stargate.sgv2.jsonapi.api.model.command.impl.FindCollectionsCommand; import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; @@ -11,7 +10,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -/** Command resolver for the {@link FindCollectionsCommand}. */ +/** Command resolver for the {@link ListTablesCommand}. */ @ApplicationScoped public class ListTablesCommandResolver implements CommandResolver { private final ObjectMapper objectMapper; From e6096a96ebe1c6b47cb18d4f5094538372c67ec6 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:14:43 -0400 Subject: [PATCH 29/37] Added tests --- .../api/model/command/CommandStatus.java | 4 + .../cqldriver/executor/TableSchemaObject.java | 13 +- .../operation/tables/ListTablesOperation.java | 10 +- .../AbstractKeyspaceIntegrationTestBase.java | 2 + .../AbstractTableIntegrationTestBase.java | 8 + .../v1/tables/ListTablesIntegrationTest.java | 181 ++++++++++++++++++ .../v1/util/DataApiKeyspaceCommandSender.java | 4 + 7 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java index bd46934b1..909707c39 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java @@ -26,6 +26,10 @@ public enum CommandStatus { /** Status for reporting existing collections. */ @JsonProperty("collections") EXISTING_COLLECTIONS, + /** Status for reporting existing collections. */ + @JsonProperty("tables") + EXISTING_TABLES, + /** * List of response entries, one for each document we tried to insert with {@code insertMany} * command. Each entry has 2 mandatory fields: {@code _id} (document id), and {@code status} (one diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 1bc73e3f5..83f8060d3 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -134,24 +134,24 @@ public static TableSchemaObject from(TableMetadata tableMetadata, ObjectMapper o * @return */ public TableResponse toTableResponse() { - String tableName = name().table(); + String tableName = removeQuotes(tableMetadata().getName()); HashMap columnsDefinition = new HashMap<>(); for (Map.Entry column : tableMetadata().getColumns().entrySet()) { ColumnType type = getColumnType(column.getKey().asInternal(), column.getValue()); - columnsDefinition.put(column.getKey().asInternal(), type); + columnsDefinition.put(removeQuotes(column.getKey()), type); } final List partitionBy = tableMetadata().getPartitionKey().stream() - .map(column -> column.getName().asInternal()) + .map(column -> removeQuotes(column.getName())) .collect(Collectors.toList()); final List partitionSort = tableMetadata().getClusteringColumns().entrySet().stream() .map( entry -> new PrimaryKey.OrderingKey( - entry.getKey().getName().asInternal(), + removeQuotes(entry.getKey().getName()), entry.getValue() == ClusteringOrder.ASC ? PrimaryKey.OrderingKey.Order.ASC : PrimaryKey.OrderingKey.Order.DESC)) @@ -231,6 +231,11 @@ private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadat } } + // Remove the quotes from the identifier + public String removeQuotes(CqlIdentifier identifier) { + return identifier.asCql(true).replaceAll("\"", ""); + } + /** * Object used to build the response for listTables command * diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java index 0696850ac..33b7f5e93 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -100,12 +100,16 @@ public CommandResult get() { .map(tableSchemaObject -> tableSchemaObject.toTableResponse()) .toList(); Map statuses = - Map.of(CommandStatus.EXISTING_COLLECTIONS, createCollectionCommands); + Map.of(CommandStatus.EXISTING_TABLES, createCollectionCommands); return new CommandResult(statuses); } else { List tables = - tables().stream().map(schemaObject -> schemaObject.name().table()).toList(); - Map statuses = Map.of(CommandStatus.EXISTING_COLLECTIONS, tables); + tables().stream() + .map( + schemaObject -> + schemaObject.removeQuotes(schemaObject.tableMetadata().getName())) + .toList(); + Map statuses = Map.of(CommandStatus.EXISTING_TABLES, tables); return new CommandResult(statuses); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java index 4c18d9df7..171bccbbb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java @@ -6,6 +6,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import com.datastax.oss.driver.api.core.CqlSession; import io.restassured.RestAssured; import io.restassured.http.ContentType; import io.restassured.specification.RequestSpecification; @@ -249,4 +250,5 @@ public static void checkIndexUsageMetrics(String commandName, boolean vector) { protected RequestSpecification givenHeadersAndJson(String json) { return given().headers(getHeaders()).contentType(ContentType.JSON).body(json); } + } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java index 922de123d..85d1e43d8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java @@ -4,6 +4,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import com.fasterxml.jackson.databind.ObjectMapper; @@ -41,6 +42,13 @@ protected DataApiResponseValidator createTable(String tableDefAsJSON) { .body("status.ok", is(1)); } + protected DataApiResponseValidator listTables(String tableDefAsJSON) { + return DataApiCommandSenders.assertNamespaceCommand(keyspaceName) + .postListTables(tableDefAsJSON) + .hasNoErrors() + .body("status.tables", notNullValue()); + } + protected DataApiResponseValidator createTableErrorValidation( String tableDefAsJSON, String errorCode, String message) { return DataApiCommandSenders.assertNamespaceCommand(keyspaceName) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java new file mode 100644 index 000000000..c9c26dca6 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -0,0 +1,181 @@ +package io.stargate.sgv2.jsonapi.api.v1.tables; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +import io.quarkus.test.common.WithTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.ClassOrderer; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestClassOrder; + +@QuarkusIntegrationTest +@WithTestResource(value = DseTestResource.class, restrictToAnnotatedClass = false) +@TestClassOrder(ClassOrderer.OrderAnnotation.class) +public class ListTablesIntegrationTest extends AbstractTableIntegrationTestBase { + @BeforeAll + public final void createDefaultTables() { + String tableData = + """ + { + "name": "allTypesTable", + "definition": { + "columns": { + "ascii_type": "ascii", + "bigint_type": "bigint", + "blob_type": "blob", + "boolean_type": "boolean", + "date_type": "date", + "decimal_type": "decimal", + "double_type": "double", + "duration_type": "duration", + "float_type": "float", + "inet_type": "inet", + "int_type": "int", + "smallint_type": "smallint", + "text_type": "text", + "time_type": "time", + "timestamp_type": "timestamp", + "tinyint_type": "tinyint", + "uuid_type": "uuid", + "varint_type": "varint", + "map_type": { + "type": "map", + "keyType": "text", + "valueType": "int" + }, + "list_type": { + "type": "list", + "valueType": "text" + }, + "set_type": { + "type": "set", + "valueType": "text" + }, + "vector_type": { + "type": "vector", + "dimension": 1024, + "service": { + "provider": "mistral", + "modelName": "mistral-embed" + } + } + }, + "primaryKey": { + "partitionBy": [ + "text_type" + ], + "partitionSort": { + "int_type": 1, + "bigint_type": -1 + } + } + }, + "options": { + "ifNotExists": true + } + } + """; + createTable(tableData); + String table2 = + """ + { + "name": "person", + "definition": { + "columns": { + "id": "text", + "age": "int", + "name": "text", + "city": "text" + }, + "primaryKey": "id" + } + } + """; + createTable(table2); + } + + @Nested + @Order(1) + class ListTables { + + @Test + public void listTablesOnly() { + String listTablesOnly = + """ + {} + """; + listTables(listTablesOnly) + .hasNoErrors() + // Validate that status.tables is not null + .body("status.tables", notNullValue()) + + // Validate the number of tables in the response + .body("status.tables", hasSize(2)) + + // Validate the specific table names in any order + .body("status.tables[0]", equalTo("allTypesTable")); + } + + @Test + public void listTablesWithSchema() { + String listTablesWithSchema = + """ + { + "options" : { + "explain" : true + } + } + """; + listTables(listTablesWithSchema) + .hasNoErrors() + // Validate that status.tables is not null and contains one table: allTypesTable + .body("status.tables", notNullValue()) + .body("status.tables[0].name", equalTo("allTypesTable")) + + // Validate that the table contains the expected columns and types + .body("status.tables[0].definition.columns.date_type.type", equalTo("date")) + .body("status.tables[0].definition.columns.time_type.type", equalTo("time")) + .body("status.tables[0].definition.columns.text_type.type", equalTo("text")) + .body("status.tables[0].definition.columns.int_type.type", equalTo("int")) + .body("status.tables[0].definition.columns.vector_type.type", equalTo("vector")) + .body( + "status.tables[0].definition.columns.vector_type.dimension", + equalTo(1024)) // Additional dimension check for vector type + .body("status.tables[0].definition.columns.duration_type.type", equalTo("duration")) + .body("status.tables[0].definition.columns.timestamp_type.type", equalTo("timestamp")) + .body("status.tables[0].definition.columns.set_type.type", equalTo("set")) + .body( + "status.tables[0].definition.columns.set_type.valueType", + equalTo("text")) // Set's valueType check + .body("status.tables[0].definition.columns.bigint_type.type", equalTo("bigint")) + .body("status.tables[0].definition.columns.boolean_type.type", equalTo("boolean")) + .body("status.tables[0].definition.columns.uuid_type.type", equalTo("uuid")) + .body("status.tables[0].definition.columns.blob_type.type", equalTo("blob")) + .body("status.tables[0].definition.columns.inet_type.type", equalTo("inet")) + .body("status.tables[0].definition.columns.list_type.type", equalTo("list")) + .body( + "status.tables[0].definition.columns.list_type.valueType", + equalTo("text")) // List's valueType check + .body("status.tables[0].definition.columns.map_type.type", equalTo("map")) + .body( + "status.tables[0].definition.columns.map_type.keyType", + equalTo("text")) // Map's keyType check + .body( + "status.tables[0].definition.columns.map_type.valueType", + equalTo("int")) // Map's valueType check + .body("status.tables[0].definition.columns.varint_type.type", equalTo("varint")) + .body("status.tables[0].definition.columns.tinyint_type.type", equalTo("tinyint")) + .body("status.tables[0].definition.columns.decimal_type.type", equalTo("decimal")) + .body("status.tables[0].definition.columns.float_type.type", equalTo("float")) + .body("status.tables[0].definition.columns.ascii_type.type", equalTo("ascii")) + .body("status.tables[0].definition.columns.double_type.type", equalTo("double")) + .body("status.tables[0].definition.columns.smallint_type.type", equalTo("smallint")); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java index 32e569761..449362997 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java @@ -16,4 +16,8 @@ protected io.restassured.response.Response postInternal(RequestSpecification req public DataApiResponseValidator postCreateTable(String tableDefAsJSON) { return postCommand("createTable", tableDefAsJSON); } + + public DataApiResponseValidator postListTables(String listDefinition) { + return postCommand("listTables", listDefinition); + } } From 3a49f2e8081f16e13677fd5cbf5df271c697691e Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:20:04 -0400 Subject: [PATCH 30/37] Formatted the files --- .../jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java index 171bccbbb..4c18d9df7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/AbstractKeyspaceIntegrationTestBase.java @@ -6,7 +6,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import com.datastax.oss.driver.api.core.CqlSession; import io.restassured.RestAssured; import io.restassured.http.ContentType; import io.restassured.specification.RequestSpecification; @@ -250,5 +249,4 @@ public static void checkIndexUsageMetrics(String commandName, boolean vector) { protected RequestSpecification givenHeadersAndJson(String json) { return given().headers(getHeaders()).contentType(ContentType.JSON).body(json); } - } From b98a2160b5223f6c99c6673634edfc4fcf6392d1 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:31:25 -0400 Subject: [PATCH 31/37] Added vector service and primary key definition --- .../api/v1/tables/ListTablesIntegrationTest.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java index c9c26dca6..d05f137d2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -147,6 +147,12 @@ public void listTablesWithSchema() { .body( "status.tables[0].definition.columns.vector_type.dimension", equalTo(1024)) // Additional dimension check for vector type + .body( + "status.tables[0].definition.columns.vector_type.service.provider", + equalTo("mistral")) + .body( + "status.tables[0].definition.columns.vector_type.service.modelName", + equalTo("mistral-embed")) .body("status.tables[0].definition.columns.duration_type.type", equalTo("duration")) .body("status.tables[0].definition.columns.timestamp_type.type", equalTo("timestamp")) .body("status.tables[0].definition.columns.set_type.type", equalTo("set")) @@ -175,7 +181,12 @@ public void listTablesWithSchema() { .body("status.tables[0].definition.columns.float_type.type", equalTo("float")) .body("status.tables[0].definition.columns.ascii_type.type", equalTo("ascii")) .body("status.tables[0].definition.columns.double_type.type", equalTo("double")) - .body("status.tables[0].definition.columns.smallint_type.type", equalTo("smallint")); + .body("status.tables[0].definition.columns.smallint_type.type", equalTo("smallint")) + + // Validate the primary key; + .body("status.tables[0].definition.primaryKey.partitionBy[0]", equalTo("text_type")) + .body("status.tables[0].definition.primaryKey.partitionSort.int_type", equalTo(1)) + .body("status.tables[0].definition.primaryKey.partitionSort.bigint_type", equalTo(-1)); } } } From e173c803df89fba31286ac09e9f0ba883db5d01f Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:00:08 -0400 Subject: [PATCH 32/37] Added serializer for Primary key ordering columns --- .../ColumnDefinitionSerializer.java | 2 +- .../serializer/OrderingKeySerializer.java | 33 +++++++++++++++++++ .../command/table/definition/PrimaryKey.java | 3 ++ .../table/definition/datatype/ColumnType.java | 5 ++- .../definition/datatype/ComplexTypes.java | 25 -------------- 5 files changed, 39 insertions(+), 29 deletions(-) rename src/main/java/io/stargate/sgv2/jsonapi/api/model/command/{deserializers => serializer}/ColumnDefinitionSerializer.java (96%) create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java similarity index 96% rename from src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java rename to src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java index 076b6ed73..5adf04bc4 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/ColumnDefinitionSerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.model.command.deserializers; +package io.stargate.sgv2.jsonapi.api.model.command.serializer; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java new file mode 100644 index 000000000..4a0324958 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java @@ -0,0 +1,33 @@ +package io.stargate.sgv2.jsonapi.api.model.command.serializer; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.PrimaryKey; +import java.io.IOException; + +/** + * Custom serializer to encode the column type to the JSON payload This is required because + * composite and custom column types may need additional properties to be serialized + */ +public class OrderingKeySerializer extends JsonSerializer { + + @Override + public void serialize( + PrimaryKey.OrderingKey[] orderingKeys, + JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) + throws IOException { + jsonGenerator.writeStartObject(); + if (orderingKeys != null) { + for (PrimaryKey.OrderingKey orderingKey : orderingKeys) { + jsonGenerator.writeNumberField( + orderingKey.column(), orderingKey.order() == PrimaryKey.OrderingKey.Order.ASC ? 1 : -1); + } + } + jsonGenerator.writeEndObject(); + } + + public record ApiSupport( + boolean createTable, boolean insert, boolean read, String cqlDefinition) {} +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java index a96881e2d..199e80c25 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java @@ -3,7 +3,9 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.PrimaryKeyDeserializer; +import io.stargate.sgv2.jsonapi.api.model.command.serializer.OrderingKeySerializer; import jakarta.annotation.Nullable; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; @@ -25,6 +27,7 @@ public record PrimaryKey( @Nullable @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) @JsonProperty("partitionSort") + @JsonSerialize(using = OrderingKeySerializer.class) OrderingKey[] orderingKeys) { public record OrderingKey(String column, Order order) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index 110df39f6..5fb7a5f82 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -3,8 +3,8 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionDeserializer; -import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionSerializer; import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; +import io.stargate.sgv2.jsonapi.api.model.command.serializer.ColumnDefinitionSerializer; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; import java.util.List; @@ -17,8 +17,7 @@ public interface ColumnType { // Returns api data type. ApiDataType getApiDataType(); - public String name(); - + // Returns the name of the column type to be used in the API request. default String getApiName() { return getApiDataType().getApiName(); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index e4ad0c1c8..cfa974d71 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -25,11 +25,6 @@ public ApiDataType getApiDataType() { (PrimitiveApiDataType) valueType.getApiDataType()); } - @Override - public String name() { - return getApiDataType().getApiName(); - } - public String keyType() { return keyType.getApiDataType().getApiName(); } @@ -52,11 +47,6 @@ public ApiDataType getApiDataType() { return new ComplexApiDataType.ListType((PrimitiveApiDataType) valueType.getApiDataType()); } - @Override - public String name() { - return "list"; - } - public String valueType() { return valueType.getApiDataType().getApiName(); } @@ -75,11 +65,6 @@ public ApiDataType getApiDataType() { return new ComplexApiDataType.SetType((PrimitiveApiDataType) valueType.getApiDataType()); } - @Override - public String name() { - return "set"; - } - public String valueType() { return valueType.getApiDataType().getApiName(); } @@ -108,11 +93,6 @@ public VectorizeConfig getVectorConfig() { return vectorConfig; } - @Override - public String name() { - return "vector"; - } - public int getDimension() { return vectorSize; } @@ -134,11 +114,6 @@ public ApiDataType getApiDataType() { throw new UnsupportedOperationException("Unsupported type"); } - @Override - public String name() { - return "UNSUPPORTED"; - } - @Override public String getApiName() { return "UNSUPPORTED"; From 0ffba5f338474be6862c884bac61fc7be95773be Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:02:32 -0400 Subject: [PATCH 33/37] Refactor as per the review --- .../command/serializer/ColumnDefinitionSerializer.java | 8 ++++---- .../command/table/definition/datatype/ComplexTypes.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java index 5adf04bc4..704f757de 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java @@ -20,12 +20,12 @@ public void serialize( jsonGenerator.writeStartObject(); jsonGenerator.writeStringField("type", columnType.getApiName()); if (columnType instanceof ComplexTypes.MapType mt) { - jsonGenerator.writeStringField("keyType", mt.keyType()); - jsonGenerator.writeStringField("valueType", mt.valueType()); + jsonGenerator.writeStringField("keyType", mt.keyTypeName()); + jsonGenerator.writeStringField("valueType", mt.valueTypeName()); } else if (columnType instanceof ComplexTypes.ListType lt) { - jsonGenerator.writeStringField("valueType", lt.valueType()); + jsonGenerator.writeStringField("valueType", lt.valueTypeName()); } else if (columnType instanceof ComplexTypes.SetType st) { - jsonGenerator.writeStringField("valueType", st.valueType()); + jsonGenerator.writeStringField("valueType", st.valueTypeName()); } else if (columnType instanceof ComplexTypes.VectorType vt) { jsonGenerator.writeNumberField("dimension", vt.getDimension()); if (vt.getVectorConfig() != null) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index cfa974d71..e8a0add66 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -25,11 +25,11 @@ public ApiDataType getApiDataType() { (PrimitiveApiDataType) valueType.getApiDataType()); } - public String keyType() { + public String keyTypeName() { return keyType.getApiDataType().getApiName(); } - public String valueType() { + public String valueTypeName() { return valueType.getApiDataType().getApiName(); } } @@ -47,7 +47,7 @@ public ApiDataType getApiDataType() { return new ComplexApiDataType.ListType((PrimitiveApiDataType) valueType.getApiDataType()); } - public String valueType() { + public String valueTypeName() { return valueType.getApiDataType().getApiName(); } } @@ -65,7 +65,7 @@ public ApiDataType getApiDataType() { return new ComplexApiDataType.SetType((PrimitiveApiDataType) valueType.getApiDataType()); } - public String valueType() { + public String valueTypeName() { return valueType.getApiDataType().getApiName(); } } From f7103aaf68f0ee715e8584755f07a9e79b55bb90 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:53:52 -0400 Subject: [PATCH 34/37] Refactor as per the review comments --- ...ializer.java => OrderingKeysSerializer.java} | 13 +++++++++++-- .../command/table/definition/PrimaryKey.java | 4 ++-- .../table/definition/datatype/ColumnType.java | 4 +++- .../definition/datatype/PrimitiveTypes.java | 9 +++------ .../cqldriver/executor/TableSchemaObject.java | 17 ++++++----------- .../operation/tables/ListTablesOperation.java | 9 ++++++--- .../sgv2/jsonapi/util/CqlIdentifierUtil.java | 5 +++++ .../v1/tables/ListTablesIntegrationTest.java | 6 ++++-- 8 files changed, 40 insertions(+), 27 deletions(-) rename src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/{OrderingKeySerializer.java => OrderingKeysSerializer.java} (74%) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeysSerializer.java similarity index 74% rename from src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java rename to src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeysSerializer.java index 4a0324958..ef219e33f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeySerializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeysSerializer.java @@ -10,7 +10,7 @@ * Custom serializer to encode the column type to the JSON payload This is required because * composite and custom column types may need additional properties to be serialized */ -public class OrderingKeySerializer extends JsonSerializer { +public class OrderingKeysSerializer extends JsonSerializer { @Override public void serialize( @@ -28,6 +28,15 @@ public void serialize( jsonGenerator.writeEndObject(); } - public record ApiSupport( + /** + * This is used when a unsupported type column is present in a table. How to use this class will + * evolve as different unsupported types are analyzed. + * + * @param createTable + * @param insert + * @param read + * @param cqlDefinition + */ + private record ApiSupport( boolean createTable, boolean insert, boolean read, String cqlDefinition) {} } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java index 199e80c25..145b88c71 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.PrimaryKeyDeserializer; -import io.stargate.sgv2.jsonapi.api.model.command.serializer.OrderingKeySerializer; +import io.stargate.sgv2.jsonapi.api.model.command.serializer.OrderingKeysSerializer; import jakarta.annotation.Nullable; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; @@ -27,7 +27,7 @@ public record PrimaryKey( @Nullable @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) @JsonProperty("partitionSort") - @JsonSerialize(using = OrderingKeySerializer.class) + @JsonSerialize(using = OrderingKeysSerializer.class) OrderingKey[] orderingKeys) { public record OrderingKey(String column, Order order) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index 5fb7a5f82..23fb1c17f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -17,7 +17,9 @@ public interface ColumnType { // Returns api data type. ApiDataType getApiDataType(); - // Returns the name of the column type to be used in the API request. + /* + Returns the name of the column type to be used in the API request. + */ default String getApiName() { return getApiDataType().getApiName(); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java index ccdb62ebc..fb50df05e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java @@ -7,9 +7,6 @@ /** Interface for primitive column types similar to what is defined in cassandra java driver. */ public enum PrimitiveTypes implements ColumnType { - - // TODO: add a private ctor to stop this class from being instantiated or make abstract - ASCII(PrimitiveApiDataType.ASCII), BIGINT(PrimitiveApiDataType.BIGINT), BINARY(PrimitiveApiDataType.BINARY), @@ -40,15 +37,15 @@ public ApiDataType getApiDataType() { private ApiDataType getApiDataType; - private static Map columnTypeMap = new HashMap<>(); + private static Map primitiveTypes = new HashMap<>(); static { for (PrimitiveTypes type : PrimitiveTypes.values()) { - columnTypeMap.put(type.getApiDataType().getApiName(), type); + primitiveTypes.put(type.getApiDataType().getApiName(), type); } } public static ColumnType fromString(String type) { - return columnTypeMap.get(type); + return primitiveTypes.get(type); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 83f8060d3..58dcce499 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -22,6 +22,7 @@ import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDef; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDefs; +import io.stargate.sgv2.jsonapi.util.CqlIdentifierUtil; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -134,24 +135,24 @@ public static TableSchemaObject from(TableMetadata tableMetadata, ObjectMapper o * @return */ public TableResponse toTableResponse() { - String tableName = removeQuotes(tableMetadata().getName()); + String tableName = CqlIdentifierUtil.externalRepresentation(tableMetadata().getName()); HashMap columnsDefinition = new HashMap<>(); for (Map.Entry column : tableMetadata().getColumns().entrySet()) { ColumnType type = getColumnType(column.getKey().asInternal(), column.getValue()); - columnsDefinition.put(removeQuotes(column.getKey()), type); + columnsDefinition.put(CqlIdentifierUtil.externalRepresentation(column.getKey()), type); } final List partitionBy = tableMetadata().getPartitionKey().stream() - .map(column -> removeQuotes(column.getName())) + .map(column -> CqlIdentifierUtil.externalRepresentation(column.getName())) .collect(Collectors.toList()); final List partitionSort = tableMetadata().getClusteringColumns().entrySet().stream() .map( entry -> new PrimaryKey.OrderingKey( - removeQuotes(entry.getKey().getName()), + CqlIdentifierUtil.externalRepresentation(entry.getKey().getName()), entry.getValue() == ClusteringOrder.ASC ? PrimaryKey.OrderingKey.Order.ASC : PrimaryKey.OrderingKey.Order.DESC)) @@ -164,7 +165,6 @@ public TableResponse toTableResponse() { tableName, new TableResponse.TableDefinition(columnsDefinition, primaryKey)); } - // Need to handle frozen types private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadata) { if (columnMetadata.getType() instanceof VectorType vt) { // Schema will always have VectorConfig for vector type @@ -226,16 +226,11 @@ private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadat return PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName()); else { // Need to return unsupported type - throw new RuntimeException("Unknown data type: " + columnMetadata.getType()); + return new ComplexTypes.UnsupportedType(columnMetadata.getType().asCql(true, false)); } } } - // Remove the quotes from the identifier - public String removeQuotes(CqlIdentifier identifier) { - return identifier.asCql(true).replaceAll("\"", ""); - } - /** * Object used to build the response for listTables command * diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java index 33b7f5e93..7405c331c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.service.operation.tables; -import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; import com.fasterxml.jackson.databind.ObjectMapper; import io.smallrye.mutiny.Uni; @@ -15,6 +14,7 @@ import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; import io.stargate.sgv2.jsonapi.service.operation.Operation; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher; +import io.stargate.sgv2.jsonapi.util.CqlIdentifierUtil; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -60,7 +60,9 @@ public Uni> execute( .getSession(dataApiRequestInfo) .getMetadata() .getKeyspaces() - .get(CqlIdentifier.fromInternal(commandContext.schemaObject().name().keyspace())); + .get( + CqlIdentifierUtil.cqlIdentifierFromUserInput( + commandContext.schemaObject().name().keyspace())); if (keyspaceMetadata == null) { return Uni.createFrom() .failure( @@ -107,7 +109,8 @@ public CommandResult get() { tables().stream() .map( schemaObject -> - schemaObject.removeQuotes(schemaObject.tableMetadata().getName())) + CqlIdentifierUtil.externalRepresentation( + schemaObject.tableMetadata().getName())) .toList(); Map statuses = Map.of(CommandStatus.EXISTING_TABLES, tables); return new CommandResult(statuses); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java b/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java index 8ed18ad6b..86ed65717 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java @@ -26,4 +26,9 @@ public static CqlIdentifier cqlIdentifierFromIndexTarget(String name) { public static String cqlIdentifierToStringForUser(CqlIdentifier identifier) { return identifier.asCql(true); } + + /** Remove the quotes from the identifier */ + public static String externalRepresentation(CqlIdentifier identifier) { + return identifier.asInternal(); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java index d05f137d2..519d2d05b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -118,8 +118,9 @@ public void listTablesOnly() { // Validate the number of tables in the response .body("status.tables", hasSize(2)) - // Validate the specific table names in any order - .body("status.tables[0]", equalTo("allTypesTable")); + // Validate the specific table names in the position + .body("status.tables[0]", equalTo("allTypesTable")) + .body("status.tables[1]", equalTo("person")); } @Test @@ -136,6 +137,7 @@ public void listTablesWithSchema() { .hasNoErrors() // Validate that status.tables is not null and contains one table: allTypesTable .body("status.tables", notNullValue()) + .body("status.tables", hasSize(2)) .body("status.tables[0].name", equalTo("allTypesTable")) // Validate that the table contains the expected columns and types From 57c3c69d0ee475a56e099c725560216212596018 Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:04:12 -0400 Subject: [PATCH 35/37] Added test to confirm listTables doesn't return collections --- .../v1/tables/ListTablesIntegrationTest.java | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java index 519d2d05b..42a0e2dc5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -1,18 +1,23 @@ package io.stargate.sgv2.jsonapi.api.v1.tables; +import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; import io.quarkus.test.common.WithTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.restassured.http.ContentType; +import io.stargate.sgv2.jsonapi.api.v1.KeyspaceResource; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.ClassOrderer; +import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestClassOrder; +import org.junit.jupiter.api.TestMethodOrder; @QuarkusIntegrationTest @WithTestResource(value = DseTestResource.class, restrictToAnnotatedClass = false) @@ -102,9 +107,11 @@ public final void createDefaultTables() { @Nested @Order(1) + @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class ListTables { @Test + @Order(1) public void listTablesOnly() { String listTablesOnly = """ @@ -124,6 +131,7 @@ public void listTablesOnly() { } @Test + @Order(2) public void listTablesWithSchema() { String listTablesWithSchema = """ @@ -188,7 +196,64 @@ public void listTablesWithSchema() { // Validate the primary key; .body("status.tables[0].definition.primaryKey.partitionBy[0]", equalTo("text_type")) .body("status.tables[0].definition.primaryKey.partitionSort.int_type", equalTo(1)) - .body("status.tables[0].definition.primaryKey.partitionSort.bigint_type", equalTo(-1)); + .body("status.tables[0].definition.primaryKey.partitionSort.bigint_type", equalTo(-1)) + + // Check the second table name + .body("status.tables[1].name", equalTo("person")); + } + + @Test + @Order(3) + public void ignoreCollections() { + String simpleCollectionToIgnore = + """ + { + "createCollection": { + "name": "simple_collection_to_ignore" + } + } + """; + + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(simpleCollectionToIgnore) + .when() + .post(KeyspaceResource.BASE_PATH, keyspaceName) + .then() + .statusCode(200); + + String listTablesOnly = + """ + {} + """; + listTables(listTablesOnly) + .hasNoErrors() + // Validate that status.tables is not null + .body("status.tables", notNullValue()) + + // Validate the number of tables in the response + .body("status.tables", hasSize(2)) + + // Validate the specific table names in the position + .body("status.tables[0]", equalTo("allTypesTable")) + .body("status.tables[1]", equalTo("person")); + + String listTablesWithSchema = + """ + { + "options" : { + "explain" : true + } + } + """; + listTables(listTablesWithSchema) + .hasNoErrors() + // Validate that status.tables is not null and contains one table: allTypesTable + .body("status.tables", notNullValue()) + .body("status.tables", hasSize(2)) + .body("status.tables[0].name", equalTo("allTypesTable")) + .body("status.tables[1].name", equalTo("person")); } } } From 6d9de5aa347dce0a2f68caf5ce09ef9eeb6b095c Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:07:50 -0400 Subject: [PATCH 36/37] Added comment as suggested. --- .../sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java index 42a0e2dc5..af3588519 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -200,6 +200,7 @@ public void listTablesWithSchema() { // Check the second table name .body("status.tables[1].name", equalTo("person")); + // Not checking second table types as the data types used are same as first table } @Test From 6b6ccc89cdc610ccda74c8bd5c76c365410572be Mon Sep 17 00:00:00 2001 From: maheshrajamani <99678631+maheshrajamani@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:08:04 -0400 Subject: [PATCH 37/37] Added comment as suggested. --- .../sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java index af3588519..56c1e351b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -200,7 +200,7 @@ public void listTablesWithSchema() { // Check the second table name .body("status.tables[1].name", equalTo("person")); - // Not checking second table types as the data types used are same as first table + // Not checking second table types as the data types used are same as first table } @Test