diff --git a/pom.xml b/pom.xml index d7a1d49fc..a22595c82 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ sgv2-jsonapi 1.0.17-SNAPSHOT - v2.1.0-BETA-17 + 2.1.0-BETA-18 2.17.2 @@ -29,7 +29,7 @@ ${cassandra.version} stargateio/coordinator-dse-next - ${stargate.version} + v${stargate.version} dse-next-${stargate.int-test.cassandra.image-tag}-cluster persistence-dse-next false diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java index 7640337c8..bc18ad581 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/Command.java @@ -64,6 +64,7 @@ enum CommandName { FIND_ONE("findOne"), INSERT_MANY("insertMany"), INSERT_ONE("insertOne"), + LIST_TABLES("listTables"), UPDATE_MANY("updateMany"), UPDATE_ONE("updateOne"), BEGIN_OFFLINE_SESSION("beginOfflineSession"), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java index bd46934b1..909707c39 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java @@ -26,6 +26,10 @@ public enum CommandStatus { /** Status for reporting existing collections. */ @JsonProperty("collections") EXISTING_COLLECTIONS, + /** Status for reporting existing collections. */ + @JsonProperty("tables") + EXISTING_TABLES, + /** * List of response entries, one for each document we tried to insert with {@code insertMany} * command. Each entry has 2 mandatory fields: {@code _id} (document id), and {@code status} (one diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java index 23b48456d..5e9e61a0c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/TableOnlyCommand.java @@ -4,11 +4,13 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.stargate.sgv2.jsonapi.api.model.command.impl.CreateTableCommand; import io.stargate.sgv2.jsonapi.api.model.command.impl.DropTableCommand; +import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; /** Interface for all commands executed against a keyspace. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonSubTypes({ @JsonSubTypes.Type(value = CreateTableCommand.class), - @JsonSubTypes.Type(value = DropTableCommand.class) + @JsonSubTypes.Type(value = DropTableCommand.class), + @JsonSubTypes.Type(value = ListTablesCommand.class), }) public interface TableOnlyCommand extends KeyspaceCommand {} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java new file mode 100644 index 000000000..8f0759ba7 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/ListTablesCommand.java @@ -0,0 +1,23 @@ +package io.stargate.sgv2.jsonapi.api.model.command.impl; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.stargate.sgv2.jsonapi.api.model.command.TableOnlyCommand; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.media.Schema; + +@Schema(description = "Command that lists all available tables in a namespace.") +@JsonTypeName("listTables") +public record ListTablesCommand(Options options) implements TableOnlyCommand { + public record Options( + @Schema( + description = "include table properties.", + type = SchemaType.BOOLEAN, + implementation = Boolean.class) + boolean explain) {} + + /** {@inheritDoc} */ + @Override + public CommandName commandName() { + return CommandName.LIST_TABLES; + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java new file mode 100644 index 000000000..704f757de --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/ColumnDefinitionSerializer.java @@ -0,0 +1,42 @@ +package io.stargate.sgv2.jsonapi.api.model.command.serializer; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ColumnType; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ComplexTypes; +import java.io.IOException; + +/** + * Custom serializer to encode the column type to the JSON payload This is required because + * composite and custom column types may need additional properties to be serialized + */ +public class ColumnDefinitionSerializer extends JsonSerializer { + + @Override + public void serialize( + ColumnType columnType, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeStringField("type", columnType.getApiName()); + if (columnType instanceof ComplexTypes.MapType mt) { + jsonGenerator.writeStringField("keyType", mt.keyTypeName()); + jsonGenerator.writeStringField("valueType", mt.valueTypeName()); + } else if (columnType instanceof ComplexTypes.ListType lt) { + jsonGenerator.writeStringField("valueType", lt.valueTypeName()); + } else if (columnType instanceof ComplexTypes.SetType st) { + jsonGenerator.writeStringField("valueType", st.valueTypeName()); + } else if (columnType instanceof ComplexTypes.VectorType vt) { + jsonGenerator.writeNumberField("dimension", vt.getDimension()); + if (vt.getVectorConfig() != null) + jsonGenerator.writeObjectField("service", vt.getVectorConfig()); + } else if (columnType instanceof ComplexTypes.UnsupportedType ut) { + jsonGenerator.writeObjectField( + "apiSupport", new ApiSupport(false, false, false, ut.cqlFormat())); + } + jsonGenerator.writeEndObject(); + } + + public record ApiSupport( + boolean createTable, boolean insert, boolean read, String cqlDefinition) {} +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeysSerializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeysSerializer.java new file mode 100644 index 000000000..ef219e33f --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/serializer/OrderingKeysSerializer.java @@ -0,0 +1,42 @@ +package io.stargate.sgv2.jsonapi.api.model.command.serializer; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.PrimaryKey; +import java.io.IOException; + +/** + * Custom serializer to encode the column type to the JSON payload This is required because + * composite and custom column types may need additional properties to be serialized + */ +public class OrderingKeysSerializer extends JsonSerializer { + + @Override + public void serialize( + PrimaryKey.OrderingKey[] orderingKeys, + JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) + throws IOException { + jsonGenerator.writeStartObject(); + if (orderingKeys != null) { + for (PrimaryKey.OrderingKey orderingKey : orderingKeys) { + jsonGenerator.writeNumberField( + orderingKey.column(), orderingKey.order() == PrimaryKey.OrderingKey.Order.ASC ? 1 : -1); + } + } + jsonGenerator.writeEndObject(); + } + + /** + * This is used when a unsupported type column is present in a table. How to use this class will + * evolve as different unsupported types are analyzed. + * + * @param createTable + * @param insert + * @param read + * @param cqlDefinition + */ + private record ApiSupport( + boolean createTable, boolean insert, boolean read, String cqlDefinition) {} +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java index abfd0d14b..145b88c71 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/PrimaryKey.java @@ -1,8 +1,11 @@ package io.stargate.sgv2.jsonapi.api.model.command.table.definition; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.PrimaryKeyDeserializer; +import io.stargate.sgv2.jsonapi.api.model.command.serializer.OrderingKeysSerializer; import jakarta.annotation.Nullable; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; @@ -16,9 +19,15 @@ // implementation = Object.class, // description = "Represents the table primary key") public record PrimaryKey( - @NotNull @Schema(description = "Columns that make the partition keys", type = SchemaType.ARRAY) + @NotNull + @Schema(description = "Columns that make the partition keys", type = SchemaType.ARRAY) + @JsonProperty("partitionBy") + @JsonInclude(JsonInclude.Include.NON_NULL) String[] keys, - @Nullable @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) + @Nullable + @Schema(description = "Columns that make the ordering keys", type = SchemaType.ARRAY) + @JsonProperty("partitionSort") + @JsonSerialize(using = OrderingKeysSerializer.class) OrderingKey[] orderingKeys) { public record OrderingKey(String column, Order order) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java index d2da91411..23fb1c17f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ColumnType.java @@ -1,8 +1,10 @@ package io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.stargate.sgv2.jsonapi.api.model.command.deserializers.ColumnDefinitionDeserializer; import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; +import io.stargate.sgv2.jsonapi.api.model.command.serializer.ColumnDefinitionSerializer; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; import java.util.List; @@ -10,10 +12,18 @@ /** Interface for column types. This is used to define the type of a column in a table. */ @JsonDeserialize(using = ColumnDefinitionDeserializer.class) +@JsonSerialize(using = ColumnDefinitionSerializer.class) public interface ColumnType { // Returns api data type. ApiDataType getApiDataType(); + /* + Returns the name of the column type to be used in the API request. + */ + default String getApiName() { + return getApiDataType().getApiName(); + } + static List getSupportedTypes() { return List.of( "ascii", @@ -46,42 +56,6 @@ static ColumnType fromString( // TODO: the name of the type should be a part of the ColumnType interface, and use a map for // the lookup switch (type) { - case "ascii": - return PrimitiveTypes.ASCII; - case "bigint": - return PrimitiveTypes.BIGINT; - case "blob": - return PrimitiveTypes.BINARY; - case "boolean": - return PrimitiveTypes.BOOLEAN; - case "date": - return PrimitiveTypes.DATE; - case "decimal": - return PrimitiveTypes.DECIMAL; - case "double": - return PrimitiveTypes.DOUBLE; - case "duration": - return PrimitiveTypes.DURATION; - case "float": - return PrimitiveTypes.FLOAT; - case "inet": - return PrimitiveTypes.INET; - case "int": - return PrimitiveTypes.INT; - case "smallint": - return PrimitiveTypes.SMALLINT; - case "text": - return PrimitiveTypes.TEXT; - case "time": - return PrimitiveTypes.TIME; - case "timestamp": - return PrimitiveTypes.TIMESTAMP; - case "tinyint": - return PrimitiveTypes.TINYINT; - case "uuid": - return PrimitiveTypes.UUID; - case "varint": - return PrimitiveTypes.VARINT; case "map": { if (keyType == null || valueType == null) { @@ -134,6 +108,10 @@ static ColumnType fromString( } default: { + ColumnType columnType = PrimitiveTypes.fromString(type); + if (columnType != null) { + return columnType; + } Map errorMessageFormattingValues = Map.of( "type", diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java index 4bb640961..e8a0add66 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/ComplexTypes.java @@ -24,6 +24,14 @@ public ApiDataType getApiDataType() { (PrimitiveApiDataType) keyType.getApiDataType(), (PrimitiveApiDataType) valueType.getApiDataType()); } + + public String keyTypeName() { + return keyType.getApiDataType().getApiName(); + } + + public String valueTypeName() { + return valueType.getApiDataType().getApiName(); + } } /** A list type implementation */ @@ -38,6 +46,10 @@ public ListType(ColumnType valueType) { public ApiDataType getApiDataType() { return new ComplexApiDataType.ListType((PrimitiveApiDataType) valueType.getApiDataType()); } + + public String valueTypeName() { + return valueType.getApiDataType().getApiName(); + } } /** A set type implementation */ @@ -52,6 +64,10 @@ public SetType(ColumnType valueType) { public ApiDataType getApiDataType() { return new ComplexApiDataType.SetType((PrimitiveApiDataType) valueType.getApiDataType()); } + + public String valueTypeName() { + return valueType.getApiDataType().getApiName(); + } } /* Vector type */ @@ -81,4 +97,30 @@ public int getDimension() { return vectorSize; } } + + /** + * Unsupported type implementation, returned in response when cql table has unsupported format + * column + */ + public static class UnsupportedType implements ColumnType { + private final String cqlFormat; + + public UnsupportedType(String cqlFormat) { + this.cqlFormat = cqlFormat; + } + + @Override + public ApiDataType getApiDataType() { + throw new UnsupportedOperationException("Unsupported type"); + } + + @Override + public String getApiName() { + return "UNSUPPORTED"; + } + + public String cqlFormat() { + return cqlFormat; + } + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java index 630bb798c..fb50df05e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/table/definition/datatype/PrimitiveTypes.java @@ -2,154 +2,50 @@ import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataType; import io.stargate.sgv2.jsonapi.service.schema.tables.PrimitiveApiDataType; +import java.util.HashMap; +import java.util.Map; /** Interface for primitive column types similar to what is defined in cassandra java driver. */ -public class PrimitiveTypes { +public enum PrimitiveTypes implements ColumnType { + ASCII(PrimitiveApiDataType.ASCII), + BIGINT(PrimitiveApiDataType.BIGINT), + BINARY(PrimitiveApiDataType.BINARY), + BOOLEAN(PrimitiveApiDataType.BOOLEAN), + DATE(PrimitiveApiDataType.DATE), + DECIMAL(PrimitiveApiDataType.DECIMAL), + DOUBLE(PrimitiveApiDataType.DOUBLE), + DURATION(PrimitiveApiDataType.DURATION), + FLOAT(PrimitiveApiDataType.FLOAT), + INET(PrimitiveApiDataType.INET), + INT(PrimitiveApiDataType.INT), + SMALLINT(PrimitiveApiDataType.SMALLINT), + TEXT(PrimitiveApiDataType.TEXT), + TIME(PrimitiveApiDataType.TIME), + TIMESTAMP(PrimitiveApiDataType.TIMESTAMP), + TINYINT(PrimitiveApiDataType.TINYINT), + UUID(PrimitiveApiDataType.UUID), + VARINT(PrimitiveApiDataType.VARINT); - // TODO: add a private ctor to stop this class from being instantiated or make abstract - - public static final ColumnType ASCII = new Ascii(); - public static final ColumnType BIGINT = new BigInt(); - public static final ColumnType BINARY = new Binary(); - public static final ColumnType BOOLEAN = new Boolean(); - public static final ColumnType DATE = new Date(); - public static final ColumnType DECIMAL = new Decimal(); - public static final ColumnType DOUBLE = new Double(); - public static final ColumnType DURATION = new Duration(); - public static final ColumnType FLOAT = new Float(); - public static final ColumnType INET = new Inet(); - public static final ColumnType INT = new Int(); - public static final ColumnType SMALLINT = new SmallInt(); - public static final ColumnType TEXT = new Text(); - public static final ColumnType TIME = new Time(); - public static final ColumnType TIMESTAMP = new Timestamp(); - public static final ColumnType TINYINT = new TinyInt(); - public static final ColumnType UUID = new Uuid(); - public static final ColumnType VARINT = new VarInt(); - - private static class Text implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TEXT; - } - } - - private static class Int implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.INT; - } + @Override + public ApiDataType getApiDataType() { + return getApiDataType; } - private static class Boolean implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BOOLEAN; - } - } - - private static class BigInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BIGINT; - } + PrimitiveTypes(ApiDataType getApiDataType) { + this.getApiDataType = getApiDataType; } - private static class Decimal implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DECIMAL; - } - } + private ApiDataType getApiDataType; - private static class Double implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DOUBLE; - } - } - - private static class Float implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.FLOAT; - } - } - - private static class SmallInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.SMALLINT; - } - } - - private static class TinyInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TINYINT; - } - } + private static Map primitiveTypes = new HashMap<>(); - private static class VarInt implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.VARINT; + static { + for (PrimitiveTypes type : PrimitiveTypes.values()) { + primitiveTypes.put(type.getApiDataType().getApiName(), type); } } - private static class Ascii implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.ASCII; - } - } - - private static class Binary implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.BINARY; - } - } - - private static class Date implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DATE; - } - } - - private static class Duration implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.DURATION; - } - } - - private static class Time implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TIME; - } - } - - private static class Timestamp implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.TIMESTAMP; - } - } - - private static class Inet implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.INET; - } - } - - private static class Uuid implements ColumnType { - @Override - public ApiDataType getApiDataType() { - return PrimitiveApiDataType.UUID; - } + public static ColumnType fromString(String type) { + return primitiveTypes.get(type); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java index 95000bb42..ec46b6aa2 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/CollectionResource.java @@ -197,8 +197,16 @@ public Uni> postCommand( } // TODO: refactor this code to be cleaner so it assigns on one line EmbeddingProvider embeddingProvider = null; - final VectorConfig.VectorizeConfig vectorizeConfig = - schemaObject.vectorConfig().vectorizeConfig(); + VectorConfig vectorConfig = schemaObject.vectorConfig(); + final VectorConfig.ColumnVectorDefinition columnVectorDefinition = + vectorConfig.columnVectorDefinitions() == null + || vectorConfig.columnVectorDefinitions().isEmpty() + ? null + : vectorConfig.columnVectorDefinitions().get(0); + final VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = + columnVectorDefinition != null + ? columnVectorDefinition.vectorizeConfig() + : null; if (vectorizeConfig != null) { embeddingProvider = embeddingProviderFactory.getConfiguration( @@ -206,7 +214,7 @@ public Uni> postCommand( dataApiRequestInfo.getCassandraToken(), vectorizeConfig.provider(), vectorizeConfig.modelName(), - schemaObject.vectorConfig().vectorSize(), + columnVectorDefinition.vectorSize(), vectorizeConfig.parameters(), vectorizeConfig.authentication(), command.getClass().getSimpleName()); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java b/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java index b2a5355c3..ff9ec460f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/exception/SchemaException.java @@ -17,6 +17,8 @@ public enum Code implements ErrorCode { COLUMN_DEFINITION_MISSING, COLUMN_TYPE_INCORRECT, COLUMN_TYPE_UNSUPPORTED, + INVALID_CONFIGURATION, + INVALID_VECTORIZE_CONFIGURATION, LIST_TYPE_INCORRECT_DEFINITION, MAP_TYPE_INCORRECT_DEFINITION, MISSING_PRIMARY_KEYS, diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java index 3e0439575..f106b87ce 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/NamespaceCache.java @@ -103,8 +103,7 @@ private Uni loadSchemaObject( optionalTable.get(), objectMapper); } - // 04-Sep-2024, tatu: Used to check that API Tables enabled; no longer checked here - return new TableSchemaObject(table); + return TableSchemaObject.from(table, objectMapper); }); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java index bd147a509..239a67d73 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/SchemaObject.java @@ -29,8 +29,8 @@ public SchemaObjectName name() { } /** - * Subclasses must always return an instance of VectorConfig, if there is no vector config they - * should return VectorConfig.notEnabledVectorConfig() + * Subclasses must always return VectorConfig, if there is no vector config they should return + * VectorConfig.notEnabledVectorConfig(). * * @return */ diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java index 3e4ef7d62..58dcce499 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/TableSchemaObject.java @@ -1,22 +1,248 @@ package io.stargate.sgv2.jsonapi.service.cqldriver.executor; +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.data.ByteUtils; +import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; +import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.IndexMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.api.core.type.MapType; +import com.datastax.oss.driver.api.core.type.VectorType; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.model.command.impl.VectorizeConfig; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.PrimaryKey; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ColumnType; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.ComplexTypes; +import io.stargate.sgv2.jsonapi.api.model.command.table.definition.datatype.PrimitiveTypes; +import io.stargate.sgv2.jsonapi.exception.SchemaException; +import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDef; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDefs; +import io.stargate.sgv2.jsonapi.util.CqlIdentifierUtil; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; public class TableSchemaObject extends TableBasedSchemaObject { public static final SchemaObjectType TYPE = SchemaObjectType.TABLE; - public TableSchemaObject(TableMetadata tableMetadata) { + private final VectorConfig vectorConfig; + + private TableSchemaObject(TableMetadata tableMetadata, VectorConfig vectorConfig) { super(TYPE, tableMetadata); + this.vectorConfig = vectorConfig; } @Override public VectorConfig vectorConfig() { - return VectorConfig.notEnabledVectorConfig(); + return vectorConfig; } @Override public IndexUsage newIndexUsage() { return IndexUsage.NO_OP; } + + /** + * Get table schema object from table metadata + * + * @param tableMetadata + * @param objectMapper + * @return + */ + public static TableSchemaObject from(TableMetadata tableMetadata, ObjectMapper objectMapper) { + Map extensions = + (Map) + tableMetadata.getOptions().get(CqlIdentifier.fromInternal("extensions")); + String vectorizeJson = null; + if (extensions != null) { + ByteBuffer vectorizeBuffer = + (ByteBuffer) extensions.get("com.datastax.data-api.vectorize-config"); + vectorizeJson = + vectorizeBuffer != null + ? new String(ByteUtils.getArray(vectorizeBuffer.duplicate()), StandardCharsets.UTF_8) + : null; + } + Map vectorizeConfigMap = + new HashMap<>(); + if (vectorizeJson != null) { + try { + JsonNode vectorizeByColumns = objectMapper.readTree(vectorizeJson); + Iterator> it = vectorizeByColumns.fields(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + try { + VectorConfig.ColumnVectorDefinition.VectorizeConfig vectorizeConfig = + objectMapper.treeToValue( + entry.getValue(), VectorConfig.ColumnVectorDefinition.VectorizeConfig.class); + vectorizeConfigMap.put(entry.getKey(), vectorizeConfig); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw SchemaException.Code.INVALID_VECTORIZE_CONFIGURATION.get( + Map.of("field", entry.getKey())); + } + } + } catch (JsonProcessingException e) { + throw SchemaException.Code.INVALID_CONFIGURATION.get(); + } + } + VectorConfig vectorConfig; + List columnVectorDefinitions = new ArrayList<>(); + for (Map.Entry column : tableMetadata.getColumns().entrySet()) { + if (column.getValue().getType() instanceof VectorType vectorType) { + final Optional index = tableMetadata.getIndex(column.getKey()); + SimilarityFunction similarityFunction = SimilarityFunction.COSINE; + if (index.isPresent()) { + final IndexMetadata indexMetadata = index.get(); + final Map indexOptions = indexMetadata.getOptions(); + final String similarityFunctionValue = indexOptions.get("similarity_function"); + if (similarityFunctionValue != null) { + similarityFunction = SimilarityFunction.fromString(similarityFunctionValue); + } + } + int dimension = vectorType.getDimensions(); + VectorConfig.ColumnVectorDefinition columnVectorDefinition = + new VectorConfig.ColumnVectorDefinition( + column.getKey().asInternal(), + dimension, + similarityFunction, + vectorizeConfigMap.get(column.getKey().asInternal())); + columnVectorDefinitions.add(columnVectorDefinition); + } + } + if (columnVectorDefinitions.isEmpty()) { + vectorConfig = VectorConfig.notEnabledVectorConfig(); + } else { + vectorConfig = new VectorConfig(true, Collections.unmodifiableList(columnVectorDefinitions)); + } + return new TableSchemaObject(tableMetadata, vectorConfig); + } + + /** + * Convert table schema object to table response which is returned as response for `listTables` + * + * @return + */ + public TableResponse toTableResponse() { + String tableName = CqlIdentifierUtil.externalRepresentation(tableMetadata().getName()); + HashMap columnsDefinition = new HashMap<>(); + for (Map.Entry column : + tableMetadata().getColumns().entrySet()) { + ColumnType type = getColumnType(column.getKey().asInternal(), column.getValue()); + columnsDefinition.put(CqlIdentifierUtil.externalRepresentation(column.getKey()), type); + } + + final List partitionBy = + tableMetadata().getPartitionKey().stream() + .map(column -> CqlIdentifierUtil.externalRepresentation(column.getName())) + .collect(Collectors.toList()); + final List partitionSort = + tableMetadata().getClusteringColumns().entrySet().stream() + .map( + entry -> + new PrimaryKey.OrderingKey( + CqlIdentifierUtil.externalRepresentation(entry.getKey().getName()), + entry.getValue() == ClusteringOrder.ASC + ? PrimaryKey.OrderingKey.Order.ASC + : PrimaryKey.OrderingKey.Order.DESC)) + .collect(Collectors.toList()); + PrimaryKey primaryKey = + new PrimaryKey( + partitionBy.toArray(new String[0]), + partitionSort.toArray(new PrimaryKey.OrderingKey[0])); + return new TableResponse( + tableName, new TableResponse.TableDefinition(columnsDefinition, primaryKey)); + } + + private ColumnType getColumnType(String columnName, ColumnMetadata columnMetadata) { + if (columnMetadata.getType() instanceof VectorType vt) { + // Schema will always have VectorConfig for vector type + VectorConfig.ColumnVectorDefinition columnVectorDefinition = + vectorConfig.columnVectorDefinitions().stream() + .filter(vc -> vc.fieldName().equals(columnName)) + .findFirst() + .get(); + VectorizeConfig vectorizeConfig = + columnVectorDefinition.vectorizeConfig() == null + ? null + : new VectorizeConfig( + columnVectorDefinition.vectorizeConfig().provider(), + columnVectorDefinition.vectorizeConfig().modelName(), + columnVectorDefinition.vectorizeConfig().authentication(), + columnVectorDefinition.vectorizeConfig().parameters()); + return new ComplexTypes.VectorType(PrimitiveTypes.FLOAT, vt.getDimensions(), vectorizeConfig); + } else if (columnMetadata.getType() instanceof MapType mt) { + if (!mt.isFrozen()) { + final Optional apiDataTypeDefKey = ApiDataTypeDefs.from(mt.getKeyType()); + final Optional apiDataTypeDefValue = + ApiDataTypeDefs.from(mt.getValueType()); + if (apiDataTypeDefKey.isPresent() && apiDataTypeDefValue.isPresent()) { + return new ComplexTypes.MapType( + PrimitiveTypes.fromString(apiDataTypeDefKey.get().getApiType().getApiName()), + PrimitiveTypes.fromString(apiDataTypeDefValue.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(mt.asCql(true, false)); + + } else if (columnMetadata.getType() + instanceof com.datastax.oss.driver.api.core.type.ListType lt) { + if (!lt.isFrozen()) { + final Optional apiDataTypeDef = ApiDataTypeDefs.from(lt.getElementType()); + if (apiDataTypeDef.isPresent()) { + return new ComplexTypes.ListType( + PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(lt.asCql(true, false)); + + } else if (columnMetadata.getType() + instanceof com.datastax.oss.driver.api.core.type.SetType st) { + if (!st.isFrozen()) { + final Optional apiDataTypeDef = ApiDataTypeDefs.from(st.getElementType()); + if (apiDataTypeDef.isPresent()) { + return new ComplexTypes.SetType( + PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName())); + } + } + // return unsupported format + return new ComplexTypes.UnsupportedType(st.asCql(true, false)); + } else { + final Optional apiDataTypeDef = + ApiDataTypeDefs.from(columnMetadata.getType()); + if (apiDataTypeDef.isPresent()) + return PrimitiveTypes.fromString(apiDataTypeDef.get().getApiType().getApiName()); + else { + // Need to return unsupported type + return new ComplexTypes.UnsupportedType(columnMetadata.getType().asCql(true, false)); + } + } + } + + /** + * Object used to build the response for listTables command + * + * @param name + * @param definition + */ + @JsonPropertyOrder({"name", "definition"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record TableResponse(String name, TableDefinition definition) { + + @JsonPropertyOrder({"columns", "primaryKey"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + record TableDefinition(Map columns, PrimaryKey primaryKey) {} + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java index dfb52b543..229d6d0de 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/VectorConfig.java @@ -2,66 +2,106 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.List; import java.util.Map; /** * incorporates vectorizeConfig into vectorConfig * - * @param vectorEnabled - * @param vectorSize - * @param similarityFunction - * @param vectorizeConfig + * @param vectorEnabled - If vector field is available for the table + * @param columnVectorDefinitions - List of columnVectorDefinitions each with respect to a + * column/field */ public record VectorConfig( - boolean vectorEnabled, - int vectorSize, - SimilarityFunction similarityFunction, - VectorizeConfig vectorizeConfig) { + boolean vectorEnabled, List columnVectorDefinitions) { // TODO: this is an immutable record, this can be singleton // TODO: Remove the use of NULL for the objects like vectorizeConfig public static VectorConfig notEnabledVectorConfig() { - return new VectorConfig(false, -1, null, null); + return new VectorConfig(false, null); } - // convert a vector jsonNode from table comment to vectorConfig - public static VectorConfig fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { - // dimension, similarityFunction, must exist - int dimension = jsonNode.get("dimension").asInt(); - SimilarityFunction similarityFunction = - SimilarityFunction.fromString(jsonNode.get("metric").asText()); + /** + * Configuration for a column, In case of collection this will be of size one + * + * @param fieldName + * @param vectorSize + * @param similarityFunction + * @param vectorizeConfig + */ + public record ColumnVectorDefinition( + String fieldName, + int vectorSize, + SimilarityFunction similarityFunction, + VectorizeConfig vectorizeConfig) { - VectorizeConfig vectorizeConfig = null; - // construct vectorizeConfig - JsonNode vectorizeServiceNode = jsonNode.get("service"); - if (vectorizeServiceNode != null) { - // provider, modelName, must exist - String provider = vectorizeServiceNode.get("provider").asText(); - String modelName = vectorizeServiceNode.get("modelName").asText(); - // construct VectorizeConfig.authentication, can be null - JsonNode vectorizeServiceAuthenticationNode = vectorizeServiceNode.get("authentication"); - Map vectorizeServiceAuthentication = - vectorizeServiceAuthenticationNode == null - ? null - : objectMapper.convertValue(vectorizeServiceAuthenticationNode, Map.class); - // construct VectorizeConfig.parameters, can be null - JsonNode vectorizeServiceParameterNode = vectorizeServiceNode.get("parameters"); - Map vectorizeServiceParameter = - vectorizeServiceParameterNode == null - ? null - : objectMapper.convertValue(vectorizeServiceParameterNode, Map.class); - vectorizeConfig = - new VectorizeConfig( - provider, modelName, vectorizeServiceAuthentication, vectorizeServiceParameter); + // convert a vector jsonNode from comment option to vectorConfig, used for collection + public static ColumnVectorDefinition fromJson(JsonNode jsonNode, ObjectMapper objectMapper) { + // dimension, similarityFunction, must exist + int dimension = jsonNode.get("dimension").asInt(); + SimilarityFunction similarityFunction = + SimilarityFunction.fromString(jsonNode.get("metric").asText()); + + return fromJson( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + dimension, + similarityFunction, + jsonNode, + objectMapper); } - return new VectorConfig(true, dimension, similarityFunction, vectorizeConfig); - } + // convert a vector jsonNode from table extension to vectorConfig, used for tables + public static ColumnVectorDefinition fromJson( + String fieldName, + int dimension, + SimilarityFunction similarityFunction, + JsonNode jsonNode, + ObjectMapper objectMapper) { + VectorizeConfig vectorizeConfig = null; + // construct vectorizeConfig + JsonNode vectorizeServiceNode = jsonNode.get("service"); + if (vectorizeServiceNode != null) { + vectorizeConfig = VectorizeConfig.fromJson(vectorizeServiceNode, objectMapper); + } + return new ColumnVectorDefinition(fieldName, dimension, similarityFunction, vectorizeConfig); + } - public record VectorizeConfig( - String provider, - String modelName, - Map authentication, - Map parameters) {} + /** + * Represent the vectorize configuration defined for a column + * + * @param provider + * @param modelName + * @param authentication + * @param parameters + */ + public record VectorizeConfig( + String provider, + String modelName, + Map authentication, + Map parameters) { + + protected static VectorizeConfig fromJson( + JsonNode vectorizeServiceNode, ObjectMapper objectMapper) { + // provider, modelName, must exist + String provider = vectorizeServiceNode.get("provider").asText(); + String modelName = vectorizeServiceNode.get("modelName").asText(); + // construct VectorizeConfig.authentication, can be null + JsonNode vectorizeServiceAuthenticationNode = vectorizeServiceNode.get("authentication"); + Map vectorizeServiceAuthentication = + vectorizeServiceAuthenticationNode == null + ? null + : objectMapper.convertValue(vectorizeServiceAuthenticationNode, Map.class); + // construct VectorizeConfig.parameters, can be null + JsonNode vectorizeServiceParameterNode = vectorizeServiceNode.get("parameters"); + Map vectorizeServiceParameter = + vectorizeServiceParameterNode == null + ? null + : objectMapper.convertValue(vectorizeServiceParameterNode, Map.class); + return new VectorizeConfig( + provider, modelName, vectorizeServiceAuthentication, vectorizeServiceParameter); + } + } + } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java index 0557ae329..1eca64f39 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/embedding/DataVectorizer.java @@ -14,6 +14,7 @@ import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.SchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.embedding.operation.EmbeddingProvider; import java.util.ArrayList; import java.util.HashMap; @@ -110,11 +111,16 @@ public Uni vectorize(List documents) { .onItem() .transform( vectorData -> { + final VectorConfig vectorConfig = schemaObject.vectorConfig(); + // This will be the first element for collection + final VectorConfig.ColumnVectorDefinition collectionVectorDefinition = + vectorConfig.columnVectorDefinitions().get(0); + // check if we get back the same number of vectors that we asked for if (vectorData.size() != vectorizeTexts.size()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' didn't return the expected number of embeddings. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), + collectionVectorDefinition.vectorizeConfig().provider(), vectorizeTexts.size(), vectorData.size()); } @@ -125,11 +131,11 @@ public Uni vectorize(List documents) { JsonNode document = documents.get(position); float[] vector = vectorData.get(vectorPosition); // check if all vectors have the expected size - if (vector.length != schemaObject.vectorConfig().vectorSize()) { + if (vector.length != collectionVectorDefinition.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), - schemaObject.vectorConfig().vectorSize(), + collectionVectorDefinition.vectorizeConfig().provider(), + collectionVectorDefinition.vectorSize(), vector.length); } final ArrayNode arrayNode = nodeFactory.arrayNode(vector.length); @@ -168,13 +174,17 @@ public Uni vectorize(String vectorizeContent) { .onItem() .transform( vectorData -> { + final VectorConfig vectorConfig = schemaObject.vectorConfig(); + // This will be the first element for collection + final VectorConfig.ColumnVectorDefinition collectionVectorDefinition = + vectorConfig.columnVectorDefinitions().get(0); float[] vector = vectorData.get(0); // check if vector have the expected size - if (vector.length != schemaObject.vectorConfig().vectorSize()) { + if (vector.length != collectionVectorDefinition.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), - schemaObject.vectorConfig().vectorSize(), + collectionVectorDefinition.vectorizeConfig().provider(), + collectionVectorDefinition.vectorSize(), vector.length); } return vector; @@ -211,12 +221,16 @@ public Uni vectorize(SortClause sortClause) { .transform( vectorData -> { float[] vector = vectorData.get(0); + final VectorConfig vectorConfig = schemaObject.vectorConfig(); + // This will be the first element for collection + final VectorConfig.ColumnVectorDefinition collectionVectorDefinition = + vectorConfig.columnVectorDefinitions().get(0); // check if vector have the expected size - if (vector.length != schemaObject.vectorConfig().vectorSize()) { + if (vector.length != collectionVectorDefinition.vectorSize()) { throw EMBEDDING_PROVIDER_UNEXPECTED_RESPONSE.toApiException( "Embedding provider '%s' did not return expected embedding length. Expect: '%d'. Actual: '%d'", - schemaObject.vectorConfig().vectorizeConfig().provider(), - schemaObject.vectorConfig().vectorSize(), + collectionVectorDefinition.vectorizeConfig().provider(), + collectionVectorDefinition.vectorSize(), vector.length); } sortExpressions.clear(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistries.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistries.java index 83ab57f75..a2b0330a0 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistries.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistries.java @@ -48,6 +48,10 @@ public abstract class JSONCodecRegistries { JSONCodecs.DURATION_FROM_STRING, JSONCodecs.TIME_FROM_STRING, JSONCodecs.TIMESTAMP_FROM_STRING, + // UUID codecs + JSONCodecs.UUID_FROM_STRING, + JSONCodecs.TIMEUUID_FROM_STRING, + // Other codecs JSONCodecs.BINARY, JSONCodecs.BOOLEAN)); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecs.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecs.java index 6323165f4..9854d1531 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecs.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecs.java @@ -11,6 +11,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; +import java.util.UUID; /** * Defines the {@link JSONCodec} instances that are added to the {@link @@ -275,4 +276,22 @@ public abstract class JSONCodecs { DataTypes.TEXT, JSONCodec.ToCQL.unsafeIdentity(), JSONCodec.ToJSON.unsafeNodeFactory(JsonNodeFactory.instance::textNode)); + + // UUID Codecs + + public static final JSONCodec UUID_FROM_STRING = + new JSONCodec<>( + GenericType.STRING, + DataTypes.UUID, + JSONCodec.ToCQL.safeFromString(UUID::fromString), + JSONCodec.ToJSON.toJSONUsingToString()); + + // While not allowed to be created as column type, we do support reading/writing + // of columns of this type in existing tables. + public static final JSONCodec TIMEUUID_FROM_STRING = + new JSONCodec<>( + GenericType.STRING, + DataTypes.TIMEUUID, + JSONCodec.ToCQL.safeFromString(UUID::fromString), + JSONCodec.ToJSON.toJSONUsingToString()); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java new file mode 100644 index 000000000..7405c331c --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/tables/ListTablesOperation.java @@ -0,0 +1,120 @@ +package io.stargate.sgv2.jsonapi.service.operation.tables; + +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; +import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.api.request.DataApiRequestInfo; +import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; +import io.stargate.sgv2.jsonapi.service.operation.Operation; +import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher; +import io.stargate.sgv2.jsonapi.util.CqlIdentifierUtil; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * List tables operation. Uses {@link CQLSessionCache} to fetch all valid tables for a namespace. + * The schema check against the table is done in the {@link CollectionTableMatcher} and ignores + * them. + * + * @param explain - returns tables schema if `true`; returns only tables names if `false` + * @param objectMapper {@link ObjectMapper} + * @param cqlSessionCache {@link CQLSessionCache} + * @param tableMatcher {@link CollectionTableMatcher} + * @param commandContext {@link CommandContext} + */ +public record ListTablesOperation( + boolean explain, + ObjectMapper objectMapper, + CQLSessionCache cqlSessionCache, + CollectionTableMatcher tableMatcher, + CommandContext commandContext) + implements Operation { + + // shared table matcher instance + // TODO: if this is static why does the record that have an instance variable passed by the ctor + // below ? + private static final CollectionTableMatcher TABLE_MATCHER = new CollectionTableMatcher(); + + public ListTablesOperation( + boolean explain, + ObjectMapper objectMapper, + CQLSessionCache cqlSessionCache, + CommandContext commandContext) { + this(explain, objectMapper, cqlSessionCache, TABLE_MATCHER, commandContext); + } + + /** {@inheritDoc} */ + @Override + public Uni> execute( + DataApiRequestInfo dataApiRequestInfo, QueryExecutor queryExecutor) { + KeyspaceMetadata keyspaceMetadata = + cqlSessionCache + .getSession(dataApiRequestInfo) + .getMetadata() + .getKeyspaces() + .get( + CqlIdentifierUtil.cqlIdentifierFromUserInput( + commandContext.schemaObject().name().keyspace())); + if (keyspaceMetadata == null) { + return Uni.createFrom() + .failure( + ErrorCodeV1.KEYSPACE_DOES_NOT_EXIST.toApiException( + "Unknown keyspace '%s', you must create it first", + commandContext.schemaObject().name().keyspace())); + } + return Uni.createFrom() + .item( + () -> { + List properties = + keyspaceMetadata + // get all tables + .getTables() + .values() + .stream() + // filter for valid collections + .filter(tableMatcher.negate()) + // map to name + .map(table -> TableSchemaObject.from(table, objectMapper)) + // get as list + .toList(); + // Wrap the properties list into a command result + return new Result(explain, properties); + }); + } + + // simple result wrapper + private record Result(boolean explain, List tables) + implements Supplier { + + @Override + public CommandResult get() { + if (explain) { + final List createCollectionCommands = + tables().stream() + .map(tableSchemaObject -> tableSchemaObject.toTableResponse()) + .toList(); + Map statuses = + Map.of(CommandStatus.EXISTING_TABLES, createCollectionCommands); + return new CommandResult(statuses); + } else { + List tables = + tables().stream() + .map( + schemaObject -> + CqlIdentifierUtil.externalRepresentation( + schemaObject.tableMetadata().getName())) + .toList(); + Map statuses = Map.of(CommandStatus.EXISTING_TABLES, tables); + return new CommandResult(statuses); + } + } + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index c994f4910..fa173dc44 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -239,6 +239,7 @@ private Tags getCustomTags( result.errors().get(0).fieldsForMetricsTag().getOrDefault("errorCode", UNKNOWN_VALUE); errorCodeTag = Tag.of(jsonApiMetricsConfig.errorCode(), errorCode); } + Tag vectorEnabled = commandContext.schemaObject().vectorConfig().vectorEnabled() ? Tag.of(jsonApiMetricsConfig.vectorEnabled(), "true") diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java index 7f25c46ed..b117cbcad 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/CreateTableCommandResolver.java @@ -11,6 +11,7 @@ import io.stargate.sgv2.jsonapi.config.OperationsConfig; import io.stargate.sgv2.jsonapi.exception.SchemaException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.operation.*; import io.stargate.sgv2.jsonapi.service.operation.Operation; import io.stargate.sgv2.jsonapi.service.operation.tables.CreateTableAttemptBuilder; @@ -39,7 +40,7 @@ public Operation resolveKeyspaceCommand( .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getApiDataType())); List partitionKeys = Arrays.stream(command.definition().primaryKey().keys()).toList(); - Map vectorizeConfigMap = + Map vectorizeConfigMap = command.definition().columns().entrySet().stream() .filter( e -> @@ -50,9 +51,15 @@ public Operation resolveKeyspaceCommand( Map.Entry::getKey, e -> { ComplexTypes.VectorType vectorType = ((ComplexTypes.VectorType) e.getValue()); - final VectorizeConfig vectorConfig = vectorType.getVectorConfig(); - validateVectorize.validateService(vectorConfig, vectorType.getDimension()); - return vectorConfig; + final VectorizeConfig vectorizeConfig = vectorType.getVectorConfig(); + validateVectorize.validateService(vectorizeConfig, vectorType.getDimension()); + VectorConfig.ColumnVectorDefinition.VectorizeConfig dbVectorConfig = + new VectorConfig.ColumnVectorDefinition.VectorizeConfig( + vectorizeConfig.provider(), + vectorizeConfig.modelName(), + vectorizeConfig.authentication(), + vectorizeConfig.parameters()); + return dbVectorConfig; })); if (partitionKeys.isEmpty()) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java new file mode 100644 index 000000000..6f62db096 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ListTablesCommandResolver.java @@ -0,0 +1,39 @@ +package io.stargate.sgv2.jsonapi.service.resolver; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.api.model.command.impl.ListTablesCommand; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.KeyspaceSchemaObject; +import io.stargate.sgv2.jsonapi.service.operation.Operation; +import io.stargate.sgv2.jsonapi.service.operation.tables.ListTablesOperation; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +/** Command resolver for the {@link ListTablesCommand}. */ +@ApplicationScoped +public class ListTablesCommandResolver implements CommandResolver { + private final ObjectMapper objectMapper; + private final CQLSessionCache cqlSessionCache; + + @Inject + public ListTablesCommandResolver(ObjectMapper objectMapper, CQLSessionCache cqlSessionCache) { + this.objectMapper = objectMapper; + this.cqlSessionCache = cqlSessionCache; + } + + /** {@inheritDoc} */ + @Override + public Class getCommandClass() { + return ListTablesCommand.class; + } + + /** {@inheritDoc} */ + @Override + public Operation resolveKeyspaceCommand( + CommandContext ctx, ListTablesCommand command) { + + boolean explain = command.options() != null ? command.options().explain() : false; + return new ListTablesOperation(explain, objectMapper, cqlSessionCache, ctx); + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java index 3c054d51c..2fe10c8e6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSchemaObject.java @@ -17,6 +17,7 @@ import io.stargate.sgv2.jsonapi.service.cqldriver.executor.*; import io.stargate.sgv2.jsonapi.service.projection.IndexingProjector; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -220,7 +221,14 @@ private static CollectionSchemaObject createCollectionSettings( collectionName, tableMetadata, IdConfig.defaultIdConfig(), - new VectorConfig(true, vectorSize, function, null), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + vectorSize, + function, + null))), null); } else { return new CollectionSchemaObject( @@ -272,6 +280,22 @@ private static CollectionSchemaObject createCollectionSettings( } } + // convert a vector jsonNode from cql table comment to vectorConfig, used for collection + private static VectorConfig.ColumnVectorDefinition fromJson( + JsonNode jsonNode, ObjectMapper objectMapper) { + // dimension, similarityFunction, must exist + int dimension = jsonNode.get("dimension").asInt(); + SimilarityFunction similarityFunction = + SimilarityFunction.fromString(jsonNode.get("metric").asText()); + + return VectorConfig.ColumnVectorDefinition.fromJson( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + dimension, + similarityFunction, + jsonNode, + objectMapper); + } + public static CreateCollectionCommand collectionSettingToCreateCollectionCommand( CollectionSchemaObject collectionSetting) { @@ -279,25 +303,28 @@ public static CreateCollectionCommand collectionSettingToCreateCollectionCommand CreateCollectionCommand.Options options = null; CreateCollectionCommand.Options.VectorSearchConfig vectorSearchConfig = null; CreateCollectionCommand.Options.IndexingConfig indexingConfig = null; - // populate the vectorSearchConfig - if (collectionSetting.vectorConfig().vectorEnabled()) { + // populate the vectorSearchConfig, Default will be the index 0 since there is only one vector + // column supported for collection + final VectorConfig vectorConfig = collectionSetting.vectorConfig(); + if (vectorConfig.vectorEnabled()) { + // This will be size 1 for collection + VectorConfig.ColumnVectorDefinition vectorConfigColumn = + vectorConfig.columnVectorDefinitions().get(0); VectorizeConfig vectorizeConfig = null; - if (collectionSetting.vectorConfig().vectorizeConfig() != null) { - Map authentication = - collectionSetting.vectorConfig().vectorizeConfig().authentication(); - Map parameters = - collectionSetting.vectorConfig().vectorizeConfig().parameters(); + if (vectorConfigColumn.vectorizeConfig() != null) { + Map authentication = vectorConfigColumn.vectorizeConfig().authentication(); + Map parameters = vectorConfigColumn.vectorizeConfig().parameters(); vectorizeConfig = new VectorizeConfig( - collectionSetting.vectorConfig().vectorizeConfig().provider(), - collectionSetting.vectorConfig().vectorizeConfig().modelName(), + vectorConfigColumn.vectorizeConfig().provider(), + vectorConfigColumn.vectorizeConfig().modelName(), authentication == null ? null : Map.copyOf(authentication), parameters == null ? null : Map.copyOf(parameters)); } vectorSearchConfig = new CreateCollectionCommand.Options.VectorSearchConfig( - collectionSetting.vectorConfig().vectorSize(), - collectionSetting.vectorConfig().similarityFunction().name().toLowerCase(), + vectorConfigColumn.vectorSize(), + vectorConfigColumn.similarityFunction().name().toLowerCase(), vectorizeConfig); } // populate the indexingConfig @@ -331,11 +358,11 @@ public CollectionIndexingConfig indexingConfig() { // TODO: these helper functions break encapsulation for very little benefit public SimilarityFunction similarityFunction() { - return vectorConfig().similarityFunction(); + return vectorConfig().columnVectorDefinitions().get(0).similarityFunction(); } public boolean isVectorEnabled() { - return vectorConfig() != null && vectorConfig().vectorEnabled(); + return vectorConfig().vectorEnabled(); } // TODO: the overrides below were auto added when migrating from a record to a class, not sure diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java index 0cf67b0be..753827934 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV0Reader.java @@ -3,9 +3,11 @@ import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.config.constants.TableCommentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; +import java.util.List; /** * schema_version 0 is before we introduce schema_version into the C* table comment of data api @@ -26,7 +28,15 @@ public CollectionSchemaObject readCollectionSettings( int vectorSize, SimilarityFunction function) { - VectorConfig vectorConfig = new VectorConfig(vectorEnabled, vectorSize, function, null); + VectorConfig vectorConfig = + new VectorConfig( + vectorEnabled, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + vectorSize, + function, + null))); CollectionIndexingConfig indexingConfig = null; JsonNode indexing = commentConfigNode.path(TableCommentConstants.COLLECTION_INDEXING_KEY); if (!indexing.isMissingNode()) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java index de0c3a7e6..9d577a9e9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/collections/CollectionSettingsV1Reader.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.config.constants.TableCommentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; +import java.util.List; /** * schema_version 1 sample: @@ -25,7 +26,9 @@ public CollectionSchemaObject readCollectionSettings( VectorConfig vectorConfig = VectorConfig.notEnabledVectorConfig(); JsonNode vector = collectionOptionsNode.path(TableCommentConstants.COLLECTION_VECTOR_KEY); if (!vector.isMissingNode()) { - vectorConfig = VectorConfig.fromJson(vector, objectMapper); + VectorConfig.ColumnVectorDefinition columnVectorDefinition = + VectorConfig.ColumnVectorDefinition.fromJson(vector, objectMapper); + vectorConfig = new VectorConfig(true, List.of(columnVectorDefinition)); } // construct collectionSettings IndexingConfig CollectionIndexingConfig indexingConfig = null; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java index 4bbd6f0d3..c3d02b5a5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ApiDataTypeDef.java @@ -3,7 +3,6 @@ import com.datastax.oss.driver.api.core.type.*; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.Objects; -import java.util.Optional; /** * The definition of a type the API supports for a table column. @@ -42,18 +41,6 @@ public boolean isContainer() { return cqlType instanceof ContainerType; } - public Optional cqlTypeAsList() { - return cqlType instanceof ListType listType ? Optional.of(listType) : Optional.empty(); - } - - public Optional cqlTypeAsSet() { - return cqlType instanceof SetType setType ? Optional.of(setType) : Optional.empty(); - } - - public Optional cqlTypeAsMap() { - return cqlType instanceof MapType mapType ? Optional.of(mapType) : Optional.empty(); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java index a27eb0418..28afa0397 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/schema/tables/ComplexApiDataType.java @@ -9,17 +9,14 @@ public abstract class ComplexApiDataType implements ApiDataType { private final String apiName; private final PrimitiveApiDataType keyType; private final PrimitiveApiDataType valueType; - private final int vectorSize; + private final int dimension; public ComplexApiDataType( - String apiName, - PrimitiveApiDataType keyType, - PrimitiveApiDataType valueType, - int vectorSize) { + String apiName, PrimitiveApiDataType keyType, PrimitiveApiDataType valueType, int dimension) { this.apiName = apiName; this.keyType = keyType; this.valueType = valueType; - this.vectorSize = vectorSize; + this.dimension = dimension; } public PrimitiveApiDataType getKeyType() { @@ -30,8 +27,8 @@ public PrimitiveApiDataType getValueType() { return valueType; } - public int getVectorSize() { - return vectorSize; + public int getDimension() { + return dimension; } public abstract DataType getCqlType(); @@ -77,14 +74,14 @@ public DataType getCqlType() { } public static class VectorType extends ComplexApiDataType { - public VectorType(PrimitiveApiDataType valueType, int vectorSize) { - super("vector", null, valueType, vectorSize); + public VectorType(PrimitiveApiDataType valueType, int dimension) { + super("vector", null, valueType, dimension); } @Override public DataType getCqlType() { return new ExtendedVectorType( - ApiDataTypeDefs.from(getValueType()).get().getCqlType(), getVectorSize()); + ApiDataTypeDefs.from(getValueType()).get().getCqlType(), getDimension()); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java b/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java index 8ed18ad6b..86ed65717 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/util/CqlIdentifierUtil.java @@ -26,4 +26,9 @@ public static CqlIdentifier cqlIdentifierFromIndexTarget(String name) { public static String cqlIdentifierToStringForUser(CqlIdentifier identifier) { return identifier.asCql(true); } + + /** Remove the quotes from the identifier */ + public static String externalRepresentation(CqlIdentifier identifier) { + return identifier.asInternal(); + } } diff --git a/src/main/resources/errors.yaml b/src/main/resources/errors.yaml index 13d8ce0a6..7c15315fe 100644 --- a/src/main/resources/errors.yaml +++ b/src/main/resources/errors.yaml @@ -382,6 +382,18 @@ request-errors: "modelName": "NV-Embed-QA" } } + - scope: SCHEMA + code: INVALID_CONFIGURATION + title: Unable to parse configuration, schema invalid. + body: |- + Unable to parse configuration, schema invalid. + + - scope: SCHEMA + code: INVALID_VECTORIZE_CONFIGURATION + title: Unable to parse vectorize configuration, schema invalid. + body: |- + Unable to parse vectorize configuration, schema invalid for field ${field}. + # ================================================================================================================ # Server Errors diff --git a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java index f13ab7a04..e990ea1f9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/TestConstants.java @@ -1,11 +1,13 @@ package io.stargate.sgv2.jsonapi; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.config.feature.ApiFeatures; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.*; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; import io.stargate.sgv2.jsonapi.service.schema.collections.IdConfig; +import java.util.List; import org.apache.commons.lang3.RandomStringUtils; /** @@ -34,7 +36,14 @@ public final class TestConstants { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null); public static final KeyspaceSchemaObject KEYSPACE_SCHEMA_OBJECT = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java index 922de123d..85d1e43d8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/AbstractTableIntegrationTestBase.java @@ -4,6 +4,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import com.fasterxml.jackson.databind.ObjectMapper; @@ -41,6 +42,13 @@ protected DataApiResponseValidator createTable(String tableDefAsJSON) { .body("status.ok", is(1)); } + protected DataApiResponseValidator listTables(String tableDefAsJSON) { + return DataApiCommandSenders.assertNamespaceCommand(keyspaceName) + .postListTables(tableDefAsJSON) + .hasNoErrors() + .body("status.tables", notNullValue()); + } + protected DataApiResponseValidator createTableErrorValidation( String tableDefAsJSON, String errorCode, String message) { return DataApiCommandSenders.assertNamespaceCommand(keyspaceName) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/InsertOneTableIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/InsertOneTableIntegrationTest.java index 8648c567d..553d89450 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/InsertOneTableIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/InsertOneTableIntegrationTest.java @@ -18,13 +18,14 @@ @WithTestResource(value = DseTestResource.class, restrictToAnnotatedClass = false) @TestClassOrder(ClassOrderer.OrderAnnotation.class) public class InsertOneTableIntegrationTest extends AbstractTableIntegrationTestBase { - static final String TABLE_WITH_TEXT_COLUMNS = "findOneTextColumnsTable"; - static final String TABLE_WITH_INT_COLUMNS = "findOneIntColumnsTable"; - static final String TABLE_WITH_FP_COLUMNS = "findOneFpColumnsTable"; - static final String TABLE_WITH_BINARY_COLUMN = "findOneBinaryColumnsTable"; - static final String TABLE_WITH_DATETIME_COLUMNS = "findOneDateTimeColumnsTable"; - static final String TABLE_WITH_LIST_COLUMNS = "findOneListColumnsTable"; - static final String TABLE_WITH_SET_COLUMNS = "findOneSetColumnsTable"; + static final String TABLE_WITH_TEXT_COLUMNS = "insertOneTextColumnsTable"; + static final String TABLE_WITH_INT_COLUMNS = "insertOneIntColumnsTable"; + static final String TABLE_WITH_FP_COLUMNS = "insertOneFpColumnsTable"; + static final String TABLE_WITH_BINARY_COLUMN = "insertOneBinaryColumnsTable"; + static final String TABLE_WITH_DATETIME_COLUMNS = "insertOneDateTimeColumnsTable"; + static final String TABLE_WITH_UUID_COLUMN = "insertOneUuidColumnTable"; + static final String TABLE_WITH_LIST_COLUMNS = "insertOneListColumnsTable"; + static final String TABLE_WITH_SET_COLUMNS = "insertOneSetColumnsTable"; final JSONCodecRegistryTestData codecTestData = new JSONCodecRegistryTestData(); @@ -68,6 +69,13 @@ public final void createDefaultTables() { "timestampValue", "timestamp"), "id"); + createTableWithColumns( + TABLE_WITH_UUID_COLUMN, + Map.of( + "id", "text", + "uuidValue", "uuid"), + "id"); + createTableWithColumns( TABLE_WITH_LIST_COLUMNS, Map.of( @@ -468,6 +476,56 @@ private String quote(String s) { @Nested @Order(6) + class InsertUUIDColumns { + @Test + void insertValidUUIDValue() { + final String docJSON = uuidDoc("uuidValid", "\"123e4567-e89b-12d3-a456-426614174000\""); + insertOneInTable(TABLE_WITH_UUID_COLUMN, docJSON); + DataApiCommandSenders.assertTableCommand(keyspaceName, TABLE_WITH_UUID_COLUMN) + .postFindOne("{ \"filter\": { \"id\": \"uuidValid\" } }") + .hasNoErrors() + .hasJSONField("data.document", docJSON); + } + + @Test + void failOnInvalidUUIDString() { + DataApiCommandSenders.assertTableCommand(keyspaceName, TABLE_WITH_UUID_COLUMN) + .postInsertOne(uuidDoc("uuidInvalid", "\"xxx\"")) + .hasSingleApiError( + DocumentException.Code.INVALID_COLUMN_VALUES, + DocumentException.class, + "Only values that are supported by", + "Error trying to convert to targetCQLType `UUID` from", + "problem: Invalid UUID string: xxx"); + } + + // Test for non-String input + @Test + void failOnInvalidUUIDArray() { + DataApiCommandSenders.assertTableCommand(keyspaceName, TABLE_WITH_UUID_COLUMN) + .postInsertOne(uuidDoc("uuidInvalid", "[1, 2, 3, 4]")) + .hasSingleApiError( + DocumentException.Code.INVALID_COLUMN_VALUES, + DocumentException.class, + "Only values that are supported by", + "Error trying to convert to targetCQLType `UUID` from", + "Root cause: no codec matching value type"); + } + + private String uuidDoc(String id, String uuidValueStr) { + return + """ + { + "id": "%s", + "uuidValue": %s + } + """ + .formatted(id, uuidValueStr); + } + } + + @Nested + @Order(7) class InsertListColumns { @Test void insertValidListValues() { @@ -566,7 +624,7 @@ void failOnWrongListElementValue() { } @Nested - @Order(7) + @Order(8) class InsertSetColumns { @Test void insertValidSetValues() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java new file mode 100644 index 000000000..56c1e351b --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/ListTablesIntegrationTest.java @@ -0,0 +1,260 @@ +package io.stargate.sgv2.jsonapi.api.v1.tables; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +import io.quarkus.test.common.WithTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.restassured.http.ContentType; +import io.stargate.sgv2.jsonapi.api.v1.KeyspaceResource; +import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.ClassOrderer; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestClassOrder; +import org.junit.jupiter.api.TestMethodOrder; + +@QuarkusIntegrationTest +@WithTestResource(value = DseTestResource.class, restrictToAnnotatedClass = false) +@TestClassOrder(ClassOrderer.OrderAnnotation.class) +public class ListTablesIntegrationTest extends AbstractTableIntegrationTestBase { + @BeforeAll + public final void createDefaultTables() { + String tableData = + """ + { + "name": "allTypesTable", + "definition": { + "columns": { + "ascii_type": "ascii", + "bigint_type": "bigint", + "blob_type": "blob", + "boolean_type": "boolean", + "date_type": "date", + "decimal_type": "decimal", + "double_type": "double", + "duration_type": "duration", + "float_type": "float", + "inet_type": "inet", + "int_type": "int", + "smallint_type": "smallint", + "text_type": "text", + "time_type": "time", + "timestamp_type": "timestamp", + "tinyint_type": "tinyint", + "uuid_type": "uuid", + "varint_type": "varint", + "map_type": { + "type": "map", + "keyType": "text", + "valueType": "int" + }, + "list_type": { + "type": "list", + "valueType": "text" + }, + "set_type": { + "type": "set", + "valueType": "text" + }, + "vector_type": { + "type": "vector", + "dimension": 1024, + "service": { + "provider": "mistral", + "modelName": "mistral-embed" + } + } + }, + "primaryKey": { + "partitionBy": [ + "text_type" + ], + "partitionSort": { + "int_type": 1, + "bigint_type": -1 + } + } + }, + "options": { + "ifNotExists": true + } + } + """; + createTable(tableData); + String table2 = + """ + { + "name": "person", + "definition": { + "columns": { + "id": "text", + "age": "int", + "name": "text", + "city": "text" + }, + "primaryKey": "id" + } + } + """; + createTable(table2); + } + + @Nested + @Order(1) + @TestMethodOrder(MethodOrderer.OrderAnnotation.class) + class ListTables { + + @Test + @Order(1) + public void listTablesOnly() { + String listTablesOnly = + """ + {} + """; + listTables(listTablesOnly) + .hasNoErrors() + // Validate that status.tables is not null + .body("status.tables", notNullValue()) + + // Validate the number of tables in the response + .body("status.tables", hasSize(2)) + + // Validate the specific table names in the position + .body("status.tables[0]", equalTo("allTypesTable")) + .body("status.tables[1]", equalTo("person")); + } + + @Test + @Order(2) + public void listTablesWithSchema() { + String listTablesWithSchema = + """ + { + "options" : { + "explain" : true + } + } + """; + listTables(listTablesWithSchema) + .hasNoErrors() + // Validate that status.tables is not null and contains one table: allTypesTable + .body("status.tables", notNullValue()) + .body("status.tables", hasSize(2)) + .body("status.tables[0].name", equalTo("allTypesTable")) + + // Validate that the table contains the expected columns and types + .body("status.tables[0].definition.columns.date_type.type", equalTo("date")) + .body("status.tables[0].definition.columns.time_type.type", equalTo("time")) + .body("status.tables[0].definition.columns.text_type.type", equalTo("text")) + .body("status.tables[0].definition.columns.int_type.type", equalTo("int")) + .body("status.tables[0].definition.columns.vector_type.type", equalTo("vector")) + .body( + "status.tables[0].definition.columns.vector_type.dimension", + equalTo(1024)) // Additional dimension check for vector type + .body( + "status.tables[0].definition.columns.vector_type.service.provider", + equalTo("mistral")) + .body( + "status.tables[0].definition.columns.vector_type.service.modelName", + equalTo("mistral-embed")) + .body("status.tables[0].definition.columns.duration_type.type", equalTo("duration")) + .body("status.tables[0].definition.columns.timestamp_type.type", equalTo("timestamp")) + .body("status.tables[0].definition.columns.set_type.type", equalTo("set")) + .body( + "status.tables[0].definition.columns.set_type.valueType", + equalTo("text")) // Set's valueType check + .body("status.tables[0].definition.columns.bigint_type.type", equalTo("bigint")) + .body("status.tables[0].definition.columns.boolean_type.type", equalTo("boolean")) + .body("status.tables[0].definition.columns.uuid_type.type", equalTo("uuid")) + .body("status.tables[0].definition.columns.blob_type.type", equalTo("blob")) + .body("status.tables[0].definition.columns.inet_type.type", equalTo("inet")) + .body("status.tables[0].definition.columns.list_type.type", equalTo("list")) + .body( + "status.tables[0].definition.columns.list_type.valueType", + equalTo("text")) // List's valueType check + .body("status.tables[0].definition.columns.map_type.type", equalTo("map")) + .body( + "status.tables[0].definition.columns.map_type.keyType", + equalTo("text")) // Map's keyType check + .body( + "status.tables[0].definition.columns.map_type.valueType", + equalTo("int")) // Map's valueType check + .body("status.tables[0].definition.columns.varint_type.type", equalTo("varint")) + .body("status.tables[0].definition.columns.tinyint_type.type", equalTo("tinyint")) + .body("status.tables[0].definition.columns.decimal_type.type", equalTo("decimal")) + .body("status.tables[0].definition.columns.float_type.type", equalTo("float")) + .body("status.tables[0].definition.columns.ascii_type.type", equalTo("ascii")) + .body("status.tables[0].definition.columns.double_type.type", equalTo("double")) + .body("status.tables[0].definition.columns.smallint_type.type", equalTo("smallint")) + + // Validate the primary key; + .body("status.tables[0].definition.primaryKey.partitionBy[0]", equalTo("text_type")) + .body("status.tables[0].definition.primaryKey.partitionSort.int_type", equalTo(1)) + .body("status.tables[0].definition.primaryKey.partitionSort.bigint_type", equalTo(-1)) + + // Check the second table name + .body("status.tables[1].name", equalTo("person")); + // Not checking second table types as the data types used are same as first table + } + + @Test + @Order(3) + public void ignoreCollections() { + String simpleCollectionToIgnore = + """ + { + "createCollection": { + "name": "simple_collection_to_ignore" + } + } + """; + + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(simpleCollectionToIgnore) + .when() + .post(KeyspaceResource.BASE_PATH, keyspaceName) + .then() + .statusCode(200); + + String listTablesOnly = + """ + {} + """; + listTables(listTablesOnly) + .hasNoErrors() + // Validate that status.tables is not null + .body("status.tables", notNullValue()) + + // Validate the number of tables in the response + .body("status.tables", hasSize(2)) + + // Validate the specific table names in the position + .body("status.tables[0]", equalTo("allTypesTable")) + .body("status.tables[1]", equalTo("person")); + + String listTablesWithSchema = + """ + { + "options" : { + "explain" : true + } + } + """; + listTables(listTablesWithSchema) + .hasNoErrors() + // Validate that status.tables is not null and contains one table: allTypesTable + .body("status.tables", notNullValue()) + .body("status.tables", hasSize(2)) + .body("status.tables[0].name", equalTo("allTypesTable")) + .body("status.tables[1].name", equalTo("person")); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java index 32e569761..449362997 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiKeyspaceCommandSender.java @@ -16,4 +16,8 @@ protected io.restassured.response.Response postInternal(RequestSpecification req public DataApiResponseValidator postCreateTable(String tableDefAsJSON) { return postCommand("createTable", tableDefAsJSON); } + + public DataApiResponseValidator postListTables(String listDefinition) { + return postCommand("listTables", listDefinition); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java index 28d0e3b66..6a267769e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/CqlFixture.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.fixtures; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.fixtures.data.DefaultData; import io.stargate.sgv2.jsonapi.fixtures.data.FixtureData; import io.stargate.sgv2.jsonapi.fixtures.identifiers.BaseFixtureIdentifiers; @@ -62,7 +63,7 @@ public CqlFixture( this.cqlData = cqlData; this.tableFixture = tableFixture; this.tableMetadata = tableFixture.tableMetadata(identifiers); - this.tableSchemaObject = new TableSchemaObject(tableMetadata); + this.tableSchemaObject = TableSchemaObject.from(tableMetadata, new ObjectMapper()); } public FixtureIdentifiers identifiers() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java index b13fde1ef..2dbfaebf3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/SchemaObjectTestData.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.fixtures.testdata; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; public class SchemaObjectTestData extends TestDataSuplier { @@ -9,6 +10,6 @@ public SchemaObjectTestData(TestData testData) { } public TableSchemaObject emptyTableSchemaObject() { - return new TableSchemaObject(testData.tableMetadata().empty()); + return TableSchemaObject.from(testData.tableMetadata().empty(), new ObjectMapper()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java index c734859c1..531388e26 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/fixtures/testdata/WhereAnalyzerTestData.java @@ -10,6 +10,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.exception.FilterException; import io.stargate.sgv2.jsonapi.exception.WarningException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.TableSchemaObject; @@ -58,8 +59,9 @@ public WhereAnalyzerFixture( this.message = message; this.tableMetadata = tableMetadata; - this.analyzer = new WhereCQLClauseAnalyzer(new TableSchemaObject(tableMetadata)); - this.tableSchemaObject = new TableSchemaObject(tableMetadata); + this.analyzer = + new WhereCQLClauseAnalyzer(TableSchemaObject.from(tableMetadata, new ObjectMapper())); + this.tableSchemaObject = TableSchemaObject.from(tableMetadata, new ObjectMapper()); this.expression = new LogicalExpressionTestData.ExpressionBuilder<>(this, expression, tableMetadata); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java index d80645866..4db187633 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTestData.java @@ -2,6 +2,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.internal.core.metadata.schema.DefaultTableMetadata; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import java.util.UUID; @@ -32,6 +33,6 @@ public DefaultDriverExceptionHandlerTestData() { Map.of(), Map.of(), Map.of()); - TABLE_SCHEMA_OBJECT = new TableSchemaObject(tableMetadata); + TABLE_SCHEMA_OBJECT = TableSchemaObject.from(tableMetadata, new ObjectMapper()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java index 1ac981fc7..e61fd4510 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/DataVectorizerTest.java @@ -13,6 +13,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortClause; import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortExpression; import io.stargate.sgv2.jsonapi.api.request.EmbeddingCredentials; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; @@ -232,9 +233,13 @@ public void testWithUnmatchedVectorSize() { IdConfig.defaultIdConfig(), new VectorConfig( true, - 4, - SimilarityFunction.COSINE, - new VectorConfig.VectorizeConfig("custom", "custom", null, null)), + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + 4, + SimilarityFunction.COSINE, + new VectorConfig.ColumnVectorDefinition.VectorizeConfig( + "custom", "custom", null, null)))), null); List documents = new ArrayList<>(); for (int i = 0; i < 2; i++) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java index ac27f0f20..df37dbd63 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/embedding/operation/TestEmbeddingProvider.java @@ -4,6 +4,7 @@ import io.stargate.sgv2.jsonapi.TestConstants; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.request.EmbeddingCredentials; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.schema.SimilarityFunction; import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject; @@ -21,9 +22,13 @@ public class TestEmbeddingProvider extends EmbeddingProvider { IdConfig.defaultIdConfig(), new VectorConfig( true, - 3, - SimilarityFunction.COSINE, - new VectorConfig.VectorizeConfig("custom", "custom", null, null)), + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + 3, + SimilarityFunction.COSINE, + new VectorConfig.ColumnVectorDefinition.VectorizeConfig( + "custom", "custom", null, null)))), null), new TestEmbeddingProvider(), "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java index 76c1d8154..7cab15ccb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperationTest.java @@ -25,6 +25,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.mappers.ThrowableToErrorMapper; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; @@ -82,7 +83,14 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java index 1ad31e4b2..d9f88307b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/InsertCollectionOperationTest.java @@ -20,6 +20,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; @@ -98,7 +99,14 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java index 570d48b86..a0b782b37 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/collections/ReadAndUpdateCollectionOperationTest.java @@ -23,6 +23,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateClause; import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateOperator; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.QueryExecutor; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.VectorConfig; import io.stargate.sgv2.jsonapi.service.cqldriver.serializer.CQLBindValues; @@ -117,7 +118,14 @@ public void init() { SCHEMA_OBJECT_NAME, null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, "testCommand", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTest.java index cdaaacd91..59254f6c4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTest.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -113,6 +114,12 @@ public void codecToCQLDatetime(DataType cqlType, Object fromValue, Object expect _codecToCQL(cqlType, fromValue, expectedCqlValue); } + @ParameterizedTest + @MethodSource("validCodecToCQLTestCasesUuid") + public void codecToCQLUuid(DataType cqlType, Object fromValue, Object expectedCqlValue) { + _codecToCQL(cqlType, fromValue, expectedCqlValue); + } + @ParameterizedTest @MethodSource("validCodecToCQLTestCasesOther") public void codecToCQLOther(DataType cqlType, Object fromValue, Object expectedCqlValue) { @@ -244,6 +251,27 @@ private static Stream validCodecToCQLTestCasesDatetime() { Instant.parse(TEST_DATA.TIMESTAMP_VALID_STR))); } + private static Stream validCodecToCQLTestCasesUuid() { + // Arguments: (CQL-type, from-caller, bound-by-driver-for-cql) + return Stream.of( + Arguments.of( + DataTypes.UUID, + TEST_DATA.UUID_VALID_STR_LC, + java.util.UUID.fromString(TEST_DATA.UUID_VALID_STR_LC)), + Arguments.of( + DataTypes.UUID, + TEST_DATA.UUID_VALID_STR_UC, + java.util.UUID.fromString(TEST_DATA.UUID_VALID_STR_UC)), + Arguments.of( + DataTypes.TIMEUUID, + TEST_DATA.UUID_VALID_STR_LC, + java.util.UUID.fromString(TEST_DATA.UUID_VALID_STR_LC)), + Arguments.of( + DataTypes.TIMEUUID, + TEST_DATA.UUID_VALID_STR_UC, + java.util.UUID.fromString(TEST_DATA.UUID_VALID_STR_UC))); + } + private static Stream validCodecToCQLTestCasesOther() { // Arguments: (CQL-type, from-caller, bound-by-driver-for-cql) return Stream.of( @@ -337,6 +365,12 @@ public void codecToJSONDatetime(DataType cqlType, Object fromValue, JsonNode exp _codecToJSON(cqlType, fromValue, expectedJsonValue); } + @ParameterizedTest + @MethodSource("validCodecToJSONTestCasesUuid") + public void codecToJSONUuid(DataType cqlType, Object fromValue, JsonNode expectedJsonValue) { + _codecToJSON(cqlType, fromValue, expectedJsonValue); + } + @ParameterizedTest @MethodSource("validCodecToJSONTestCasesOther") public void codecToJSONOther(DataType cqlType, Object fromValue, JsonNode expectedJsonValue) { @@ -458,7 +492,31 @@ private static Stream validCodecToJSONTestCasesDatetime() { JSONS.textNode(TEST_DATA.TIMESTAMP_VALID_STR))); } - private static Stream validCodecToJSONTestCasesOther() throws IOException { + private static Stream validCodecToJSONTestCasesUuid() { + // Arguments: (CQL-type, from-CQL-result-set, JsonNode-to-serialize) + return Stream.of( + // Short regular base64-encoded string + Arguments.of( + DataTypes.UUID, + UUID.fromString(TEST_DATA.UUID_VALID_STR_LC), + JSONS.textNode(TEST_DATA.UUID_VALID_STR_LC)), + Arguments.of( + DataTypes.UUID, + UUID.fromString(TEST_DATA.UUID_VALID_STR_UC), + // JSON codec accepts either casing but always writes lowercase UUIDs + JSONS.textNode(TEST_DATA.UUID_VALID_STR_UC.toLowerCase())), + Arguments.of( + DataTypes.TIMEUUID, + UUID.fromString(TEST_DATA.UUID_VALID_STR_LC), + JSONS.textNode(TEST_DATA.UUID_VALID_STR_LC)), + Arguments.of( + DataTypes.TIMEUUID, + UUID.fromString(TEST_DATA.UUID_VALID_STR_UC), + // JSON codec accepts either casing but always writes lowercase UUIDs + JSONS.textNode(TEST_DATA.UUID_VALID_STR_UC.toLowerCase()))); + } + + private static Stream validCodecToJSONTestCasesOther() { // Arguments: (CQL-type, from-CQL-result-set, JsonNode-to-serialize) return Stream.of( // Short regular base64-encoded string diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTestData.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTestData.java index cc214c916..7ee75be61 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTestData.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/filters/table/codecs/JSONCodecRegistryTestData.java @@ -39,6 +39,9 @@ public class JSONCodecRegistryTestData { public final BigDecimal NOT_EXACT_AS_INTEGER = new BigDecimal("1.25"); + public final String UUID_VALID_STR_LC = "123e4567-e89b-12d3-a456-426614174000"; + public final String UUID_VALID_STR_UC = "A34FACED-F158-4FDB-AA32-C4128D25A20F"; + // From https://en.wikipedia.org/wiki/Base64 -- 10-to-16 character sample case, with padding public final String BASE64_PADDED_DECODED_STR = "light work"; public final byte[] BASE64_PADDED_DECODED_BYTES = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java index e1a7de40d..503b7b863 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/CommandResolverWithVectorizerTest.java @@ -19,6 +19,7 @@ import io.stargate.sgv2.jsonapi.api.model.command.impl.UpdateOneCommand; import io.stargate.sgv2.jsonapi.api.request.DataApiRequestInfo; import io.stargate.sgv2.jsonapi.config.OperationsConfig; +import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.config.feature.ApiFeatures; import io.stargate.sgv2.jsonapi.exception.ErrorCodeV1; import io.stargate.sgv2.jsonapi.exception.JsonApiException; @@ -41,6 +42,7 @@ import io.stargate.sgv2.jsonapi.service.updater.DocumentUpdater; import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import jakarta.inject.Inject; +import java.util.List; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -83,7 +85,14 @@ class Resolve { new SchemaObjectName(KEYSPACE_NAME, COLLECTION_NAME), null, IdConfig.defaultIdConfig(), - new VectorConfig(true, -1, SimilarityFunction.COSINE, null), + new VectorConfig( + true, + List.of( + new VectorConfig.ColumnVectorDefinition( + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + -1, + SimilarityFunction.COSINE, + null))), null), null, null,