diff --git a/client/src/components/status/Types.ts b/client/src/components/status/Types.ts index 92780ba3..8e7ba313 100644 --- a/client/src/components/status/Types.ts +++ b/client/src/components/status/Types.ts @@ -8,6 +8,7 @@ export type StatusActionType = { action?: Function; onIndexCreate?: Function; showExtraAction?: boolean; + showLoadButton?: boolean; collection: CollectionObject; createIndexElement?: React.ReactNode; }; diff --git a/client/src/consts/Milvus.ts b/client/src/consts/Milvus.ts index 7f898f8c..a1a5e172 100644 --- a/client/src/consts/Milvus.ts +++ b/client/src/consts/Milvus.ts @@ -7,29 +7,34 @@ export const MILVUS_DATABASE = export const DYNAMIC_FIELD = `$meta`; export enum DataTypeEnum { + None = 0, Bool = 1, Int8 = 2, Int16 = 3, Int32 = 4, Int64 = 5, + Float = 10, Double = 11, - String = 20, - VarChar = 21, + + // String = 20, + VarChar = 21, // variable-length strings with a specified maximum length + Array = 22, JSON = 23, + BinaryVector = 100, FloatVector = 101, Float16Vector = 102, - SparseFloatVector = 104, BFloat16Vector = 103, - Array = 22, + SparseFloatVector = 104, + VarCharBM25 = 1000, } export const VectorTypes = [ - DataTypeEnum.BinaryVector, DataTypeEnum.FloatVector, - DataTypeEnum.BFloat16Vector, + DataTypeEnum.BinaryVector, DataTypeEnum.Float16Vector, + DataTypeEnum.BFloat16Vector, DataTypeEnum.SparseFloatVector, ]; @@ -64,6 +69,7 @@ export enum METRIC_TYPES_VALUES { TANIMOTO = 'TANIMOTO', SUBSTRUCTURE = 'SUBSTRUCTURE', SUPERSTRUCTURE = 'SUPERSTRUCTURE', + BM25 = 'BM25', } export const METRIC_TYPES = [ @@ -99,6 +105,10 @@ export const METRIC_TYPES = [ value: METRIC_TYPES_VALUES.TANIMOTO, label: 'TANIMOTO', }, + { + value: METRIC_TYPES_VALUES.BM25, + label: 'BM25', + }, ]; export type MetricType = @@ -392,6 +402,13 @@ export enum DataTypeStringEnum { Array = 'Array', None = 'None', } +export const VectorTypesString: DataTypeStringEnum[] = [ + DataTypeStringEnum.BinaryVector, + DataTypeStringEnum.FloatVector, + DataTypeStringEnum.BFloat16Vector, + DataTypeStringEnum.Float16Vector, + DataTypeStringEnum.SparseFloatVector, +]; export const NONE_INDEXABLE_DATA_TYPES = [DataTypeStringEnum.JSON]; @@ -454,3 +471,8 @@ export const databaseDefaults: Property[] = [ { key: 'database.max.collections', value: '', desc: '', type: 'number' }, { key: 'database.force.deny.writing', value: '', desc: '', type: 'boolean' }, ]; + +export enum FunctionType { + Unknown = 0, + BM25 = 1, +} diff --git a/client/src/i18n/cn/button.ts b/client/src/i18n/cn/button.ts index 5b76f10f..9b37ceac 100644 --- a/client/src/i18n/cn/button.ts +++ b/client/src/i18n/cn/button.ts @@ -25,7 +25,7 @@ const btnTrans = { importSampleData: '插入样本数据', loading: '加载中...', importing: '导入中...', - example: '生成随机向量', + example: '生成随机数据', rename: '重命名', duplicate: '复制', export: '导出', @@ -36,6 +36,8 @@ const btnTrans = { star: '给我一颗小星星', applyFilter: '应用过滤器', createIndex: '创建索引', + createVectorIndex: '创建向量索引', + createScalarIndex: '创建标量索引', edit: '编辑', explore: '探索', close: '关闭', diff --git a/client/src/i18n/cn/collection.ts b/client/src/i18n/cn/collection.ts index ba613187..57aebb25 100644 --- a/client/src/i18n/cn/collection.ts +++ b/client/src/i18n/cn/collection.ts @@ -22,6 +22,9 @@ const collectionTrans = { createdTime: '创建时间', maxLength: '最大长度', dynamicSchema: '动态schema', + function: 'Function', + functionInput: 'Function输入', + functionOutput: 'Function输出', // table tooltip aliasInfo: '别名可以在向量搜索中用作Collection名称。', @@ -34,7 +37,8 @@ const collectionTrans = { // create dialog createTitle: '创建Collection', - general: '一般信息', + idAndVectorFields: 'ID、向量或可用 BM25 算法处理的文本字段', + scalarFields: '标量字段', schema: 'schema', consistency: '一致性', consistencyLevel: '一致性级别', diff --git a/client/src/i18n/cn/dialog.ts b/client/src/i18n/cn/dialog.ts index ba6890e6..5642dcfe 100644 --- a/client/src/i18n/cn/dialog.ts +++ b/client/src/i18n/cn/dialog.ts @@ -14,7 +14,7 @@ const dialogTrans = { loadTitle: `加载 {{type}}`, editEntityTitle: `编辑 Entity`, modifyReplicaTitle: `修改 {{type}} 的副本`, - editAnalyzerTitle: `编辑 {{type}} 分析器`, + editAnalyzerTitle: `编辑分析器`, loadContent: `您正在尝试加载带有数据的 {{type}}。只有已加载的 {{type}} 可以被搜索。`, releaseContent: `您正在尝试发布带有数据的 {{type}}。请注意,数据将不再可用于搜索。`, diff --git a/client/src/i18n/cn/search.ts b/client/src/i18n/cn/search.ts index cf9740fe..55dbe7a5 100644 --- a/client/src/i18n/cn/search.ts +++ b/client/src/i18n/cn/search.ts @@ -2,7 +2,6 @@ const searchTrans = { firstTip: '2. 输入搜索向量 {{dimensionTip}}', secondTip: '1. 选择Collection和字段', thirdTip: '搜索参数 {{metricType}}', - vectorPlaceholder: '请在此输入您的向量值,例如 [1, 2, 3, 4]', collection: '已加载的Collection', noCollection: '没有已加载的Collection', field: '向量字段', @@ -29,6 +28,7 @@ const searchTrans = { consistency: '一致性', graphNodeHoverTip: '双击以查看更多', inputVectorPlaceHolder: '向量或实体ID', + textPlaceHolder: '请在此输入您的文本', partitionFilter: '分区过滤', loading: '加载中...', }; diff --git a/client/src/i18n/en/button.ts b/client/src/i18n/en/button.ts index 7baebf19..f69365e6 100644 --- a/client/src/i18n/en/button.ts +++ b/client/src/i18n/en/button.ts @@ -25,7 +25,7 @@ const btnTrans = { importSampleData: 'Insert Sample Data', loading: 'Loading...', importing: 'Importing...', - example: 'Generate Random Vector', + example: 'Generate Random Data', rename: 'Rename', duplicate: 'Duplicate', export: 'Export', @@ -36,6 +36,8 @@ const btnTrans = { star: 'Give me a Star', applyFilter: 'Apply Filters', createIndex: 'Create Index', + createVectorIndex: 'Vector Index', + createScalarIndex: 'Scalar Index', edit: 'Edit', explore: 'Explore', close: 'Close', diff --git a/client/src/i18n/en/collection.ts b/client/src/i18n/en/collection.ts index 60b6a2cf..6d4abcfa 100644 --- a/client/src/i18n/en/collection.ts +++ b/client/src/i18n/en/collection.ts @@ -22,6 +22,9 @@ const collectionTrans = { createdTime: 'Created Time', maxLength: 'Max Length', dynamicSchema: 'Dynamic Schema', + function: 'Function', + functionInput: 'Input', + functionOutput: 'Output', // table tooltip aliasInfo: 'Alias can be used as collection name in vector search.', @@ -35,7 +38,8 @@ const collectionTrans = { // create dialog createTitle: 'Create Collection', - general: 'General information', + idAndVectorFields: 'ID, Vector, or VarChar Fields for BM25 Processing', + scalarFields: 'Scalar Fields', schema: 'Schema', consistency: 'Consistency', consistencyLevel: 'Consistency Level', diff --git a/client/src/i18n/en/dialog.ts b/client/src/i18n/en/dialog.ts index 9c64a880..44497e32 100644 --- a/client/src/i18n/en/dialog.ts +++ b/client/src/i18n/en/dialog.ts @@ -11,7 +11,7 @@ const dialogTrans = { flush: `Flush data for {{type}}`, loadTitle: `Load {{type}}`, editEntityTitle: `Edit Entity(JSON)`, - editAnalyzerTitle: `Edit analyzer for {{type}}`, + editAnalyzerTitle: `Edit Analyzer`, modifyReplicaTitle: `Modify replica for {{type}}`, loadContent: `You are trying to load a {{type}} with data. Only loaded {{type}} can be searched.`, diff --git a/client/src/i18n/en/search.ts b/client/src/i18n/en/search.ts index 7cfbf058..694c4e21 100644 --- a/client/src/i18n/en/search.ts +++ b/client/src/i18n/en/search.ts @@ -2,7 +2,6 @@ const searchTrans = { firstTip: '2. Enter search vector {{dimensionTip}}', secondTip: '1. Choose collection and field', thirdTip: 'Search Parameters {{metricType}}', - vectorPlaceholder: 'Please input your vector value here, e.g. [1, 2, 3, 4]', collection: 'loaded collection', noCollection: 'No loaded collection', field: 'Vector field', @@ -29,6 +28,7 @@ const searchTrans = { consistency: 'Consistency', graphNodeHoverTip: 'Double click to explore more', inputVectorPlaceHolder: 'Vector or entity id', + textPlaceHolder: 'Please input your text here', partitionFilter: 'Partition Filter', loading: 'Loading...', }; diff --git a/client/src/pages/databases/Databases.tsx b/client/src/pages/databases/Databases.tsx index 0d0e9ee1..1c8ce40a 100644 --- a/client/src/pages/databases/Databases.tsx +++ b/client/src/pages/databases/Databases.tsx @@ -82,7 +82,7 @@ const useStyles = makeStyles((theme: Theme) => ({ // Databases page(tree and tabs) const Databases = () => { // context - const { database, collections, loading, fetchCollection, ui, setUIPref } = + const { database, collections, loading, ui, setUIPref } = useContext(dataContext); // UI state diff --git a/client/src/pages/databases/collections/StatusAction.tsx b/client/src/pages/databases/collections/StatusAction.tsx index 732c8bf3..2b9e32c0 100644 --- a/client/src/pages/databases/collections/StatusAction.tsx +++ b/client/src/pages/databases/collections/StatusAction.tsx @@ -45,6 +45,7 @@ const useStyles = makeStyles((theme: Theme) => ({ }, extraBtn: { height: 24, + padding: '0 8px', }, })); @@ -55,6 +56,7 @@ const StatusAction: FC = props => { collection, action = () => {}, showExtraAction, + showLoadButton, createIndexElement, } = props; @@ -134,6 +136,33 @@ const StatusAction: FC = props => { }); }; + if ( + collection.schema && + status === LOADING_STATE.UNLOADED && + collection.schema.hasVectorIndex && + showLoadButton + ) { + return ( + } + className={classes.extraBtn} + variant="contained" + tooltip={collectionTrans('clickToLoad')} + onClick={() => { + setDialog({ + open: true, + type: 'custom', + params: { + component: , + }, + }); + }} + > + {collectionTrans('loadTitle')} + + ); + } + return (
@@ -166,6 +195,7 @@ const StatusAction: FC = props => { {btnTrans('vectorSearch')} )} + {!collection.schema.hasVectorIndex && createIndexElement} )} diff --git a/client/src/pages/databases/collections/Types.ts b/client/src/pages/databases/collections/Types.ts index 523152ae..b82ea9d1 100644 --- a/client/src/pages/databases/collections/Types.ts +++ b/client/src/pages/databases/collections/Types.ts @@ -1,16 +1,26 @@ import { Dispatch, SetStateAction } from 'react'; -import { DataTypeEnum } from '@/consts'; +import { DataTypeEnum, FunctionType } from '@/consts'; export interface CollectionCreateProps { onCreate?: () => void; } +export type FunctionConfig = { + name: string; + description: string; + type: FunctionType; + input_field_names: string[]; + output_field_names: string[]; + params: Record; +}; + export interface CollectionCreateParam { collection_name: string; description: string; autoID: boolean; fields: CreateField[]; consistency_level: string; + functions: FunctionConfig[]; } export type AnalyzerType = 'standard' | 'english' | 'chinese'; diff --git a/client/src/pages/databases/collections/data/CollectionData.tsx b/client/src/pages/databases/collections/data/CollectionData.tsx index 9525043d..bf9e538b 100644 --- a/client/src/pages/databases/collections/data/CollectionData.tsx +++ b/client/src/pages/databases/collections/data/CollectionData.tsx @@ -152,7 +152,7 @@ const CollectionData = (props: CollectionDataProps) => { } = useQuery({ collection, consistencyLevel, - fields, + fields: fields.filter(f => !f.is_function_output), onQueryStart: (expr: string = '') => { setTableLoading(true); if (expr === '') { @@ -422,15 +422,17 @@ const CollectionData = (props: CollectionDataProps) => {
{ - return { - label: - f.name === DYNAMIC_FIELD - ? searchTrans('dynamicFields') - : f.name, - value: f.name, - }; - })} + options={fields + .filter(f => !f.is_function_output) + .map(f => { + return { + label: + f.name === DYNAMIC_FIELD + ? searchTrans('dynamicFields') + : f.name, + value: f.name, + }; + })} values={outputFields} renderValue={selected => ( {`${(selected as string[]).length} ${ diff --git a/client/src/pages/databases/collections/schema/CreateIndexDialog.tsx b/client/src/pages/databases/collections/schema/CreateIndexDialog.tsx index 9f91c6f8..06fa614d 100644 --- a/client/src/pages/databases/collections/schema/CreateIndexDialog.tsx +++ b/client/src/pages/databases/collections/schema/CreateIndexDialog.tsx @@ -1,8 +1,6 @@ import { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { CodeLanguageEnum, CodeViewData } from '@/components/code/Types'; import DialogTemplate from '@/components/customDialog/DialogTemplate'; -import CustomSwitch from '@/components/customSwitch/CustomSwitch'; import { Option } from '@/components/customSelector/Types'; import { INDEX_CONFIG, @@ -11,38 +9,26 @@ import { INDEX_TYPES_ENUM, DataTypeEnum, DataTypeStringEnum, - VectorTypes, + VectorTypesString, } from '@/consts'; import { useFormValidation } from '@/hooks'; -import { getCreateIndexJSCode } from '@/utils/code/Js'; -import { getCreateIndexPYCode } from '@/utils/code/Py'; -import { getCreateIndexJavaCode } from '@/utils/code/Java'; -import { formatForm, getMetricOptions, getScalarIndexOption } from '@/utils'; +import { + formatForm, + getMetricOptions, + getScalarIndexOption, + isVectorType, +} from '@/utils'; import CreateForm from './CreateForm'; import { IndexType, IndexExtraParam } from './Types'; +import { FieldObject } from '@server/types'; const CreateIndex = (props: { collectionName: string; - fieldType: DataTypeStringEnum; - elementType?: DataTypeStringEnum; - dataType: DataTypeEnum; + field: FieldObject; handleCreate: (params: IndexExtraParam, index_name: string) => void; handleCancel: () => void; - - // used for code mode - fieldName: string; - // used for sizing info - dimension: number; }) => { - const { - collectionName, - fieldType, - elementType, - handleCreate, - handleCancel, - fieldName, - dataType, - } = props; + const { collectionName, handleCreate, handleCancel, field } = props; const { t: indexTrans } = useTranslation('index'); const { t: dialogTrans } = useTranslation('dialog'); @@ -53,7 +39,7 @@ const CreateIndex = (props: { const defaultIndexType = INDEX_TYPES_ENUM.AUTOINDEX; const defaultMetricType = useMemo(() => { - switch (fieldType) { + switch (field.data_type) { case DataTypeStringEnum.BinaryVector: return METRIC_TYPES_VALUES.HAMMING; case DataTypeStringEnum.FloatVector: @@ -61,11 +47,13 @@ const CreateIndex = (props: { case DataTypeStringEnum.BFloat16Vector: return METRIC_TYPES_VALUES.COSINE; case DataTypeStringEnum.SparseFloatVector: - return METRIC_TYPES_VALUES.IP; + return field.is_function_output + ? METRIC_TYPES_VALUES.BM25 + : METRIC_TYPES_VALUES.IP; default: return ''; } - }, [fieldType]); + }, [field.data_type]); const [indexSetting, setIndexSetting] = useState<{ index_type: IndexType; @@ -92,9 +80,6 @@ const CreateIndex = (props: { cache_dataset_on_device: 'false', }); - // control whether show code mode - const [showCode, setShowCode] = useState(false); - const indexCreateParams = useMemo(() => { if (!INDEX_CONFIG[indexSetting.index_type]) { return []; @@ -103,10 +88,10 @@ const CreateIndex = (props: { }, [indexSetting.index_type]); const metricOptions = useMemo(() => { - return VectorTypes.includes(dataType) - ? getMetricOptions(indexSetting.index_type, dataType) + return isVectorType(field) + ? getMetricOptions(indexSetting.index_type, field) : []; - }, [indexSetting.index_type, fieldType]); + }, [indexSetting.index_type, field]); const extraParams = useMemo(() => { const params: { [x: string]: string } = {}; @@ -133,8 +118,8 @@ const CreateIndex = (props: { const autoOption = getOptions('AUTOINDEX', INDEX_OPTIONS_MAP['AUTOINDEX']); let options = []; - if (VectorTypes.includes(dataType)) { - switch (fieldType) { + if (isVectorType(field)) { + switch (field.data_type) { case DataTypeStringEnum.BinaryVector: options = [ ...getOptions( @@ -166,18 +151,15 @@ const CreateIndex = (props: { } } else { options = [ - ...getOptions( - indexTrans('scalar'), - getScalarIndexOption(fieldType, elementType) - ), + ...getOptions(indexTrans('scalar'), getScalarIndexOption(field)), ]; } return [...autoOption, ...options]; - }, [fieldType, dataType, fieldName]); + }, [field]); const checkedForm = useMemo(() => { - if (!VectorTypes.includes(dataType)) { + if (!isVectorType(field)) { return []; } const paramsForm: any = { metric_type: indexSetting.metric_type }; @@ -186,47 +168,7 @@ const CreateIndex = (props: { }); const form = formatForm(paramsForm); return form; - }, [indexSetting, indexCreateParams, fieldType]); - - /** - * create index code mode - */ - const codeBlockData: CodeViewData[] = useMemo(() => { - const isScalarField = !VectorTypes.includes(dataType); - const getCodeParams = { - collectionName, - fieldName, - extraParams, - isScalarField, - indexName: indexSetting.index_name, - metricType: indexSetting.metric_type, - indexType: indexSetting.index_type, - }; - return [ - { - label: commonTrans('py'), - language: CodeLanguageEnum.python, - code: getCreateIndexPYCode(getCodeParams), - }, - { - label: commonTrans('java'), - language: CodeLanguageEnum.java, - code: getCreateIndexJavaCode(getCodeParams), - }, - { - label: commonTrans('js'), - language: CodeLanguageEnum.javascript, - code: getCreateIndexJSCode(getCodeParams), - }, - ]; - }, [ - commonTrans, - extraParams, - collectionName, - fieldName, - indexSetting.index_name, - fieldType, - ]); + }, [indexSetting, indexCreateParams, field]); const { validation, @@ -262,24 +204,16 @@ const CreateIndex = (props: { await handleCreate(extraParams, indexSetting.index_name); }; - const handleShowCode = (event: React.ChangeEvent<{ checked: boolean }>) => { - const isChecked = event.target.checked; - setShowCode(isChecked); - }; - return ( } - showCode={showCode} - codeBlocksData={codeBlockData} > <> ({ wrapper: { @@ -31,10 +28,18 @@ const useStyles = makeStyles((theme: Theme) => ({ alignItems: 'center', whiteSpace: 'nowrap', color: theme.palette.primary.main, - height: 24, + height: 26, + fontSize: 13, + border: `1px solid transparent`, '&:hover': { cursor: 'pointer', }, + '&.outline': { + border: `1px dashed ${theme.palette.primary.main}`, + }, + '& svg': { + width: 15, + }, }, btnDisabled: { color: theme.palette.text.secondary, @@ -67,7 +72,7 @@ const IndexTypeElement: FC<{ disabled?: boolean; disabledTooltip?: string; cb?: (collectionName: string) => void; -}> = ({ field, collectionName, cb, disabled, disabledTooltip }) => { +}> = ({ field, collectionName, cb, disabled }) => { const { createIndex, dropIndex } = useContext(dataContext); const classes = useStyles(); @@ -75,7 +80,6 @@ const IndexTypeElement: FC<{ const { t: indexTrans } = useTranslation('index'); const { t: dialogTrans } = useTranslation('dialog'); const { t: successTrans } = useTranslation('success'); - const { t: collectionTrans } = useTranslation('collection'); const { t: btnTrans } = useTranslation('btn'); const { setDialog, handleCloseDialog, openSnackBar } = @@ -108,11 +112,7 @@ const IndexTypeElement: FC<{ component: ( @@ -192,14 +192,14 @@ const IndexTypeElement: FC<{ } if (!field.index) { + const isVector = isVectorType(field); return ( } - className={classes.btn} - tooltip={collectionTrans('clickToCreateVectorIndex')} + startIcon={} + className={`${classes.btn}${isVector ? ' outline' : ''}`} onClick={e => handleCreate(e)} > - {btnTrans('createIndex')} + {btnTrans(isVector ? 'createVectorIndex' : 'createScalarIndex')} ); } diff --git a/client/src/pages/databases/collections/schema/Schema.tsx b/client/src/pages/databases/collections/schema/Schema.tsx index 582b6042..01d1601f 100644 --- a/client/src/pages/databases/collections/schema/Schema.tsx +++ b/client/src/pages/databases/collections/schema/Schema.tsx @@ -61,7 +61,7 @@ const Overview = () => { className={classes.primaryKeyChip} title={collectionTrans('idFieldName')} > - +
) : null} {f.is_partition_key ? ( @@ -71,13 +71,6 @@ const Overview = () => { label="Partition key" /> ) : null} - {f.autoID ? ( - - ) : null} {findKeyValue(f.type_params, 'enable_match') ? ( { /> ) : null} + + {f.function ? ( + + { + const textToCopy = JSON.stringify(f.function); + navigator.clipboard + .writeText(textToCopy as string) + .then(() => { + alert('Copied to clipboard!'); + }) + .catch(err => { + alert('Failed to copy: ' + err); + }); + }} + /> + + ) : null}
); }, @@ -239,6 +258,11 @@ const Overview = () => { const enableModifyReplica = data && data.queryNodes && data.queryNodes.length > 1; + // if is autoID enabled + const isAutoIDEnabled = collection?.schema?.fields.some( + f => f.autoID === true + ); + // get loading state label return (
@@ -304,7 +328,8 @@ const Overview = () => { status={collection.status} percentage={collection.loadedPercentage} collection={collection} - showExtraAction={true} + showExtraAction={false} + showLoadButton={true} createIndexElement={CreateIndexElement} /> @@ -362,6 +387,13 @@ const Overview = () => { {collectionTrans('features')} + {isAutoIDEnabled ? ( + + ) : null} ({ }, primaryKeyChip: { fontSize: '8px', - position: 'relative', - top: '3px', - color: 'grey', }, chip: { fontSize: '12px', color: theme.palette.text.primary, border: 'none', cursor: 'normal', + marginRight: 4, + marginLeft: 4, }, featureChip: { - marginRight: 4, border: 'none', + marginLeft: 0, }, nameWrapper: { display: 'flex', diff --git a/client/src/pages/databases/collections/search/Search.tsx b/client/src/pages/databases/collections/search/Search.tsx index 2d746132..d9dd7243 100644 --- a/client/src/pages/databases/collections/search/Search.tsx +++ b/client/src/pages/databases/collections/search/Search.tsx @@ -18,7 +18,7 @@ import { getLabelDisplayedRows } from '@/pages/search/Utils'; import { useSearchResult, usePaginationHook } from '@/hooks'; import { getQueryStyles } from './Styles'; import SearchGlobalParams from './SearchGlobalParams'; -import VectorInputBox from './VectorInputBox'; +import VectorInputBox from './SearchInputBox'; import StatusIcon, { LoadingType } from '@/components/status/StatusIcon'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import CustomInput from '@/components/customInput/CustomInput'; @@ -138,7 +138,7 @@ const Search = (props: CollectionDataProps) => { }, [JSON.stringify(searchParams)]); // on vector input change, update the search params - const onVectorInputChange = useCallback( + const onSearchInputChange = useCallback( (anns_field: string, value: string) => { const s = cloneObj(searchParams) as SearchParamsType; const target = s.searchParams.find((sp: SearchSingleParams) => { @@ -442,7 +442,11 @@ const Search = (props: CollectionDataProps) => { s.data.length > 0 ? 'bold' : '' }`} > - {field.name} + {field.is_function_output + ? `${field.name}<=${ + field.function!.input_field_names[0] + }` + : field.name} {formatFieldType(field)} @@ -454,8 +458,9 @@ const Search = (props: CollectionDataProps) => { diff --git a/client/src/pages/databases/collections/search/VectorInputBox.tsx b/client/src/pages/databases/collections/search/SearchInputBox.tsx similarity index 56% rename from client/src/pages/databases/collections/search/VectorInputBox.tsx rename to client/src/pages/databases/collections/search/SearchInputBox.tsx index b2e0a64c..9e7cdb80 100644 --- a/client/src/pages/databases/collections/search/VectorInputBox.tsx +++ b/client/src/pages/databases/collections/search/SearchInputBox.tsx @@ -7,118 +7,31 @@ import { indentUnit } from '@codemirror/language'; import { minimalSetup } from 'codemirror'; import { javascript } from '@codemirror/lang-javascript'; import { linter, Diagnostic } from '@codemirror/lint'; -import { CollectionFullObject, FieldObject } from '@server/types'; +import { CollectionFullObject } from '@server/types'; import { CollectionService } from '@/http'; import { DataTypeStringEnum } from '@/consts'; import { SearchSingleParams } from '../../types'; -import { isSparseVector, transformObjStrToJSONStr } from '@/utils'; import { getQueryStyles } from './Styles'; import { useTheme } from '@mui/material'; import { githubLight } from '@ddietr/codemirror-themes/github-light'; import { githubDark } from '@ddietr/codemirror-themes/github-dark'; +import { Validator } from './utils'; -const floatVectorValidator = (text: string, field: FieldObject) => { - try { - const value = JSON.parse(text); - const dim = field.dimension; - if (!Array.isArray(value)) { - return { - valid: false, - message: `Not an array`, - }; - } - - if (Array.isArray(value) && value.length !== dim) { - return { - valid: false, - value: undefined, - message: `Dimension ${value.length} is not equal to ${dim} `, - }; - } - - return { valid: true, message: ``, value: value }; - } catch (e: any) { - return { - valid: false, - message: `Wrong Float Vector format, it should be an array of ${field.dimension} numbers`, - }; - } -}; - -const binaryVectorValidator = (text: string, field: FieldObject) => { - try { - const value = JSON.parse(text); - const dim = field.dimension; - if (!Array.isArray(value)) { - return { - valid: false, - message: `Not an array`, - }; - } - - if (Array.isArray(value) && value.length !== dim / 8) { - return { - valid: false, - value: undefined, - message: `Dimension ${value.length} is not equal to ${dim / 8} `, - }; - } - - return { valid: true, message: ``, value: value }; - } catch (e: any) { - return { - valid: false, - message: `Wrong Binary Vector format, it should be an array of ${ - field.dimension / 8 - } numbers`, - }; - } -}; - -const sparseVectorValidator = (text: string, field: FieldObject) => { - if (!isSparseVector(text)) { - return { - valid: false, - value: undefined, - message: `Incorrect Sparse Vector format, it should be like {1: 0.1, 3: 0.2}`, - }; - } - try { - JSON.parse(transformObjStrToJSONStr(text)); - return { - valid: true, - message: ``, - }; - } catch (e: any) { - return { - valid: false, - message: `Wrong Sparse Vector format`, - }; - } -}; - -const Validator = { - [DataTypeStringEnum.FloatVector]: floatVectorValidator, - [DataTypeStringEnum.BinaryVector]: binaryVectorValidator, - [DataTypeStringEnum.Float16Vector]: floatVectorValidator, - [DataTypeStringEnum.BFloat16Vector]: floatVectorValidator, - [DataTypeStringEnum.SparseFloatVector]: sparseVectorValidator, -}; - -export type VectorInputBoxProps = { +export type SearchInputBoxProps = { onChange: (anns_field: string, value: string) => void; searchParams: SearchSingleParams; collection: CollectionFullObject; + type?: 'vector' | 'text'; }; let queryTimeout: NodeJS.Timeout; -export default function VectorInputBox(props: VectorInputBoxProps) { +export default function SearchInputBox(props: SearchInputBoxProps) { const theme = useTheme(); const { t: searchTrans } = useTranslation('search'); // props - const { searchParams, onChange, collection } = props; + const { searchParams, onChange, collection, type } = props; const { field, data } = searchParams; // classes @@ -187,12 +100,64 @@ export default function VectorInputBox(props: VectorInputBoxProps) { // create editor useEffect(() => { if (!editor.current) { - const startState = EditorState.create({ - doc: data, - extensions: [ - minimalSetup, + // update outside data timeout handler + let updateTimeout: NodeJS.Timeout; + + let extensions = [ + minimalSetup, + placeholder( + searchTrans( + type === 'text' ? 'textPlaceHolder' : 'inputVectorPlaceHolder' + ) + ), + keymap.of([{ key: 'Tab', run: insertTab }]), // fix tab behaviour + indentUnit.of(' '), // fix tab indentation + EditorView.theme({ + '&.cm-editor': { + '&.cm-focused': { + outline: 'none', + }, + }, + '.cm-content': { + fontSize: '12px', + minHeight: '124px', + }, + '.cm-gutters': { + display: 'none', + }, + }), + EditorView.lineWrapping, + EditorView.updateListener.of(update => { + if (update.docChanged) { + if (queryTimeout || updateTimeout) { + clearTimeout(queryTimeout); + clearTimeout(updateTimeout); + } + + updateTimeout = setTimeout(() => { + // get text + const text = update.state.doc.toString(); + // validate text + const { valid } = validator(text, fieldRef.current); + // if valid, update search params + if (valid || text === '' || type === 'text') { + onChangeRef.current(searchParams.anns_field, text); + } else { + getVectorById(text); + } + }, 500); + } + if (update.focusChanged) { + editorEl.current?.classList.toggle('focused', update.view.hasFocus); + } + }), + ]; + + if (type === 'vector') { + extensions = [ + ...extensions, javascript(), - placeholder(searchTrans('inputVectorPlaceHolder')), + linter(view => { const text = view.state.doc.toString(); @@ -227,67 +192,37 @@ export default function VectorInputBox(props: VectorInputBoxProps) { return []; } }), - keymap.of([{ key: 'Tab', run: insertTab }]), // fix tab behaviour - indentUnit.of(' '), // fix tab indentation - EditorView.theme({ - '&.cm-editor': { - '&.cm-focused': { - outline: 'none', - }, - }, - '.cm-content': { - fontSize: '12px', - minHeight: '124px', - }, - '.cm-gutters': { - display: 'none', - }, - }), - EditorView.lineWrapping, - EditorView.updateListener.of(update => { - if (update.docChanged) { - if (queryTimeout) { - clearTimeout(queryTimeout); - } - const text = update.state.doc.toString(); + ]; + } - const { valid } = validator(text, fieldRef.current); - if (valid || text === '') { - onChangeRef.current(searchParams.anns_field, text); - } else { - getVectorById(text); - } - } - if (update.focusChanged) { - editorEl.current?.classList.toggle( - 'focused', - update.view.hasFocus - ); - } - }), - ], + // create editor + const startState = EditorState.create({ + doc: data, + extensions, }); + // create editor view const view = new EditorView({ state: startState, parent: editorEl.current!, }); + // set editor ref editor.current = view; - - // focus editor, the cursor will be at the end of the text - const endPos = editor.current.state.doc.length; - editor.current.dispatch({ - selection: { anchor: endPos }, - }); - - editor.current.focus(); - - return () => { - view.destroy(); - editor.current = undefined; - }; + } else { + if (editor.current.state.doc.toString() !== data) { + console.log('not equal'); + editor.current.dispatch({ + changes: { + from: 0, + to: editor.current.state.doc.length, + insert: data, + }, + }); + } } + + return () => {}; }, [JSON.stringify({ field, data })]); useEffect(() => { @@ -303,5 +238,5 @@ export default function VectorInputBox(props: VectorInputBoxProps) { } }, [theme.palette.mode]); - return
; + return
; } diff --git a/client/src/pages/databases/collections/search/Styles.ts b/client/src/pages/databases/collections/search/Styles.ts index dac9cd76..f2728d6e 100644 --- a/client/src/pages/databases/collections/search/Styles.ts +++ b/client/src/pages/databases/collections/search/Styles.ts @@ -18,7 +18,7 @@ export const getQueryStyles = makeStyles((theme: Theme) => ({ accordions: { display: 'flex', - width: '220px', + width: '230px', flexDirection: 'column', flexShrink: 0, padding: '0 8px 8px 0', @@ -107,12 +107,12 @@ export const getQueryStyles = makeStyles((theme: Theme) => ({ marginLeft: '4px', fontSize: '10px', fontWeight: 600, - color: theme.palette.primary.light, + color: theme.palette.secondary.main, }, }, }, - vectorInputBox: { + searchInputBox: { height: '124px', margin: '0 0 8px 0', overflow: 'auto', diff --git a/client/src/pages/databases/collections/search/utils.ts b/client/src/pages/databases/collections/search/utils.ts new file mode 100644 index 00000000..fd774cd4 --- /dev/null +++ b/client/src/pages/databases/collections/search/utils.ts @@ -0,0 +1,91 @@ +import { isSparseVector, transformObjStrToJSONStr } from '@/utils'; +import { FieldObject } from '@server/types'; +import { DataTypeStringEnum } from '@/consts'; + +const floatVectorValidator = (text: string, field: FieldObject) => { + try { + const value = JSON.parse(text); + const dim = field.dimension; + if (!Array.isArray(value)) { + return { + valid: false, + message: `Not an array`, + }; + } + + if (Array.isArray(value) && value.length !== dim) { + return { + valid: false, + value: undefined, + message: `Dimension ${value.length} is not equal to ${dim} `, + }; + } + + return { valid: true, message: ``, value: value }; + } catch (e: any) { + return { + valid: false, + message: `Wrong Float Vector format, it should be an array of ${field.dimension} numbers`, + }; + } +}; + +const binaryVectorValidator = (text: string, field: FieldObject) => { + try { + const value = JSON.parse(text); + const dim = field.dimension; + if (!Array.isArray(value)) { + return { + valid: false, + message: `Not an array`, + }; + } + + if (Array.isArray(value) && value.length !== dim / 8) { + return { + valid: false, + value: undefined, + message: `Dimension ${value.length} is not equal to ${dim / 8} `, + }; + } + + return { valid: true, message: ``, value: value }; + } catch (e: any) { + return { + valid: false, + message: `Wrong Binary Vector format, it should be an array of ${ + field.dimension / 8 + } numbers`, + }; + } +}; + +const sparseVectorValidator = (text: string, field: FieldObject) => { + if (!isSparseVector(text)) { + return { + valid: false, + value: undefined, + message: `Incorrect Sparse Vector format, it should be like {1: 0.1, 3: 0.2}`, + }; + } + try { + JSON.parse(transformObjStrToJSONStr(text)); + return { + valid: true, + message: ``, + }; + } catch (e: any) { + return { + valid: false, + message: `Wrong Sparse Vector format`, + }; + } +}; + +export const Validator = { + [DataTypeStringEnum.FloatVector]: floatVectorValidator, + [DataTypeStringEnum.BinaryVector]: binaryVectorValidator, + [DataTypeStringEnum.Float16Vector]: floatVectorValidator, + [DataTypeStringEnum.BFloat16Vector]: floatVectorValidator, + [DataTypeStringEnum.SparseFloatVector]: sparseVectorValidator, +}; diff --git a/client/src/pages/dialogs/CreateCollectionDialog.tsx b/client/src/pages/dialogs/CreateCollectionDialog.tsx index 05550c80..fe2d07cb 100644 --- a/client/src/pages/dialogs/CreateCollectionDialog.tsx +++ b/client/src/pages/dialogs/CreateCollectionDialog.tsx @@ -8,14 +8,19 @@ import { ITextfieldConfig } from '@/components/customInput/Types'; import { rootContext, dataContext } from '@/context'; import { useFormValidation } from '@/hooks'; import { formatForm, getAnalyzerParams, TypeEnum } from '@/utils'; -import { DataTypeEnum, ConsistencyLevelEnum, DEFAULT_ATTU_DIM } from '@/consts'; -import CreateFields from '../databases/collections/CreateFields'; +import { + DataTypeEnum, + ConsistencyLevelEnum, + DEFAULT_ATTU_DIM, + FunctionType, +} from '@/consts'; +import CreateFields from './create/CreateFields'; import { CollectionCreateParam, CollectionCreateProps, CreateField, } from '../databases/collections/Types'; -import { CONSISTENCY_LEVEL_OPTIONS } from '../databases/collections/Constants'; +import { CONSISTENCY_LEVEL_OPTIONS } from './create/Constants'; import { makeStyles } from '@mui/styles'; const useStyles = makeStyles((theme: Theme) => ({ @@ -29,7 +34,6 @@ const useStyles = makeStyles((theme: Theme) => ({ marginBottom: '0', }, '& legend': { - marginBottom: theme.spacing(1), lineHeight: '20px', fontSize: '14px', }, @@ -49,7 +53,7 @@ const useStyles = makeStyles((theme: Theme) => ({ marginTop: theme.spacing(2), }, dialog: { - minWidth: 820, + minWidth: 880, }, })); @@ -191,6 +195,8 @@ const CreateCollectionDialog: FC = ({ onCreate }) => { ]; const handleCreateCollection = async () => { + // function output fields + const fnOutputFields: CreateField[] = []; const param: CollectionCreateParam = { ...form, fields: fields.map(v => { @@ -203,6 +209,9 @@ const CreateCollectionDialog: FC = ({ onCreate }) => { data_type: v.data_type, }; + // remove unused id + delete data.id; + // if we need if (typeof v.dim !== undefined && !isNaN(Number(v.dim))) { data.dim = Number(v.dim); @@ -218,23 +227,71 @@ const CreateCollectionDialog: FC = ({ onCreate }) => { data.max_capacity = Number(v.max_capacity); } - if (v.analyzer_params && v.enable_analyzer) { + // handle BM25 row + if (data.data_type === DataTypeEnum.VarCharBM25) { + data.data_type = DataTypeEnum.VarChar; + data.enable_analyzer = true; + data.analyzer_params = data.analyzer_params || 'standard'; + // create sparse field + const sparseField = { + name: `${data.name}_embeddings`, + is_primary_key: false, + data_type: DataTypeEnum.SparseFloatVector, + description: `fn BM25(${data.name}) -> embeddings`, + is_function_output: true, + }; + // push sparse field to fields + fnOutputFields.push(sparseField); + } + + if (data.analyzer_params && data.enable_analyzer) { // if analyzer_params is string, we need to use default value - data.analyzer_params = getAnalyzerParams(v.analyzer_params); + data.analyzer_params = getAnalyzerParams(data.analyzer_params); + } else { + delete data.analyzer_params; + delete data.enable_analyzer; } - v.is_primary_key && (data.autoID = form.autoID); + data.is_primary_key && (data.autoID = form.autoID); // delete sparse vector dime if (data.data_type === DataTypeEnum.SparseFloatVector) { delete data.dim; } + // delete analyzer if not varchar + if ( + data.data_type !== DataTypeEnum.VarChar && + data.data_type === DataTypeEnum.Array && + data.element_type !== DataTypeEnum.VarChar + ) { + delete data.enable_analyzer; + delete data.analyzer_params; + delete data.max_length; + } return data; }), + functions: [], consistency_level: consistencyLevel, }; + // push sparse fields to param.fields + param.fields.push(...fnOutputFields); + + // build functions + fnOutputFields.forEach((field, index) => { + const [input] = (field.name as string).split('_'); + const functionParam = { + name: `BM25_${index}`, + description: `${input} BM25 function`, + type: FunctionType.BM25, + input_field_names: [input], + output_field_names: [field.name as string], + params: {}, + }; + param.functions.push(functionParam); + }); + // create collection await createCollection({ ...param, @@ -276,7 +333,7 @@ const CreateCollectionDialog: FC = ({ onCreate }) => {
- {collectionTrans('schema')} + {/* {collectionTrans('schema')} */} ({ - optionalWrapper: { + scalarFieldsWrapper: { width: '100%', paddingRight: theme.spacing(1), overflowY: 'auto', }, + title: { + '& button': { + position: 'relative', + top: '-1px', + marginLeft: 4, + }, + }, rowWrapper: { display: 'flex', flexWrap: 'nowrap', alignItems: 'center', gap: '8px', - flex: '1 0 auto', marginBottom: 4, '& .MuiFormLabel-root': { fontSize: 14, @@ -74,6 +84,10 @@ const useStyles = makeStyles((theme: Theme) => ({ width: '150px', marginTop: '-20px', }, + smallSelect: { + width: '105px', + marginTop: '-20px', + }, autoIdSelect: { width: '120px', marginTop: '-20px', @@ -162,7 +176,7 @@ const CreateFields: FC = ({ const AddIcon = icons.addOutline; const RemoveIcon = icons.remove; - const { requiredFields, optionalFields } = useMemo( + const { requiredFields, scalarFields } = useMemo( () => fields.reduce( (acc, field) => { @@ -170,10 +184,11 @@ const CreateFields: FC = ({ const requiredTypes: CreateFieldType[] = [ 'primaryKey', 'defaultVector', + 'vector', ]; const key = requiredTypes.includes(createType) ? 'requiredFields' - : 'optionalFields'; + : 'scalarFields'; acc[key].push({ ...field, @@ -184,7 +199,7 @@ const CreateFields: FC = ({ }, { requiredFields: [] as FieldType[], - optionalFields: [] as FieldType[], + scalarFields: [] as FieldType[], } ), @@ -192,17 +207,18 @@ const CreateFields: FC = ({ ); const getSelector = ( - type: 'all' | 'vector' | 'element' | 'primaryKey', + type: 'scalar' | 'vector' | 'element' | 'primaryKey', label: string, value: number, - onChange: (value: DataTypeEnum) => void + onChange: (value: DataTypeEnum) => void, + className: string = classes.select ) => { let _options = ALL_OPTIONS; switch (type) { case 'primaryKey': _options = PRIMARY_FIELDS_OPTIONS; break; - case 'all': + case 'scalar': _options = ALL_OPTIONS; break; case 'vector': @@ -213,6 +229,7 @@ const CreateFields: FC = ({ d => d.label !== 'Array' && d.label !== 'JSON' && + d.label !== 'VarChar(BM25)' && !d.label.includes('Vector') ); break; @@ -222,7 +239,7 @@ const CreateFields: FC = ({ return ( { @@ -557,13 +574,17 @@ const CreateFields: FC = ({ return (
{ changeFields(field.id!, { enable_analyzer: !field.enable_analyzer, }); }} + disabled={field.data_type === DataTypeEnum.VarCharBM25} /> = ({ onChange={e => { changeFields(field.id!, { analyzer_params: e.target.value }); }} - disabled={!field.enable_analyzer} + disabled={ + !field.enable_analyzer && + field.data_type !== DataTypeEnum.VarCharBM25 + } value={analyzer} variant="filled" label={collectionTrans('analyzer')} /> { setDialog2({ open: true, @@ -625,6 +652,7 @@ const CreateFields: FC = ({ // remove varchar params, if not varchar if ( updatedField.data_type !== DataTypeEnum.VarChar && + updatedField.data_type !== DataTypeEnum.VarCharBM25 && updatedField.element_type !== DataTypeEnum.VarChar ) { delete updatedField.max_length; @@ -647,15 +675,17 @@ const CreateFields: FC = ({ setFields(newFields); }; - const handleAddNewField = (index: number) => { + const handleAddNewField = (index: number, type = DataTypeEnum.Int16) => { const id = generateId(); const newDefaultItem: FieldType = { name: '', - data_type: DataTypeEnum.Int16, + data_type: type, is_primary_key: false, description: '', isDefault: false, dim: DEFAULT_ATTU_DIM, + max_length: DEFAULT_ATTU_VARCHAR_MAX_LENGTH, + enable_analyzer: type === DataTypeEnum.VarCharBM25, id, }; const newValidation = { @@ -740,7 +770,7 @@ const CreateFields: FC = ({ {generateDimension(field)} {generateDesc(field)} handleAddNewField(index)} + onClick={() => handleAddNewField(index, field.data_type)} classes={{ root: classes.iconBtn }} aria-label="add" size="large" @@ -751,7 +781,7 @@ const CreateFields: FC = ({ ); }; - const generateNonRequiredRow = ( + const generateScalarFieldRow = ( field: FieldType, index: number, fields: FieldType[] @@ -776,10 +806,12 @@ const CreateFields: FC = ({
{generateFieldName(field)} {getSelector( - 'all', + 'scalar', collectionTrans('fieldType'), field.data_type, - (value: DataTypeEnum) => changeFields(field.id!, { data_type: value }) + (value: DataTypeEnum) => + changeFields(field.id!, { data_type: value }), + classes.smallSelect )} {isArray @@ -788,7 +820,8 @@ const CreateFields: FC = ({ collectionTrans('elementType'), field.element_type || DEFAULT_ATTU_ELEMENT_TYPE, (value: DataTypeEnum) => - changeFields(field.id!, { element_type: value }) + changeFields(field.id!, { element_type: value }), + classes.smallSelect ) : null} @@ -814,7 +847,7 @@ const CreateFields: FC = ({ { - handleAddNewField(index); + handleAddNewField(index, field.data_type); }} classes={{ root: classes.iconBtn }} aria-label="add" @@ -837,23 +870,36 @@ const CreateFields: FC = ({ ); }; - const generateVectorRow = (field: FieldType, index: number) => { + const generateFunctionRow = ( + field: FieldType, + index: number, + fields: FieldType[], + requiredFields: FieldType[] + ) => { return (
{generateFieldName(field)} {getSelector( - 'all', + 'vector', collectionTrans('fieldType'), field.data_type, (value: DataTypeEnum) => changeFields(field.id!, { data_type: value }) )} - {generateDimension(field)} + {generateMaxLength(field)} + {generateDefaultValue(field)} {generateDesc(field)} +
+ {generateAnalyzerCheckBox(field, fields)} + {generateTextMatchCheckBox(field, fields)} + {generatePartitionKeyCheckbox(field, fields)} + {generateNullableCheckbox(field, fields)} +
+ { - handleAddNewField(index); + handleAddNewField(index, field.data_type); }} classes={{ root: classes.iconBtn }} aria-label="add" @@ -861,17 +907,59 @@ const CreateFields: FC = ({ > + + {requiredFields.length !== 2 && ( + { + const id = field.id || ''; + handleRemoveField(id); + }} + classes={{ root: classes.iconBtn }} + aria-label="delete" + size="large" + > + + + )} +
+ ); + }; + + const generateVectorRow = (field: FieldType, index: number) => { + return ( +
+ {generateFieldName(field)} + {getSelector( + 'vector', + `${collectionTrans('vectorType')} `, + field.data_type, + (value: DataTypeEnum) => changeFields(field.id!, { data_type: value }) + )} + + {generateDimension(field)} + {generateDesc(field)} + { - const id = field.id || ''; - handleRemoveField(id); - }} + onClick={() => handleAddNewField(index, field.data_type)} classes={{ root: classes.iconBtn }} - aria-label="delete" + aria-label="add" size="large" > - + + {requiredFields.length !== 2 && ( + { + const id = field.id || ''; + handleRemoveField(id); + }} + classes={{ root: classes.iconBtn }} + aria-label="delete" + size="large" + > + + + )}
); }; @@ -879,44 +967,63 @@ const CreateFields: FC = ({ const generateRequiredFieldRow = ( field: FieldType, autoID: boolean, - index: number + index: number, + fields: FieldType[], + requiredFields: FieldType[] ) => { // required type is primaryKey or defaultVector if (field.createType === 'primaryKey') { return generatePrimaryKeyRow(field, autoID); } - // use defaultVector as default return type - return generateDefaultVectorRow(field, index); - }; - const generateOptionalFieldRow = ( - field: FieldType, - index: number, - fields: FieldType[] - ) => { - // optional type is vector or number - if (field.createType === 'vector') { - return generateVectorRow(field, index); + if (field.data_type === DataTypeEnum.VarCharBM25) { + return generateFunctionRow(field, index, fields, requiredFields); + } + + if (field.createType === 'defaultVector') { + return generateDefaultVectorRow(field, index); } - // use number as default createType - return generateNonRequiredRow(field, index, fields); + // generate other vector rows + return generateVectorRow(field, index); }; return ( <> +

+ {`${collectionTrans('idAndVectorFields')}(${requiredFields.length})`} +

{requiredFields.map((field, index) => ( - {generateRequiredFieldRow(field, autoID, index)} + {generateRequiredFieldRow( + field, + autoID, + index, + fields, + requiredFields + )} ))} -
- {optionalFields.map((field, index) => ( +

+ {`${collectionTrans('scalarFields')}(${scalarFields.length})`} + { + handleAddNewField(requiredFields.length + 1); + }} + classes={{ root: classes.iconBtn }} + aria-label="add" + size="large" + > + + +

+
+ {scalarFields.map((field, index) => ( - {generateOptionalFieldRow( + {generateScalarFieldRow( field, index + requiredFields.length, - optionalFields + fields )} ))} diff --git a/client/src/styles/theme.ts b/client/src/styles/theme.ts index 16e2b89e..6efef8df 100644 --- a/client/src/styles/theme.ts +++ b/client/src/styles/theme.ts @@ -1,5 +1,6 @@ import { PaletteMode } from '@mui/material'; + const getCommonThemes = (mode: PaletteMode) => ({ typography: { fontFamily: [ diff --git a/client/src/utils/Form.ts b/client/src/utils/Form.ts index 4c4cd65f..f9769050 100644 --- a/client/src/utils/Form.ts +++ b/client/src/utils/Form.ts @@ -1,13 +1,13 @@ import { Option } from '@/components/customSelector/Types'; import { METRIC_TYPES_VALUES, - DataTypeEnum, SCALAR_INDEX_OPTIONS, DataTypeStringEnum, INDEX_TYPES_ENUM, } from '@/consts'; import { IForm } from '@/hooks'; import { IndexType } from '@/pages/databases/collections/schema/Types'; +import { FieldObject } from '@server/types'; interface IInfo { [key: string]: any; @@ -27,7 +27,7 @@ export const formatForm = (info: IInfo): IForm[] => { export const getMetricOptions = ( indexType: IndexType, - fieldType: DataTypeEnum + field: FieldObject ): Option[] => { const baseFloatOptions = [ { @@ -59,19 +59,26 @@ export const getMetricOptions = ( }, ]; - switch (fieldType) { - case DataTypeEnum.FloatVector: - case DataTypeEnum.Float16Vector: - case DataTypeEnum.BFloat16Vector: + switch (field.data_type) { + case DataTypeStringEnum.FloatVector: + case DataTypeStringEnum.Float16Vector: + case DataTypeStringEnum.BFloat16Vector: return baseFloatOptions; - case DataTypeEnum.SparseFloatVector: - return [ - { - value: METRIC_TYPES_VALUES.IP, - label: 'IP', - }, - ]; - case DataTypeEnum.BinaryVector: + case DataTypeStringEnum.SparseFloatVector: + return field.is_function_output + ? [ + { + value: METRIC_TYPES_VALUES.BM25, + label: 'BM25', + }, + ] + : [ + { + value: METRIC_TYPES_VALUES.IP, + label: 'IP', + }, + ]; + case DataTypeStringEnum.BinaryVector: switch (indexType) { case 'BIN_FLAT': return [ @@ -95,10 +102,7 @@ export const getMetricOptions = ( } }; -export const getScalarIndexOption = ( - fieldType: DataTypeStringEnum, - elementType?: DataTypeStringEnum -): Option[] => { +export const getScalarIndexOption = (field: FieldObject): Option[] => { // Helper function to check if a type is numeric const isNumericType = (type: DataTypeStringEnum): boolean => ['Int8', 'Int16', 'Int32', 'Int64', 'Float', 'Double'].includes(type); @@ -111,7 +115,7 @@ export const getScalarIndexOption = ( const options: Option[] = []; // Add options based on fieldType - if (fieldType === DataTypeStringEnum.VarChar) { + if (field.data_type === DataTypeStringEnum.VarChar) { options.push( SCALAR_INDEX_OPTIONS.find( opt => opt.value === INDEX_TYPES_ENUM.MARISA_TRIE @@ -119,21 +123,21 @@ export const getScalarIndexOption = ( ); } - if (isNumericType(fieldType)) { + if (isNumericType(field.data_type as DataTypeStringEnum)) { options.push( SCALAR_INDEX_OPTIONS.find(opt => opt.value === INDEX_TYPES_ENUM.SORT)! ); } if ( - fieldType === DataTypeStringEnum.Array && - elementType && - isBitmapSupportedType(elementType) + field.data_type === 'Array' && + field.data_type && + isBitmapSupportedType(field.element_type as DataTypeStringEnum) ) { options.push( SCALAR_INDEX_OPTIONS.find(opt => opt.value === INDEX_TYPES_ENUM.BITMAP)! ); - } else if (isBitmapSupportedType(fieldType)) { + } else if (isBitmapSupportedType(field.data_type as DataTypeStringEnum)) { options.push( SCALAR_INDEX_OPTIONS.find(opt => opt.value === INDEX_TYPES_ENUM.BITMAP)! ); diff --git a/client/src/utils/Format.ts b/client/src/utils/Format.ts index 468da47d..7454d7a2 100644 --- a/client/src/utils/Format.ts +++ b/client/src/utils/Format.ts @@ -5,6 +5,7 @@ import { VectorTypes, DataTypeStringEnum, DEFAULT_ANALYZER_PARAMS, + DataTypeEnum, } from '@/consts'; import { CreateFieldType, @@ -102,16 +103,23 @@ export const checkIsBinarySubstructure = (metricLabel: string): boolean => { return metricLabel === 'Superstructure' || metricLabel === 'Substructure'; }; -export const getCreateFieldType = (config: CreateField): CreateFieldType => { - if (config.is_primary_key) { +export const isVectorType = (field: FieldObject): boolean => { + return VectorTypes.includes(field.dataType as any); +} + +export const getCreateFieldType = (field: CreateField): CreateFieldType => { + if (field.is_primary_key) { return 'primaryKey'; } - if (config.isDefault) { + if (field.isDefault) { return 'defaultVector'; } - if (VectorTypes.includes(config.data_type)) { + if ( + VectorTypes.includes(field.data_type) || + field.data_type === DataTypeEnum.VarCharBM25 + ) { return 'vector'; } @@ -284,10 +292,12 @@ export const generateVectorsByField = (field: FieldObject) => { ? field.dimension / 8 : field.dimension; return JSON.stringify(generateVector(dim)); - case 'SparseFloatVector': - return transformObjToStr({ - [Math.floor(Math.random() * 10)]: Math.random(), - }); + case DataTypeStringEnum.SparseFloatVector: + return field.is_function_output + ? 'fox' + : transformObjToStr({ + [Math.floor(Math.random() * 10)]: Math.random(), + }); default: return [1, 2, 3]; } diff --git a/client/src/utils/search.ts b/client/src/utils/search.ts index 439b95e3..d97378b6 100644 --- a/client/src/utils/search.ts +++ b/client/src/utils/search.ts @@ -26,7 +26,9 @@ export const buildSearchParams = (searchParams: SearchParams) => { if (s.selected) { data.push({ anns_field: s.field.name, - data: formatter(s.data), + data: s.field.is_function_output + ? s.data.replace(/\n/g, '') + : formatter(s.data), params: s.params, }); weightedParams.push( diff --git a/server/src/collections/collections.service.ts b/server/src/collections/collections.service.ts index 0f464fba..9b1af6d7 100644 --- a/server/src/collections/collections.service.ts +++ b/server/src/collections/collections.service.ts @@ -111,6 +111,28 @@ export class CollectionsService { const vectorFields: FieldObject[] = []; const scalarFields: FieldObject[] = []; + const functionFields: FieldObject[] = []; + + // assign function to field + const fieldMap = new Map( + res.schema.fields.map(field => [field.name, field]) + ); + res.schema.functions.forEach(fn => { + const assignFunction = (fieldName: string) => { + const field = fieldMap.get(fieldName); + if (field) { + field.function = fn; + } + }; + + fn.output_field_names.forEach(assignFunction); + fn.input_field_names.forEach(assignFunction); + }); + + // get function input fields + const inputFieldNames = res.schema.functions.reduce((acc, cur) => { + return acc.concat(cur.input_field_names); + }, []); // append index info to each field res.schema.fields.forEach((field: FieldObject) => { @@ -119,18 +141,11 @@ export class CollectionsService { index => index.field_name === field.name ) as IndexObject; // add dimension - field.dimension = - Number(field.type_params.find(item => item.key === 'dim')?.value) || -1; + field.dimension = Number(field.dim) || -1; // add max capacity - field.maxCapacity = - Number( - field.type_params.find(item => item.key === 'max_capacity')?.value - ) || -1; + field.maxCapacity = Number(field.max_capacity) || -1; // add max length - field.maxLength = - Number( - field.type_params.find(item => item.key === 'max_length')?.value - ) || -1; + field.maxLength = Number(field.max_length) || -1; // classify fields if (VectorTypes.includes(field.data_type)) { @@ -142,6 +157,11 @@ export class CollectionsService { if (field.is_primary_key) { res.schema.primaryField = field; } + + // add functionFields if field name included in inputFieldNames + if (inputFieldNames.includes(field.name)) { + functionFields.push(field); + } }); // add extra data to schema @@ -151,6 +171,7 @@ export class CollectionsService { ); res.schema.scalarFields = scalarFields; res.schema.vectorFields = vectorFields; + res.schema.functionFields = functionFields; res.schema.dynamicFields = res.schema.enable_dynamic_field ? [ { diff --git a/server/src/types/collections.type.ts b/server/src/types/collections.type.ts index 90fef236..f7d3321b 100644 --- a/server/src/types/collections.type.ts +++ b/server/src/types/collections.type.ts @@ -8,9 +8,19 @@ import { DescribeCollectionResponse, QuerySegmentInfo, PersistentSegmentInfo, + FunctionType, } from '@zilliz/milvus2-sdk-node'; import { WS_EVENTS, WS_EVENTS_TYPE, LOADING_STATE } from '../utils'; +export type FunctionObject = { + name: string; + description?: string; + type: FunctionType; + input_field_names: string[]; + output_field_names?: string[]; + params: Record; +}; + export interface IndexObject extends IndexDescription { indexType: string; metricType: string; @@ -22,6 +32,7 @@ export interface FieldObject extends FieldSchema { dimension: number; maxCapacity: number; maxLength: number; + function?: FunctionObject; } export interface SchemaObject extends CollectionSchema { @@ -30,6 +41,7 @@ export interface SchemaObject extends CollectionSchema { vectorFields: FieldObject[]; scalarFields: FieldObject[]; dynamicFields: FieldObject[]; + functionFields: FieldObject[]; hasVectorIndex: boolean; enablePartitionKey: boolean; } diff --git a/server/src/utils/Helper.ts b/server/src/utils/Helper.ts index 4ebcc1fb..02c28682 100644 --- a/server/src/utils/Helper.ts +++ b/server/src/utils/Helper.ts @@ -142,7 +142,7 @@ export const genRow = ( ) => { const result: any = {}; fields.forEach(field => { - if (!field.autoID) { + if (!field.autoID && !field.is_function_output) { if ((field.nullable || field.default_value) && Math.random() < 0.5) { return; }