From 9721cb78432cbd34f05802daf9b7a7e9e2f03dbc Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Fri, 28 Jun 2024 15:00:35 +0100 Subject: [PATCH 01/31] Add support for async replication and tenant offloading schema definition --- package-lock.json | 851 ++++++++++++++++++++++-- package.json | 2 + src/collections/config/types/index.ts | 1 + src/collections/config/utils.ts | 1 + src/collections/configure/index.ts | 12 +- src/collections/configure/types/base.ts | 1 + src/collections/configure/unit.test.ts | 2 + src/collections/tenants/index.ts | 48 +- src/collections/tenants/unit.test.ts | 122 ++++ src/openapi/schema.ts | 9 +- src/proto/google/health/v1/health.ts | 8 +- src/proto/google/protobuf/struct.ts | 15 +- src/proto/v1/base.ts | 6 + src/proto/v1/batch.ts | 6 + src/proto/v1/batch_delete.ts | 18 +- src/proto/v1/properties.ts | 10 +- src/proto/v1/search_get.ts | 757 +++++++++++++++++++-- src/proto/v1/tenants.ts | 30 +- src/proto/v1/weaviate.ts | 8 +- 19 files changed, 1737 insertions(+), 170 deletions(-) create mode 100644 src/collections/tenants/unit.test.ts diff --git a/package-lock.json b/package-lock.json index 9b2f1153..2ff12f18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "graphql": "^16.8.1", "graphql-request": "^6.1.0", "long": "^5.2.3", - "nice-grpc": "^2.1.7", + "nice-grpc": "^2.1.9", "nice-grpc-client-middleware-deadline": "^2.0.12", "uuid": "^9.0.1" }, @@ -24,6 +24,7 @@ "@curveball/core": "0.20.0", "@curveball/kernel": "0.20.1", "@rollup/plugin-babel": "^5.3.1", + "@types/express": "^4.17.21", "@types/isomorphic-fetch": "^0.0.36", "@types/jest": "^29.4.0", "@types/node": "^18.14.0", @@ -34,6 +35,7 @@ "eslint": "^8.35.0", "eslint-config-prettier": "^8.7.0", "eslint-plugin-prettier": "^4.2.1", + "express": "^4.19.2", "graphql-request": "^6.1.0", "grpc-tools": "^1.12.4", "husky": "^8.0.3", @@ -835,26 +837,6 @@ "node": ">=14.4" } }, - "node_modules/@curveball/core/node_modules/ws": { - "version": "8.12.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/@curveball/http-errors": { "version": "0.4.1", "dev": true, @@ -875,26 +857,6 @@ "node": ">=14.4" } }, - "node_modules/@curveball/kernel/node_modules/ws": { - "version": "8.12.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.19.12", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz", @@ -2252,16 +2214,59 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, "node_modules/@types/color-name": { "version": "1.1.1", "dev": true, "license": "MIT" }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/estree": { "version": "0.0.39", "dev": true, "license": "MIT" }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "dev": true, + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.5", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.5.tgz", + "integrity": "sha512-y6W03tvrACO72aijJ5uF02FRq5cgDR9lUxddQ8vyF+GvmjJQqbzDcJngEjURc+ZsG31VI3hODNZJ2URj86pzmg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.6", "dev": true, @@ -2270,6 +2275,12 @@ "@types/node": "*" } }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true + }, "node_modules/@types/isomorphic-fetch": { "version": "0.0.36", "dev": true, @@ -2311,6 +2322,12 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true + }, "node_modules/@types/node": { "version": "18.14.0", "dev": true, @@ -2321,12 +2338,45 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/qs": { + "version": "6.9.15", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", + "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==", + "dev": true + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true + }, "node_modules/@types/semver": { "version": "7.5.8", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", "dev": true }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "node_modules/@types/stack-utils": { "version": "2.0.1", "dev": true, @@ -2819,6 +2869,12 @@ "sprintf-js": "~1.0.2" } }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -2937,6 +2993,45 @@ "node": ">=8" } }, + "node_modules/body-parser": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, "node_modules/brace-expansion": { "version": "1.1.11", "dev": true, @@ -2947,11 +3042,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, - "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -3040,6 +3136,25 @@ "node": ">=8" } }, + "node_modules/call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "dev": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "dev": true, @@ -3084,9 +3199,10 @@ } }, "node_modules/chalk": { - "version": "4.1.0", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, - "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -3338,6 +3454,47 @@ "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", "dev": true }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-disposition/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/convert-source-map": { "version": "1.7.0", "dev": true, @@ -3346,6 +3503,12 @@ "safe-buffer": "~5.1.1" } }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -3410,6 +3573,23 @@ "node": ">=0.10.0" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dev": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -3424,6 +3604,16 @@ "node": ">= 0.8" } }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/detect-libc": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", @@ -3509,6 +3699,12 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "dev": true }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, "node_modules/electron-to-chromium": { "version": "1.4.284", "dev": true, @@ -3530,6 +3726,15 @@ "dev": true, "license": "MIT" }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/encoding": { "version": "0.1.13", "dev": true, @@ -3561,6 +3766,27 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.19.12", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.19.12.tgz", @@ -3607,6 +3833,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, "node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -3955,6 +4187,15 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/execa": { "version": "5.1.1", "dev": true, @@ -3999,6 +4240,98 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/express": { + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", + "dev": true, + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.6.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/express/node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "dev": true + }, + "node_modules/express/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4080,9 +4413,10 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, - "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -4090,6 +4424,39 @@ "node": ">=8" } }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, "node_modules/find-up": { "version": "4.1.0", "dev": true, @@ -4121,6 +4488,24 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -4169,9 +4554,13 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", "dev": true, - "license": "MIT" + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/gauge": { "version": "3.0.2", @@ -4209,6 +4598,25 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dev": true, + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-package-type": { "version": "0.1.0", "dev": true, @@ -4299,6 +4707,18 @@ "integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==", "dev": true }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.10", "dev": true, @@ -4370,12 +4790,60 @@ "node": ">=8" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dev": true, + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", "dev": true }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "dev": true, @@ -4526,6 +4994,15 @@ "dev": true, "license": "ISC" }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "dev": true, @@ -4593,8 +5070,9 @@ }, "node_modules/is-number": { "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, - "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -5732,6 +6210,21 @@ "node": ">= 12" } }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", + "dev": true + }, "node_modules/merge-stream": { "version": "2.0.0", "dev": true, @@ -5746,6 +6239,15 @@ "node": ">= 8" } }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/micromatch": { "version": "4.0.5", "dev": true, @@ -6022,14 +6524,29 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", "dev": true, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "dev": true, @@ -6198,6 +6715,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "dev": true, @@ -6237,9 +6763,10 @@ } }, "node_modules/picocolors": { - "version": "1.0.0", - "dev": true, - "license": "ISC" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==", + "dev": true }, "node_modules/picomatch": { "version": "2.3.1", @@ -6447,6 +6974,19 @@ "node": ">=12.0.0" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -6456,6 +6996,21 @@ "node": ">=6" } }, + "node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -6476,10 +7031,20 @@ } ] }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/raw-body": { - "version": "2.5.1", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dev": true, - "license": "MIT", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -6705,12 +7270,101 @@ "semver": "bin/semver.js" } }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/send/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dev": true, + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", "dev": true }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dev": true, + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/setprototypeof": { "version": "1.2.0", "dev": true, @@ -6747,6 +7401,24 @@ "vscode-textmate": "^8.0.0" } }, + "node_modules/side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "dev": true, @@ -7136,8 +7808,9 @@ }, "node_modules/to-regex-range": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, - "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -7477,6 +8150,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typedoc": { "version": "0.25.13", "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.25.13.tgz", @@ -7545,9 +8231,9 @@ } }, "node_modules/undici": { - "version": "5.27.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.27.0.tgz", - "integrity": "sha512-l3ydWhlhOJzMVOYkymLykcRRXqbUaQriERtR70B9LzNkZ4bX52Fc8wbTDneMiwo8T+AemZXvXaTx+9o5ROxrXg==", + "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "dev": true, "dependencies": { "@fastify/busboy": "^2.0.0" @@ -7604,6 +8290,15 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -7636,6 +8331,15 @@ "node": ">=10.12.0" } }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/vscode-oniguruma": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", @@ -7738,6 +8442,27 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/y18n": { "version": "5.0.8", "dev": true, diff --git a/package.json b/package.json index 2e53bdaa..96b3ff39 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@curveball/core": "0.20.0", "@curveball/kernel": "0.20.1", "@rollup/plugin-babel": "^5.3.1", + "@types/express": "^4.17.21", "@types/isomorphic-fetch": "^0.0.36", "@types/jest": "^29.4.0", "@types/node": "^18.14.0", @@ -76,6 +77,7 @@ "eslint": "^8.35.0", "eslint-config-prettier": "^8.7.0", "eslint-plugin-prettier": "^4.2.1", + "express": "^4.19.2", "graphql-request": "^6.1.0", "grpc-tools": "^1.12.4", "husky": "^8.0.3", diff --git a/src/collections/config/types/index.ts b/src/collections/config/types/index.ts index 0dacdd3d..75dc04cf 100644 --- a/src/collections/config/types/index.ts +++ b/src/collections/config/types/index.ts @@ -41,6 +41,7 @@ export type MultiTenancyConfig = { }; export type ReplicationConfig = { + asyncEnabled: boolean; factor: number; }; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 142c6e81..c96d4e0c 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -263,6 +263,7 @@ class ConfigMapping { throw new WeaviateDeserializationError('Replication factor was not returned by Weaviate'); return { factor: v.factor, + asyncEnabled: v.asyncEnabled ? v.asyncEnabled : false, }; } static sharding(v?: WeaviateShardingConfig): ShardingConfig { diff --git a/src/collections/configure/index.ts b/src/collections/configure/index.ts index 2caa4d95..777a387b 100644 --- a/src/collections/configure/index.ts +++ b/src/collections/configure/index.ts @@ -129,10 +129,11 @@ const configure = { * * See [the docs](https://weaviate.io/developers/weaviate/concepts/replication-architecture#replication-vs-sharding) for more details. * - * @param {number} options.factor The replication factor. Default is 1. + * @param {boolean} [options.asyncEnabled] Whether asynchronous replication is enabled. Default is false. + * @param {number} [options.factor] The replication factor. Default is 1. */ - replication: (options: { factor?: number }): ReplicationConfigCreate => { - return { factor: options.factor }; + replication: (options: { asyncEnabled?: boolean; factor?: number }): ReplicationConfigCreate => { + return { asyncEnabled: options.asyncEnabled, factor: options.factor }; }, /** * Create a `ShardingConfigCreate` object to be used when defining the sharding configuration of your collection. @@ -214,10 +215,11 @@ const reconfigure = { * * See [the docs](https://weaviate.io/developers/weaviate/concepts/replication-architecture#replication-vs-sharding) for more details. * + * @param {boolean} [options.asyncEnabled] Whether asynchronous replication is enabled. * @param {number} [options.factor] The replication factor. */ - replication: (options: { factor?: number }): ReplicationConfigCreate => { - return { factor: options.factor }; + replication: (options: { asyncEnabled?: boolean; factor?: number }): ReplicationConfigCreate => { + return { asyncEnabled: options.asyncEnabled, factor: options.factor }; }, }; diff --git a/src/collections/configure/types/base.ts b/src/collections/configure/types/base.ts index 4b0e4b31..32b4d171 100644 --- a/src/collections/configure/types/base.ts +++ b/src/collections/configure/types/base.ts @@ -129,6 +129,7 @@ export type ReferenceConfigCreate = export type ReplicationConfigCreate = RecursivePartial; export type ReplicationConfigUpdate = { + asyncEnabled?: boolean; factor?: number; }; diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index 35c0d319..131360b0 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -75,9 +75,11 @@ describe('Unit testing of the configure factory class', () => { it('should create the correct ReplicationConfig type with all values', () => { const config = configure.replication({ + asyncEnabled: true, factor: 2, }); expect(config).toEqual({ + asyncEnabled: true, factor: 2, }); }); diff --git a/src/collections/tenants/index.ts b/src/collections/tenants/index.ts index 1e3994da..9bf4449e 100644 --- a/src/collections/tenants/index.ts +++ b/src/collections/tenants/index.ts @@ -5,6 +5,11 @@ import { TenantsCreator, TenantsDeleter, TenantsGetter, TenantsUpdater } from '. import { DbVersionSupport } from '../../utils/dbVersion.js'; export type Tenant = { + name: string; + activityStatus: 'COLD' | 'HOT' | 'FREEZING' | 'FROZEN' | 'UNFREEZING' | 'UNFROZEN'; +}; + +export type TenantInput = { name: string; activityStatus?: 'COLD' | 'HOT'; }; @@ -14,12 +19,20 @@ export type TenantsGetOptions = { }; class ActivityStatusMapper { - static from(status: TenantActivityStatus): 'COLD' | 'HOT' { + static from( + status: TenantActivityStatus + ): 'COLD' | 'HOT' | 'FROZEN' | 'FREEZING' | 'UNFREEZING' | 'UNFROZEN' { switch (status) { case TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD: return 'COLD'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_HOT: return 'HOT'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: + return 'FREEZING'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN: + return 'FROZEN'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING: + return 'UNFREEZING'; default: throw new Error(`Unsupported tenant activity status: ${status}`); } @@ -42,10 +55,11 @@ const checkSupportForGRPCTenantsGetEndpoint = async (dbVersionSupport: DbVersion if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message); }; -const parseTenantOrTenantArray = (tenants: Tenant | Tenant[]) => +const parseTenantOrTenantArray = (tenants: TenantInput | TenantInput[]) => Array.isArray(tenants) ? tenants : [tenants]; -const parseStringOrTenant = (tenant: string | Tenant) => (typeof tenant === 'string' ? tenant : tenant.name); +const parseStringOrTenant = (tenant: string | TenantInput) => + typeof tenant === 'string' ? tenant : tenant.name; const tenants = ( connection: ConnectionGRPC, @@ -67,24 +81,24 @@ const tenants = ( return result; }); return { - create: (tenants: Tenant | Tenant[]) => + create: (tenants: TenantInput | TenantInput[]) => new TenantsCreator(connection, collection, parseTenantOrTenantArray(tenants)).do() as Promise, get: async function () { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); return check.supports ? getGRPC() : getREST(); }, - getByNames: (tenants: (string | Tenant)[]) => getGRPC(tenants.map(parseStringOrTenant)), - getByName: (tenant: string | Tenant) => { + getByNames: (tenants: (string | TenantInput)[]) => getGRPC(tenants.map(parseStringOrTenant)), + getByName: (tenant: string | TenantInput) => { const tenantName = parseStringOrTenant(tenant); return getGRPC([tenantName]).then((tenants) => tenants[tenantName] || null); }, - remove: (tenants: Tenant | Tenant[]) => + remove: (tenants: TenantInput | TenantInput[]) => new TenantsDeleter( connection, collection, parseTenantOrTenantArray(tenants).map((t) => t.name) ).do(), - update: (tenants: Tenant | Tenant[]) => + update: (tenants: TenantInput | TenantInput[]) => new TenantsUpdater(connection, collection, parseTenantOrTenantArray(tenants)).do() as Promise, }; }; @@ -104,10 +118,10 @@ export interface Tenants { * * The collection must have been created with multi-tenancy enabled. * - * @param {Tenant | Tenant[]} tenants The tenant or tenants to create. + * @param {TenantInput | TenantInput[]} tenants The tenant or tenants to create. * @returns {Promise} The created tenant(s) as a list of Tenant. */ - create: (tenants: Tenant | Tenant[]) => Promise; + create: (tenants: TenantInput | TenantInput[]) => Promise; /** * Return all tenants currently associated with a collection in Weaviate. * @@ -121,19 +135,19 @@ export interface Tenants { * * The collection must have been created with multi-tenancy enabled. * - * @param {(string | Tenant)[]} names The tenants to retrieve. + * @param {(string | TenantInput)[]} names The tenants to retrieve. * @returns {Promise} The list of tenants. If the tenant does not exist, it will not be included in the list. */ - getByNames: (names: (string | Tenant)[]) => Promise>; + getByNames: (names: (string | TenantInput)[]) => Promise>; /** * Return the specified tenant from a collection in Weaviate. * * The collection must have been created with multi-tenancy enabled. * - * @param {string | Tenant} name The name of the tenant to retrieve. + * @param {string | TenantInput} name The name of the tenant to retrieve. * @returns {Promise} The tenant as a Tenant type, or null if the tenant does not exist. */ - getByName: (name: string | Tenant) => Promise; + getByName: (name: string | TenantInput) => Promise; /** * Remove the specified tenants from a collection in Weaviate. * @@ -142,14 +156,14 @@ export interface Tenants { * @param {Tenant | Tenant[]} tenants The tenant or tenants to remove. * @returns {Promise} An empty promise. */ - remove: (tenants: Tenant | Tenant[]) => Promise; + remove: (tenants: TenantInput | TenantInput[]) => Promise; /** * Update the specified tenants for a collection in Weaviate. * * The collection must have been created with multi-tenancy enabled. * - * @param {Tenant | Tenant[]} tenants The tenant or tenants to update. + * @param {TenantInput | TenantInput[]} tenants The tenant or tenants to update. * @returns {Promise} The updated tenant(s) as a list of Tenant. */ - update: (tenants: Tenant | Tenant[]) => Promise; + update: (tenants: TenantInput | TenantInput[]) => Promise; } diff --git a/src/collections/tenants/unit.test.ts b/src/collections/tenants/unit.test.ts new file mode 100644 index 00000000..4a278632 --- /dev/null +++ b/src/collections/tenants/unit.test.ts @@ -0,0 +1,122 @@ +import express from 'express'; +import { Server as HttpServer } from 'http'; + +import { createServer, Server as GrpcServer } from 'nice-grpc'; +import { + HealthCheckRequest, + HealthCheckResponse, + HealthCheckResponse_ServingStatus, + HealthDefinition, + HealthServiceImplementation, +} from '../../proto/google/health/v1/health'; +import { TenantActivityStatus, TenantsGetReply, TenantsGetRequest } from '../../proto/v1/tenants'; +import { WeaviateDefinition, WeaviateServiceImplementation } from '../../proto/v1/weaviate'; + +import weaviate, { Tenant } from '../../index'; + +const TENANTS_GET_COLLECTION_NAME = 'TestCollectionTenants'; + +const makeRestApp = (version: string) => { + const httpApp = express(); + httpApp.get(`/v1/schema/${TENANTS_GET_COLLECTION_NAME}/tenants`, (req, res) => + res.send([ + { name: 'hot', activityStatus: 'HOT' }, + { name: 'cold', activityStatus: 'COLD' }, + { name: 'frozen', activityStatus: 'FROZEN' }, + { name: 'freezing', activityStatus: 'FREEZING' }, + { name: 'unfreezing', activityStatus: 'UNFREEZING' }, + ]) + ); + httpApp.get('/v1/meta', (req, res) => res.send({ version })); + return httpApp; +}; + +const makeGrpcApp = () => { + const weaviateMockImpl: WeaviateServiceImplementation = { + tenantsGet: (request: TenantsGetRequest): Promise => + Promise.resolve({ + took: 0.1, + tenants: [ + { name: 'hot', activityStatus: TenantActivityStatus.TENANT_ACTIVITY_STATUS_HOT }, + { name: 'cold', activityStatus: TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD }, + { name: 'frozen', activityStatus: TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN }, + { name: 'freezing', activityStatus: TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING }, + { name: 'unfreezing', activityStatus: TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING }, + ], + }), + search: jest.fn(), + batchDelete: jest.fn(), + batchObjects: jest.fn(), + }; + const healthMockImpl: HealthServiceImplementation = { + check: (request: HealthCheckRequest): Promise => + Promise.resolve(HealthCheckResponse.create({ status: HealthCheckResponse_ServingStatus.SERVING })), + watch: jest.fn(), + }; + + const grpcApp = createServer(); + grpcApp.add(WeaviateDefinition, weaviateMockImpl); + grpcApp.add(HealthDefinition, healthMockImpl); + + return grpcApp; +}; + +const makeMockServers = async (weaviateVersion: string, httpPort: number, grpcAddress: string) => { + const rest = makeRestApp(weaviateVersion); + const grpc = makeGrpcApp(); + const server = await rest.listen(httpPort); + await grpc.listen(grpcAddress); + return { rest: server, grpc }; +}; + +describe('Mock testing of tenants.get() method with a REST server', () => { + let servers: { + rest: HttpServer; + grpc: GrpcServer; + }; + + beforeAll(async () => { + servers = await makeMockServers('1.24.0', 8954, 'localhost:8955'); + }); + + it('should get mocked tenants', async () => { + const client = await weaviate.connectToLocal({ port: 8954, grpcPort: 8955 }); + const collection = client.collections.get(TENANTS_GET_COLLECTION_NAME); + const tenants = await collection.tenants.get(); + expect(tenants).toEqual>({ + hot: { name: 'hot', activityStatus: 'HOT' }, + cold: { name: 'cold', activityStatus: 'COLD' }, + frozen: { name: 'frozen', activityStatus: 'FROZEN' }, + freezing: { name: 'freezing', activityStatus: 'FREEZING' }, + unfreezing: { name: 'unfreezing', activityStatus: 'UNFREEZING' }, + }); + }); + + afterAll(() => Promise.all([servers.rest.close(), servers.grpc.shutdown()])); +}); + +describe('Mock testing of tenants.get() method with a gRPC server', () => { + let servers: { + rest: HttpServer; + grpc: GrpcServer; + }; + + beforeAll(async () => { + servers = await makeMockServers('1.25.0', 8956, 'localhost:8957'); + }); + + it('should get the mocked tenants', async () => { + const client = await weaviate.connectToLocal({ port: 8956, grpcPort: 8957 }); + const collection = client.collections.get(TENANTS_GET_COLLECTION_NAME); + const tenants = await collection.tenants.get(); + expect(tenants).toEqual>({ + hot: { name: 'hot', activityStatus: 'HOT' }, + cold: { name: 'cold', activityStatus: 'COLD' }, + frozen: { name: 'frozen', activityStatus: 'FROZEN' }, + freezing: { name: 'freezing', activityStatus: 'FREEZING' }, + unfreezing: { name: 'unfreezing', activityStatus: 'UNFREEZING' }, + }); + }); + + afterAll(() => Promise.all([servers.rest.close(), servers.grpc.shutdown()])); +}); diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index 8d574a3a..61a2a1e8 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -342,6 +342,8 @@ export interface definitions { ReplicationConfig: { /** @description Number of times a class is replicated */ factor?: number; + /** @description Enable asynchronous replication */ + asyncEnabled?: boolean; }; /** @description tuning parameters for the BM25 algorithm */ BM25Config: { @@ -510,6 +512,8 @@ export interface definitions { indexFilterable?: boolean; /** @description Optional. Should this property be indexed in the inverted index. Defaults to true. Applicable only to properties of data type text and text[]. If you choose false, you will not be able to use this property in bm25 or hybrid search. This property has no affect on vectorization decisions done by modules */ indexSearchable?: boolean; + /** @description Optional. TODO roaring-set-range */ + indexRangeable?: boolean; /** * @description Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are `word` (default; splits on any non-alphanumerical, lowercases), `lowercase` (splits on white spaces, lowercases), `whitespace` (splits on white spaces), `field` (trims). Not supported for remaining data types * @enum {string} @@ -532,6 +536,7 @@ export interface definitions { name?: string; indexFilterable?: boolean; indexSearchable?: boolean; + indexRangeable?: boolean; /** @enum {string} */ tokenization?: 'word' | 'lowercase' | 'whitespace' | 'field'; nestedProperties?: definitions['NestedProperty'][]; @@ -1284,10 +1289,10 @@ export interface definitions { /** @description name of the tenant */ name?: string; /** - * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. Allowed values are `HOT` - tenant is fully active, `WARM` - tenant is active, some restrictions are imposed (TBD; not supported yet), `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, `FROZEN` - as COLD, but files are stored on cloud storage (not supported yet) + * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. Allowed values are `HOT` - tenant is fully active, `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, `FROZEN` - as COLD, but files are stored on cloud storage * @enum {string} */ - activityStatus?: 'HOT' | 'WARM' | 'COLD' | 'FROZEN'; + activityStatus?: 'HOT' | 'COLD' | 'FROZEN'; }; } diff --git a/src/proto/google/health/v1/health.ts b/src/proto/google/health/v1/health.ts index dfd141e1..f9144464 100644 --- a/src/proto/google/health/v1/health.ts +++ b/src/proto/google/health/v1/health.ts @@ -1,5 +1,11 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: health.proto + /* eslint-disable */ -import type { CallContext, CallOptions } from "nice-grpc-common"; +import { type CallContext, type CallOptions } from "nice-grpc-common"; import _m0 from "protobufjs/minimal.js"; export const protobufPackage = "grpc.health.v1"; diff --git a/src/proto/google/protobuf/struct.ts b/src/proto/google/protobuf/struct.ts index ef64ac1b..bd64f230 100644 --- a/src/proto/google/protobuf/struct.ts +++ b/src/proto/google/protobuf/struct.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: google/protobuf/struct.proto + /* eslint-disable */ import _m0 from "protobufjs/minimal.js"; @@ -184,10 +190,11 @@ export const Struct = { wrap(object: { [key: string]: any } | undefined): Struct { const struct = createBaseStruct(); + if (object !== undefined) { - Object.keys(object).forEach((key) => { + for (const key of Object.keys(object)) { struct.fields[key] = object[key]; - }); + } } return struct; }, @@ -195,9 +202,9 @@ export const Struct = { unwrap(message: Struct): { [key: string]: any } { const object: { [key: string]: any } = {}; if (message.fields) { - Object.keys(message.fields).forEach((key) => { + for (const key of Object.keys(message.fields)) { object[key] = message.fields[key]; - }); + } } return object; }, diff --git a/src/proto/v1/base.ts b/src/proto/v1/base.ts index e27a47e9..697a265c 100644 --- a/src/proto/v1/base.ts +++ b/src/proto/v1/base.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/base.proto + /* eslint-disable */ import Long from "long"; import _m0 from "protobufjs/minimal.js"; diff --git a/src/proto/v1/batch.ts b/src/proto/v1/batch.ts index 6bfc738a..d4ca290e 100644 --- a/src/proto/v1/batch.ts +++ b/src/proto/v1/batch.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/batch.proto + /* eslint-disable */ import _m0 from "protobufjs/minimal.js"; import { Struct } from "../google/protobuf/struct.js"; diff --git a/src/proto/v1/batch_delete.ts b/src/proto/v1/batch_delete.ts index 62662c32..83c219ae 100644 --- a/src/proto/v1/batch_delete.ts +++ b/src/proto/v1/batch_delete.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/batch_delete.proto + /* eslint-disable */ import Long from "long"; import _m0 from "protobufjs/minimal.js"; @@ -48,10 +54,10 @@ export const BatchDeleteRequest = { if (message.filters !== undefined) { Filters.encode(message.filters, writer.uint32(18).fork()).ldelim(); } - if (message.verbose === true) { + if (message.verbose !== false) { writer.uint32(24).bool(message.verbose); } - if (message.dryRun === true) { + if (message.dryRun !== false) { writer.uint32(32).bool(message.dryRun); } if (message.consistencyLevel !== undefined) { @@ -140,10 +146,10 @@ export const BatchDeleteRequest = { if (message.filters !== undefined) { obj.filters = Filters.toJSON(message.filters); } - if (message.verbose === true) { + if (message.verbose !== false) { obj.verbose = message.verbose; } - if (message.dryRun === true) { + if (message.dryRun !== false) { obj.dryRun = message.dryRun; } if (message.consistencyLevel !== undefined) { @@ -302,7 +308,7 @@ export const BatchDeleteObject = { if (message.uuid.length !== 0) { writer.uint32(10).bytes(message.uuid); } - if (message.successful === true) { + if (message.successful !== false) { writer.uint32(16).bool(message.successful); } if (message.error !== undefined) { @@ -361,7 +367,7 @@ export const BatchDeleteObject = { if (message.uuid.length !== 0) { obj.uuid = base64FromBytes(message.uuid); } - if (message.successful === true) { + if (message.successful !== false) { obj.successful = message.successful; } if (message.error !== undefined) { diff --git a/src/proto/v1/properties.ts b/src/proto/v1/properties.ts index 3f21d26e..5b7dc122 100644 --- a/src/proto/v1/properties.ts +++ b/src/proto/v1/properties.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/properties.proto + /* eslint-disable */ import Long from "long"; import _m0 from "protobufjs/minimal.js"; @@ -1223,7 +1229,7 @@ export const PhoneNumber = { if (message.nationalFormatted !== "") { writer.uint32(50).string(message.nationalFormatted); } - if (message.valid === true) { + if (message.valid !== false) { writer.uint32(56).bool(message.valid); } return writer; @@ -1328,7 +1334,7 @@ export const PhoneNumber = { if (message.nationalFormatted !== "") { obj.nationalFormatted = message.nationalFormatted; } - if (message.valid === true) { + if (message.valid !== false) { obj.valid = message.valid; } return obj; diff --git a/src/proto/v1/search_get.ts b/src/proto/v1/search_get.ts index 24cbbc36..b20335b4 100644 --- a/src/proto/v1/search_get.ts +++ b/src/proto/v1/search_get.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/search_get.proto + /* eslint-disable */ import Long from "long"; import _m0 from "protobufjs/minimal.js"; @@ -19,6 +25,63 @@ import { Properties } from "./properties.js"; export const protobufPackage = "weaviate.v1"; +export enum CombinationMethod { + COMBINATION_METHOD_UNSPECIFIED = 0, + COMBINATION_METHOD_TYPE_SUM = 1, + COMBINATION_METHOD_TYPE_MIN = 2, + COMBINATION_METHOD_TYPE_AVERAGE = 3, + COMBINATION_METHOD_TYPE_RELATIVE_SCORE = 4, + COMBINATION_METHOD_TYPE_MANUAL = 5, + UNRECOGNIZED = -1, +} + +export function combinationMethodFromJSON(object: any): CombinationMethod { + switch (object) { + case 0: + case "COMBINATION_METHOD_UNSPECIFIED": + return CombinationMethod.COMBINATION_METHOD_UNSPECIFIED; + case 1: + case "COMBINATION_METHOD_TYPE_SUM": + return CombinationMethod.COMBINATION_METHOD_TYPE_SUM; + case 2: + case "COMBINATION_METHOD_TYPE_MIN": + return CombinationMethod.COMBINATION_METHOD_TYPE_MIN; + case 3: + case "COMBINATION_METHOD_TYPE_AVERAGE": + return CombinationMethod.COMBINATION_METHOD_TYPE_AVERAGE; + case 4: + case "COMBINATION_METHOD_TYPE_RELATIVE_SCORE": + return CombinationMethod.COMBINATION_METHOD_TYPE_RELATIVE_SCORE; + case 5: + case "COMBINATION_METHOD_TYPE_MANUAL": + return CombinationMethod.COMBINATION_METHOD_TYPE_MANUAL; + case -1: + case "UNRECOGNIZED": + default: + return CombinationMethod.UNRECOGNIZED; + } +} + +export function combinationMethodToJSON(object: CombinationMethod): string { + switch (object) { + case CombinationMethod.COMBINATION_METHOD_UNSPECIFIED: + return "COMBINATION_METHOD_UNSPECIFIED"; + case CombinationMethod.COMBINATION_METHOD_TYPE_SUM: + return "COMBINATION_METHOD_TYPE_SUM"; + case CombinationMethod.COMBINATION_METHOD_TYPE_MIN: + return "COMBINATION_METHOD_TYPE_MIN"; + case CombinationMethod.COMBINATION_METHOD_TYPE_AVERAGE: + return "COMBINATION_METHOD_TYPE_AVERAGE"; + case CombinationMethod.COMBINATION_METHOD_TYPE_RELATIVE_SCORE: + return "COMBINATION_METHOD_TYPE_RELATIVE_SCORE"; + case CombinationMethod.COMBINATION_METHOD_TYPE_MANUAL: + return "COMBINATION_METHOD_TYPE_MANUAL"; + case CombinationMethod.UNRECOGNIZED: + default: + return "UNRECOGNIZED"; + } +} + export interface SearchRequest { /** required */ collection: string; @@ -116,6 +179,17 @@ export interface ObjectPropertiesRequest { objectProperties: ObjectPropertiesRequest[]; } +export interface Targets { + targetVectors: string[]; + combination: CombinationMethod; + weights: { [key: string]: number }; +} + +export interface Targets_WeightsEntry { + key: string; + value: number; +} + export interface Hybrid { query: string; properties: string[]; @@ -128,13 +202,19 @@ export interface Hybrid { alpha: number; fusionType: Hybrid_FusionType; vectorBytes: Uint8Array; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; - /** target_vector in msg is ignored and should not be set for hybrid */ + /** targets in msg is ignored and should not be set for hybrid */ nearText: | NearTextSearch | undefined; /** same as above. Use the target vector in the hybrid message */ nearVector: NearVector | undefined; + targets: Targets | undefined; } export enum Hybrid_FusionType { @@ -182,8 +262,16 @@ export interface NearTextSearch { certainty?: number | undefined; distance?: number | undefined; moveTo?: NearTextSearch_Move | undefined; - moveAway?: NearTextSearch_Move | undefined; + moveAway?: + | NearTextSearch_Move + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface NearTextSearch_Move { @@ -195,43 +283,91 @@ export interface NearTextSearch_Move { export interface NearImageSearch { image: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface NearAudioSearch { audio: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface NearVideoSearch { video: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface NearDepthSearch { depth: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface NearThermalSearch { thermal: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface NearIMUSearch { imu: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface BM25 { @@ -256,14 +392,34 @@ export interface NearVector { certainty?: number | undefined; distance?: number | undefined; vectorBytes: Uint8Array; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; + vectorPerTarget: { [key: string]: Uint8Array }; +} + +export interface NearVector_VectorPerTargetEntry { + key: string; + value: Uint8Array; } export interface NearObject { id: string; certainty?: number | undefined; - distance?: number | undefined; + distance?: + | number + | undefined; + /** + * deprecated in 1.26 - use targets + * + * @deprecated + */ targetVectors: string[]; + targets: Targets | undefined; } export interface Rerank { @@ -470,10 +626,10 @@ export const SearchRequest = { if (message.rerank !== undefined) { Rerank.encode(message.rerank, writer.uint32(490).fork()).ldelim(); } - if (message.uses123Api === true) { + if (message.uses123Api !== false) { writer.uint32(800).bool(message.uses123Api); } - if (message.uses125Api === true) { + if (message.uses125Api !== false) { writer.uint32(808).bool(message.uses125Api); } return writer; @@ -793,10 +949,10 @@ export const SearchRequest = { if (message.rerank !== undefined) { obj.rerank = Rerank.toJSON(message.rerank); } - if (message.uses123Api === true) { + if (message.uses123Api !== false) { obj.uses123Api = message.uses123Api; } - if (message.uses125Api === true) { + if (message.uses125Api !== false) { obj.uses125Api = message.uses125Api; } return obj; @@ -967,7 +1123,7 @@ function createBaseSortBy(): SortBy { export const SortBy = { encode(message: SortBy, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - if (message.ascending === true) { + if (message.ascending !== false) { writer.uint32(8).bool(message.ascending); } for (const v of message.path) { @@ -1015,7 +1171,7 @@ export const SortBy = { toJSON(message: SortBy): unknown { const obj: any = {}; - if (message.ascending === true) { + if (message.ascending !== false) { obj.ascending = message.ascending; } if (message.path?.length) { @@ -1143,31 +1299,31 @@ function createBaseMetadataRequest(): MetadataRequest { export const MetadataRequest = { encode(message: MetadataRequest, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - if (message.uuid === true) { + if (message.uuid !== false) { writer.uint32(8).bool(message.uuid); } - if (message.vector === true) { + if (message.vector !== false) { writer.uint32(16).bool(message.vector); } - if (message.creationTimeUnix === true) { + if (message.creationTimeUnix !== false) { writer.uint32(24).bool(message.creationTimeUnix); } - if (message.lastUpdateTimeUnix === true) { + if (message.lastUpdateTimeUnix !== false) { writer.uint32(32).bool(message.lastUpdateTimeUnix); } - if (message.distance === true) { + if (message.distance !== false) { writer.uint32(40).bool(message.distance); } - if (message.certainty === true) { + if (message.certainty !== false) { writer.uint32(48).bool(message.certainty); } - if (message.score === true) { + if (message.score !== false) { writer.uint32(56).bool(message.score); } - if (message.explainScore === true) { + if (message.explainScore !== false) { writer.uint32(64).bool(message.explainScore); } - if (message.isConsistent === true) { + if (message.isConsistent !== false) { writer.uint32(72).bool(message.isConsistent); } for (const v of message.vectors) { @@ -1279,31 +1435,31 @@ export const MetadataRequest = { toJSON(message: MetadataRequest): unknown { const obj: any = {}; - if (message.uuid === true) { + if (message.uuid !== false) { obj.uuid = message.uuid; } - if (message.vector === true) { + if (message.vector !== false) { obj.vector = message.vector; } - if (message.creationTimeUnix === true) { + if (message.creationTimeUnix !== false) { obj.creationTimeUnix = message.creationTimeUnix; } - if (message.lastUpdateTimeUnix === true) { + if (message.lastUpdateTimeUnix !== false) { obj.lastUpdateTimeUnix = message.lastUpdateTimeUnix; } - if (message.distance === true) { + if (message.distance !== false) { obj.distance = message.distance; } - if (message.certainty === true) { + if (message.certainty !== false) { obj.certainty = message.certainty; } - if (message.score === true) { + if (message.score !== false) { obj.score = message.score; } - if (message.explainScore === true) { + if (message.explainScore !== false) { obj.explainScore = message.explainScore; } - if (message.isConsistent === true) { + if (message.isConsistent !== false) { obj.isConsistent = message.isConsistent; } if (message.vectors?.length) { @@ -1346,7 +1502,7 @@ export const PropertiesRequest = { for (const v of message.objectProperties) { ObjectPropertiesRequest.encode(v!, writer.uint32(26).fork()).ldelim(); } - if (message.returnAllNonrefProperties === true) { + if (message.returnAllNonrefProperties !== false) { writer.uint32(88).bool(message.returnAllNonrefProperties); } return writer; @@ -1424,7 +1580,7 @@ export const PropertiesRequest = { if (message.objectProperties?.length) { obj.objectProperties = message.objectProperties.map((e) => ObjectPropertiesRequest.toJSON(e)); } - if (message.returnAllNonrefProperties === true) { + if (message.returnAllNonrefProperties !== false) { obj.returnAllNonrefProperties = message.returnAllNonrefProperties; } return obj; @@ -1536,6 +1692,190 @@ export const ObjectPropertiesRequest = { }, }; +function createBaseTargets(): Targets { + return { targetVectors: [], combination: 0, weights: {} }; +} + +export const Targets = { + encode(message: Targets, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + for (const v of message.targetVectors) { + writer.uint32(10).string(v!); + } + if (message.combination !== 0) { + writer.uint32(16).int32(message.combination); + } + Object.entries(message.weights).forEach(([key, value]) => { + Targets_WeightsEntry.encode({ key: key as any, value }, writer.uint32(26).fork()).ldelim(); + }); + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): Targets { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseTargets(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.targetVectors.push(reader.string()); + continue; + case 2: + if (tag !== 16) { + break; + } + + message.combination = reader.int32() as any; + continue; + case 3: + if (tag !== 26) { + break; + } + + const entry3 = Targets_WeightsEntry.decode(reader, reader.uint32()); + if (entry3.value !== undefined) { + message.weights[entry3.key] = entry3.value; + } + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): Targets { + return { + targetVectors: globalThis.Array.isArray(object?.targetVectors) + ? object.targetVectors.map((e: any) => globalThis.String(e)) + : [], + combination: isSet(object.combination) ? combinationMethodFromJSON(object.combination) : 0, + weights: isObject(object.weights) + ? Object.entries(object.weights).reduce<{ [key: string]: number }>((acc, [key, value]) => { + acc[key] = Number(value); + return acc; + }, {}) + : {}, + }; + }, + + toJSON(message: Targets): unknown { + const obj: any = {}; + if (message.targetVectors?.length) { + obj.targetVectors = message.targetVectors; + } + if (message.combination !== 0) { + obj.combination = combinationMethodToJSON(message.combination); + } + if (message.weights) { + const entries = Object.entries(message.weights); + if (entries.length > 0) { + obj.weights = {}; + entries.forEach(([k, v]) => { + obj.weights[k] = v; + }); + } + } + return obj; + }, + + create(base?: DeepPartial): Targets { + return Targets.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): Targets { + const message = createBaseTargets(); + message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.combination = object.combination ?? 0; + message.weights = Object.entries(object.weights ?? {}).reduce<{ [key: string]: number }>((acc, [key, value]) => { + if (value !== undefined) { + acc[key] = globalThis.Number(value); + } + return acc; + }, {}); + return message; + }, +}; + +function createBaseTargets_WeightsEntry(): Targets_WeightsEntry { + return { key: "", value: 0 }; +} + +export const Targets_WeightsEntry = { + encode(message: Targets_WeightsEntry, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.key !== "") { + writer.uint32(10).string(message.key); + } + if (message.value !== 0) { + writer.uint32(21).float(message.value); + } + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): Targets_WeightsEntry { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseTargets_WeightsEntry(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.key = reader.string(); + continue; + case 2: + if (tag !== 21) { + break; + } + + message.value = reader.float(); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): Targets_WeightsEntry { + return { + key: isSet(object.key) ? globalThis.String(object.key) : "", + value: isSet(object.value) ? globalThis.Number(object.value) : 0, + }; + }, + + toJSON(message: Targets_WeightsEntry): unknown { + const obj: any = {}; + if (message.key !== "") { + obj.key = message.key; + } + if (message.value !== 0) { + obj.value = message.value; + } + return obj; + }, + + create(base?: DeepPartial): Targets_WeightsEntry { + return Targets_WeightsEntry.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): Targets_WeightsEntry { + const message = createBaseTargets_WeightsEntry(); + message.key = object.key ?? ""; + message.value = object.value ?? 0; + return message; + }, +}; + function createBaseHybrid(): Hybrid { return { query: "", @@ -1547,6 +1887,7 @@ function createBaseHybrid(): Hybrid { targetVectors: [], nearText: undefined, nearVector: undefined, + targets: undefined, }; } @@ -1581,6 +1922,9 @@ export const Hybrid = { if (message.nearVector !== undefined) { NearVector.encode(message.nearVector, writer.uint32(74).fork()).ldelim(); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(82).fork()).ldelim(); + } return writer; }, @@ -1664,6 +2008,13 @@ export const Hybrid = { message.nearVector = NearVector.decode(reader, reader.uint32()); continue; + case 10: + if (tag !== 82) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -1688,6 +2039,7 @@ export const Hybrid = { : [], nearText: isSet(object.nearText) ? NearTextSearch.fromJSON(object.nearText) : undefined, nearVector: isSet(object.nearVector) ? NearVector.fromJSON(object.nearVector) : undefined, + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -1720,6 +2072,9 @@ export const Hybrid = { if (message.nearVector !== undefined) { obj.nearVector = NearVector.toJSON(message.nearVector); } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -1741,6 +2096,9 @@ export const Hybrid = { message.nearVector = (object.nearVector !== undefined && object.nearVector !== null) ? NearVector.fromPartial(object.nearVector) : undefined; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; @@ -1753,6 +2111,7 @@ function createBaseNearTextSearch(): NearTextSearch { moveTo: undefined, moveAway: undefined, targetVectors: [], + targets: undefined, }; } @@ -1776,6 +2135,9 @@ export const NearTextSearch = { for (const v of message.targetVectors) { writer.uint32(50).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(58).fork()).ldelim(); + } return writer; }, @@ -1828,6 +2190,13 @@ export const NearTextSearch = { message.targetVectors.push(reader.string()); continue; + case 7: + if (tag !== 58) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -1847,6 +2216,7 @@ export const NearTextSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -1870,6 +2240,9 @@ export const NearTextSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -1888,6 +2261,9 @@ export const NearTextSearch = { ? NearTextSearch_Move.fromPartial(object.moveAway) : undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; @@ -1982,7 +2358,7 @@ export const NearTextSearch_Move = { }; function createBaseNearImageSearch(): NearImageSearch { - return { image: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { image: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearImageSearch = { @@ -1999,6 +2375,9 @@ export const NearImageSearch = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2037,6 +2416,13 @@ export const NearImageSearch = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2054,6 +2440,7 @@ export const NearImageSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -2071,6 +2458,9 @@ export const NearImageSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -2083,12 +2473,15 @@ export const NearImageSearch = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; function createBaseNearAudioSearch(): NearAudioSearch { - return { audio: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { audio: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearAudioSearch = { @@ -2105,6 +2498,9 @@ export const NearAudioSearch = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2143,6 +2539,13 @@ export const NearAudioSearch = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2160,6 +2563,7 @@ export const NearAudioSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -2177,6 +2581,9 @@ export const NearAudioSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -2189,12 +2596,15 @@ export const NearAudioSearch = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; function createBaseNearVideoSearch(): NearVideoSearch { - return { video: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { video: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearVideoSearch = { @@ -2211,6 +2621,9 @@ export const NearVideoSearch = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2249,6 +2662,13 @@ export const NearVideoSearch = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2266,6 +2686,7 @@ export const NearVideoSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -2283,6 +2704,9 @@ export const NearVideoSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -2295,12 +2719,15 @@ export const NearVideoSearch = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; function createBaseNearDepthSearch(): NearDepthSearch { - return { depth: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { depth: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearDepthSearch = { @@ -2317,6 +2744,9 @@ export const NearDepthSearch = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2355,6 +2785,13 @@ export const NearDepthSearch = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2372,6 +2809,7 @@ export const NearDepthSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -2389,6 +2827,9 @@ export const NearDepthSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -2401,12 +2842,15 @@ export const NearDepthSearch = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; function createBaseNearThermalSearch(): NearThermalSearch { - return { thermal: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { thermal: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearThermalSearch = { @@ -2423,6 +2867,9 @@ export const NearThermalSearch = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2461,6 +2908,13 @@ export const NearThermalSearch = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2478,6 +2932,7 @@ export const NearThermalSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -2495,6 +2950,9 @@ export const NearThermalSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -2507,12 +2965,15 @@ export const NearThermalSearch = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; function createBaseNearIMUSearch(): NearIMUSearch { - return { imu: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { imu: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearIMUSearch = { @@ -2529,6 +2990,9 @@ export const NearIMUSearch = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2567,6 +3031,13 @@ export const NearIMUSearch = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2584,6 +3055,7 @@ export const NearIMUSearch = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -2601,6 +3073,9 @@ export const NearIMUSearch = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -2613,6 +3088,9 @@ export const NearIMUSearch = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; @@ -2802,7 +3280,15 @@ export const RefPropertiesRequest = { }; function createBaseNearVector(): NearVector { - return { vector: [], certainty: undefined, distance: undefined, vectorBytes: new Uint8Array(0), targetVectors: [] }; + return { + vector: [], + certainty: undefined, + distance: undefined, + vectorBytes: new Uint8Array(0), + targetVectors: [], + targets: undefined, + vectorPerTarget: {}, + }; } export const NearVector = { @@ -2824,6 +3310,12 @@ export const NearVector = { for (const v of message.targetVectors) { writer.uint32(42).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(50).fork()).ldelim(); + } + Object.entries(message.vectorPerTarget).forEach(([key, value]) => { + NearVector_VectorPerTargetEntry.encode({ key: key as any, value }, writer.uint32(58).fork()).ldelim(); + }); return writer; }, @@ -2879,6 +3371,23 @@ export const NearVector = { message.targetVectors.push(reader.string()); continue; + case 6: + if (tag !== 50) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; + case 7: + if (tag !== 58) { + break; + } + + const entry7 = NearVector_VectorPerTargetEntry.decode(reader, reader.uint32()); + if (entry7.value !== undefined) { + message.vectorPerTarget[entry7.key] = entry7.value; + } + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -2897,6 +3406,13 @@ export const NearVector = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, + vectorPerTarget: isObject(object.vectorPerTarget) + ? Object.entries(object.vectorPerTarget).reduce<{ [key: string]: Uint8Array }>((acc, [key, value]) => { + acc[key] = bytesFromBase64(value as string); + return acc; + }, {}) + : {}, }; }, @@ -2917,6 +3433,18 @@ export const NearVector = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } + if (message.vectorPerTarget) { + const entries = Object.entries(message.vectorPerTarget); + if (entries.length > 0) { + obj.vectorPerTarget = {}; + entries.forEach(([k, v]) => { + obj.vectorPerTarget[k] = base64FromBytes(v); + }); + } + } return obj; }, @@ -2930,12 +3458,98 @@ export const NearVector = { message.distance = object.distance ?? undefined; message.vectorBytes = object.vectorBytes ?? new Uint8Array(0); message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; + message.vectorPerTarget = Object.entries(object.vectorPerTarget ?? {}).reduce<{ [key: string]: Uint8Array }>( + (acc, [key, value]) => { + if (value !== undefined) { + acc[key] = value; + } + return acc; + }, + {}, + ); + return message; + }, +}; + +function createBaseNearVector_VectorPerTargetEntry(): NearVector_VectorPerTargetEntry { + return { key: "", value: new Uint8Array(0) }; +} + +export const NearVector_VectorPerTargetEntry = { + encode(message: NearVector_VectorPerTargetEntry, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.key !== "") { + writer.uint32(10).string(message.key); + } + if (message.value.length !== 0) { + writer.uint32(18).bytes(message.value); + } + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): NearVector_VectorPerTargetEntry { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseNearVector_VectorPerTargetEntry(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.key = reader.string(); + continue; + case 2: + if (tag !== 18) { + break; + } + + message.value = reader.bytes(); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): NearVector_VectorPerTargetEntry { + return { + key: isSet(object.key) ? globalThis.String(object.key) : "", + value: isSet(object.value) ? bytesFromBase64(object.value) : new Uint8Array(0), + }; + }, + + toJSON(message: NearVector_VectorPerTargetEntry): unknown { + const obj: any = {}; + if (message.key !== "") { + obj.key = message.key; + } + if (message.value.length !== 0) { + obj.value = base64FromBytes(message.value); + } + return obj; + }, + + create(base?: DeepPartial): NearVector_VectorPerTargetEntry { + return NearVector_VectorPerTargetEntry.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): NearVector_VectorPerTargetEntry { + const message = createBaseNearVector_VectorPerTargetEntry(); + message.key = object.key ?? ""; + message.value = object.value ?? new Uint8Array(0); return message; }, }; function createBaseNearObject(): NearObject { - return { id: "", certainty: undefined, distance: undefined, targetVectors: [] }; + return { id: "", certainty: undefined, distance: undefined, targetVectors: [], targets: undefined }; } export const NearObject = { @@ -2952,6 +3566,9 @@ export const NearObject = { for (const v of message.targetVectors) { writer.uint32(34).string(v!); } + if (message.targets !== undefined) { + Targets.encode(message.targets, writer.uint32(42).fork()).ldelim(); + } return writer; }, @@ -2990,6 +3607,13 @@ export const NearObject = { message.targetVectors.push(reader.string()); continue; + case 5: + if (tag !== 42) { + break; + } + + message.targets = Targets.decode(reader, reader.uint32()); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -3007,6 +3631,7 @@ export const NearObject = { targetVectors: globalThis.Array.isArray(object?.targetVectors) ? object.targetVectors.map((e: any) => globalThis.String(e)) : [], + targets: isSet(object.targets) ? Targets.fromJSON(object.targets) : undefined, }; }, @@ -3024,6 +3649,9 @@ export const NearObject = { if (message.targetVectors?.length) { obj.targetVectors = message.targetVectors; } + if (message.targets !== undefined) { + obj.targets = Targets.toJSON(message.targets); + } return obj; }, @@ -3036,6 +3664,9 @@ export const NearObject = { message.certainty = object.certainty ?? undefined; message.distance = object.distance ?? undefined; message.targetVectors = object.targetVectors?.map((e) => e) || []; + message.targets = (object.targets !== undefined && object.targets !== null) + ? Targets.fromPartial(object.targets) + : undefined; return message; }, }; @@ -3620,37 +4251,37 @@ export const MetadataResult = { if (message.creationTimeUnix !== 0) { writer.uint32(24).int64(message.creationTimeUnix); } - if (message.creationTimeUnixPresent === true) { + if (message.creationTimeUnixPresent !== false) { writer.uint32(32).bool(message.creationTimeUnixPresent); } if (message.lastUpdateTimeUnix !== 0) { writer.uint32(40).int64(message.lastUpdateTimeUnix); } - if (message.lastUpdateTimeUnixPresent === true) { + if (message.lastUpdateTimeUnixPresent !== false) { writer.uint32(48).bool(message.lastUpdateTimeUnixPresent); } if (message.distance !== 0) { writer.uint32(61).float(message.distance); } - if (message.distancePresent === true) { + if (message.distancePresent !== false) { writer.uint32(64).bool(message.distancePresent); } if (message.certainty !== 0) { writer.uint32(77).float(message.certainty); } - if (message.certaintyPresent === true) { + if (message.certaintyPresent !== false) { writer.uint32(80).bool(message.certaintyPresent); } if (message.score !== 0) { writer.uint32(93).float(message.score); } - if (message.scorePresent === true) { + if (message.scorePresent !== false) { writer.uint32(96).bool(message.scorePresent); } if (message.explainScore !== "") { writer.uint32(106).string(message.explainScore); } - if (message.explainScorePresent === true) { + if (message.explainScorePresent !== false) { writer.uint32(112).bool(message.explainScorePresent); } if (message.isConsistent !== undefined) { @@ -3659,10 +4290,10 @@ export const MetadataResult = { if (message.generative !== "") { writer.uint32(130).string(message.generative); } - if (message.generativePresent === true) { + if (message.generativePresent !== false) { writer.uint32(136).bool(message.generativePresent); } - if (message.isConsistentPresent === true) { + if (message.isConsistentPresent !== false) { writer.uint32(144).bool(message.isConsistentPresent); } if (message.vectorBytes.length !== 0) { @@ -3674,7 +4305,7 @@ export const MetadataResult = { if (message.rerankScore !== 0) { writer.uint32(169).double(message.rerankScore); } - if (message.rerankScorePresent === true) { + if (message.rerankScorePresent !== false) { writer.uint32(176).bool(message.rerankScorePresent); } for (const v of message.vectors) { @@ -3913,37 +4544,37 @@ export const MetadataResult = { if (message.creationTimeUnix !== 0) { obj.creationTimeUnix = Math.round(message.creationTimeUnix); } - if (message.creationTimeUnixPresent === true) { + if (message.creationTimeUnixPresent !== false) { obj.creationTimeUnixPresent = message.creationTimeUnixPresent; } if (message.lastUpdateTimeUnix !== 0) { obj.lastUpdateTimeUnix = Math.round(message.lastUpdateTimeUnix); } - if (message.lastUpdateTimeUnixPresent === true) { + if (message.lastUpdateTimeUnixPresent !== false) { obj.lastUpdateTimeUnixPresent = message.lastUpdateTimeUnixPresent; } if (message.distance !== 0) { obj.distance = message.distance; } - if (message.distancePresent === true) { + if (message.distancePresent !== false) { obj.distancePresent = message.distancePresent; } if (message.certainty !== 0) { obj.certainty = message.certainty; } - if (message.certaintyPresent === true) { + if (message.certaintyPresent !== false) { obj.certaintyPresent = message.certaintyPresent; } if (message.score !== 0) { obj.score = message.score; } - if (message.scorePresent === true) { + if (message.scorePresent !== false) { obj.scorePresent = message.scorePresent; } if (message.explainScore !== "") { obj.explainScore = message.explainScore; } - if (message.explainScorePresent === true) { + if (message.explainScorePresent !== false) { obj.explainScorePresent = message.explainScorePresent; } if (message.isConsistent !== undefined) { @@ -3952,10 +4583,10 @@ export const MetadataResult = { if (message.generative !== "") { obj.generative = message.generative; } - if (message.generativePresent === true) { + if (message.generativePresent !== false) { obj.generativePresent = message.generativePresent; } - if (message.isConsistentPresent === true) { + if (message.isConsistentPresent !== false) { obj.isConsistentPresent = message.isConsistentPresent; } if (message.vectorBytes.length !== 0) { @@ -3967,7 +4598,7 @@ export const MetadataResult = { if (message.rerankScore !== 0) { obj.rerankScore = message.rerankScore; } - if (message.rerankScorePresent === true) { + if (message.rerankScorePresent !== false) { obj.rerankScorePresent = message.rerankScorePresent; } if (message.vectors?.length) { @@ -4060,7 +4691,7 @@ export const PropertiesResult = { if (message.nonRefProps !== undefined) { Properties.encode(message.nonRefProps, writer.uint32(90).fork()).ldelim(); } - if (message.refPropsRequested === true) { + if (message.refPropsRequested !== false) { writer.uint32(96).bool(message.refPropsRequested); } return writer; @@ -4232,7 +4863,7 @@ export const PropertiesResult = { if (message.nonRefProps !== undefined) { obj.nonRefProps = Properties.toJSON(message.nonRefProps); } - if (message.refPropsRequested === true) { + if (message.refPropsRequested !== false) { obj.refPropsRequested = message.refPropsRequested; } return obj; diff --git a/src/proto/v1/tenants.ts b/src/proto/v1/tenants.ts index 4b292921..e35e570b 100644 --- a/src/proto/v1/tenants.ts +++ b/src/proto/v1/tenants.ts @@ -1,3 +1,9 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/tenants.proto + /* eslint-disable */ import _m0 from "protobufjs/minimal.js"; @@ -7,8 +13,10 @@ export enum TenantActivityStatus { TENANT_ACTIVITY_STATUS_UNSPECIFIED = 0, TENANT_ACTIVITY_STATUS_HOT = 1, TENANT_ACTIVITY_STATUS_COLD = 2, - TENANT_ACTIVITY_STATUS_WARM = 3, TENANT_ACTIVITY_STATUS_FROZEN = 4, + TENANT_ACTIVITY_STATUS_UNFREEZING = 5, + TENANT_ACTIVITY_STATUS_FREEZING = 6, + TENANT_ACTIVITY_STATUS_UNFROZEN = 7, UNRECOGNIZED = -1, } @@ -23,12 +31,18 @@ export function tenantActivityStatusFromJSON(object: any): TenantActivityStatus case 2: case "TENANT_ACTIVITY_STATUS_COLD": return TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD; - case 3: - case "TENANT_ACTIVITY_STATUS_WARM": - return TenantActivityStatus.TENANT_ACTIVITY_STATUS_WARM; case 4: case "TENANT_ACTIVITY_STATUS_FROZEN": return TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN; + case 5: + case "TENANT_ACTIVITY_STATUS_UNFREEZING": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING; + case 6: + case "TENANT_ACTIVITY_STATUS_FREEZING": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING; + case 7: + case "TENANT_ACTIVITY_STATUS_UNFROZEN": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFROZEN; case -1: case "UNRECOGNIZED": default: @@ -44,10 +58,14 @@ export function tenantActivityStatusToJSON(object: TenantActivityStatus): string return "TENANT_ACTIVITY_STATUS_HOT"; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD: return "TENANT_ACTIVITY_STATUS_COLD"; - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_WARM: - return "TENANT_ACTIVITY_STATUS_WARM"; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN: return "TENANT_ACTIVITY_STATUS_FROZEN"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING: + return "TENANT_ACTIVITY_STATUS_UNFREEZING"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: + return "TENANT_ACTIVITY_STATUS_FREEZING"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFROZEN: + return "TENANT_ACTIVITY_STATUS_UNFROZEN"; case TenantActivityStatus.UNRECOGNIZED: default: return "UNRECOGNIZED"; diff --git a/src/proto/v1/weaviate.ts b/src/proto/v1/weaviate.ts index aeb4a23c..aecfe694 100644 --- a/src/proto/v1/weaviate.ts +++ b/src/proto/v1/weaviate.ts @@ -1,5 +1,11 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v1.176.0 +// protoc v3.19.1 +// source: v1/weaviate.proto + /* eslint-disable */ -import type { CallContext, CallOptions } from "nice-grpc-common"; +import { type CallContext, type CallOptions } from "nice-grpc-common"; import { BatchObjectsReply, BatchObjectsRequest } from "./batch.js"; import { BatchDeleteReply, BatchDeleteRequest } from "./batch_delete.js"; import { SearchReply, SearchRequest } from "./search_get.js"; From 71ed3d2b815e62543b1d5202863fd5f392609b24 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Fri, 28 Jun 2024 16:56:58 +0100 Subject: [PATCH 02/31] Fix TenantInput typings and test without asyncEnabled --- src/collections/collection/index.ts | 28 +++++++++++----------------- src/collections/journey.test.ts | 1 + 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/collections/collection/index.ts b/src/collections/collection/index.ts index 6433ddef..00ab4329 100644 --- a/src/collections/collection/index.ts +++ b/src/collections/collection/index.ts @@ -13,7 +13,7 @@ import generate, { Generate } from '../generate/index.js'; import { Iterator } from '../iterator/index.js'; import query, { Query } from '../query/index.js'; import sort, { Sort } from '../sort/index.js'; -import tenants, { Tenant, Tenants } from '../tenants/index.js'; +import tenants, { TenantInput, Tenants } from '../tenants/index.js'; import { QueryMetadata, QueryProperty, QueryReference } from '../types/index.js'; export interface Collection { @@ -78,10 +78,10 @@ export interface Collection { * * This method does not send a request to Weaviate. It only returns a new collection object that is specific to the tenant you specify. * - * @param {string | Tenant} tenant The tenant name or tenant object to use. + * @param {string | TenantInput} tenant The tenant name or tenant object to use. * @returns {Collection} A new collection object specific to the tenant you specified. */ - withTenant: (tenant: string | Tenant) => Collection; + withTenant: (tenant: string | TenantInput) => Collection; } export type IteratorOptions = { @@ -101,26 +101,20 @@ const collection = ( name: N, dbVersionSupport: DbVersionSupport, consistencyLevel?: ConsistencyLevel, - tenant?: Tenant + tenant?: string ): Collection => { if (!isString(name)) { throw new WeaviateInvalidInputError(`The collection name must be a string, got: ${typeof name}`); } const capitalizedName = capitalizeCollectionName(name); - const queryCollection = query( - connection, - capitalizedName, - dbVersionSupport, - consistencyLevel, - tenant?.name - ); + const queryCollection = query(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant); return { - aggregate: aggregate(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant?.name), + aggregate: aggregate(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), backup: backupCollection(connection, capitalizedName), - config: config(connection, capitalizedName, dbVersionSupport, tenant?.name), - data: data(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant?.name), + config: config(connection, capitalizedName, dbVersionSupport, tenant), + data: data(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), filter: filter(), - generate: generate(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant?.name), + generate: generate(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), metrics: metrics(), name: name, query: queryCollection, @@ -142,13 +136,13 @@ const collection = ( ), withConsistency: (consistencyLevel: ConsistencyLevel) => collection(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), - withTenant: (tenant: string | Tenant) => + withTenant: (tenant: string | TenantInput) => collection( connection, capitalizedName, dbVersionSupport, consistencyLevel, - typeof tenant === 'string' ? { name: tenant } : tenant + typeof tenant === 'string' ? tenant : tenant.name ), }; }; diff --git a/src/collections/journey.test.ts b/src/collections/journey.test.ts index e76b41a9..dc379020 100644 --- a/src/collections/journey.test.ts +++ b/src/collections/journey.test.ts @@ -140,6 +140,7 @@ describe('Journey testing of the client using a WCD cluster', () => { ], references: [], replication: { + asyncEnabled: false, factor: 1, }, reranker: { From bd79545c5b3da57a1f5b7ce62fd9a961f5b05aef Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Fri, 28 Jun 2024 17:02:02 +0100 Subject: [PATCH 03/31] Update CI versions and number of checks --- .github/workflows/main.yaml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3b38b421..a1fdb89f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,23 +9,16 @@ on: env: WEAVIATE_124: 1.24.19 WEAVIATE_125: 1.25.5 + WEAVIATE_126: 1.26.0-rc.0 jobs: checks: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - node: [ - "18.x", - "20.x", - "22.x" - ] steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: ${{ matrix.node }} + node-version: '18.x' - name: "Run checks" run: | npm ci @@ -40,12 +33,11 @@ jobs: fail-fast: false matrix: versions: [ - { node: "18.x", weaviate: $WEAVIATE_124}, - { node: "20.x", weaviate: $WEAVIATE_124}, { node: "22.x", weaviate: $WEAVIATE_124}, - { node: "18.x", weaviate: $WEAVIATE_125}, - { node: "20.x", weaviate: $WEAVIATE_125}, - { node: "22.x", weaviate: $WEAVIATE_125} + { node: "22.x", weaviate: $WEAVIATE_125}, + { node: "18.x", weaviate: $WEAVIATE_126}, + { node: "20.x", weaviate: $WEAVIATE_126}, + { node: "22.x", weaviate: $WEAVIATE_126} ] steps: - uses: actions/checkout@v3 From 71a98ae3df8714bcb8458e908fd41cdba579f7c7 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Fri, 28 Jun 2024 17:39:49 +0100 Subject: [PATCH 04/31] Refactor v2 test for new indexRangeable prop config --- src/schema/journey.test.ts | 71 +++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index 21b0dfa8..f0f16bcc 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -11,8 +11,11 @@ const isVer = (client: WeaviateClient, minor: number, patch: number) => if (!version) { return false; } - const semver = version.split('.').map((v) => parseInt(v, 10)); - return semver[1] >= minor && semver[2] >= patch; + const semver = version + .split('-')[0] + .split('.') + .map((v) => parseInt(v, 10)); + return semver[1] > minor ? true : semver[1] === minor && semver[2] >= patch; }); describe('schema', () => { @@ -21,7 +24,12 @@ describe('schema', () => { host: 'localhost:8080', }); - const classObjPromise = newClassObject('MyThingClass', isVer(client, 25, 0), isVer(client, 25, 2)); + const classObjPromise = newClassObject( + 'MyThingClass', + isVer(client, 25, 0), + isVer(client, 25, 2), + isVer(client, 26, 0) + ); it('creates a thing class (implicitly)', async () => { const classObj = await classObjPromise; @@ -61,6 +69,7 @@ describe('schema', () => { name: 'anotherProp', tokenization: 'field', indexFilterable: true, + indexRangeable: false, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { @@ -166,7 +175,12 @@ describe('schema', () => { it('updates all shards in a class', async () => { const shardCount = 3; - const newClass: any = await newClassObject('NewClass', isVer(client, 25, 0), isVer(client, 25, 2)); + const newClass: any = await newClassObject( + 'NewClass', + isVer(client, 25, 0), + isVer(client, 25, 2), + isVer(client, 26, 0) + ); newClass.shardingConfig.desiredCount = shardCount; await client.schema @@ -209,7 +223,12 @@ describe('schema', () => { }); it('has updated values of bm25 config', async () => { - const newClass: any = await newClassObject('NewClass', isVer(client, 25, 0), isVer(client, 25, 2)); + const newClass: any = await newClassObject( + 'NewClass', + isVer(client, 25, 0), + isVer(client, 25, 2), + isVer(client, 26, 0) + ); const bm25Config = { k1: 1.13, b: 0.222 }; newClass.invertedIndexConfig.bm25 = bm25Config; @@ -226,7 +245,12 @@ describe('schema', () => { }); it('has updated values of stopwords config', async () => { - const newClass: any = await newClassObject('SpaceClass', isVer(client, 25, 0), isVer(client, 25, 2)); + const newClass: any = await newClassObject( + 'SpaceClass', + isVer(client, 25, 0), + isVer(client, 25, 2), + isVer(client, 26, 0) + ); const stopwordConfig: any = { preset: 'en', additions: ['star', 'nebula'], @@ -278,7 +302,12 @@ describe('schema', () => { it('creates a class with explicit replication config', async () => { const replicationFactor = 1; - const newClass: any = await newClassObject('SomeClass', isVer(client, 25, 0), isVer(client, 25, 2)); + const newClass: any = await newClassObject( + 'SomeClass', + isVer(client, 25, 0), + isVer(client, 25, 2), + isVer(client, 26, 0) + ); newClass.replicationConfig.factor = replicationFactor; await client.schema @@ -293,7 +322,12 @@ describe('schema', () => { }); it('creates a class with implicit replication config', async () => { - const newClass: any = await newClassObject('SomeClass', isVer(client, 25, 0), isVer(client, 25, 2)); + const newClass: any = await newClassObject( + 'SomeClass', + isVer(client, 25, 0), + isVer(client, 25, 2), + isVer(client, 26, 0) + ); delete newClass.replicationConfig; await client.schema @@ -311,12 +345,14 @@ describe('schema', () => { const newClass: any = await newClassObject( 'LetsDeleteThisClass', isVer(client, 25, 0), - isVer(client, 25, 2) + isVer(client, 25, 2), + isVer(client, 26, 0) ); const newClass2: any = await newClassObject( 'LetsDeleteThisClassToo', isVer(client, 25, 0), - isVer(client, 25, 2) + isVer(client, 25, 2), + isVer(client, 26, 0) ); const classNames = [newClass.class, newClass2.class]; Promise.all([ @@ -502,7 +538,7 @@ describe('property setting defaults and migrations', () => { ); const errMsg1 = - '`indexInverted` is deprecated and can not be set together with `indexFilterable` or `indexSearchable`'; + '`indexInverted` is deprecated and can not be set together with `indexFilterable`, `indexSearchable` or `indexRangeable`'; const errMsg2 = '`indexSearchable`'; test.each([ ['text', false, null, false, errMsg1], @@ -681,7 +717,8 @@ describe('multi tenancy', () => { const classObjWithoutMultiTenancyConfig = newClassObject( 'NoMultiTenancy', isVer(client, 25, 0), - isVer(client, 25, 2) + isVer(client, 25, 2), + isVer(client, 26, 0) ); it('creates a NoMultiTenancy class', async () => { @@ -711,7 +748,8 @@ describe('multi tenancy', () => { async function newClassObject( className: string, is1250Promise: Promise, - is1252Promise: Promise + is1252Promise: Promise, + is1260Promise: Promise ) { return { class: className, @@ -721,6 +759,7 @@ async function newClassObject( name: 'stringProp', tokenization: 'word', indexFilterable: true, + indexRangeable: false, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { @@ -754,6 +793,11 @@ async function newClassObject( bq: { enabled: false, }, + sq: { + enabled: false, + rescoreLimit: 20, + trainingLimit: 100000, + }, skip: false, efConstruction: 128, vectorCacheMaxObjects: 500000, @@ -792,6 +836,7 @@ async function newClassObject( virtualPerPhysical: 128, }, replicationConfig: { + asyncEnabled: (await is1260Promise) ? false : undefined, factor: 1, }, }; From 2a22e9246ca609fd2672378587c4060bb9690d9f Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Fri, 28 Jun 2024 17:54:43 +0100 Subject: [PATCH 05/31] Fix highly coupled tests with old Weaviate vers --- src/schema/journey.test.ts | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index f0f16bcc..59a82484 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -62,14 +62,14 @@ describe('schema', () => { return client.schema.exists('NonExistingClass').then((res) => expect(res).toEqual(false)); }); - it('extends the thing class with a new property', () => { + it('extends the thing class with a new property', async () => { const className = 'MyThingClass'; const prop: Property = { dataType: ['text'], name: 'anotherProp', tokenization: 'field', indexFilterable: true, - indexRangeable: false, + indexRangeable: (await isVer(client, 26, 0)) ? false : undefined, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { @@ -537,9 +537,12 @@ describe('property setting defaults and migrations', () => { } ); - const errMsg1 = - '`indexInverted` is deprecated and can not be set together with `indexFilterable`, `indexSearchable` or `indexRangeable`'; - const errMsg2 = '`indexSearchable`'; + const errMsg1 = isVer(client, 26, 0).then((yes) => + yes + ? '`indexInverted` is deprecated and can not be set together with `indexFilterable`, `indexSearchable` or `indexRangeable`' + : '`indexInverted` is deprecated and can not be set together with `indexFilterable` or `indexSearchable`' + ); + const errMsg2 = Promise.resolve('`indexSearchable`'); test.each([ ['text', false, null, false, errMsg1], ['text', false, null, true, errMsg1], @@ -585,7 +588,7 @@ describe('property setting defaults and migrations', () => { inverted: boolean | null, filterable: boolean | null, searchable: boolean | null, - errMsg: string + errMsg: Promise ) => { await client.schema .classCreator() @@ -602,8 +605,8 @@ describe('property setting defaults and migrations', () => { ], }) .do() - .catch((e: Error) => { - expect(e.message).toContain(errMsg); + .catch(async (e: Error) => { + expect(e.message).toContain(await errMsg); }); } ); @@ -759,7 +762,7 @@ async function newClassObject( name: 'stringProp', tokenization: 'word', indexFilterable: true, - indexRangeable: false, + indexRangeable: (await is1260Promise) ? false : undefined, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { @@ -793,11 +796,13 @@ async function newClassObject( bq: { enabled: false, }, - sq: { - enabled: false, - rescoreLimit: 20, - trainingLimit: 100000, - }, + sq: (await is1260Promise) + ? { + enabled: false, + rescoreLimit: 20, + trainingLimit: 100000, + } + : undefined, skip: false, efConstruction: 128, vectorCacheMaxObjects: 500000, From c83e6e70194858c48c1521308688cf0ce08537c1 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 3 Jul 2024 13:20:59 +0100 Subject: [PATCH 06/31] Update WEAVIATE_126 CI var to latest build --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a1fdb89f..ba4f90f4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.19 WEAVIATE_125: 1.25.5 - WEAVIATE_126: 1.26.0-rc.0 + WEAVIATE_126: preview--13e65f1 jobs: checks: From 4f13fd7dec37b5db8137ddbabb33a5ddf68758a6 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 3 Jul 2024 15:42:03 +0100 Subject: [PATCH 07/31] Add initial implementation --- src/collections/index.ts | 1 + src/collections/query/index.ts | 7 +- src/collections/query/types.ts | 17 +-- src/collections/query/utils.ts | 30 ++++++ src/collections/serialize/index.ts | 165 +++++++++++++++++++++++++---- src/collections/vectors/index.ts | 61 +++++++++++ 6 files changed, 253 insertions(+), 28 deletions(-) create mode 100644 src/collections/query/utils.ts create mode 100644 src/collections/vectors/index.ts diff --git a/src/collections/index.ts b/src/collections/index.ts index 4acab7ef..691dcbb8 100644 --- a/src/collections/index.ts +++ b/src/collections/index.ts @@ -306,3 +306,4 @@ export * from './references/index.js'; export * from './sort/index.js'; export * from './tenants/index.js'; export * from './types/index.js'; +export * from './vectors/index.js'; diff --git a/src/collections/query/index.ts b/src/collections/query/index.ts index 7a67aff0..c72a180e 100644 --- a/src/collections/query/index.ts +++ b/src/collections/query/index.ts @@ -27,6 +27,7 @@ import { NearMediaType, NearOptions, NearTextOptions, + NearVectorInputType, Query, QueryReturn, SearchOptions, @@ -255,9 +256,9 @@ class QueryManager implements Query { .then((reply) => this.parseGroupByReply(opts, reply)); } - public nearVector(vector: number[], opts?: BaseNearOptions): Promise>; - public nearVector(vector: number[], opts: GroupByNearOptions): Promise>; - public nearVector(vector: number[], opts?: NearOptions): QueryReturn { + public nearVector(vector: NearVectorInputType, opts?: BaseNearOptions): Promise>; + public nearVector(vector: NearVectorInputType, opts: GroupByNearOptions): Promise>; + public nearVector(vector: NearVectorInputType, opts?: NearOptions): QueryReturn { return this.checkSupportForNamedVectors(opts) .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) .then((search) => diff --git a/src/collections/query/types.ts b/src/collections/query/types.ts index 268ce107..e6bede9b 100644 --- a/src/collections/query/types.ts +++ b/src/collections/query/types.ts @@ -1,4 +1,5 @@ import { FilterValue } from '../filters/index.js'; +import { MultiTargetVectorJoin } from '../index.js'; import { Sorting } from '../sort/classes.js'; import { GroupByOptions, @@ -165,6 +166,10 @@ export type GroupByNearTextOptions = BaseNearTextOptions & { /** The type of the media to search for in the `query.nearMedia` method */ export type NearMediaType = 'audio' | 'depth' | 'image' | 'imu' | 'thermal' | 'video'; +export type NearVectorInputType = number[] | number[][] | Record; + +export type TargetVectorInputType = string | string[] | MultiTargetVectorJoin; + interface Bm25 { /** * Search for objects in this collection using the keyword-based BM25 algorithm. @@ -433,11 +438,11 @@ interface NearVector { * * This overload is for performing a search without the `groupBy` param. * - * @param {number[]} vector - The vector to search for. + * @param {NearVectorInputType} vector - The vector(s) to search on. * @param {BaseNearOptions} [opts] - The available options for the search excluding the `groupBy` param. * @returns {Promise>} - The result of the search within the fetched collection. */ - nearVector(vector: number[], opts?: BaseNearOptions): Promise>; + nearVector(vector: NearVectorInputType, opts?: BaseNearOptions): Promise>; /** * Search for objects by vector in this collection using a vector-based similarity search. * @@ -445,11 +450,11 @@ interface NearVector { * * This overload is for performing a search with the `groupBy` param. * - * @param {number[]} vector - The vector to search for. + * @param {NearVectorInputType} vector - The vector(s) to search for. * @param {GroupByNearOptions} opts - The available options for the search including the `groupBy` param. * @returns {Promise>} - The group by result of the search within the fetched collection. */ - nearVector(vector: number[], opts: GroupByNearOptions): Promise>; + nearVector(vector: NearVectorInputType, opts: GroupByNearOptions): Promise>; /** * Search for objects by vector in this collection using a vector-based similarity search. * @@ -457,11 +462,11 @@ interface NearVector { * * This overload is for performing a search with a programmatically defined `opts` param. * - * @param {number[]} vector - The vector to search for. + * @param {NearVectorInputType} vector - The vector(s) to search for. * @param {NearOptions} [opts] - The available options for the search. * @returns {QueryReturn} - The result of the search within the fetched collection. */ - nearVector(vector: number[], opts?: NearOptions): QueryReturn; + nearVector(vector: NearVectorInputType, opts?: NearOptions): QueryReturn; } /** All the available methods on the `.query` namespace. */ diff --git a/src/collections/query/utils.ts b/src/collections/query/utils.ts new file mode 100644 index 00000000..74629068 --- /dev/null +++ b/src/collections/query/utils.ts @@ -0,0 +1,30 @@ +import { MultiTargetVectorJoin } from '../index.js'; +import { NearVectorInputType, TargetVectorInputType } from './types.js'; + +export class NearVectorInputGuards { + public static is1DArray(input: NearVectorInputType): input is number[] { + return Array.isArray(input) && input.length > 0 && !Array.isArray(input[0]); + } + + public static is2DArray(input: NearVectorInputType): input is number[][] { + return Array.isArray(input) && input.length > 0 && Array.isArray(input[0]); + } + + public static isObject(input: NearVectorInputType): input is Record { + return !Array.isArray(input); + } +} + +export class TargetVectorInputGuards { + public static isSingle(input: TargetVectorInputType): input is string { + return typeof input === 'string'; + } + + public static isMulti(input: TargetVectorInputType): input is string[] { + return Array.isArray(input); + } + + public static isMultiJoin(input: TargetVectorInputType): input is MultiTargetVectorJoin { + return (input as MultiTargetVectorJoin).combination !== undefined; + } +} diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index 2dd40387..8056554d 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -8,6 +8,7 @@ import { } from '../../proto/v1/batch.js'; import { BM25, + CombinationMethod, GenerativeSearch, GroupBy, Hybrid, @@ -27,6 +28,7 @@ import { PropertiesRequest, Rerank, SortBy as SortByGrpc, + Targets, } from '../../proto/v1/search_get.js'; import { WeaviateInvalidInputError, WeaviateSerializationError } from '../../errors.js'; @@ -66,6 +68,7 @@ import { PrimitiveFilterValueType, PrimitiveListFilterValueType, } from '../filters/types.js'; +import { MultiTargetVectorJoin } from '../index.js'; import { BaseHybridOptions, BaseNearOptions, @@ -77,8 +80,11 @@ import { HybridOptions, NearOptions, NearTextOptions, + NearVectorInputType, SearchOptions, + TargetVectorInputType, } from '../query/types.js'; +import { NearVectorInputGuards, TargetVectorInputGuards } from '../query/utils.js'; import { ReferenceGuards } from '../references/classes.js'; import { Beacon } from '../references/index.js'; import { uuidToBeacon } from '../references/utils.js'; @@ -406,7 +412,7 @@ export class Serialize { return Hybrid_FusionType.FUSION_TYPE_UNSPECIFIED; } }; - + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), hybridSearch: Hybrid.fromPartial({ @@ -415,7 +421,8 @@ export class Serialize { properties: args.queryProperties, vectorBytes: Serialize.hybridVector(args.vector), fusionType: fusionType(args.fusionType), - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, nearText: Serialize.hybridNearText(args.vector), nearVector: Serialize.hybridNearVector(args.vector), }), @@ -424,65 +431,75 @@ export class Serialize { }; public static nearAudio = (args: { audio: string } & NearOptions): SearchNearAudioArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearAudio: NearAudioSearch.fromPartial({ audio: args.audio, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; }; public static nearDepth = (args: { depth: string } & NearOptions): SearchNearDepthArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearDepth: NearDepthSearch.fromPartial({ depth: args.depth, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; }; public static nearImage = (args: { image: string } & NearOptions): SearchNearImageArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearImage: NearImageSearch.fromPartial({ image: args.image, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; }; public static nearIMU = (args: { imu: string } & NearOptions): SearchNearIMUArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearIMU: NearIMUSearch.fromPartial({ imu: args.imu, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; }; public static nearObject = (args: { id: string } & NearOptions): SearchNearObjectArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearObject: NearObject.fromPartial({ id: args.id, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; @@ -496,11 +513,13 @@ export class Serialize { moveAway?: { concepts?: string[]; force?: number; objects?: string[] }; moveTo?: { concepts?: string[]; force?: number; objects?: string[] }; }) => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return NearTextSearch.fromPartial({ query: typeof args.query === 'string' ? [args.query] : args.query, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, moveAway: args.moveAway ? NearTextSearch_Move.fromPartial({ concepts: args.moveAway.concepts, @@ -529,13 +548,15 @@ export class Serialize { }; public static nearThermal = (args: { thermal: string } & NearOptions): SearchNearThermalArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearThermal: NearThermalSearch.fromPartial({ thermal: args.thermal, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; @@ -546,20 +567,124 @@ export class Serialize { }; private static nearVectorSearch = (args: { - vector: number[]; + vector: NearVectorInputType; certainty?: number; distance?: number; - targetVector?: string; + targetVector?: TargetVectorInputType; }) => { - return NearVector.fromPartial({ - vectorBytes: Serialize.vectorToBytes(args.vector), - certainty: args.certainty, - distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, - }); + const invalidNearVectorError = new WeaviateInvalidInputError(`near vector argument can be: + - an array of numbers + - an array of arrays of numbers for multi target search + - an object with target names as keys and arrays of numbers as values + received: ${args.vector}`); + + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + if (NearVectorInputGuards.isObject(args.vector)) { + if (targets === undefined || targets.targetVectors.length != Object.keys(args.vector).length) { + throw new WeaviateInvalidInputError( + 'The number of target vectors must be equal to the number of search vectors.' + ); + } + const vectorPerTarget: Record = {}; + Object.entries(args.vector).forEach(([k, v]) => { + vectorPerTarget[k] = Serialize.vectorToBytes(v); + }); + return NearVector.fromPartial({ + certainty: args.certainty, + distance: args.distance, + targetVectors, + targets, + vectorPerTarget: vectorPerTarget, + }); + } else { + if (args.vector.length === 0) { + throw invalidNearVectorError; + } + if (NearVectorInputGuards.is1DArray(args.vector)) { + const vectorBytes = Serialize.vectorToBytes(args.vector); + return NearVector.fromPartial({ + certainty: args.certainty, + distance: args.distance, + targetVectors, + targets, + vectorBytes, + }); + } + if (NearVectorInputGuards.is2DArray(args.vector)) { + if (targets === undefined || targets.targetVectors.length != args.vector.length) { + throw new WeaviateInvalidInputError( + 'The number of target vectors must be equal to the number of search vectors.' + ); + } + const vectorPerTarget: Record = {}; + args.vector.forEach((v, i) => { + vectorPerTarget[targets.targetVectors[i]] = Serialize.vectorToBytes(v); + }); + return NearVector.fromPartial({ + certainty: args.certainty, + distance: args.distance, + targetVectors, + targets, + vectorPerTarget, + }); + } + throw invalidNearVectorError; + } + }; + + private static targetVector = ( + targetVector?: TargetVectorInputType + ): { targets?: Targets; targetVectors?: string[] } => { + if (targetVector === undefined) { + return {}; + } else if (TargetVectorInputGuards.isSingle(targetVector)) { + return { + targets: Targets.fromPartial({ + targetVectors: [targetVector], + }), + }; + } else if (TargetVectorInputGuards.isMulti(targetVector)) { + return { + targets: Targets.fromPartial({ + targetVectors: targetVector, + }), + }; + } else { + return Serialize.targets(targetVector); + } + }; + + private static targets = (targets: MultiTargetVectorJoin) => { + let combination: CombinationMethod; + switch (targets.combination) { + case 'sum': + combination = CombinationMethod.COMBINATION_METHOD_TYPE_SUM; + break; + case 'average': + combination = CombinationMethod.COMBINATION_METHOD_TYPE_AVERAGE; + break; + case 'minimum': + combination = CombinationMethod.COMBINATION_METHOD_TYPE_MIN; + break; + case 'relative-score': + combination = CombinationMethod.COMBINATION_METHOD_TYPE_RELATIVE_SCORE; + break; + case 'manual-weights': + combination = CombinationMethod.COMBINATION_METHOD_TYPE_MANUAL; + break; + default: + throw new Error('Invalid combination method'); + } + return { + combination, + targetVectors: targets.targetVectors, + weights: targets.weights ? targets.weights : {}, + }; }; - public static nearVector = (args: { vector: number[] } & NearOptions): SearchNearVectorArgs => { + public static nearVector = ( + args: { vector: NearVectorInputType } & NearOptions + ): SearchNearVectorArgs => { return { ...Serialize.common(args), nearVector: Serialize.nearVectorSearch(args), @@ -568,13 +693,15 @@ export class Serialize { }; public static nearVideo = (args: { video: string } & NearOptions): SearchNearVideoArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.targetVector); return { ...Serialize.common(args), nearVideo: NearVideoSearch.fromPartial({ video: args.video, certainty: args.certainty, distance: args.distance, - targetVectors: args.targetVector ? [args.targetVector] : undefined, + targetVectors, + targets, }), autocut: args.autoLimit, }; diff --git a/src/collections/vectors/index.ts b/src/collections/vectors/index.ts new file mode 100644 index 00000000..974929b1 --- /dev/null +++ b/src/collections/vectors/index.ts @@ -0,0 +1,61 @@ +export type MultiTargetVectorJoinCombination = + | 'sum' + | 'average' + | 'minimum' + | 'relative-score' + | 'manual-weights'; + +export type MultiTargetVectorJoin = { + combination: MultiTargetVectorJoinCombination; + targetVectors: string[]; + weights?: Record; + + // constructor(combination: MultiTargetVectorJoinCombination, targetVectors: string[], weights?: Record) { + // this.combination = combination + // this.targetVectors = targetVectors; + // this.weights = weights; + // } + + // private mapCombination(): CombinationMethod { + // switch (this.combination) { + // case 'sum': + // return CombinationMethod.COMBINATION_METHOD_TYPE_SUM; + // case 'average': + // return CombinationMethod.COMBINATION_METHOD_TYPE_AVERAGE; + // case 'minimum': + // return CombinationMethod.COMBINATION_METHOD_TYPE_MIN; + // case 'relative-score': + // return CombinationMethod.COMBINATION_METHOD_TYPE_RELATIVE_SCORE; + // case 'manual-weights': + // return CombinationMethod.COMBINATION_METHOD_TYPE_MANUAL; + // default: + // throw new Error('Invalid combination method'); + // } + // } + + // public toGrpcTargetVector(): Targets { + // return { + // combination: this.mapCombination(), + // targetVectors: this.targetVectors, + // weights: this.weights ? this.weights : {} + // } + // } +}; + +export const targetVectors = { + sum: (targetVectors: string[]) => { + return { combination: 'sum', targetVectors }; + }, + average: (targetVectors: string[]) => { + return { combination: 'average', targetVectors }; + }, + minimum: (targetVectors: string[]) => { + return { combination: 'minimum', targetVectors }; + }, + relativeScore: (weights: Record) => { + return { combination: 'relative-score', targetVectors: Object.keys(weights), weights }; + }, + manualWeights: (weights: Record) => { + return { combiantion: 'manual-weights', targetVectors: Object.keys(weights), weights }; + }, +}; From f22f113c4d91ad5785656e023a8b499876961f84 Mon Sep 17 00:00:00 2001 From: JP Hwang Date: Thu, 4 Jul 2024 09:38:32 +0100 Subject: [PATCH 08/31] Add support for the Kagome (KR) tokenizer --- src/collections/configure/index.ts | 1 + src/openapi/schema.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/collections/configure/index.ts b/src/collections/configure/index.ts index 777a387b..3ad323b8 100644 --- a/src/collections/configure/index.ts +++ b/src/collections/configure/index.ts @@ -43,6 +43,7 @@ const tokenization = { FIELD: 'field' as const, TRIGRAM: 'trigram' as const, GSE: 'gse' as const, + KAGOME_KR: 'kagome_kr' as const, }; const vectorDistances = { diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index 61a2a1e8..c151d1bd 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -518,7 +518,7 @@ export interface definitions { * @description Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are `word` (default; splits on any non-alphanumerical, lowercases), `lowercase` (splits on white spaces, lowercases), `whitespace` (splits on white spaces), `field` (trims). Not supported for remaining data types * @enum {string} */ - tokenization?: 'word' | 'lowercase' | 'whitespace' | 'field' | 'trigram' | 'gse'; + tokenization?: 'word' | 'lowercase' | 'whitespace' | 'field' | 'trigram' | 'gse' | 'kagome_kr'; /** @description The properties of the nested object(s). Applies to object and object[] data types. */ nestedProperties?: definitions['NestedProperty'][]; }; From b94d15d9ed8ad8d7207f74bd9338f6b007aaec84 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 4 Jul 2024 16:27:23 +0100 Subject: [PATCH 09/31] Use new OFFLOAD nomenclature and updated OpenAPI schema --- src/collections/collection/index.ts | 9 +- src/collections/deserialize/index.ts | 48 +++++++ src/collections/serialize/index.ts | 72 ++++++++++- src/collections/tenants/index.ts | 135 +++++++++----------- src/collections/tenants/integration.test.ts | 24 +++- src/collections/tenants/types.ts | 39 ++++++ src/collections/tenants/unit.test.ts | 20 +-- src/openapi/schema.ts | 4 +- src/openapi/types.ts | 1 + 9 files changed, 257 insertions(+), 95 deletions(-) create mode 100644 src/collections/tenants/types.ts diff --git a/src/collections/collection/index.ts b/src/collections/collection/index.ts index 00ab4329..79a177e3 100644 --- a/src/collections/collection/index.ts +++ b/src/collections/collection/index.ts @@ -13,7 +13,7 @@ import generate, { Generate } from '../generate/index.js'; import { Iterator } from '../iterator/index.js'; import query, { Query } from '../query/index.js'; import sort, { Sort } from '../sort/index.js'; -import tenants, { TenantInput, Tenants } from '../tenants/index.js'; +import tenants, { TenantBase, Tenants } from '../tenants/index.js'; import { QueryMetadata, QueryProperty, QueryReference } from '../types/index.js'; export interface Collection { @@ -78,10 +78,11 @@ export interface Collection { * * This method does not send a request to Weaviate. It only returns a new collection object that is specific to the tenant you specify. * - * @param {string | TenantInput} tenant The tenant name or tenant object to use. + * @typedef {TenantBase} TT A type that extends TenantBase. + * @param {string | TT} tenant The tenant name or tenant object to use. * @returns {Collection} A new collection object specific to the tenant you specified. */ - withTenant: (tenant: string | TenantInput) => Collection; + withTenant: (tenant: string | TT) => Collection; } export type IteratorOptions = { @@ -136,7 +137,7 @@ const collection = ( ), withConsistency: (consistencyLevel: ConsistencyLevel) => collection(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), - withTenant: (tenant: string | TenantInput) => + withTenant: (tenant: string | TT) => collection( connection, capitalizedName, diff --git a/src/collections/deserialize/index.ts b/src/collections/deserialize/index.ts index 91c38817..e1130a59 100644 --- a/src/collections/deserialize/index.ts +++ b/src/collections/deserialize/index.ts @@ -1,10 +1,13 @@ import { WeaviateDeserializationError } from '../../errors.js'; +import { Tenant as TenantREST } from '../../openapi/types.js'; import { BatchObject as BatchObjectGRPC, BatchObjectsReply } from '../../proto/v1/batch.js'; import { BatchDeleteReply } from '../../proto/v1/batch_delete.js'; import { ListValue, Properties as PropertiesGrpc, Value } from '../../proto/v1/properties.js'; import { MetadataResult, PropertiesResult, SearchReply } from '../../proto/v1/search_get.js'; +import { TenantActivityStatus, TenantsGetReply } from '../../proto/v1/tenants.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; import { referenceFromObjects } from '../references/utils.js'; +import { Tenant } from '../tenants/index.js'; import { BatchObject, BatchObjectsReturn, @@ -298,4 +301,49 @@ export class Deserialize { : (undefined as any), }; } + + private static activityStatusGRPC(status: TenantActivityStatus): Tenant['activityStatus'] { + switch (status) { + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD: + return 'COLD'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_HOT: + return 'HOT'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN: + return 'OFFLOADED'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: + return 'OFFLOADING'; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING: + return 'ONLOADING'; + default: + throw new Error(`Unsupported tenant activity status: ${status}`); + } + } + + public static activityStatusREST(status: TenantREST['activityStatus']): Tenant['activityStatus'] { + switch (status) { + case 'COLD': + return 'COLD'; + case 'HOT': + return 'HOT'; + case 'FROZEN': + return 'OFFLOADED'; + case 'FREEZING': + return 'OFFLOADING'; + case 'UNFREEZING': + return 'ONLOADING'; + default: + throw new Error(`Unsupported tenant activity status: ${status}`); + } + } + + public static tenantsGet(reply: TenantsGetReply) { + const tenants: Record = {}; + reply.tenants.forEach((t) => { + tenants[t.name] = { + name: t.name, + activityStatus: Deserialize.activityStatusGRPC(t.activityStatus), + }; + }); + return tenants; + } } diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index 2dd40387..468b1554 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -1,5 +1,5 @@ import { v4 as uuidv4 } from 'uuid'; -import { WhereFilter } from '../../openapi/types.js'; +import { TenantActivityStatus, WhereFilter } from '../../openapi/types.js'; import { BatchObject as BatchObjectGRPC, BatchObject_MultiTargetRefProps, @@ -82,6 +82,7 @@ import { import { ReferenceGuards } from '../references/classes.js'; import { Beacon } from '../references/index.js'; import { uuidToBeacon } from '../references/utils.js'; +import { Tenant, TenantCreate, TenantUpdate } from '../tenants/types.js'; import { BatchObject, BatchObjects, @@ -1135,4 +1136,73 @@ export class Serialize { return { batch: batch, mapped: objs }; }); }; + + public static tenantsCreate(tenant: Tenant | TenantCreate): { + name: string; + activityStatus?: 'HOT' | 'COLD'; + } { + let activityStatus: TenantActivityStatus; + switch (tenant.activityStatus) { + case 'ACTIVE': + activityStatus = 'HOT'; + break; + case 'INACTIVE': + activityStatus = 'COLD'; + break; + case 'HOT': + case 'COLD': + case undefined: + activityStatus = tenant.activityStatus; + break; + case 'OFFLOADED': + throw new WeaviateInvalidInputError( + 'Cannot create a tenant with activity status OFFLOADED. Add objects to the tenant first and then you can update it to OFFLOADED.' + ); + case 'OFFLOADING': + throw new WeaviateInvalidInputError( + 'Cannot create a tenant with activity status OFFLOADING. This status is a read-only value that the server sets in the processing of making a tenant OFFLOADED.' + ); + case 'ONLOADING': + throw new WeaviateInvalidInputError( + 'Cannot create a tenant with activity status ONLOADING. This status is a read-only value that the server sets in the processing of making a tenant HOT.' + ); + } + return { + name: tenant.name, + activityStatus, + }; + } + + public static tenantUpdate = ( + tenant: Tenant | TenantUpdate + ): { name: string; activityStatus: 'HOT' | 'COLD' | 'FROZEN' } => { + let activityStatus: TenantActivityStatus; + switch (tenant.activityStatus) { + case 'ACTIVE': + activityStatus = 'HOT'; + break; + case 'INACTIVE': + activityStatus = 'COLD'; + break; + case 'OFFLOADED': + activityStatus = 'FROZEN'; + break; + case 'HOT': + case 'COLD': + activityStatus = tenant.activityStatus; + break; + case 'OFFLOADING': + throw new WeaviateInvalidInputError( + 'Cannot create a tenant with activity status OFFLOADING. This status is a read-only value that the server sets in the processing of making a tenant OFFLOADED.' + ); + case 'ONLOADING': + throw new WeaviateInvalidInputError( + 'Cannot create a tenant with activity status ONLOADING. This status is a read-only value that the server sets in the processing of making a tenant HOT.' + ); + } + return { + name: tenant.name, + activityStatus, + }; + }; } diff --git a/src/collections/tenants/index.ts b/src/collections/tenants/index.ts index 9bf4449e..8c62d8d4 100644 --- a/src/collections/tenants/index.ts +++ b/src/collections/tenants/index.ts @@ -1,64 +1,19 @@ import { ConnectionGRPC } from '../../connection/index.js'; import { WeaviateUnsupportedFeatureError } from '../../errors.js'; -import { TenantActivityStatus, TenantsGetReply } from '../../proto/v1/tenants.js'; import { TenantsCreator, TenantsDeleter, TenantsGetter, TenantsUpdater } from '../../schema/index.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; - -export type Tenant = { - name: string; - activityStatus: 'COLD' | 'HOT' | 'FREEZING' | 'FROZEN' | 'UNFREEZING' | 'UNFROZEN'; -}; - -export type TenantInput = { - name: string; - activityStatus?: 'COLD' | 'HOT'; -}; - -export type TenantsGetOptions = { - tenants?: string; -}; - -class ActivityStatusMapper { - static from( - status: TenantActivityStatus - ): 'COLD' | 'HOT' | 'FROZEN' | 'FREEZING' | 'UNFREEZING' | 'UNFROZEN' { - switch (status) { - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD: - return 'COLD'; - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_HOT: - return 'HOT'; - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: - return 'FREEZING'; - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN: - return 'FROZEN'; - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING: - return 'UNFREEZING'; - default: - throw new Error(`Unsupported tenant activity status: ${status}`); - } - } -} - -const mapReply = (reply: TenantsGetReply): Record => { - const tenants: Record = {}; - reply.tenants.forEach((t) => { - tenants[t.name] = { - name: t.name, - activityStatus: ActivityStatusMapper.from(t.activityStatus), - }; - }); - return tenants; -}; +import { Deserialize } from '../deserialize/index.js'; +import { Serialize } from '../serialize/index.js'; +import { Tenant, TenantBase, TenantCreate, TenantUpdate } from './types.js'; const checkSupportForGRPCTenantsGetEndpoint = async (dbVersionSupport: DbVersionSupport) => { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message); }; -const parseTenantOrTenantArray = (tenants: TenantInput | TenantInput[]) => - Array.isArray(tenants) ? tenants : [tenants]; +const parseValueOrValueArray = (value: V | V[]) => (Array.isArray(value) ? value : [value]); -const parseStringOrTenant = (tenant: string | TenantInput) => +const parseStringOrTenant = (tenant: string | T) => typeof tenant === 'string' ? tenant : tenant.name; const tenants = ( @@ -70,100 +25,132 @@ const tenants = ( checkSupportForGRPCTenantsGetEndpoint(dbVersionSupport) .then(() => connection.tenants(collection)) .then((builder) => builder.withGet({ names })) - .then(mapReply); + .then(Deserialize.tenantsGet); const getREST = () => new TenantsGetter(connection, collection).do().then((tenants) => { const result: Record = {}; tenants.forEach((tenant) => { if (!tenant.name) return; - result[tenant.name] = tenant as Tenant; + result[tenant.name] = { + name: tenant.name!, + activityStatus: Deserialize.activityStatusREST(tenant.activityStatus), + }; }); return result; }); return { - create: (tenants: TenantInput | TenantInput[]) => - new TenantsCreator(connection, collection, parseTenantOrTenantArray(tenants)).do() as Promise, + create: (tenants: Tenant | TenantCreate | (Tenant | TenantCreate)[]) => + new TenantsCreator( + connection, + collection, + parseValueOrValueArray(tenants).map(Serialize.tenantsCreate) + ).do() as Promise, get: async function () { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); return check.supports ? getGRPC() : getREST(); }, - getByNames: (tenants: (string | TenantInput)[]) => getGRPC(tenants.map(parseStringOrTenant)), - getByName: (tenant: string | TenantInput) => { + getByNames: (tenants: (string | T)[]) => getGRPC(tenants.map(parseStringOrTenant)), + getByName: (tenant: string | T) => { const tenantName = parseStringOrTenant(tenant); return getGRPC([tenantName]).then((tenants) => tenants[tenantName] || null); }, - remove: (tenants: TenantInput | TenantInput[]) => + remove: (tenants: string | T | (string | T)[]) => new TenantsDeleter( connection, collection, - parseTenantOrTenantArray(tenants).map((t) => t.name) + parseValueOrValueArray(tenants).map(parseStringOrTenant) ).do(), - update: (tenants: TenantInput | TenantInput[]) => - new TenantsUpdater(connection, collection, parseTenantOrTenantArray(tenants)).do() as Promise, + update: (tenants: Tenant | TenantUpdate | (Tenant | TenantUpdate)[]) => + new TenantsUpdater( + connection, + collection, + parseValueOrValueArray(tenants).map(Serialize.tenantUpdate) + ).do() as Promise, }; }; export default tenants; +export { Tenant, TenantBase, TenantCreate, TenantUpdate }; + /** * Represents all the CRUD methods available on a collection's multi-tenancy specification within Weaviate. * The collection must have been created with multi-tenancy enabled in order to use any of these methods. This class * should not be instantiated directly, but is available as a property of the `Collection` class under * the `collection.tenants` class attribute. + * + * Starting from Weaviate v1.26, the naming convention around tenant activitiy statuses is changing. + * The changing nomenclature is as follows: + * - `HOT` is now `ACTIVE`, which means loaded fully into memory and ready for use. + * - `COLD` is now `INACTIVE`, which means not loaded into memory with files stored on disk. + * + * With this change, new statuses are being added. One is mutable and the other two are immutable. They are: + * - `OFFLOADED`, which means the tenant is not loaded into memory with files stored on the cloud. + * - `OFFLOADING`, which means the tenant is transitioning to the `OFFLOADED` status. + * - `ONLOADING`, which means the tenant is transitioning from the `OFFLOADED` status. */ export interface Tenants { /** * Create the specified tenants for a collection in Weaviate. - * * The collection must have been created with multi-tenancy enabled. * - * @param {TenantInput | TenantInput[]} tenants The tenant or tenants to create. + * For details on the new activity statuses, see the docstring for the `Tenants` interface type. + * + * @param {Tenant | TenantCreate | (Tenant | TenantCreate)[]} tenants The tenant or tenants to create. * @returns {Promise} The created tenant(s) as a list of Tenant. */ - create: (tenants: TenantInput | TenantInput[]) => Promise; + create: (tenants: Tenant | TenantCreate | (Tenant | TenantCreate)[]) => Promise; /** * Return all tenants currently associated with a collection in Weaviate. - * * The collection must have been created with multi-tenancy enabled. * + * For details on the new activity statuses, see the docstring for the `Tenants` interface type. + * * @returns {Promise>} A list of tenants as an object of Tenant types, where the key is the tenant name. */ get: () => Promise>; /** * Return the specified tenants from a collection in Weaviate. - * * The collection must have been created with multi-tenancy enabled. * - * @param {(string | TenantInput)[]} names The tenants to retrieve. + * For details on the new activity statuses, see the docstring for the `Tenants` interface type. + * + * @typedef {TenantBase} T A type that extends TenantBase. + * @param {(string | T)[]} names The tenants to retrieve. * @returns {Promise} The list of tenants. If the tenant does not exist, it will not be included in the list. */ - getByNames: (names: (string | TenantInput)[]) => Promise>; + getByNames: (names: (string | T)[]) => Promise>; /** * Return the specified tenant from a collection in Weaviate. - * * The collection must have been created with multi-tenancy enabled. * - * @param {string | TenantInput} name The name of the tenant to retrieve. + * For details on the new activity statuses, see the docstring for the `Tenants` interface type. + * + * @typedef {TenantBase} T A type that extends TenantBase. + * @param {string | T} name The name of the tenant to retrieve. * @returns {Promise} The tenant as a Tenant type, or null if the tenant does not exist. */ - getByName: (name: string | TenantInput) => Promise; + getByName: (name: string | T) => Promise; /** * Remove the specified tenants from a collection in Weaviate. - * * The collection must have been created with multi-tenancy enabled. * + * For details on the new activity statuses, see the docstring for the `Tenants` interface type. + * + * @typedef {TenantBase} T A type that extends TenantBase. * @param {Tenant | Tenant[]} tenants The tenant or tenants to remove. * @returns {Promise} An empty promise. */ - remove: (tenants: TenantInput | TenantInput[]) => Promise; + remove: (tenants: string | T | (string | T)[]) => Promise; /** * Update the specified tenants for a collection in Weaviate. - * * The collection must have been created with multi-tenancy enabled. * + * For details on the new activity statuses, see the docstring for the `Tenants` interface type. + * * @param {TenantInput | TenantInput[]} tenants The tenant or tenants to update. * @returns {Promise} The updated tenant(s) as a list of Tenant. */ - update: (tenants: TenantInput | TenantInput[]) => Promise; + update: (tenants: Tenant | TenantUpdate | (Tenant | TenantUpdate)[]) => Promise; } diff --git a/src/collections/tenants/integration.test.ts b/src/collections/tenants/integration.test.ts index 96542124..98423028 100644 --- a/src/collections/tenants/integration.test.ts +++ b/src/collections/tenants/integration.test.ts @@ -26,13 +26,14 @@ describe('Testing of the collection.tenants methods', () => { .then(() => collection.tenants.create([ { name: 'hot', activityStatus: 'HOT' }, - { name: 'cold', activityStatus: 'COLD' }, + { name: 'cold-old', activityStatus: 'COLD' }, + { name: 'cold-new', activityStatus: 'COLD' }, { name: 'remove-me', activityStatus: 'HOT' }, ]) ); }); - it('should be able to create a tenant', async () => { + it('should be able to create a tenant with old nomenclature', async () => { const tenant = 'tenant'; const result = await collection.tenants.create([{ name: tenant, activityStatus: 'HOT' }]); expect(result.length).toBe(1); @@ -40,6 +41,14 @@ describe('Testing of the collection.tenants methods', () => { expect(result[0].activityStatus).toBe('HOT'); }); + it('should be able to create a tenant with new nomenclature', async () => { + const tenant = 'tenant'; + const result = await collection.tenants.create([{ name: tenant, activityStatus: 'ACTIVE' }]); + expect(result.length).toBe(1); + expect(result[0].name).toBe(tenant); + expect(result[0].activityStatus).toBe('HOT'); + }); + it('should be able to get existing tenants', async () => { const result = await collection.tenants.get(); @@ -59,8 +68,15 @@ describe('Testing of the collection.tenants methods', () => { expect(result).not.toHaveProperty('remove-me'); }); - it('should be able to update a tenant', async () => { - const result = await collection.tenants.update([{ name: 'cold', activityStatus: 'HOT' }]); + it('should be able to update a tenant with old nomenclature', async () => { + const result = await collection.tenants.update([{ name: 'cold-old', activityStatus: 'HOT' }]); + expect(result.length).toBe(1); + expect(result[0].name).toBe('cold'); + expect(result[0].activityStatus).toBe('HOT'); + }); + + it('should be able to update a tenant with new nomenclature', async () => { + const result = await collection.tenants.update([{ name: 'cold-new', activityStatus: 'ACTIVE' }]); expect(result.length).toBe(1); expect(result[0].name).toBe('cold'); expect(result[0].activityStatus).toBe('HOT'); diff --git a/src/collections/tenants/types.ts b/src/collections/tenants/types.ts new file mode 100644 index 00000000..eefa99c1 --- /dev/null +++ b/src/collections/tenants/types.ts @@ -0,0 +1,39 @@ +/** The base type for a tenant. Only the name is required. */ +export type TenantBase = { + /** The name of the tenant. */ + name: string; +}; + +/** The expected type when creating a tenant. */ +export type TenantCreate = TenantBase & { + /** The activity status of the tenant. Defaults to 'ACTIVE' if not provided. */ + activityStatus?: 'ACTIVE' | 'INACTIVE'; +}; + +/** The expected type when updating a tenant. */ +export type TenantUpdate = TenantBase & { + /** The activity status of the tenant. Must be set to one of the options. */ + activityStatus: 'ACTIVE' | 'INACTIVE' | 'OFFLOADED'; +}; + +/** The expected type when getting tenants. */ +export type TenantsGetOptions = { + tenants?: string; +}; + +/** + * The expected type returned by all tenant methods. + * + * WARNING: The `COLD` and `HOT` statuses are deprecated and will be replaced in a future release. + * See the docstring for the `activityStatus` field in this type for more information. + */ +export type Tenant = TenantBase & { + /** + * `COLD` and `HOT` are included for backwards compatability purposes and are deprecated. + * + * In a future release, these will be removed in favour of the new statuses as so: + * - `HOT` -> `ACTIVE` + * - `COLD` -> `INACTIVE` + */ + activityStatus: 'COLD' | 'HOT' | 'OFFLOADED' | 'OFFLOADING' | 'ONLOADING'; +}; diff --git a/src/collections/tenants/unit.test.ts b/src/collections/tenants/unit.test.ts index 4a278632..24ae87a5 100644 --- a/src/collections/tenants/unit.test.ts +++ b/src/collections/tenants/unit.test.ts @@ -14,11 +14,11 @@ import { WeaviateDefinition, WeaviateServiceImplementation } from '../../proto/v import weaviate, { Tenant } from '../../index'; -const TENANTS_GET_COLLECTION_NAME = 'TestCollectionTenants'; +const TENANTS_COLLECTION_NAME = 'TestCollectionTenants'; const makeRestApp = (version: string) => { const httpApp = express(); - httpApp.get(`/v1/schema/${TENANTS_GET_COLLECTION_NAME}/tenants`, (req, res) => + httpApp.get(`/v1/schema/${TENANTS_COLLECTION_NAME}/tenants`, (req, res) => res.send([ { name: 'hot', activityStatus: 'HOT' }, { name: 'cold', activityStatus: 'COLD' }, @@ -81,14 +81,14 @@ describe('Mock testing of tenants.get() method with a REST server', () => { it('should get mocked tenants', async () => { const client = await weaviate.connectToLocal({ port: 8954, grpcPort: 8955 }); - const collection = client.collections.get(TENANTS_GET_COLLECTION_NAME); + const collection = client.collections.get(TENANTS_COLLECTION_NAME); const tenants = await collection.tenants.get(); expect(tenants).toEqual>({ hot: { name: 'hot', activityStatus: 'HOT' }, cold: { name: 'cold', activityStatus: 'COLD' }, - frozen: { name: 'frozen', activityStatus: 'FROZEN' }, - freezing: { name: 'freezing', activityStatus: 'FREEZING' }, - unfreezing: { name: 'unfreezing', activityStatus: 'UNFREEZING' }, + frozen: { name: 'frozen', activityStatus: 'OFFLOADED' }, + freezing: { name: 'freezing', activityStatus: 'OFFLOADING' }, + unfreezing: { name: 'unfreezing', activityStatus: 'ONLOADING' }, }); }); @@ -107,14 +107,14 @@ describe('Mock testing of tenants.get() method with a gRPC server', () => { it('should get the mocked tenants', async () => { const client = await weaviate.connectToLocal({ port: 8956, grpcPort: 8957 }); - const collection = client.collections.get(TENANTS_GET_COLLECTION_NAME); + const collection = client.collections.get(TENANTS_COLLECTION_NAME); const tenants = await collection.tenants.get(); expect(tenants).toEqual>({ hot: { name: 'hot', activityStatus: 'HOT' }, cold: { name: 'cold', activityStatus: 'COLD' }, - frozen: { name: 'frozen', activityStatus: 'FROZEN' }, - freezing: { name: 'freezing', activityStatus: 'FREEZING' }, - unfreezing: { name: 'unfreezing', activityStatus: 'UNFREEZING' }, + frozen: { name: 'frozen', activityStatus: 'OFFLOADED' }, + freezing: { name: 'freezing', activityStatus: 'OFFLOADING' }, + unfreezing: { name: 'unfreezing', activityStatus: 'ONLOADING' }, }); }); diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index c151d1bd..04c50908 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -1289,10 +1289,10 @@ export interface definitions { /** @description name of the tenant */ name?: string; /** - * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. Allowed values are `HOT` - tenant is fully active, `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, `FROZEN` - as COLD, but files are stored on cloud storage + * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. For creation, allowed values are `HOT` - tenant is fully active and `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally. For updating, `HOT`, `COLD` and also `FROZEN` - as COLD, but files are stored on cloud storage. The following values are read-only and are set by the server for internal use: `FREEZING` - tenant is transitioning from HOT/COLD to FROZEN, `UNFREEZING` - tenant is transitioning from FROZEN to HOT/COLD * @enum {string} */ - activityStatus?: 'HOT' | 'COLD' | 'FROZEN'; + activityStatus?: 'HOT' | 'COLD' | 'FROZEN' | 'FREEZING' | 'UNFREEZING'; }; } diff --git a/src/openapi/types.ts b/src/openapi/types.ts index 01dbc77a..b1b27824 100644 --- a/src/openapi/types.ts +++ b/src/openapi/types.ts @@ -44,6 +44,7 @@ export type WeaviateNestedProperty = definitions['NestedProperty']; export type ShardStatus = definitions['ShardStatus']; export type ShardStatusList = definitions['ShardStatusList']; export type Tenant = definitions['Tenant']; +export type TenantActivityStatus = Tenant['activityStatus']; export type SchemaClusterStatus = definitions['SchemaClusterStatus']; export type WeaviateModuleConfig = WeaviateClass['moduleConfig']; export type WeaviateInvertedIndexConfig = WeaviateClass['invertedIndexConfig']; From ab8aa5315833a518dee2d8eb1cca5f3684f3edf9 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 4 Jul 2024 16:50:16 +0100 Subject: [PATCH 10/31] Fix broken tests --- src/collections/tenants/integration.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/collections/tenants/integration.test.ts b/src/collections/tenants/integration.test.ts index 98423028..5c24c8d0 100644 --- a/src/collections/tenants/integration.test.ts +++ b/src/collections/tenants/integration.test.ts @@ -26,7 +26,7 @@ describe('Testing of the collection.tenants methods', () => { .then(() => collection.tenants.create([ { name: 'hot', activityStatus: 'HOT' }, - { name: 'cold-old', activityStatus: 'COLD' }, + { name: 'cold', activityStatus: 'COLD' }, { name: 'cold-new', activityStatus: 'COLD' }, { name: 'remove-me', activityStatus: 'HOT' }, ]) @@ -69,7 +69,7 @@ describe('Testing of the collection.tenants methods', () => { }); it('should be able to update a tenant with old nomenclature', async () => { - const result = await collection.tenants.update([{ name: 'cold-old', activityStatus: 'HOT' }]); + const result = await collection.tenants.update([{ name: 'cold', activityStatus: 'HOT' }]); expect(result.length).toBe(1); expect(result[0].name).toBe('cold'); expect(result[0].activityStatus).toBe('HOT'); @@ -78,7 +78,7 @@ describe('Testing of the collection.tenants methods', () => { it('should be able to update a tenant with new nomenclature', async () => { const result = await collection.tenants.update([{ name: 'cold-new', activityStatus: 'ACTIVE' }]); expect(result.length).toBe(1); - expect(result[0].name).toBe('cold'); + expect(result[0].name).toBe('cold-new'); expect(result[0].activityStatus).toBe('HOT'); }); From 94bea82c2512b99054f4b4b25d453045c45c7b54 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 4 Jul 2024 20:08:23 +0100 Subject: [PATCH 11/31] Handle multi target vectors in searches --- src/collections/collection/index.ts | 4 + src/collections/generate/index.ts | 66 ++++++++------ src/collections/index.ts | 2 +- src/collections/query/index.ts | 66 ++++++++------ src/collections/query/integration.test.ts | 42 ++++++++- src/collections/query/types.ts | 17 +++- src/collections/serialize/index.ts | 93 +++++++++++++------- src/collections/vectors/index.ts | 61 ------------- src/collections/vectors/multiTargetVector.ts | 48 ++++++++++ src/grpc/searcher.ts | 5 +- src/utils/dbVersion.ts | 10 +++ 11 files changed, 261 insertions(+), 153 deletions(-) delete mode 100644 src/collections/vectors/index.ts create mode 100644 src/collections/vectors/multiTargetVector.ts diff --git a/src/collections/collection/index.ts b/src/collections/collection/index.ts index 00ab4329..b328efe2 100644 --- a/src/collections/collection/index.ts +++ b/src/collections/collection/index.ts @@ -15,6 +15,7 @@ import query, { Query } from '../query/index.js'; import sort, { Sort } from '../sort/index.js'; import tenants, { TenantInput, Tenants } from '../tenants/index.js'; import { QueryMetadata, QueryProperty, QueryReference } from '../types/index.js'; +import multiTargetVector, { MultiTargetVector } from '../vectors/multiTargetVector.js'; export interface Collection { /** This namespace includes all the querying methods available to you when using Weaviate's standard aggregation capabilities. */ @@ -39,6 +40,8 @@ export interface Collection { sort: Sort; /** This namespace includes all the CRUD methods available to you when modifying the tenants of a multi-tenancy-enabled collection in Weaviate. */ tenants: Tenants; + /** This namespaces includes the methods by which you cna create the `MultiTargetVectorJoin` values for use when performing multi-target vector searches over your collection. */ + multiTargetVector: MultiTargetVector; /** * Use this method to check if the collection exists in Weaviate. * @@ -116,6 +119,7 @@ const collection = ( filter: filter(), generate: generate(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), metrics: metrics(), + multiTargetVector: multiTargetVector(), name: name, query: queryCollection, sort: sort(), diff --git a/src/collections/generate/index.ts b/src/collections/generate/index.ts index f3c227dd..a84d458a 100644 --- a/src/collections/generate/index.ts +++ b/src/collections/generate/index.ts @@ -76,6 +76,24 @@ class GenerateManager implements Generate { if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message(query)); }; + private checkSupportForMultiTargetVectorSearch = async (opts?: BaseNearOptions) => { + if (!Serialize.isMultiTargetVector(opts)) return false; + const check = await this.dbVersionSupport.supportsMultiTargetVectorSearch(); + if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message); + return check.supports; + }; + + private nearSearch = async (opts?: BaseNearOptions) => { + const [_, supportsTargets] = await Promise.all([ + this.checkSupportForNamedVectors(opts), + this.checkSupportForMultiTargetVectorSearch(opts), + ]); + return { + search: await this.connection.search(this.name, this.consistencyLevel, this.tenant), + supportsTargets, + }; + }; + private async parseReply(reply: SearchReply) { const deserialize = await Deserialize.use(this.dbVersionSupport); return deserialize.generate(reply); @@ -150,7 +168,7 @@ class GenerateManager implements Generate { .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) .then((search) => search.withHybrid({ - ...Serialize.hybrid({ query, ...opts }), + ...Serialize.hybrid({ query, supportsTargets: false, ...opts }), generative: Serialize.generative(generate), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) @@ -175,12 +193,11 @@ class GenerateManager implements Generate { generate: GenerateOptions, opts?: NearOptions ): GenerateReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => toBase64FromMedia(image).then((image) => search.withNearImage({ - ...Serialize.nearImage({ image, ...(opts ? opts : {}) }), + ...Serialize.nearImage({ image, supportsTargets, ...(opts ? opts : {}) }), generative: Serialize.generative(generate), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) @@ -202,11 +219,10 @@ class GenerateManager implements Generate { opts: GroupByNearOptions ): Promise>; public nearObject(id: string, generate: GenerateOptions, opts?: NearOptions): GenerateReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => search.withNearObject({ - ...Serialize.nearObject({ id, ...(opts ? opts : {}) }), + ...Serialize.nearObject({ id, supportsTargets, ...(opts ? opts : {}) }), generative: Serialize.generative(generate), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) @@ -231,11 +247,10 @@ class GenerateManager implements Generate { generate: GenerateOptions, opts?: NearOptions ): GenerateReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => search.withNearText({ - ...Serialize.nearText({ query, ...(opts ? opts : {}) }), + ...Serialize.nearText({ query, supportsTargets, ...(opts ? opts : {}) }), generative: Serialize.generative(generate), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) @@ -260,11 +275,10 @@ class GenerateManager implements Generate { generate: GenerateOptions, opts?: NearOptions ): GenerateReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => search.withNearVector({ - ...Serialize.nearVector({ vector, ...(opts ? opts : {}) }), + ...Serialize.nearVector({ vector, supportsTargets, ...(opts ? opts : {}) }), generative: Serialize.generative(generate), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) @@ -292,10 +306,10 @@ class GenerateManager implements Generate { generate: GenerateOptions, opts?: NearOptions ): GenerateReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => { + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => { let reply: Promise; + const args = { supportsTargets, ...(opts ? opts : {}) }; const generative = Serialize.generative(generate); const groupBy = Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) @@ -304,7 +318,7 @@ class GenerateManager implements Generate { case 'audio': reply = toBase64FromMedia(media).then((media) => search.withNearAudio({ - ...Serialize.nearAudio({ audio: media, ...(opts ? opts : {}) }), + ...Serialize.nearAudio({ audio: media, ...args }), generative, groupBy, }) @@ -313,7 +327,7 @@ class GenerateManager implements Generate { case 'depth': reply = toBase64FromMedia(media).then((media) => search.withNearDepth({ - ...Serialize.nearDepth({ depth: media, ...(opts ? opts : {}) }), + ...Serialize.nearDepth({ depth: media, ...args }), generative, groupBy, }) @@ -322,7 +336,7 @@ class GenerateManager implements Generate { case 'image': reply = toBase64FromMedia(media).then((media) => search.withNearImage({ - ...Serialize.nearImage({ image: media, ...(opts ? opts : {}) }), + ...Serialize.nearImage({ image: media, ...args }), generative, groupBy, }) @@ -331,7 +345,7 @@ class GenerateManager implements Generate { case 'imu': reply = toBase64FromMedia(media).then((media) => search.withNearIMU({ - ...Serialize.nearIMU({ imu: media, ...(opts ? opts : {}) }), + ...Serialize.nearIMU({ imu: media, ...args }), generative, groupBy, }) @@ -340,7 +354,7 @@ class GenerateManager implements Generate { case 'thermal': reply = toBase64FromMedia(media).then((media) => search.withNearThermal({ - ...Serialize.nearThermal({ thermal: media, ...(opts ? opts : {}) }), + ...Serialize.nearThermal({ thermal: media, ...args }), generative, groupBy, }) @@ -349,7 +363,7 @@ class GenerateManager implements Generate { case 'video': reply = toBase64FromMedia(media).then((media) => search.withNearVideo({ - ...Serialize.nearVideo({ video: media, ...(opts ? opts : {}) }), + ...Serialize.nearVideo({ video: media, ...args }), generative, groupBy, }) diff --git a/src/collections/index.ts b/src/collections/index.ts index 691dcbb8..e9e9a437 100644 --- a/src/collections/index.ts +++ b/src/collections/index.ts @@ -306,4 +306,4 @@ export * from './references/index.js'; export * from './sort/index.js'; export * from './tenants/index.js'; export * from './types/index.js'; -export * from './vectors/index.js'; +export * from './vectors/multiTargetVector.js'; diff --git a/src/collections/query/index.ts b/src/collections/query/index.ts index c72a180e..410bd797 100644 --- a/src/collections/query/index.ts +++ b/src/collections/query/index.ts @@ -85,6 +85,24 @@ class QueryManager implements Query { if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message); }; + private checkSupportForMultiTargetVectorSearch = async (opts?: BaseNearOptions) => { + if (!Serialize.isMultiTargetVector(opts)) return false; + const check = await this.dbVersionSupport.supportsMultiTargetVectorSearch(); + if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message); + return check.supports; + }; + + private nearSearch = async (opts?: BaseNearOptions) => { + const [_, supportsTargets] = await Promise.all([ + this.checkSupportForNamedVectors(opts), + this.checkSupportForMultiTargetVectorSearch(opts), + ]); + return { + search: await this.connection.search(this.name, this.consistencyLevel, this.tenant), + supportsTargets, + }; + }; + private async parseReply(reply: SearchReply) { const deserialize = await Deserialize.use(this.dbVersionSupport); return deserialize.query(reply); @@ -143,7 +161,7 @@ class QueryManager implements Query { .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) .then((search) => search.withHybrid({ - ...Serialize.hybrid({ query, ...opts }), + ...Serialize.hybrid({ query, supportsTargets: false, ...opts }), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) : undefined, @@ -155,12 +173,11 @@ class QueryManager implements Query { public nearImage(image: string | Buffer, opts?: BaseNearOptions): Promise>; public nearImage(image: string | Buffer, opts: GroupByNearOptions): Promise>; public nearImage(image: string | Buffer, opts?: NearOptions): QueryReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => { + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => { return toBase64FromMedia(image).then((image) => search.withNearImage({ - ...Serialize.nearImage({ image, ...(opts ? opts : {}) }), + ...Serialize.nearImage({ image, supportsTargets, ...(opts ? opts : {}) }), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) : undefined, @@ -181,39 +198,39 @@ class QueryManager implements Query { opts: GroupByNearOptions ): Promise>; public nearMedia(media: string | Buffer, type: NearMediaType, opts?: NearOptions): QueryReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => { + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => { + const args = { supportsTargets, ...(opts ? opts : {}) }; let reply: Promise; switch (type) { case 'audio': reply = toBase64FromMedia(media).then((media) => - search.withNearAudio(Serialize.nearAudio({ audio: media, ...(opts ? opts : {}) })) + search.withNearAudio(Serialize.nearAudio({ audio: media, ...args })) ); break; case 'depth': reply = toBase64FromMedia(media).then((media) => - search.withNearDepth(Serialize.nearDepth({ depth: media, ...(opts ? opts : {}) })) + search.withNearDepth(Serialize.nearDepth({ depth: media, ...args })) ); break; case 'image': reply = toBase64FromMedia(media).then((media) => - search.withNearImage(Serialize.nearImage({ image: media, ...(opts ? opts : {}) })) + search.withNearImage(Serialize.nearImage({ image: media, ...args })) ); break; case 'imu': reply = toBase64FromMedia(media).then((media) => - search.withNearIMU(Serialize.nearIMU({ imu: media, ...(opts ? opts : {}) })) + search.withNearIMU(Serialize.nearIMU({ imu: media, ...args })) ); break; case 'thermal': reply = toBase64FromMedia(media).then((media) => - search.withNearThermal(Serialize.nearThermal({ thermal: media, ...(opts ? opts : {}) })) + search.withNearThermal(Serialize.nearThermal({ thermal: media, ...args })) ); break; case 'video': reply = toBase64FromMedia(media).then((media) => - search.withNearVideo(Serialize.nearVideo({ video: media, ...(opts ? opts : {}) })) + search.withNearVideo(Serialize.nearVideo({ video: media, ...args })) ); break; default: @@ -227,11 +244,10 @@ class QueryManager implements Query { public nearObject(id: string, opts?: BaseNearOptions): Promise>; public nearObject(id: string, opts: GroupByNearOptions): Promise>; public nearObject(id: string, opts?: NearOptions): QueryReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => search.withNearObject({ - ...Serialize.nearObject({ id, ...(opts ? opts : {}) }), + ...Serialize.nearObject({ id, supportsTargets, ...(opts ? opts : {}) }), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) : undefined, @@ -243,11 +259,10 @@ class QueryManager implements Query { public nearText(query: string | string[], opts?: BaseNearTextOptions): Promise>; public nearText(query: string | string[], opts: GroupByNearTextOptions): Promise>; public nearText(query: string | string[], opts?: NearTextOptions): QueryReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => search.withNearText({ - ...Serialize.nearText({ query, ...(opts ? opts : {}) }), + ...Serialize.nearText({ query, supportsTargets, ...(opts ? opts : {}) }), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) : undefined, @@ -259,11 +274,10 @@ class QueryManager implements Query { public nearVector(vector: NearVectorInputType, opts?: BaseNearOptions): Promise>; public nearVector(vector: NearVectorInputType, opts: GroupByNearOptions): Promise>; public nearVector(vector: NearVectorInputType, opts?: NearOptions): QueryReturn { - return this.checkSupportForNamedVectors(opts) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.nearSearch(opts) + .then(({ search, supportsTargets }) => search.withNearVector({ - ...Serialize.nearVector({ vector, ...(opts ? opts : {}) }), + ...Serialize.nearVector({ vector, supportsTargets, ...(opts ? opts : {}) }), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) : undefined, diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 482475f7..856f0c2f 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -529,7 +529,7 @@ describe('Testing of the collection.query methods with a collection with a refer }); }); - describe('Testing of the collection.query methods with a collection with multiple vectors', () => { + describe('Testing of the collection.query methods with a collection with a multiple vectors', () => { let client: WeaviateClient; let collection: Collection; const collectionName = 'TestCollectionQueryWithMultiVector'; @@ -567,6 +567,10 @@ describe('Testing of the collection.query methods with a collection with a refer name: 'title', sourceProperties: ['title'], }), + weaviate.configure.vectorizer.text2VecContextionary({ + name: 'title2', + sourceProperties: ['title'], + }), ], }) .then(async () => { @@ -588,7 +592,7 @@ describe('Testing of the collection.query methods with a collection with a refer return query(); }); - it('should query returning the named vector', async () => { + it('should query returning a named vector', async () => { const query = () => collection.query.fetchObjects({ returnProperties: ['title'], @@ -607,7 +611,7 @@ describe('Testing of the collection.query methods with a collection with a refer expect(ret.objects[1].vectors.title).toBeDefined(); }); - it('should query without searching returning named vector', async () => { + it('should query without searching returning a named vector', async () => { if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 24, 0))) { return; } @@ -635,6 +639,38 @@ describe('Testing of the collection.query methods with a collection with a refer expect(ret.objects[0].properties.title).toEqual('test'); expect(ret.objects[1].properties.title).toEqual('other'); }); + + it('should query a multi-target vector search over the named vector spaces', async () => { + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + return; + } + const ret = await collection.query.nearObject(id1, { + returnProperties: ['title'], + targetVector: ['title', 'title2'], + }); + expect(ret.objects.length).toEqual(2); + expect(ret.objects[0].properties.title).toEqual('test'); + expect(ret.objects[1].properties.title).toEqual('other'); + }); + + it('should query a weighted multi-target vector search over the named vector spaces', async () => { + const query = () => + collection.query.nearObject(id1, { + distance: 0.01, + returnProperties: ['title'], + targetVector: collection.multiTargetVector.manualWeights({ + title: 5, + title2: 0.1, + }), + }); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(1); + expect(ret.objects[0].properties.title).toEqual('test'); + }); }); }); diff --git a/src/collections/query/types.ts b/src/collections/query/types.ts index e6bede9b..a085b760 100644 --- a/src/collections/query/types.ts +++ b/src/collections/query/types.ts @@ -136,7 +136,7 @@ export type BaseNearOptions = SearchOptions & { /** The maximum distance to search. Incompatible with the `certainty` param. */ distance?: number; /** Specify which vector to search on if using named vectors. */ - targetVector?: string; + targetVector?: TargetVectorInputType; }; /** Options available in the near search queries when specifying the `groupBy` parameter. */ @@ -166,8 +166,23 @@ export type GroupByNearTextOptions = BaseNearTextOptions & { /** The type of the media to search for in the `query.nearMedia` method */ export type NearMediaType = 'audio' | 'depth' | 'image' | 'imu' | 'thermal' | 'video'; +/** + * The vector(s) to search for in the `query.nearVector` method. One of: + * - a single vector, in which case pass a single number array. + * - multiple unnamed vectors, in which case pass an array of number arrays. + * - multiple named vectors, in which case pass an object of type `Record`. + * + * If you pass multiple unnamed vectors here then you must specify the names of the vector spaces in the `targetVector` parameter. + * If these two arrays are of different lengths, the method will throw an error. + */ export type NearVectorInputType = number[] | number[][] | Record; +/** + * Over which vector spaces to perform the vector search query in the `nearX` search method. One of: + * - a single vector space search, in which case pass a string with the name of the vector space to search in. + * - a multi-vector space search, in which case pass an array of strings with the names of the vector spaces to search in. + * - a weighted multi-vector space search, in which case pass an object of type `MultiTargetVectorJoin` detailing the vector spaces to search in. + */ export type TargetVectorInputType = string | string[] | MultiTargetVectorJoin; interface Bm25 { diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index 8056554d..b81791ca 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -316,6 +316,10 @@ export class Serialize { return Array.isArray(opts?.includeVector) || opts?.targetVector !== undefined; }; + public static isMultiTargetVector = (opts?: BaseNearOptions): boolean => { + return opts?.targetVector !== undefined && !TargetVectorInputGuards.isSingle(opts.targetVector); + }; + private static common = (args?: SearchOptions): BaseSearchArgs => { const out: BaseSearchArgs = { limit: args?.limit, @@ -388,6 +392,7 @@ export class Serialize { return Serialize.isHybridNearTextSearch(vector) ? Serialize.nearTextSearch({ ...vector, + supportsTargets: false, query: vector.query, }) : undefined; @@ -401,7 +406,9 @@ export class Serialize { : undefined; }; - public static hybrid = (args: { query: string } & HybridOptions): SearchHybridArgs => { + public static hybrid = ( + args: { query: string; supportsTargets: boolean } & HybridOptions + ): SearchHybridArgs => { const fusionType = (fusionType?: string): Hybrid_FusionType => { switch (fusionType) { case 'Ranked': @@ -412,7 +419,7 @@ export class Serialize { return Hybrid_FusionType.FUSION_TYPE_UNSPECIFIED; } }; - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), hybridSearch: Hybrid.fromPartial({ @@ -430,8 +437,10 @@ export class Serialize { }; }; - public static nearAudio = (args: { audio: string } & NearOptions): SearchNearAudioArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearAudio = ( + args: { audio: string; supportsTargets: boolean } & NearOptions + ): SearchNearAudioArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), nearAudio: NearAudioSearch.fromPartial({ @@ -445,8 +454,10 @@ export class Serialize { }; }; - public static nearDepth = (args: { depth: string } & NearOptions): SearchNearDepthArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearDepth = ( + args: { depth: string; supportsTargets: boolean } & NearOptions + ): SearchNearDepthArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), nearDepth: NearDepthSearch.fromPartial({ @@ -460,8 +471,10 @@ export class Serialize { }; }; - public static nearImage = (args: { image: string } & NearOptions): SearchNearImageArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearImage = ( + args: { image: string; supportsTargets: boolean } & NearOptions + ): SearchNearImageArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), nearImage: NearImageSearch.fromPartial({ @@ -475,8 +488,10 @@ export class Serialize { }; }; - public static nearIMU = (args: { imu: string } & NearOptions): SearchNearIMUArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearIMU = ( + args: { imu: string; supportsTargets: boolean } & NearOptions + ): SearchNearIMUArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), nearIMU: NearIMUSearch.fromPartial({ @@ -490,8 +505,11 @@ export class Serialize { }; }; - public static nearObject = (args: { id: string } & NearOptions): SearchNearObjectArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearObject = ( + args: { id: string; supportsTargets: boolean } & NearOptions + ): SearchNearObjectArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + console.log(targets, targetVectors); return { ...Serialize.common(args), nearObject: NearObject.fromPartial({ @@ -507,13 +525,14 @@ export class Serialize { private static nearTextSearch = (args: { query: string | string[]; + supportsTargets: boolean; certainty?: number; distance?: number; - targetVector?: string; + targetVector?: TargetVectorInputType; moveAway?: { concepts?: string[]; force?: number; objects?: string[] }; moveTo?: { concepts?: string[]; force?: number; objects?: string[] }; }) => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return NearTextSearch.fromPartial({ query: typeof args.query === 'string' ? [args.query] : args.query, certainty: args.certainty, @@ -538,7 +557,7 @@ export class Serialize { }; public static nearText = ( - args: { query: string | string[] } & NearTextOptions + args: { query: string | string[]; supportsTargets: boolean } & NearTextOptions ): SearchNearTextArgs => { return { ...Serialize.common(args), @@ -547,8 +566,10 @@ export class Serialize { }; }; - public static nearThermal = (args: { thermal: string } & NearOptions): SearchNearThermalArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearThermal = ( + args: { thermal: string; supportsTargets: boolean } & NearOptions + ): SearchNearThermalArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), nearThermal: NearThermalSearch.fromPartial({ @@ -568,6 +589,7 @@ export class Serialize { private static nearVectorSearch = (args: { vector: NearVectorInputType; + supportsTargets: boolean; certainty?: number; distance?: number; targetVector?: TargetVectorInputType; @@ -578,7 +600,7 @@ export class Serialize { - an object with target names as keys and arrays of numbers as values received: ${args.vector}`); - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); if (NearVectorInputGuards.isObject(args.vector)) { if (targets === undefined || targets.targetVectors.length != Object.keys(args.vector).length) { throw new WeaviateInvalidInputError( @@ -633,24 +655,29 @@ export class Serialize { }; private static targetVector = ( + supportsTargets: boolean, targetVector?: TargetVectorInputType ): { targets?: Targets; targetVectors?: string[] } => { if (targetVector === undefined) { return {}; } else if (TargetVectorInputGuards.isSingle(targetVector)) { - return { - targets: Targets.fromPartial({ - targetVectors: [targetVector], - }), - }; + return supportsTargets + ? { + targets: Targets.fromPartial({ + targetVectors: [targetVector], + }), + } + : { targetVectors: [targetVector] }; } else if (TargetVectorInputGuards.isMulti(targetVector)) { - return { - targets: Targets.fromPartial({ - targetVectors: targetVector, - }), - }; + return supportsTargets + ? { + targets: Targets.fromPartial({ + targetVectors: targetVector, + }), + } + : { targetVectors: targetVector }; } else { - return Serialize.targets(targetVector); + return { targets: Serialize.targets(targetVector) }; } }; @@ -683,7 +710,7 @@ export class Serialize { }; public static nearVector = ( - args: { vector: NearVectorInputType } & NearOptions + args: { vector: NearVectorInputType; supportsTargets: boolean } & NearOptions ): SearchNearVectorArgs => { return { ...Serialize.common(args), @@ -692,8 +719,10 @@ export class Serialize { }; }; - public static nearVideo = (args: { video: string } & NearOptions): SearchNearVideoArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.targetVector); + public static nearVideo = ( + args: { video: string; supportsTargets: boolean } & NearOptions + ): SearchNearVideoArgs => { + const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); return { ...Serialize.common(args), nearVideo: NearVideoSearch.fromPartial({ diff --git a/src/collections/vectors/index.ts b/src/collections/vectors/index.ts deleted file mode 100644 index 974929b1..00000000 --- a/src/collections/vectors/index.ts +++ /dev/null @@ -1,61 +0,0 @@ -export type MultiTargetVectorJoinCombination = - | 'sum' - | 'average' - | 'minimum' - | 'relative-score' - | 'manual-weights'; - -export type MultiTargetVectorJoin = { - combination: MultiTargetVectorJoinCombination; - targetVectors: string[]; - weights?: Record; - - // constructor(combination: MultiTargetVectorJoinCombination, targetVectors: string[], weights?: Record) { - // this.combination = combination - // this.targetVectors = targetVectors; - // this.weights = weights; - // } - - // private mapCombination(): CombinationMethod { - // switch (this.combination) { - // case 'sum': - // return CombinationMethod.COMBINATION_METHOD_TYPE_SUM; - // case 'average': - // return CombinationMethod.COMBINATION_METHOD_TYPE_AVERAGE; - // case 'minimum': - // return CombinationMethod.COMBINATION_METHOD_TYPE_MIN; - // case 'relative-score': - // return CombinationMethod.COMBINATION_METHOD_TYPE_RELATIVE_SCORE; - // case 'manual-weights': - // return CombinationMethod.COMBINATION_METHOD_TYPE_MANUAL; - // default: - // throw new Error('Invalid combination method'); - // } - // } - - // public toGrpcTargetVector(): Targets { - // return { - // combination: this.mapCombination(), - // targetVectors: this.targetVectors, - // weights: this.weights ? this.weights : {} - // } - // } -}; - -export const targetVectors = { - sum: (targetVectors: string[]) => { - return { combination: 'sum', targetVectors }; - }, - average: (targetVectors: string[]) => { - return { combination: 'average', targetVectors }; - }, - minimum: (targetVectors: string[]) => { - return { combination: 'minimum', targetVectors }; - }, - relativeScore: (weights: Record) => { - return { combination: 'relative-score', targetVectors: Object.keys(weights), weights }; - }, - manualWeights: (weights: Record) => { - return { combiantion: 'manual-weights', targetVectors: Object.keys(weights), weights }; - }, -}; diff --git a/src/collections/vectors/multiTargetVector.ts b/src/collections/vectors/multiTargetVector.ts new file mode 100644 index 00000000..edf8ccf5 --- /dev/null +++ b/src/collections/vectors/multiTargetVector.ts @@ -0,0 +1,48 @@ +export type MultiTargetVectorJoinCombination = + | 'sum' + | 'average' + | 'minimum' + | 'relative-score' + | 'manual-weights'; + +export type MultiTargetVectorJoin = { + combination: MultiTargetVectorJoinCombination; + targetVectors: string[]; + weights?: Record; +}; + +export default () => { + return { + sum: (targetVectors: string[]) => { + return { combination: 'sum' as MultiTargetVectorJoinCombination, targetVectors }; + }, + average: (targetVectors: string[]) => { + return { combination: 'average' as MultiTargetVectorJoinCombination, targetVectors }; + }, + minimum: (targetVectors: string[]) => { + return { combination: 'minimum' as MultiTargetVectorJoinCombination, targetVectors }; + }, + relativeScore: (weights: Record) => { + return { + combination: 'relative-score' as MultiTargetVectorJoinCombination, + targetVectors: Object.keys(weights), + weights, + }; + }, + manualWeights: (weights: Record) => { + return { + combination: 'manual-weights' as MultiTargetVectorJoinCombination, + targetVectors: Object.keys(weights), + weights, + }; + }, + }; +}; + +export interface MultiTargetVector { + sum: (targetVectors: string[]) => MultiTargetVectorJoin; + average: (targetVectors: string[]) => MultiTargetVectorJoin; + minimum: (targetVectors: string[]) => MultiTargetVectorJoin; + relativeScore: (weights: Record) => MultiTargetVectorJoin; + manualWeights: (weights: Record) => MultiTargetVectorJoin; +} diff --git a/src/grpc/searcher.ts b/src/grpc/searcher.ts index 1a7bb754..82300b9a 100644 --- a/src/grpc/searcher.ts +++ b/src/grpc/searcher.ts @@ -136,8 +136,8 @@ export default class Searcher extends Base implements Search { public withNearVector = (args: SearchNearVectorArgs) => this.call(SearchRequest.fromPartial(args)); public withNearVideo = (args: SearchNearVideoArgs) => this.call(SearchRequest.fromPartial(args)); - private call(message: SearchRequest) { - return this.sendWithTimeout((signal: AbortSignal) => + private call = (message: SearchRequest) => + this.sendWithTimeout((signal: AbortSignal) => this.connection .search( { @@ -157,5 +157,4 @@ export default class Searcher extends Base implements Search { throw new WeaviateQueryError(err.message, 'gRPC'); }) ); - } } diff --git a/src/utils/dbVersion.ts b/src/utils/dbVersion.ts index 7de53d15..fd53ef33 100644 --- a/src/utils/dbVersion.ts +++ b/src/utils/dbVersion.ts @@ -147,6 +147,16 @@ export class DbVersionSupport { }; }); }; + + supportsMultiTargetVectorSearch = () => { + return this.dbVersionProvider.getVersion().then((version) => { + return { + version: version, + supports: version.isAtLeast(1, 26, 0), + message: this.errorMessage('Multi-target vector search', version.show(), '1.26.0'), + }; + }); + }; } const EMPTY_VERSION = ''; From ed69dac892a4691b927c4a462c417c00987e396c Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 4 Jul 2024 20:16:03 +0100 Subject: [PATCH 12/31] Fix hard coupled tests using latest build image --- .github/workflows/main.yaml | 2 +- src/collections/aggregate/integration.test.ts | 2 +- src/collections/config/integration.test.ts | 6 +++--- src/collections/journey.test.ts | 6 ++++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ba4f90f4..8f2839c6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.19 WEAVIATE_125: 1.25.5 - WEAVIATE_126: preview--13e65f1 + WEAVIATE_126: preview--4e2eb3a jobs: checks: diff --git a/src/collections/aggregate/integration.test.ts b/src/collections/aggregate/integration.test.ts index 06f1f506..172e83df 100644 --- a/src/collections/aggregate/integration.test.ts +++ b/src/collections/aggregate/integration.test.ts @@ -145,7 +145,7 @@ describe('Testing of the collection.aggregate methods', () => { it('should aggregate grouped by data with a near text search and no property metrics', async () => { const result = await collection.aggregate.groupBy.nearText('test', { groupBy: 'text', - certainty: 0.1, + certainty: 0.01, }); expect(result.length).toEqual(1); expect(result[0].totalCount).toEqual(100); diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index 1df6a634..c8cc7721 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -53,7 +53,7 @@ describe('Testing of the collection.config namespace', () => { expect(config.vectorizers.default.indexConfig).toEqual({ skip: false, cleanupIntervalSeconds: 300, - maxConnections: 64, + maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) ? 64 : 32, efConstruction: 128, ef: -1, dynamicEfMin: 100, @@ -106,7 +106,7 @@ describe('Testing of the collection.config namespace', () => { expect(config.vectorizers.default.indexConfig).toEqual({ skip: false, cleanupIntervalSeconds: 300, - maxConnections: 64, + maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) ? 64 : 32, efConstruction: 128, ef: -1, dynamicEfMin: 100, @@ -475,7 +475,7 @@ describe('Testing of the collection.config namespace', () => { expect(config.vectorizers.default.indexConfig).toEqual({ skip: false, cleanupIntervalSeconds: 300, - maxConnections: 64, + maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) ? 64 : 32, efConstruction: 128, ef: 4, dynamicEfMin: 100, diff --git a/src/collections/journey.test.ts b/src/collections/journey.test.ts index dc379020..44ae6cc4 100644 --- a/src/collections/journey.test.ts +++ b/src/collections/journey.test.ts @@ -53,7 +53,7 @@ describe('Journey testing of the client using a WCD cluster', () => { return client.collections .get(collectionName) .config.get() - .then((config) => { + .then(async (config) => { expect(config).toEqual({ name: collectionName, generative: { @@ -172,7 +172,9 @@ describe('Journey testing of the client using a WCD cluster', () => { ef: -1, efConstruction: 128, flatSearchCutoff: 40000, - maxConnections: 64, + maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) + ? 64 + : 32, skip: false, vectorCacheMaxObjects: 1000000000000, quantizer: undefined, From 338ce5e3f42ad60cb862a30b5f084d348055eb94 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 4 Jul 2024 20:30:23 +0100 Subject: [PATCH 13/31] Fix change for indexRangeable field --- src/openapi/schema.ts | 10 +++++----- src/schema/journey.test.ts | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index c151d1bd..9ce33dd0 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -512,8 +512,8 @@ export interface definitions { indexFilterable?: boolean; /** @description Optional. Should this property be indexed in the inverted index. Defaults to true. Applicable only to properties of data type text and text[]. If you choose false, you will not be able to use this property in bm25 or hybrid search. This property has no affect on vectorization decisions done by modules */ indexSearchable?: boolean; - /** @description Optional. TODO roaring-set-range */ - indexRangeable?: boolean; + /** @description Optional. Should this property be indexed in the inverted index. Defaults to false. Provides better performance for range queries compared to filterable index in large datasets. Applicable only to properties of data type int, number, date. */ + indexRangeFilters?: boolean; /** * @description Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are `word` (default; splits on any non-alphanumerical, lowercases), `lowercase` (splits on white spaces, lowercases), `whitespace` (splits on white spaces), `field` (trims). Not supported for remaining data types * @enum {string} @@ -536,7 +536,7 @@ export interface definitions { name?: string; indexFilterable?: boolean; indexSearchable?: boolean; - indexRangeable?: boolean; + indexRangeFilters?: boolean; /** @enum {string} */ tokenization?: 'word' | 'lowercase' | 'whitespace' | 'field'; nestedProperties?: definitions['NestedProperty'][]; @@ -1289,10 +1289,10 @@ export interface definitions { /** @description name of the tenant */ name?: string; /** - * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. Allowed values are `HOT` - tenant is fully active, `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, `FROZEN` - as COLD, but files are stored on cloud storage + * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. For creation, allowed values are `HOT` - tenant is fully active and `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally. For updating, `HOT`, `COLD` and also `FROZEN` - as COLD, but files are stored on cloud storage. The following values are read-only and are set by the server for internal use: `FREEZING` - tenant is transitioning from HOT/COLD to FROZEN, `UNFREEZING` - tenant is transitioning from FROZEN to HOT/COLD * @enum {string} */ - activityStatus?: 'HOT' | 'COLD' | 'FROZEN'; + activityStatus?: 'HOT' | 'COLD' | 'FROZEN' | 'FREEZING' | 'UNFREEZING'; }; } diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index 59a82484..0f19c2ee 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -69,7 +69,7 @@ describe('schema', () => { name: 'anotherProp', tokenization: 'field', indexFilterable: true, - indexRangeable: (await isVer(client, 26, 0)) ? false : undefined, + indexRangeFilters: (await isVer(client, 26, 0)) ? false : undefined, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { @@ -539,7 +539,7 @@ describe('property setting defaults and migrations', () => { const errMsg1 = isVer(client, 26, 0).then((yes) => yes - ? '`indexInverted` is deprecated and can not be set together with `indexFilterable`, `indexSearchable` or `indexRangeable`' + ? '`indexInverted` is deprecated and can not be set together with `indexFilterable`, `indexSearchable` or `indexRangeFilters`' : '`indexInverted` is deprecated and can not be set together with `indexFilterable` or `indexSearchable`' ); const errMsg2 = Promise.resolve('`indexSearchable`'); @@ -762,7 +762,7 @@ async function newClassObject( name: 'stringProp', tokenization: 'word', indexFilterable: true, - indexRangeable: (await is1260Promise) ? false : undefined, + indexRangeFilters: (await is1260Promise) ? false : undefined, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { From 4f6c04825000972263c1a2b8fa259b22a7185b1a Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 4 Jul 2024 20:51:07 +0100 Subject: [PATCH 14/31] Fix unit tests --- src/collections/serialize/index.ts | 1 - src/collections/serialize/unit.test.ts | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index b81791ca..340b8de5 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -509,7 +509,6 @@ export class Serialize { args: { id: string; supportsTargets: boolean } & NearOptions ): SearchNearObjectArgs => { const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); - console.log(targets, targetVectors); return { ...Serialize.common(args), nearObject: NearObject.fromPartial({ diff --git a/src/collections/serialize/unit.test.ts b/src/collections/serialize/unit.test.ts index 6ec41cd7..ddf26306 100644 --- a/src/collections/serialize/unit.test.ts +++ b/src/collections/serialize/unit.test.ts @@ -149,6 +149,7 @@ describe('Unit testing of Serialize', () => { vector: [1, 2, 3], targetVector: 'title', fusionType: 'Ranked', + supportsTargets: false, }); expect(args).toEqual({ hybridSearch: Hybrid.fromPartial({ @@ -169,6 +170,7 @@ describe('Unit testing of Serialize', () => { certainty: 0.6, distance: 0.4, targetVector: 'audio', + supportsTargets: false, }); expect(args).toEqual({ nearAudio: NearAudioSearch.fromPartial({ @@ -184,6 +186,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearDepth', () => { const args = Serialize.nearDepth({ depth: 'depth', + supportsTargets: false, }); expect(args).toEqual({ nearDepth: NearDepthSearch.fromPartial({ @@ -196,6 +199,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearIMU', () => { const args = Serialize.nearIMU({ imu: 'imu', + supportsTargets: false, }); expect(args).toEqual({ nearIMU: NearIMUSearch.fromPartial({ @@ -208,6 +212,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearImage', () => { const args = Serialize.nearImage({ image: 'image', + supportsTargets: false, }); expect(args).toEqual({ nearImage: NearImageSearch.fromPartial({ @@ -220,6 +225,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearObject', () => { const args = Serialize.nearObject({ id: 'id', + supportsTargets: false, }); expect(args).toEqual({ nearObject: NearObject.fromPartial({ @@ -242,6 +248,7 @@ describe('Unit testing of Serialize', () => { concepts: ['good'], force: 0.6, }, + supportsTargets: false, }); expect(args).toEqual({ nearText: NearTextSearch.fromPartial({ @@ -264,6 +271,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearThermal', () => { const args = Serialize.nearThermal({ thermal: 'thermal', + supportsTargets: false, }); expect(args).toEqual({ nearThermal: NearThermalSearch.fromPartial({ @@ -276,6 +284,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearVector', () => { const args = Serialize.nearVector({ vector: [1, 2, 3], + supportsTargets: false, }); expect(args).toEqual({ nearVector: NearVector.fromPartial({ @@ -288,6 +297,7 @@ describe('Unit testing of Serialize', () => { it('should parse args for nearVideo', () => { const args = Serialize.nearVideo({ video: 'video', + supportsTargets: false, }); expect(args).toEqual({ nearVideo: NearVideoSearch.fromPartial({ From 92dbaae1c2ecb55a46c4e34b863ab31ab425a989 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Sun, 7 Jul 2024 17:14:16 +0100 Subject: [PATCH 15/31] Fix BC for queries, add generative support, at hybrid support, use latest preview image --- .github/workflows/main.yaml | 2 +- src/collections/generate/index.ts | 29 ++- src/collections/generate/integration.test.ts | 138 +++++++++++-- src/collections/generate/types.ts | 17 +- src/collections/query/index.ts | 26 ++- src/collections/query/integration.test.ts | 92 +++++++++ src/collections/query/types.ts | 8 +- src/collections/serialize/index.ts | 201 +++++++++++-------- 8 files changed, 389 insertions(+), 124 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8f2839c6..9082217c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.19 WEAVIATE_125: 1.25.5 - WEAVIATE_126: preview--4e2eb3a + WEAVIATE_126: preview--c2bfc40 jobs: checks: diff --git a/src/collections/generate/index.ts b/src/collections/generate/index.ts index a84d458a..065fae9d 100644 --- a/src/collections/generate/index.ts +++ b/src/collections/generate/index.ts @@ -76,6 +76,12 @@ class GenerateManager implements Generate { if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message(query)); }; + private checkSupportForHybridNearTextAndNearVectorSubSearches = async (opts?: HybridOptions) => { + if (opts?.vector === undefined || Array.isArray(opts.vector)) return; + const check = await this.dbVersionSupport.supportsHybridNearTextAndNearVectorSubsearchQueries(); + if (!check.supports) throw new WeaviateUnsupportedFeatureError(check.message); + }; + private checkSupportForMultiTargetVectorSearch = async (opts?: BaseNearOptions) => { if (!Serialize.isMultiTargetVector(opts)) return false; const check = await this.dbVersionSupport.supportsMultiTargetVectorSearch(); @@ -94,6 +100,19 @@ class GenerateManager implements Generate { }; }; + private hybridSearch = async (opts?: BaseHybridOptions) => { + const [supportsTargets] = await Promise.all([ + this.checkSupportForMultiTargetVectorSearch(opts), + this.checkSupportForNamedVectors(opts), + this.checkSupportForBm25AndHybridGroupByQueries('Hybrid', opts), + this.checkSupportForHybridNearTextAndNearVectorSubSearches(opts), + ]); + return { + search: await this.connection.search(this.name, this.consistencyLevel, this.tenant), + supportsTargets, + }; + }; + private async parseReply(reply: SearchReply) { const deserialize = await Deserialize.use(this.dbVersionSupport); return deserialize.generate(reply); @@ -161,14 +180,10 @@ class GenerateManager implements Generate { opts: GroupByHybridOptions ): Promise>; public hybrid(query: string, generate: GenerateOptions, opts?: HybridOptions): GenerateReturn { - return Promise.all([ - this.checkSupportForNamedVectors(opts), - this.checkSupportForBm25AndHybridGroupByQueries('Bm25', opts), - ]) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.hybridSearch(opts) + .then(({ search, supportsTargets }) => search.withHybrid({ - ...Serialize.hybrid({ query, supportsTargets: false, ...opts }), + ...Serialize.hybrid({ query, supportsTargets, ...opts }), generative: Serialize.generative(generate), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) diff --git a/src/collections/generate/integration.test.ts b/src/collections/generate/integration.test.ts index 75660a01..93d722f2 100644 --- a/src/collections/generate/integration.test.ts +++ b/src/collections/generate/integration.test.ts @@ -7,6 +7,15 @@ import { GenerateOptions, GroupByOptions } from '../types/index.js'; const maybe = process.env.OPENAI_APIKEY ? describe : describe.skip; +const makeOpenAIClient = () => + weaviate.connectToLocal({ + port: 8086, + grpcPort: 50057, + headers: { + 'X-Openai-Api-Key': process.env.OPENAI_APIKEY!, + }, + }); + maybe('Testing of the collection.generate methods with a simple collection', () => { let client: WeaviateClient; let collection: Collection; @@ -32,13 +41,7 @@ maybe('Testing of the collection.generate methods with a simple collection', () }); beforeAll(async () => { - client = await weaviate.connectToLocal({ - port: 8086, - grpcPort: 50057, - headers: { - 'X-Openai-Api-Key': process.env.OPENAI_APIKEY!, - }, - }); + client = await makeOpenAIClient(); collection = client.collections.get(collectionName); id = await client.collections .create({ @@ -179,13 +182,7 @@ maybe('Testing of the groupBy collection.generate methods with a simple collecti }); beforeAll(async () => { - client = await weaviate.connectToLocal({ - port: 8086, - grpcPort: 50057, - headers: { - 'X-Openai-Api-Key': process.env.OPENAI_APIKEY!, - }, - }); + client = await makeOpenAIClient(); collection = client.collections.get(collectionName); id = await client.collections .create({ @@ -314,3 +311,116 @@ maybe('Testing of the groupBy collection.generate methods with a simple collecti expect(ret.objects[0].belongsToGroup).toEqual('test'); }); }); + +maybe('Testing of the collection.generate methods with a multi vector collection', () => { + let client: WeaviateClient; + let collection: Collection; + const collectionName = 'TestCollectionQueryWithMultiVector'; + + let id1: string; + let id2: string; + let titleVector: number[]; + let title2Vector: number[]; + + afterAll(() => { + return client.collections.delete(collectionName).catch((err) => { + console.error(err); + throw err; + }); + }); + + beforeAll(async () => { + client = await makeOpenAIClient(); + collection = client.collections.get(collectionName); + const query = () => + client.collections + .create({ + name: collectionName, + properties: [ + { + name: 'title', + dataType: 'text', + vectorizePropertyName: false, + }, + ], + vectorizers: [ + weaviate.configure.vectorizer.text2VecOpenAI({ + name: 'title', + sourceProperties: ['title'], + }), + weaviate.configure.vectorizer.text2VecOpenAI({ + name: 'title2', + sourceProperties: ['title'], + }), + ], + }) + .then(async () => { + id1 = await collection.data.insert({ + properties: { + title: 'test', + }, + }); + id2 = await collection.data.insert({ + properties: { + title: 'other', + }, + }); + const res = await collection.query.fetchObjectById(id1, { includeVector: true }); + titleVector = res!.vectors.title!; + title2Vector = res!.vectors.title2!; + }); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 24, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + return query(); + }); + + it('should generate with a near vector search on multi vectors', async () => { + const query = () => + collection.generate.nearVector( + [titleVector, title2Vector], + { + groupedTask: 'What is the value of title here? {title}', + groupedProperties: ['title'], + singlePrompt: 'Write a haiku about ducks for {title}', + }, + { + targetVector: ['title', 'title2'], + } + ); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(2); + expect(ret.generated).toBeDefined(); + expect(ret.objects[0].generated).toBeDefined(); + expect(ret.objects[1].generated).toBeDefined(); + }); + + it('should generate with a near vector search on multi vectors', async () => { + const query = () => + collection.generate.nearVector( + { title: titleVector, title2: title2Vector }, + { + groupedTask: 'What is the value of title here? {title}', + groupedProperties: ['title'], + singlePrompt: 'Write a haiku about ducks for {title}', + }, + { + targetVector: ['title', 'title2'], + } + ); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(2); + expect(ret.generated).toBeDefined(); + expect(ret.objects[0].generated).toBeDefined(); + expect(ret.objects[1].generated).toBeDefined(); + }); +}); diff --git a/src/collections/generate/types.ts b/src/collections/generate/types.ts index 24cccacf..b211a46a 100644 --- a/src/collections/generate/types.ts +++ b/src/collections/generate/types.ts @@ -13,6 +13,7 @@ import { NearMediaType, NearOptions, NearTextOptions, + NearVectorInputType, } from '../query/types.js'; import { GenerateOptions, @@ -301,13 +302,13 @@ interface NearVector { * * This overload is for performing a search without the `groupBy` param. * - * @param {number[]} vector - The vector to search for. + * @param {NearVectorInputType} vector - The vector(s) to search for. * @param {GenerateOptions} generate - The available options for performing the generation. * @param {BaseNearOptions} [opts] - The available options for performing the near-vector search. * @return {Promise>} - The results of the search including the generated data. */ nearVector( - vector: number[], + vector: NearVectorInputType, generate: GenerateOptions, opts?: BaseNearOptions ): Promise>; @@ -318,13 +319,13 @@ interface NearVector { * * This overload is for performing a search with the `groupBy` param. * - * @param {number[]} vector - The vector to search for. + * @param {NearVectorInputType} vector - The vector(s) to search for. * @param {GenerateOptions} generate - The available options for performing the generation. * @param {GroupByNearOptions} opts - The available options for performing the near-vector search. * @return {Promise>} - The results of the search including the generated data grouped by the specified properties. */ nearVector( - vector: number[], + vector: NearVectorInputType, generate: GenerateOptions, opts: GroupByNearOptions ): Promise>; @@ -335,12 +336,16 @@ interface NearVector { * * This overload is for performing a search with a programmatically defined `opts` param. * - * @param {number[]} vector - The vector to search for. + * @param {NearVectorInputType} vector - The vector(s) to search for. * @param {GenerateOptions} generate - The available options for performing the generation. * @param {NearOptions} [opts] - The available options for performing the near-vector search. * @return {GenerateReturn} - The results of the search including the generated data. */ - nearVector(vector: number[], generate: GenerateOptions, opts?: NearOptions): GenerateReturn; + nearVector( + vector: NearVectorInputType, + generate: GenerateOptions, + opts?: NearOptions + ): GenerateReturn; } export interface Generate diff --git a/src/collections/query/index.ts b/src/collections/query/index.ts index 410bd797..d261f9a9 100644 --- a/src/collections/query/index.ts +++ b/src/collections/query/index.ts @@ -93,9 +93,22 @@ class QueryManager implements Query { }; private nearSearch = async (opts?: BaseNearOptions) => { - const [_, supportsTargets] = await Promise.all([ + const [supportsTargets] = await Promise.all([ + this.checkSupportForMultiTargetVectorSearch(opts), this.checkSupportForNamedVectors(opts), + ]); + return { + search: await this.connection.search(this.name, this.consistencyLevel, this.tenant), + supportsTargets, + }; + }; + + private hybridSearch = async (opts?: BaseHybridOptions) => { + const [supportsTargets] = await Promise.all([ this.checkSupportForMultiTargetVectorSearch(opts), + this.checkSupportForNamedVectors(opts), + this.checkSupportForBm25AndHybridGroupByQueries('Hybrid', opts), + this.checkSupportForHybridNearTextAndNearVectorSubSearches(opts), ]); return { search: await this.connection.search(this.name, this.consistencyLevel, this.tenant), @@ -153,15 +166,10 @@ class QueryManager implements Query { public hybrid(query: string, opts?: BaseHybridOptions): Promise>; public hybrid(query: string, opts: GroupByHybridOptions): Promise>; public hybrid(query: string, opts?: HybridOptions): QueryReturn { - return Promise.all([ - this.checkSupportForNamedVectors(opts), - this.checkSupportForBm25AndHybridGroupByQueries('Hybrid', opts), - this.checkSupportForHybridNearTextAndNearVectorSubSearches(opts), - ]) - .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant)) - .then((search) => + return this.hybridSearch(opts) + .then(({ search, supportsTargets }) => search.withHybrid({ - ...Serialize.hybrid({ query, supportsTargets: false, ...opts }), + ...Serialize.hybrid({ query, supportsTargets, ...opts }), groupBy: Serialize.isGroupBy>(opts) ? Serialize.groupBy(opts.groupBy) : undefined, diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 856f0c2f..6ac2c7e1 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -653,6 +653,24 @@ describe('Testing of the collection.query methods with a collection with a refer expect(ret.objects[1].properties.title).toEqual('other'); }); + it('should group-by query a multi-target vector search over the named vector spaces', async () => { + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + return; + } + const ret = await collection.query.nearObject(id1, { + returnProperties: ['title'], + targetVector: ['title', 'title2'], + groupBy: { + numberOfGroups: 2, + objectsPerGroup: 1, + property: 'title', + }, + }); + expect(ret.objects.length).toEqual(2); + expect(ret.objects[0].belongsToGroup).toEqual('test'); + expect(ret.objects[1].belongsToGroup).toEqual('other'); + }); + it('should query a weighted multi-target vector search over the named vector spaces', async () => { const query = () => collection.query.nearObject(id1, { @@ -671,6 +689,80 @@ describe('Testing of the collection.query methods with a collection with a refer expect(ret.objects.length).toEqual(1); expect(ret.objects[0].properties.title).toEqual('test'); }); + + it('should group-by query a weighted multi-target vector search over the named vector spaces', async () => { + const query = () => + collection.query.nearObject(id1, { + returnProperties: ['title'], + targetVector: collection.multiTargetVector.sum(['title', 'title2']), + groupBy: { + numberOfGroups: 2, + objectsPerGroup: 1, + property: 'title', + }, + }); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(2); + expect(ret.objects[0].belongsToGroup).toEqual('test'); + expect(ret.objects[1].belongsToGroup).toEqual('other'); + }); + + it('should perform a hybrid query over the named vector spaces', async () => { + const query = () => + collection.query.hybrid('test', { + returnProperties: ['title'], + targetVector: ['title', 'title2'], + }); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(2); + expect(ret.objects[0].properties.title).toEqual('test'); + expect(ret.objects[1].properties.title).toEqual('other'); + }); + + it('should perform a group-by hybrid query over the named vector spaces', async () => { + const query = () => + collection.query.hybrid('test', { + returnProperties: ['title'], + targetVector: ['title', 'title2'], + groupBy: { + numberOfGroups: 2, + objectsPerGroup: 1, + property: 'title', + }, + }); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(2); + expect(ret.objects[0].belongsToGroup).toEqual('test'); + expect(ret.objects[1].belongsToGroup).toEqual('other'); + }); + + it('should perform a weighted hybrid query over the named vector spaces', async () => { + const query = () => + collection.query.hybrid('test', { + returnProperties: ['title'], + targetVector: collection.multiTargetVector.sum(['title', 'title2']), + }); + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) { + await expect(query()).rejects.toThrow(WeaviateUnsupportedFeatureError); + return; + } + const ret = await query(); + expect(ret.objects.length).toEqual(2); + expect(ret.objects[0].properties.title).toEqual('test'); + expect(ret.objects[1].properties.title).toEqual('other'); + }); }); }); diff --git a/src/collections/query/types.ts b/src/collections/query/types.ts index a085b760..33d1706b 100644 --- a/src/collections/query/types.ts +++ b/src/collections/query/types.ts @@ -96,13 +96,13 @@ export type BaseHybridOptions = SearchOptions & { /** The weight of the BM25 score. If not specified, the default weight specified by the server is used. */ alpha?: number; /** The specific vector to search for or a specific vector subsearch. If not specified, the query is vectorized and used in the similarity search. */ - vector?: number[] | HybridNearTextSubSearch | HybridNearVectorSubSearch; + vector?: NearVectorInputType | HybridNearTextSubSearch | HybridNearVectorSubSearch; /** The properties to search in. If not specified, all properties are searched. */ queryProperties?: PrimitiveKeys[]; /** The type of fusion to apply. If not specified, the default fusion type specified by the server is used. */ fusionType?: 'Ranked' | 'RelativeScore'; - /** Specify which vector to search on if using named vectors. */ - targetVector?: string; + /** Specify which vector(s) to search on if using named vectors. */ + targetVector?: TargetVectorInputType; }; export type HybridSubSearchBase = { @@ -117,7 +117,7 @@ export type HybridNearTextSubSearch = HybridSubSearchBase & { }; export type HybridNearVectorSubSearch = HybridSubSearchBase & { - vector: number[]; + vector: NearVectorInputType; }; /** Options available in the `query.hybrid` method when specifying the `groupBy` parameter. */ diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index 340b8de5..00e7539e 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -368,8 +368,10 @@ export class Serialize { }; }; - private static isHybridVectorSearch = (vector: BaseHybridOptions['vector']): vector is number[] => { - return Array.isArray(vector); + private static isHybridVectorSearch = ( + vector: BaseHybridOptions['vector'] + ): vector is number[] | number[][] | Record => { + return !Serialize.isHybridNearTextSearch(vector) && !Serialize.isHybridNearVectorSearch(vector); }; private static isHybridNearTextSearch = ( @@ -384,26 +386,46 @@ export class Serialize { return (vector as HybridNearVectorSubSearch)?.vector !== undefined; }; - private static hybridVector = (vector: BaseHybridOptions['vector']): Uint8Array | undefined => { - return Serialize.isHybridVectorSearch(vector) ? Serialize.vectorToBytes(vector) : undefined; - }; - - private static hybridNearText = (vector: BaseHybridOptions['vector']): NearTextSearch | undefined => { - return Serialize.isHybridNearTextSearch(vector) - ? Serialize.nearTextSearch({ - ...vector, - supportsTargets: false, - query: vector.query, - }) - : undefined; - }; - - private static hybridNearVector = (vector: BaseHybridOptions['vector']): NearVector | undefined => { - return Serialize.isHybridNearVectorSearch(vector) - ? NearVector.fromPartial({ - vectorBytes: Serialize.vectorToBytes(vector.vector), - }) - : undefined; + private static hybridVector = (args: { + supportsTargets: boolean; + vector?: BaseHybridOptions['vector']; + }) => { + const vector = args.vector; + if (Serialize.isHybridVectorSearch(vector)) { + const { targets, targetVectors, vectorBytes } = Serialize.vectors({ ...args, vector: vector }); + return { vectorBytes, targetVectors, targets }; + } else if (Serialize.isHybridNearTextSearch(vector)) { + const { targetVectors, targets } = Serialize.vectors({ ...args, vector: undefined }); + return { + targets, + targetVectors, + nearText: NearTextSearch.fromPartial({ + query: typeof vector.query === 'string' ? [vector.query] : vector.query, + certainty: vector.certainty, + distance: vector.distance, + moveAway: vector.moveAway ? NearTextSearch_Move.fromPartial(vector.moveAway) : undefined, + moveTo: vector.moveTo ? NearTextSearch_Move.fromPartial(vector.moveTo) : undefined, + }), + }; + } else if (Serialize.isHybridNearVectorSearch(vector)) { + const { targetVectors, targets, vectorBytes, vectorPerTarget } = Serialize.vectors({ + ...args, + vector: vector.vector, + }); + return { + targetVectors, + targets, + nearVector: NearVector.fromPartial({ + certainty: vector.certainty, + distance: vector.distance, + vectorBytes, + vectorPerTarget, + }), + }; + } else { + const { targets, targetVectors } = Serialize.targetVector(args); + return { targets, targetVectors }; + } }; public static hybrid = ( @@ -419,19 +441,19 @@ export class Serialize { return Hybrid_FusionType.FUSION_TYPE_UNSPECIFIED; } }; - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors, vectorBytes, nearText, nearVector } = Serialize.hybridVector(args); return { ...Serialize.common(args), hybridSearch: Hybrid.fromPartial({ query: args.query, alpha: args.alpha ? args.alpha : 0.5, properties: args.queryProperties, - vectorBytes: Serialize.hybridVector(args.vector), + vectorBytes: vectorBytes, fusionType: fusionType(args.fusionType), targetVectors, targets, - nearText: Serialize.hybridNearText(args.vector), - nearVector: Serialize.hybridNearVector(args.vector), + nearText, + nearVector, }), autocut: args.autoLimit, }; @@ -440,7 +462,7 @@ export class Serialize { public static nearAudio = ( args: { audio: string; supportsTargets: boolean } & NearOptions ): SearchNearAudioArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearAudio: NearAudioSearch.fromPartial({ @@ -457,7 +479,7 @@ export class Serialize { public static nearDepth = ( args: { depth: string; supportsTargets: boolean } & NearOptions ): SearchNearDepthArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearDepth: NearDepthSearch.fromPartial({ @@ -474,7 +496,7 @@ export class Serialize { public static nearImage = ( args: { image: string; supportsTargets: boolean } & NearOptions ): SearchNearImageArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearImage: NearImageSearch.fromPartial({ @@ -491,7 +513,7 @@ export class Serialize { public static nearIMU = ( args: { imu: string; supportsTargets: boolean } & NearOptions ): SearchNearIMUArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearIMU: NearIMUSearch.fromPartial({ @@ -508,7 +530,7 @@ export class Serialize { public static nearObject = ( args: { id: string; supportsTargets: boolean } & NearOptions ): SearchNearObjectArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearObject: NearObject.fromPartial({ @@ -525,19 +547,19 @@ export class Serialize { private static nearTextSearch = (args: { query: string | string[]; supportsTargets: boolean; + targetVector?: TargetVectorInputType; certainty?: number; distance?: number; - targetVector?: TargetVectorInputType; moveAway?: { concepts?: string[]; force?: number; objects?: string[] }; moveTo?: { concepts?: string[]; force?: number; objects?: string[] }; }) => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return NearTextSearch.fromPartial({ query: typeof args.query === 'string' ? [args.query] : args.query, certainty: args.certainty, distance: args.distance, - targetVectors, targets, + targetVectors, moveAway: args.moveAway ? NearTextSearch_Move.fromPartial({ concepts: args.moveAway.concepts, @@ -560,7 +582,7 @@ export class Serialize { ): SearchNearTextArgs => { return { ...Serialize.common(args), - nearText: Serialize.nearTextSearch(args), + nearText: this.nearTextSearch(args), autocut: args.autoLimit, }; }; @@ -568,7 +590,7 @@ export class Serialize { public static nearThermal = ( args: { thermal: string; supportsTargets: boolean } & NearOptions ): SearchNearThermalArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearThermal: NearThermalSearch.fromPartial({ @@ -593,13 +615,59 @@ export class Serialize { distance?: number; targetVector?: TargetVectorInputType; }) => { - const invalidNearVectorError = new WeaviateInvalidInputError(`near vector argument can be: - - an array of numbers - - an array of arrays of numbers for multi target search - - an object with target names as keys and arrays of numbers as values - received: ${args.vector}`); + const { targetVectors, targets, vectorBytes, vectorPerTarget } = Serialize.vectors(args); + return NearVector.fromPartial({ + certainty: args.certainty, + distance: args.distance, + targetVectors, + targets, + vectorPerTarget, + vectorBytes, + }); + }; - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + private static targetVector = (args: { + supportsTargets: boolean; + targetVector?: TargetVectorInputType; + }): { targets?: Targets; targetVectors?: string[] } => { + if (args.targetVector === undefined) { + return {}; + } else if (TargetVectorInputGuards.isSingle(args.targetVector)) { + return args.supportsTargets + ? { + targets: Targets.fromPartial({ + targetVectors: [args.targetVector], + }), + } + : { targetVectors: [args.targetVector] }; + } else if (TargetVectorInputGuards.isMulti(args.targetVector)) { + return args.supportsTargets + ? { + targets: Targets.fromPartial({ + targetVectors: args.targetVector, + }), + } + : { targetVectors: args.targetVector }; + } else { + return { targets: Serialize.targets(args.targetVector) }; + } + }; + + private static vectors = (args: { + supportsTargets: boolean; + targetVector?: TargetVectorInputType; + vector?: NearVectorInputType; + }) => { + const invalidNearVectorError = new WeaviateInvalidInputError(`near vector argument can be: + - an array of numbers + - an array of arrays of numbers for multi target search + - an object with target names as keys and arrays of numbers as values + received: ${args.vector}`); + + const { targets, targetVectors } = Serialize.targetVector(args); + if (args.vector === undefined) { + return { targetVectors, targets }; + } if (NearVectorInputGuards.isObject(args.vector)) { if (targets === undefined || targets.targetVectors.length != Object.keys(args.vector).length) { throw new WeaviateInvalidInputError( @@ -610,26 +678,22 @@ export class Serialize { Object.entries(args.vector).forEach(([k, v]) => { vectorPerTarget[k] = Serialize.vectorToBytes(v); }); - return NearVector.fromPartial({ - certainty: args.certainty, - distance: args.distance, + return { targetVectors, targets, - vectorPerTarget: vectorPerTarget, - }); + vectorPerTarget, + }; } else { if (args.vector.length === 0) { throw invalidNearVectorError; } if (NearVectorInputGuards.is1DArray(args.vector)) { const vectorBytes = Serialize.vectorToBytes(args.vector); - return NearVector.fromPartial({ - certainty: args.certainty, - distance: args.distance, + return { targetVectors, targets, vectorBytes, - }); + }; } if (NearVectorInputGuards.is2DArray(args.vector)) { if (targets === undefined || targets.targetVectors.length != args.vector.length) { @@ -641,45 +705,16 @@ export class Serialize { args.vector.forEach((v, i) => { vectorPerTarget[targets.targetVectors[i]] = Serialize.vectorToBytes(v); }); - return NearVector.fromPartial({ - certainty: args.certainty, - distance: args.distance, + return { targetVectors, targets, vectorPerTarget, - }); + }; } throw invalidNearVectorError; } }; - private static targetVector = ( - supportsTargets: boolean, - targetVector?: TargetVectorInputType - ): { targets?: Targets; targetVectors?: string[] } => { - if (targetVector === undefined) { - return {}; - } else if (TargetVectorInputGuards.isSingle(targetVector)) { - return supportsTargets - ? { - targets: Targets.fromPartial({ - targetVectors: [targetVector], - }), - } - : { targetVectors: [targetVector] }; - } else if (TargetVectorInputGuards.isMulti(targetVector)) { - return supportsTargets - ? { - targets: Targets.fromPartial({ - targetVectors: targetVector, - }), - } - : { targetVectors: targetVector }; - } else { - return { targets: Serialize.targets(targetVector) }; - } - }; - private static targets = (targets: MultiTargetVectorJoin) => { let combination: CombinationMethod; switch (targets.combination) { @@ -721,7 +756,7 @@ export class Serialize { public static nearVideo = ( args: { video: string; supportsTargets: boolean } & NearOptions ): SearchNearVideoArgs => { - const { targets, targetVectors } = Serialize.targetVector(args.supportsTargets, args.targetVector); + const { targets, targetVectors } = Serialize.targetVector(args); return { ...Serialize.common(args), nearVideo: NearVideoSearch.fromPartial({ From 0541640cb240ea873e95ead8954df79624f5d261 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 8 Jul 2024 16:42:19 +0100 Subject: [PATCH 16/31] Make breaking change to introduce ACTIVE/INACTIVE in returns but allow HOT/COLD in create/update --- src/collections/deserialize/index.ts | 10 +++-- src/collections/query/integration.test.ts | 2 +- src/collections/serialize/index.ts | 33 +++++++---------- src/collections/tenants/index.ts | 41 +++++++++++---------- src/collections/tenants/integration.test.ts | 14 +++---- src/collections/tenants/types.ts | 23 +++++++----- src/collections/tenants/unit.test.ts | 8 ++-- 7 files changed, 65 insertions(+), 66 deletions(-) diff --git a/src/collections/deserialize/index.ts b/src/collections/deserialize/index.ts index e1130a59..00a9e45f 100644 --- a/src/collections/deserialize/index.ts +++ b/src/collections/deserialize/index.ts @@ -305,9 +305,9 @@ export class Deserialize { private static activityStatusGRPC(status: TenantActivityStatus): Tenant['activityStatus'] { switch (status) { case TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD: - return 'COLD'; + return 'INACTIVE'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_HOT: - return 'HOT'; + return 'ACTIVE'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN: return 'OFFLOADED'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: @@ -322,15 +322,17 @@ export class Deserialize { public static activityStatusREST(status: TenantREST['activityStatus']): Tenant['activityStatus'] { switch (status) { case 'COLD': - return 'COLD'; + return 'INACTIVE'; case 'HOT': - return 'HOT'; + return 'ACTIVE'; case 'FROZEN': return 'OFFLOADED'; case 'FREEZING': return 'OFFLOADING'; case 'UNFREEZING': return 'ONLOADING'; + case undefined: + return 'ACTIVE'; default: throw new Error(`Unsupported tenant activity status: ${status}`); } diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 482475f7..f9e5e6e8 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -909,7 +909,7 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio expect(obj2.objects[0].uuid).toEqual(id2); }); - it.skip('should find the objects in their tenants by nearObject', async () => { + it('should find the objects in their tenants by nearObject', async () => { const obj1 = await collection.withTenant(tenantOne).query.nearObject(id1); const obj2 = await collection.withTenant(tenantTwo).query.nearObject(id2); expect(obj1.objects.length).toEqual(1); diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index 468b1554..ce7ef963 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -1,5 +1,5 @@ import { v4 as uuidv4 } from 'uuid'; -import { TenantActivityStatus, WhereFilter } from '../../openapi/types.js'; +import { WhereFilter } from '../../openapi/types.js'; import { BatchObject as BatchObjectGRPC, BatchObject_MultiTargetRefProps, @@ -82,7 +82,7 @@ import { import { ReferenceGuards } from '../references/classes.js'; import { Beacon } from '../references/index.js'; import { uuidToBeacon } from '../references/utils.js'; -import { Tenant, TenantCreate, TenantUpdate } from '../tenants/types.js'; +import { TenantBC, TenantCreate, TenantUpdate } from '../tenants/types.js'; import { BatchObject, BatchObjects, @@ -1137,11 +1137,11 @@ export class Serialize { }); }; - public static tenantsCreate(tenant: Tenant | TenantCreate): { + public static tenantsCreate(tenant: TenantBC | TenantCreate): { name: string; activityStatus?: 'HOT' | 'COLD'; } { - let activityStatus: TenantActivityStatus; + let activityStatus: 'HOT' | 'COLD' | undefined; switch (tenant.activityStatus) { case 'ACTIVE': activityStatus = 'HOT'; @@ -1154,17 +1154,13 @@ export class Serialize { case undefined: activityStatus = tenant.activityStatus; break; - case 'OFFLOADED': - throw new WeaviateInvalidInputError( - 'Cannot create a tenant with activity status OFFLOADED. Add objects to the tenant first and then you can update it to OFFLOADED.' - ); - case 'OFFLOADING': + case 'FROZEN': throw new WeaviateInvalidInputError( - 'Cannot create a tenant with activity status OFFLOADING. This status is a read-only value that the server sets in the processing of making a tenant OFFLOADED.' + 'Invalid activity status. Please provide one of the following: ACTIVE, INACTIVE, HOT, COLD.' ); - case 'ONLOADING': + default: throw new WeaviateInvalidInputError( - 'Cannot create a tenant with activity status ONLOADING. This status is a read-only value that the server sets in the processing of making a tenant HOT.' + 'Invalid activity status. Please provide one of the following: ACTIVE, INACTIVE, HOT, COLD.' ); } return { @@ -1174,9 +1170,9 @@ export class Serialize { } public static tenantUpdate = ( - tenant: Tenant | TenantUpdate + tenant: TenantBC | TenantUpdate ): { name: string; activityStatus: 'HOT' | 'COLD' | 'FROZEN' } => { - let activityStatus: TenantActivityStatus; + let activityStatus: 'HOT' | 'COLD' | 'FROZEN'; switch (tenant.activityStatus) { case 'ACTIVE': activityStatus = 'HOT'; @@ -1189,15 +1185,12 @@ export class Serialize { break; case 'HOT': case 'COLD': + case 'FROZEN': activityStatus = tenant.activityStatus; break; - case 'OFFLOADING': - throw new WeaviateInvalidInputError( - 'Cannot create a tenant with activity status OFFLOADING. This status is a read-only value that the server sets in the processing of making a tenant OFFLOADED.' - ); - case 'ONLOADING': + default: throw new WeaviateInvalidInputError( - 'Cannot create a tenant with activity status ONLOADING. This status is a read-only value that the server sets in the processing of making a tenant HOT.' + 'Invalid activity status. Please provide one of the following: ACTIVE, INACTIVE, HOT, COLD, OFFLOADED.' ); } return { diff --git a/src/collections/tenants/index.ts b/src/collections/tenants/index.ts index 8c62d8d4..5390ac66 100644 --- a/src/collections/tenants/index.ts +++ b/src/collections/tenants/index.ts @@ -1,10 +1,11 @@ import { ConnectionGRPC } from '../../connection/index.js'; import { WeaviateUnsupportedFeatureError } from '../../errors.js'; +import { Tenant as TenantREST } from '../../openapi/types.js'; import { TenantsCreator, TenantsDeleter, TenantsGetter, TenantsUpdater } from '../../schema/index.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; import { Deserialize } from '../deserialize/index.js'; import { Serialize } from '../serialize/index.js'; -import { Tenant, TenantBase, TenantCreate, TenantUpdate } from './types.js'; +import { Tenant, TenantBC, TenantBase, TenantCreate, TenantUpdate } from './types.js'; const checkSupportForGRPCTenantsGetEndpoint = async (dbVersionSupport: DbVersionSupport) => { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); @@ -16,6 +17,13 @@ const parseValueOrValueArray = (value: V | V[]) => (Array.isArray(value) ? va const parseStringOrTenant = (tenant: string | T) => typeof tenant === 'string' ? tenant : tenant.name; +const parseTenantREST = (tenant: TenantREST): Tenant => { + return { + name: tenant.name!, + activityStatus: Deserialize.activityStatusREST(tenant.activityStatus), + }; +}; + const tenants = ( connection: ConnectionGRPC, collection: string, @@ -31,20 +39,15 @@ const tenants = ( const result: Record = {}; tenants.forEach((tenant) => { if (!tenant.name) return; - result[tenant.name] = { - name: tenant.name!, - activityStatus: Deserialize.activityStatusREST(tenant.activityStatus), - }; + result[tenant.name] = parseTenantREST(tenant); }); return result; }); return { - create: (tenants: Tenant | TenantCreate | (Tenant | TenantCreate)[]) => - new TenantsCreator( - connection, - collection, - parseValueOrValueArray(tenants).map(Serialize.tenantsCreate) - ).do() as Promise, + create: (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => + new TenantsCreator(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantsCreate)) + .do() + .then((res) => res.map(parseTenantREST)), get: async function () { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); return check.supports ? getGRPC() : getREST(); @@ -60,12 +63,10 @@ const tenants = ( collection, parseValueOrValueArray(tenants).map(parseStringOrTenant) ).do(), - update: (tenants: Tenant | TenantUpdate | (Tenant | TenantUpdate)[]) => - new TenantsUpdater( - connection, - collection, - parseValueOrValueArray(tenants).map(Serialize.tenantUpdate) - ).do() as Promise, + update: (tenants: TenantBC | TenantUpdate | (TenantBC | TenantUpdate)[]) => + new TenantsUpdater(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantUpdate)) + .do() + .then((res) => res.map(parseTenantREST)), }; }; @@ -97,10 +98,10 @@ export interface Tenants { * * For details on the new activity statuses, see the docstring for the `Tenants` interface type. * - * @param {Tenant | TenantCreate | (Tenant | TenantCreate)[]} tenants The tenant or tenants to create. + * @param {TenantCreate | TenantCreate[]} tenants The tenant or tenants to create. * @returns {Promise} The created tenant(s) as a list of Tenant. */ - create: (tenants: Tenant | TenantCreate | (Tenant | TenantCreate)[]) => Promise; + create: (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => Promise; /** * Return all tenants currently associated with a collection in Weaviate. * The collection must have been created with multi-tenancy enabled. @@ -152,5 +153,5 @@ export interface Tenants { * @param {TenantInput | TenantInput[]} tenants The tenant or tenants to update. * @returns {Promise} The updated tenant(s) as a list of Tenant. */ - update: (tenants: Tenant | TenantUpdate | (Tenant | TenantUpdate)[]) => Promise; + update: (tenants: TenantBC | TenantUpdate | (TenantBC | TenantUpdate)[]) => Promise; } diff --git a/src/collections/tenants/integration.test.ts b/src/collections/tenants/integration.test.ts index 5c24c8d0..aa623e5f 100644 --- a/src/collections/tenants/integration.test.ts +++ b/src/collections/tenants/integration.test.ts @@ -38,7 +38,7 @@ describe('Testing of the collection.tenants methods', () => { const result = await collection.tenants.create([{ name: tenant, activityStatus: 'HOT' }]); expect(result.length).toBe(1); expect(result[0].name).toBe(tenant); - expect(result[0].activityStatus).toBe('HOT'); + expect(result[0].activityStatus).toBe('ACTIVE'); }); it('should be able to create a tenant with new nomenclature', async () => { @@ -46,7 +46,7 @@ describe('Testing of the collection.tenants methods', () => { const result = await collection.tenants.create([{ name: tenant, activityStatus: 'ACTIVE' }]); expect(result.length).toBe(1); expect(result[0].name).toBe(tenant); - expect(result[0].activityStatus).toBe('HOT'); + expect(result[0].activityStatus).toBe('ACTIVE'); }); it('should be able to get existing tenants', async () => { @@ -54,7 +54,7 @@ describe('Testing of the collection.tenants methods', () => { expect(result).toHaveProperty('hot'); expect(result.hot.name).toBe('hot'); - expect(result.hot.activityStatus).toBe('HOT'); + expect(result.hot.activityStatus).toBe('ACTIVE'); expect(result).toHaveProperty('cold'); expect(result.cold.name).toBe('cold'); @@ -72,14 +72,14 @@ describe('Testing of the collection.tenants methods', () => { const result = await collection.tenants.update([{ name: 'cold', activityStatus: 'HOT' }]); expect(result.length).toBe(1); expect(result[0].name).toBe('cold'); - expect(result[0].activityStatus).toBe('HOT'); + expect(result[0].activityStatus).toBe('ACTIVE'); }); it('should be able to update a tenant with new nomenclature', async () => { const result = await collection.tenants.update([{ name: 'cold-new', activityStatus: 'ACTIVE' }]); expect(result.length).toBe(1); expect(result[0].name).toBe('cold-new'); - expect(result[0].activityStatus).toBe('HOT'); + expect(result[0].activityStatus).toBe('ACTIVE'); }); describe('getByName and getByNames', () => { @@ -91,7 +91,7 @@ describe('Testing of the collection.tenants methods', () => { } const result = await query(); expect(result).toHaveProperty('name', 'hot'); - expect(result).toHaveProperty('activityStatus', 'HOT'); + expect(result).toHaveProperty('activityStatus', 'ACTIVE'); }); it('should be able to get a tenant by tenant object', async () => { @@ -102,7 +102,7 @@ describe('Testing of the collection.tenants methods', () => { } const result = await query(); expect(result).toHaveProperty('name', 'hot'); - expect(result).toHaveProperty('activityStatus', 'HOT'); + expect(result).toHaveProperty('activityStatus', 'ACTIVE'); }); it('should fail to get a non-existing tenant', async () => { diff --git a/src/collections/tenants/types.ts b/src/collections/tenants/types.ts index eefa99c1..35024537 100644 --- a/src/collections/tenants/types.ts +++ b/src/collections/tenants/types.ts @@ -23,17 +23,20 @@ export type TenantsGetOptions = { /** * The expected type returned by all tenant methods. - * - * WARNING: The `COLD` and `HOT` statuses are deprecated and will be replaced in a future release. - * See the docstring for the `activityStatus` field in this type for more information. */ export type Tenant = TenantBase & { - /** - * `COLD` and `HOT` are included for backwards compatability purposes and are deprecated. - * - * In a future release, these will be removed in favour of the new statuses as so: - * - `HOT` -> `ACTIVE` - * - `COLD` -> `INACTIVE` + /** There are two statuses that are immutable: `OFFLOADED` and `ONLOADING, which are set by the server: + * - `ONLOADING`, which means the tenant is transitioning from the `OFFLOADED` status to `ACTIVE/INACTIVE`. + * - `OFFLOADING`, which means the tenant is transitioning from `ACTIVE/INACTIVE` to the `OFFLOADED` status. + * The other three statuses are mutable within the `.create` and `.update`, methods: + * - `ACTIVE`, which means loaded fully into memory and ready for use. + * - `INACTIVE`, which means not loaded into memory with files stored on disk. + * - `OFFLOADED`, which means not loaded into memory with files stored on the cloud. */ - activityStatus: 'COLD' | 'HOT' | 'OFFLOADED' | 'OFFLOADING' | 'ONLOADING'; + activityStatus: 'ACTIVE' | 'INACTIVE' | 'OFFLOADED' | 'OFFLOADING' | 'ONLOADING'; +}; + +/** This is the type of the Tenant as defined in Weaviate's OpenAPI schema. It is included here for Backwards Compatibility. */ +export type TenantBC = TenantBase & { + activityStatus?: 'HOT' | 'COLD' | 'FROZEN'; }; diff --git a/src/collections/tenants/unit.test.ts b/src/collections/tenants/unit.test.ts index 24ae87a5..b1b2e2c5 100644 --- a/src/collections/tenants/unit.test.ts +++ b/src/collections/tenants/unit.test.ts @@ -84,8 +84,8 @@ describe('Mock testing of tenants.get() method with a REST server', () => { const collection = client.collections.get(TENANTS_COLLECTION_NAME); const tenants = await collection.tenants.get(); expect(tenants).toEqual>({ - hot: { name: 'hot', activityStatus: 'HOT' }, - cold: { name: 'cold', activityStatus: 'COLD' }, + hot: { name: 'hot', activityStatus: 'ACTIVE' }, + cold: { name: 'cold', activityStatus: 'INACTIVE' }, frozen: { name: 'frozen', activityStatus: 'OFFLOADED' }, freezing: { name: 'freezing', activityStatus: 'OFFLOADING' }, unfreezing: { name: 'unfreezing', activityStatus: 'ONLOADING' }, @@ -110,8 +110,8 @@ describe('Mock testing of tenants.get() method with a gRPC server', () => { const collection = client.collections.get(TENANTS_COLLECTION_NAME); const tenants = await collection.tenants.get(); expect(tenants).toEqual>({ - hot: { name: 'hot', activityStatus: 'HOT' }, - cold: { name: 'cold', activityStatus: 'COLD' }, + hot: { name: 'hot', activityStatus: 'ACTIVE' }, + cold: { name: 'cold', activityStatus: 'INACTIVE' }, frozen: { name: 'frozen', activityStatus: 'OFFLOADED' }, freezing: { name: 'freezing', activityStatus: 'OFFLOADING' }, unfreezing: { name: 'unfreezing', activityStatus: 'ONLOADING' }, From 4037d8df8d1a4e58876460a7ea06e7bf59d39bd5 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 10:09:59 +0100 Subject: [PATCH 17/31] Add support for configuring and reconfiguring SQ --- src/collections/config/classes.ts | 15 +++- src/collections/config/types/vectorIndex.ts | 8 +- src/collections/config/unit.test.ts | 82 +++++++++++-------- src/collections/config/utils.ts | 18 +++- src/collections/configure/parsing.ts | 23 +++++- .../configure/types/vectorIndex.ts | 13 ++- src/collections/configure/unit.test.ts | 19 +++++ src/collections/configure/vectorIndex.ts | 52 ++++++++++-- 8 files changed, 180 insertions(+), 50 deletions(-) diff --git a/src/collections/config/classes.ts b/src/collections/config/classes.ts index 2269780f..0966a84d 100644 --- a/src/collections/config/classes.ts +++ b/src/collections/config/classes.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { WeaviateInvalidInputError } from '../../errors.js'; import { WeaviateClass, WeaviateInvertedIndexConfig, @@ -118,10 +119,14 @@ export class MergeWithExisting { ): WeaviateVectorIndexConfig { if (update === undefined) return current; if ( - (QuantizerGuards.isBQUpdate(update.quantizer) && ((current?.bq as any) || {}).enabled) || - (QuantizerGuards.isPQUpdate(update.quantizer) && ((current?.pq as any) || {}).enabled) + (QuantizerGuards.isBQUpdate(update.quantizer) && + (((current?.pq as any) || {}).enabled || ((current?.sq as any) || {}).enabled)) || + (QuantizerGuards.isPQUpdate(update.quantizer) && + (((current?.bq as any) || {}).enabled || ((current?.sq as any) || {}).enabled)) || + (QuantizerGuards.isSQUpdate(update.quantizer) && + (((current?.pq as any) || {}).enabled || ((current?.bq as any) || {}).enabled)) ) - throw Error(`Cannot update the quantizer type of an enabled vector index.`); + throw new WeaviateInvalidInputError(`Cannot update the quantizer type of an enabled vector index.`); const { quantizer, ...rest } = update; const merged: WeaviateVectorIndexConfig = { ...current, ...rest }; if (QuantizerGuards.isBQUpdate(quantizer)) { @@ -132,6 +137,10 @@ export class MergeWithExisting { const { type, ...quant } = quantizer; merged.pq = { ...current!.pq!, ...quant, enabled: true }; } + if (QuantizerGuards.isSQUpdate(quantizer)) { + const { type, ...quant } = quantizer; + merged.sq = { ...current!.sq!, ...quant, enabled: true }; + } return merged; } } diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index 9622469e..85e77e42 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -8,7 +8,7 @@ export type VectorIndexConfigHNSW = { ef: number; flatSearchCutoff: number; maxConnections: number; - quantizer: PQConfig | BQConfig | undefined; + quantizer: PQConfig | BQConfig | SQConfig | undefined; skip: boolean; vectorCacheMaxObjects: number; type: 'hnsw'; @@ -45,6 +45,12 @@ export type BQConfig = { type: 'bq'; }; +export type SQConfig = { + rescoreLimit: number; + trainingLimit: number; + type: 'sq'; +}; + export type PQConfig = { bitCompression: boolean; centroids: number; diff --git a/src/collections/config/unit.test.ts b/src/collections/config/unit.test.ts index d7cd4a47..fd044c88 100644 --- a/src/collections/config/unit.test.ts +++ b/src/collections/config/unit.test.ts @@ -43,6 +43,9 @@ describe('Unit testing of the MergeWithExisting class', () => { bq: { enabled: false, }, + sq: { + enabled: false, + }, }, vectorIndexType: 'hnsw', vectorizer: { @@ -55,7 +58,7 @@ describe('Unit testing of the MergeWithExisting class', () => { }; it('should merge a full invertedIndexUpdate with existing schema', () => { - const merged = MergeWithExisting.invertedIndex(Object.assign({}, invertedIndex), { + const merged = MergeWithExisting.invertedIndex(JSON.parse(JSON.stringify(invertedIndex)), { bm25: { b: 0.9, k1: 1.4, @@ -110,7 +113,7 @@ describe('Unit testing of the MergeWithExisting class', () => { }; it('should merge a partial invertedIndexUpdate with existing schema', () => { - const merged = MergeWithExisting.invertedIndex(Object.assign({}, invertedIndex), { + const merged = MergeWithExisting.invertedIndex(JSON.parse(JSON.stringify(invertedIndex)), { bm25: { b: 0.9, }, @@ -134,7 +137,7 @@ describe('Unit testing of the MergeWithExisting class', () => { }); it('should merge a no quantizer HNSW vectorIndexConfig with existing schema', () => { - const merged = MergeWithExisting.vectors(Object.assign({}, hnswVectorConfig), [ + const merged = MergeWithExisting.vectors(JSON.parse(JSON.stringify(hnswVectorConfig)), [ { name: 'name', vectorIndex: { @@ -158,6 +161,7 @@ describe('Unit testing of the MergeWithExisting class', () => { expect(merged).toEqual({ name: { vectorIndexConfig: { + ...hnswVectorConfig.name.vectorIndexConfig, skip: true, cleanupIntervalSeconds: 301, maxConnections: 65, @@ -169,20 +173,6 @@ describe('Unit testing of the MergeWithExisting class', () => { vectorCacheMaxObjects: 1000000000001, flatSearchCutoff: 40001, distance: 'euclidean', - pq: { - enabled: false, - bitCompression: false, - segments: 0, - centroids: 256, - trainingLimit: 100000, - encoder: { - type: 'kmeans', - distribution: 'log-normal', - }, - }, - bq: { - enabled: false, - }, }, vectorIndexType: 'hnsw', vectorizer: { @@ -196,7 +186,7 @@ describe('Unit testing of the MergeWithExisting class', () => { }); it('should merge a PQ quantizer HNSW vectorIndexConfig with existing schema', () => { - const merged = MergeWithExisting.vectors(Object.assign({}, hnswVectorConfig), [ + const merged = MergeWithExisting.vectors(JSON.parse(JSON.stringify(hnswVectorConfig)), [ { name: 'name', vectorIndex: { @@ -220,17 +210,7 @@ describe('Unit testing of the MergeWithExisting class', () => { expect(merged).toEqual({ name: { vectorIndexConfig: { - skip: true, - cleanupIntervalSeconds: 301, - maxConnections: 65, - efConstruction: 129, - ef: -2, - dynamicEfMin: 101, - dynamicEfMax: 501, - dynamicEfFactor: 9, - vectorCacheMaxObjects: 1000000000001, - flatSearchCutoff: 40001, - distance: 'euclidean', + ...hnswVectorConfig.name.vectorIndexConfig, pq: { enabled: true, bitCompression: true, @@ -242,9 +222,6 @@ describe('Unit testing of the MergeWithExisting class', () => { distribution: 'normal', }, }, - bq: { - enabled: false, - }, }, vectorIndexType: 'hnsw', vectorizer: { @@ -258,7 +235,7 @@ describe('Unit testing of the MergeWithExisting class', () => { }); it('should merge a BQ quantizer HNSW vectorIndexConfig with existing schema', () => { - const merged = MergeWithExisting.vectors(Object.assign({}, hnswVectorConfig), [ + const merged = MergeWithExisting.vectors(JSON.parse(JSON.stringify(hnswVectorConfig)), [ { name: 'name', vectorIndex: { @@ -292,8 +269,45 @@ describe('Unit testing of the MergeWithExisting class', () => { }); }); + it('should merge a SQ quantizer HNSW vectorIndexConfig with existing schema', () => { + const merged = MergeWithExisting.vectors(JSON.parse(JSON.stringify(hnswVectorConfig)), [ + { + name: 'name', + vectorIndex: { + name: 'hnsw', + config: { + quantizer: { + type: 'sq', + rescoreLimit: 1000, + trainingLimit: 10000, + }, + }, + }, + }, + ]); + expect(merged).toEqual({ + name: { + vectorIndexConfig: { + ...hnswVectorConfig.name.vectorIndexConfig, + sq: { + enabled: true, + rescoreLimit: 1000, + trainingLimit: 10000, + }, + }, + vectorIndexType: 'hnsw', + vectorizer: { + 'text2vec-contextionary': { + properties: ['name'], + vectorizeCollectionName: false, + }, + }, + }, + }); + }); + it('should merge a BQ quantizer Flat vectorIndexConfig with existing schema', () => { - const merged = MergeWithExisting.vectors(Object.assign({}, flatVectorConfig), [ + const merged = MergeWithExisting.vectors(JSON.parse(JSON.stringify(flatVectorConfig)), [ { name: 'name', vectorIndex: { diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index c96d4e0c..a5b791eb 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -37,6 +37,7 @@ import { ReplicationConfig, Reranker, RerankerConfig, + SQConfig, ShardingConfig, VectorConfig, VectorDistance, @@ -356,11 +357,13 @@ class ConfigMapping { throw new WeaviateDeserializationError( 'Vector index vector cache max objects was not returned by Weaviate' ); - let quantizer: PQConfig | BQConfig | undefined; + let quantizer: PQConfig | BQConfig | SQConfig | undefined; if (exists>(v.pq) && v.pq.enabled === true) { quantizer = ConfigMapping.pq(v.pq); } else if (exists>(v.bq) && v.bq.enabled === true) { quantizer = ConfigMapping.bq(v.bq); + } else if (exists>(v.sq) && v.sq.enabled === true) { + quantizer = ConfigMapping.sq(v.sq); } else { quantizer = undefined; } @@ -393,6 +396,19 @@ class ConfigMapping { type: 'bq', }; } + static sq(v?: Record): SQConfig | undefined { + if (v === undefined) throw new WeaviateDeserializationError('SQ was not returned by Weaviate'); + if (!exists(v.enabled)) + throw new WeaviateDeserializationError('SQ enabled was not returned by Weaviate'); + if (v.enabled === false) return undefined; + const rescoreLimit = v.rescoreLimit === undefined ? 1000 : (v.rescoreLimit as number); + const trainingLimit = v.trainingLimit === undefined ? 100000 : (v.trainingLimit as number); + return { + rescoreLimit, + trainingLimit, + type: 'sq', + }; + } static vectorIndexFlat(v: WeaviateVectorIndexConfig): VectorIndexConfigFlat { if (v === undefined) throw new WeaviateDeserializationError('Vector index was not returned by Weaviate'); if (!exists(v.vectorCacheMaxObjects)) diff --git a/src/collections/configure/parsing.ts b/src/collections/configure/parsing.ts index b858d3d6..3a1bae20 100644 --- a/src/collections/configure/parsing.ts +++ b/src/collections/configure/parsing.ts @@ -1,6 +1,19 @@ -import { BQConfigCreate, BQConfigUpdate, PQConfigCreate, PQConfigUpdate } from './types/index.js'; +import { + BQConfigCreate, + BQConfigUpdate, + PQConfigCreate, + PQConfigUpdate, + SQConfigCreate, + SQConfigUpdate, +} from './types/index.js'; -type QuantizerConfig = PQConfigCreate | PQConfigUpdate | BQConfigCreate | BQConfigUpdate; +type QuantizerConfig = + | PQConfigCreate + | PQConfigUpdate + | BQConfigCreate + | BQConfigUpdate + | SQConfigCreate + | SQConfigUpdate; export class QuantizerGuards { static isPQCreate(config?: QuantizerConfig): config is PQConfigCreate { @@ -15,6 +28,12 @@ export class QuantizerGuards { static isBQUpdate(config?: QuantizerConfig): config is BQConfigUpdate { return (config as BQConfigUpdate)?.type === 'bq'; } + static isSQCreate(config?: QuantizerConfig): config is SQConfigCreate { + return (config as SQConfigCreate)?.type === 'sq'; + } + static isSQUpdate(config?: QuantizerConfig): config is SQConfigUpdate { + return (config as SQConfigUpdate)?.type === 'sq'; + } } export function parseWithDefault(value: D | undefined, defaultValue: D): D { diff --git a/src/collections/configure/types/vectorIndex.ts b/src/collections/configure/types/vectorIndex.ts index a703b2d8..275447fa 100644 --- a/src/collections/configure/types/vectorIndex.ts +++ b/src/collections/configure/types/vectorIndex.ts @@ -4,6 +4,7 @@ import { PQConfig, PQEncoderDistribution, PQEncoderType, + SQConfig, VectorDistance, VectorIndexConfigDynamic, VectorIndexConfigFlat, @@ -36,6 +37,14 @@ export type BQConfigUpdate = { type: 'bq'; }; +export type SQConfigCreate = QuantizerRecursivePartial; + +export type SQConfigUpdate = { + rescoreLimit?: number; + trainingLimit?: number; + type: 'sq'; +}; + export type VectorIndexConfigHNSWCreate = RecursivePartial; export type VectorIndexConfigDynamicCreate = RecursivePartial; @@ -48,7 +57,7 @@ export type VectorIndexConfigHNSWUpdate = { dynamicEfFactor?: number; ef?: number; flatSearchCutoff?: number; - quantizer?: PQConfigUpdate | BQConfigUpdate; + quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate; vectorCacheMaxObjects?: number; }; @@ -118,7 +127,7 @@ export type VectorIndexConfigHNSWCreateOptions = { /** The maximum number of connections. Default is 64. */ maxConnections?: number; /** The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one. */ - quantizer?: PQConfigCreate | BQConfigCreate; + quantizer?: PQConfigCreate | BQConfigCreate | SQConfigCreate; /** Whether to skip the index. Default is false. */ skip?: boolean; /** The maximum number of objects to cache in the vector cache. Default is 1000000000000. */ diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index 131360b0..a948cd0b 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -198,6 +198,25 @@ describe('Unit testing of the configure factory class', () => { }, }); }); + + it('should create an hnsw VectorIndexConfig type with SQ quantizer', () => { + const config = configure.vectorIndex.hnsw({ + quantizer: configure.vectorIndex.quantizer.sq({ + rescoreLimit: 100, + trainingLimit: 200, + }), + }); + expect(config).toEqual>({ + name: 'hnsw', + config: { + quantizer: { + rescoreLimit: 100, + trainingLimit: 200, + type: 'sq', + }, + }, + }); + }); }); describe('Unit testing of the vectorizer factory class', () => { diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index 8ca7b07a..7783a841 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -4,6 +4,8 @@ import { BQConfigUpdate, PQConfigCreate, PQConfigUpdate, + SQConfigCreate, + SQConfigUpdate, VectorIndexConfigDynamicCreate, VectorIndexConfigDynamicCreateOptions, VectorIndexConfigFlatCreate, @@ -93,11 +95,11 @@ const configure = { */ quantizer: { /** - * Create a `BQConfigCreate` object to be used when defining the quantizer configuration of a vector index. + * Create an object of type `BQConfigCreate` to be used when defining the quantizer configuration of a vector index. * * @param {boolean} [options.cache] Whether to cache the quantizer. Default is false. * @param {number} [options.rescoreLimit] The rescore limit. Default is 1000. - * @returns {BQConfigCreate} The `BQConfigCreate` object. + * @returns {BQConfigCreate} The object of type `BQConfigCreate`. */ bq: (options?: { cache?: boolean; rescoreLimit?: number }): BQConfigCreate => { return { @@ -107,7 +109,7 @@ const configure = { }; }, /** - * Create a `PQConfigCreate` object to be used when defining the quantizer configuration of a vector index. + * Create an object of type `PQConfigCreate` to be used when defining the quantizer configuration of a vector index. * * @param {boolean} [options.bitCompression] Whether to use bit compression. * @param {number} [options.centroids] The number of centroids[. @@ -115,7 +117,7 @@ const configure = { * @param {PQEncoderType} [options.encoder.type] The encoder type. * @param {number} [options.segments] The number of segments. * @param {number} [options.trainingLimit] The training limit. - * @returns {PQConfigCreate} The `PQConfigCreate` object. + * @returns {PQConfigCreate} The object of type `PQConfigCreate`. */ pq: (options?: { bitCompression?: boolean; @@ -141,6 +143,20 @@ const configure = { type: 'pq', }; }, + /** + * Create an object of type `SQConfigCreate` to be used when defining the quantizer configuration of a vector index. + * + * @param {number} [options.rescoreLimit] The rescore limit. + * @param {number} [options.trainingLimit] The training limit. + * @returns {SQConfigCreate} The object of type `SQConfigCreate`. + */ + sq: (options?: { rescoreLimit?: number; trainingLimit?: number }): SQConfigCreate => { + return { + rescoreLimit: options?.rescoreLimit, + trainingLimit: options?.trainingLimit, + type: 'sq', + }; + }, }, }; @@ -187,7 +203,7 @@ const reconfigure = { dynamicEfMin?: number; ef?: number; flatSearchCutoff?: number; - quantizer?: PQConfigUpdate | BQConfigUpdate; + quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate; vectorCacheMaxObjects?: number; }): ModuleConfig<'hnsw', VectorIndexConfigHNSWUpdate> => { return { @@ -200,7 +216,10 @@ const reconfigure = { */ quantizer: { /** - * Create a `BQConfigUpdate` object to be used when defining the quantizer configuration of a vector index. + * Create an object of type `BQConfigUpdate` to be used when updating the quantizer configuration of a vector index. + * + * NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values. + * So if you want to change the quantizer type, you must recreate the collection. * * @param {boolean} [options.cache] Whether to cache the quantizer. Default is false. * @param {number} [options.rescoreLimit] The rescore limit. Default is 1000. @@ -213,7 +232,10 @@ const reconfigure = { }; }, /** - * Create a `PQConfigCreate` object to be used when defining the quantizer configuration of a vector index. + * Create an object of type `PQConfigUpdate` to be used when updating the quantizer configuration of a vector index. + * + * NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values. + * So if you want to change the quantizer type, you must recreate the collection. * * @param {number} [options.centroids] The number of centroids. Default is 256. * @param {PQEncoderDistribution} [options.pqEncoderDistribution] The encoder distribution. Default is 'log-normal'. @@ -242,6 +264,22 @@ const reconfigure = { type: 'pq', }; }, + /** + * Create an object of type `SQConfigUpdate` to be used when updating the quantizer configuration of a vector index. + * + * NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values. + * So if you want to change the quantizer type, you must recreate the collection. + * + * @param {number} [options.rescoreLimit] The rescore limit. Default is 1000. + * @param {number} [options.trainingLimit] The training limit. Default is 100000. + * @returns {SQConfigUpdate} The configuration object. + */ + sq: (options?: { rescoreLimit?: number; trainingLimit?: number }): SQConfigUpdate => { + return { + ...options, + type: 'sq', + }; + }, }, }; From c3f8c23bfaed3d5b2aaa779b1be839d4418c71cb Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 11:37:24 +0100 Subject: [PATCH 18/31] Enforce maxConnections: 64 to fix flake in test --- src/collections/aggregate/integration.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/collections/aggregate/integration.test.ts b/src/collections/aggregate/integration.test.ts index 172e83df..b7b3a418 100644 --- a/src/collections/aggregate/integration.test.ts +++ b/src/collections/aggregate/integration.test.ts @@ -91,6 +91,7 @@ describe('Testing of the collection.aggregate methods', () => { ], vectorizers: weaviate.configure.vectorizer.text2VecContextionary({ vectorizeCollectionName: false, + vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ maxConnections: 64 }), }), }) .then(async () => { From 4cf9ef880892cc5d66ad776d9897a55445d553b3 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 16:22:30 +0100 Subject: [PATCH 19/31] Update to latest images, allow release on branch as next tag --- .github/workflows/main.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8f2839c6..fad869d1 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,9 +7,9 @@ on: pull_request: env: - WEAVIATE_124: 1.24.19 - WEAVIATE_125: 1.25.5 - WEAVIATE_126: preview--4e2eb3a + WEAVIATE_124: 1.24.20 + WEAVIATE_125: 1.25.7 + WEAVIATE_126: preview--30d0b3d jobs: checks: @@ -83,7 +83,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - run: npm ci - run: npm run build - - run: npm publish + - run: npm publish --tag next env: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTOMATION_TOKEN }} - run: npm run docs From fae8d6c0c6c185ab8c07b504df98123680bf42a0 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 16:42:19 +0100 Subject: [PATCH 20/31] Remove reranker compose stack due to size --- ci/compose.sh | 2 +- ci/docker-compose-rerank.yml | 30 --- src/collections/query/integration.test.ts | 258 +++++++++++----------- 3 files changed, 130 insertions(+), 160 deletions(-) delete mode 100644 ci/docker-compose-rerank.yml diff --git a/ci/compose.sh b/ci/compose.sh index a841e738..ec9a788d 100644 --- a/ci/compose.sh +++ b/ci/compose.sh @@ -21,5 +21,5 @@ function compose_down_all { } function all_weaviate_ports { - echo "8078 8079 8080 8081 8082 8083 8085 8086 8087 8088 8089 8090" + echo "8078 8080 8081 8082 8083 8085 8086 8087 8088 8089 8090" } diff --git a/ci/docker-compose-rerank.yml b/ci/docker-compose-rerank.yml deleted file mode 100644 index 302bee1d..00000000 --- a/ci/docker-compose-rerank.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -version: '3.4' -services: - weaviate-reranker: - command: - - --host - - 0.0.0.0 - - --port - - '8080' - - --scheme - - http - image: semitechnologies/weaviate:${WEAVIATE_VERSION} - ports: - - 8079:8080 - - 50050:50051 - restart: on-failure:0 - environment: - RERANKER_INFERENCE_API: http://reranker-transformers:8080 - QUERY_DEFAULTS_LIMIT: 25 - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' - PERSISTENCE_DATA_PATH: "./data" - DEFAULT_VECTORIZER_MODULE: 'text2vec-openai' - ENABLE_MODULES: 'text2vec-openai,reranker-transformers,generative-openai' - CLUSTER_HOSTNAME: 'node1' - DISABLE_TELEMETRY: 'true' - reranker-transformers: - image: semitechnologies/reranker-transformers:cross-encoder-ms-marco-MiniLM-L-6-v2 - environment: - ENABLE_CUDA: '0' -... \ No newline at end of file diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 482475f7..dcb96ac7 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -949,132 +949,132 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio }); }); -const maybe = process.env.OPENAI_APIKEY ? describe : describe.skip; - -maybe('Testing of collection.query using rerank functionality', () => { - let client: WeaviateClient; - let collection: Collection; - const collectionName = 'TestCollectionRerank'; - let id1: string; - let id2: string; - - afterAll(() => { - return client.collections.delete(collectionName).catch((err) => { - console.error(err); - throw err; - }); - }); - - beforeAll(async () => { - client = await weaviate.connectToLocal({ - port: 8079, - grpcPort: 50050, - headers: { - 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, - }, - }); - collection = client.collections.get(collectionName); - [id1, id2] = await client.collections - .create({ - name: collectionName, - properties: [ - { - name: 'text', - dataType: 'text', - }, - ], - reranker: weaviate.configure.reranker.transformers(), - vectorizers: weaviate.configure.vectorizer.text2VecOpenAI(), - }) - .then(() => - Promise.all([ - collection.data.insert({ - properties: { - text: 'This is a test', - }, - }), - collection.data.insert({ - properties: { - text: 'This is another test', - }, - }), - ]) - ); - }); - - it('should rerank the results in a bm25 query', async () => { - const ret = await collection.query.bm25('test', { - rerank: { - property: 'text', - query: 'another', - }, - }); - const objects = ret.objects; - expect(objects.length).toEqual(2); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[0].properties.text).toEqual('This is another test'); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[1].properties.text).toEqual('This is a test'); - }); - - it('should rerank the results in a hybrid query', async () => { - const ret = await collection.query.hybrid('test', { - rerank: { - property: 'text', - query: 'another', - }, - }); - const objects = ret.objects; - expect(objects.length).toEqual(2); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[0].properties.text).toEqual('This is another test'); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[1].properties.text).toEqual('This is a test'); - }); - - it.skip('should rerank the results in a nearObject query', async () => { - const ret = await collection.query.nearObject(id1, { - rerank: { - property: 'text', - query: 'another', - }, - }); - const objects = ret.objects; - expect(objects.length).toEqual(2); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[0].properties.text).toEqual('This is another test'); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[1].properties.text).toEqual('This is a test'); - }); - - it('should rerank the results in a nearText query', async () => { - const ret = await collection.query.nearText('text', { - rerank: { - property: 'text', - query: 'another', - }, - }); - const objects = ret.objects; - expect(objects.length).toEqual(2); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[0].properties.text).toEqual('This is another test'); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[1].properties.text).toEqual('This is a test'); - }); - - it.skip('should rerank the results in a nearObject query', async () => { - const obj = await collection.query.fetchObjectById(id1, { includeVector: true }); - const ret = await collection.query.nearVector(obj?.vectors.default!, { - rerank: { - property: 'text', - query: 'another', - }, - }); - const objects = ret.objects; - expect(objects.length).toEqual(2); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[0].properties.text).toEqual('This is another test'); - expect(objects[0].metadata?.rerankScore).toBeDefined(); - expect(objects[1].properties.text).toEqual('This is a test'); - }); -}); +// const maybe = process.env.OPENAI_APIKEY ? describe : describe.skip; + +// maybe('Testing of collection.query using rerank functionality', () => { +// let client: WeaviateClient; +// let collection: Collection; +// const collectionName = 'TestCollectionRerank'; +// let id1: string; +// let id2: string; + +// afterAll(() => { +// return client.collections.delete(collectionName).catch((err) => { +// console.error(err); +// throw err; +// }); +// }); + +// beforeAll(async () => { +// client = await weaviate.connectToLocal({ +// port: 8079, +// grpcPort: 50050, +// headers: { +// 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, +// }, +// }); +// collection = client.collections.get(collectionName); +// [id1, id2] = await client.collections +// .create({ +// name: collectionName, +// properties: [ +// { +// name: 'text', +// dataType: 'text', +// }, +// ], +// reranker: weaviate.configure.reranker.transformers(), +// vectorizers: weaviate.configure.vectorizer.text2VecOpenAI(), +// }) +// .then(() => +// Promise.all([ +// collection.data.insert({ +// properties: { +// text: 'This is a test', +// }, +// }), +// collection.data.insert({ +// properties: { +// text: 'This is another test', +// }, +// }), +// ]) +// ); +// }); + +// it('should rerank the results in a bm25 query', async () => { +// const ret = await collection.query.bm25('test', { +// rerank: { +// property: 'text', +// query: 'another', +// }, +// }); +// const objects = ret.objects; +// expect(objects.length).toEqual(2); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[0].properties.text).toEqual('This is another test'); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[1].properties.text).toEqual('This is a test'); +// }); + +// it('should rerank the results in a hybrid query', async () => { +// const ret = await collection.query.hybrid('test', { +// rerank: { +// property: 'text', +// query: 'another', +// }, +// }); +// const objects = ret.objects; +// expect(objects.length).toEqual(2); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[0].properties.text).toEqual('This is another test'); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[1].properties.text).toEqual('This is a test'); +// }); + +// it.skip('should rerank the results in a nearObject query', async () => { +// const ret = await collection.query.nearObject(id1, { +// rerank: { +// property: 'text', +// query: 'another', +// }, +// }); +// const objects = ret.objects; +// expect(objects.length).toEqual(2); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[0].properties.text).toEqual('This is another test'); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[1].properties.text).toEqual('This is a test'); +// }); + +// it('should rerank the results in a nearText query', async () => { +// const ret = await collection.query.nearText('text', { +// rerank: { +// property: 'text', +// query: 'another', +// }, +// }); +// const objects = ret.objects; +// expect(objects.length).toEqual(2); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[0].properties.text).toEqual('This is another test'); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[1].properties.text).toEqual('This is a test'); +// }); + +// it.skip('should rerank the results in a nearObject query', async () => { +// const obj = await collection.query.fetchObjectById(id1, { includeVector: true }); +// const ret = await collection.query.nearVector(obj?.vectors.default!, { +// rerank: { +// property: 'text', +// query: 'another', +// }, +// }); +// const objects = ret.objects; +// expect(objects.length).toEqual(2); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[0].properties.text).toEqual('This is another test'); +// expect(objects[0].metadata?.rerankScore).toBeDefined(); +// expect(objects[1].properties.text).toEqual('This is a test'); +// }); +// }); From df1806d0916687b81b8effc13fc7d99b66611ab4 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 17:13:51 +0100 Subject: [PATCH 21/31] Comment out WCD reranker tests --- src/collections/query/integration.test.ts | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 0a8f2eab..40a3b189 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -1077,7 +1077,7 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio }); }); -// const maybe = process.env.OPENAI_APIKEY ? describe : describe.skip; +// const maybe = process.env.COHERE_APIKEY ? describe : describe.skip; // maybe('Testing of collection.query using rerank functionality', () => { // let client: WeaviateClient; @@ -1094,13 +1094,15 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio // }); // beforeAll(async () => { -// client = await weaviate.connectToLocal({ -// port: 8079, -// grpcPort: 50050, -// headers: { -// 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, -// }, -// }); +// client = await weaviate.connectToWeaviateCloud( +// 'https://piblpmmdsiknacjnm1ltla.c1.europe-west3.gcp.weaviate.cloud', +// { +// authCredentials: 'NOg5AliYnrN6z7dZDuGv7SLVKhTabAaSTKS7', +// headers: { +// 'X-Cohere-Api-Key': process.env.COHERE_APIKEY as string, +// }, +// } +// ); // collection = client.collections.get(collectionName); // [id1, id2] = await client.collections // .create({ @@ -1111,8 +1113,8 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio // dataType: 'text', // }, // ], -// reranker: weaviate.configure.reranker.transformers(), -// vectorizers: weaviate.configure.vectorizer.text2VecOpenAI(), +// reranker: weaviate.configure.reranker.cohere(), +// vectorizers: weaviate.configure.vectorizer.text2VecCohere(), // }) // .then(() => // Promise.all([ @@ -1160,7 +1162,7 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio // expect(objects[1].properties.text).toEqual('This is a test'); // }); -// it.skip('should rerank the results in a nearObject query', async () => { +// it('should rerank the results in a nearObject query', async () => { // const ret = await collection.query.nearObject(id1, { // rerank: { // property: 'text', @@ -1190,7 +1192,7 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio // expect(objects[1].properties.text).toEqual('This is a test'); // }); -// it.skip('should rerank the results in a nearObject query', async () => { +// it('should rerank the results in a nearObject query', async () => { // const obj = await collection.query.fetchObjectById(id1, { includeVector: true }); // const ret = await collection.query.nearVector(obj?.vectors.default!, { // rerank: { From 911ee1401de59eef4e03b5601bbbfb01b54d3402 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 17:15:18 +0100 Subject: [PATCH 22/31] 3.1.0-rc.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2ff12f18..e75725a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "weaviate-client", - "version": "3.0.8", + "version": "3.1.0-rc.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "weaviate-client", - "version": "3.0.8", + "version": "3.1.0-rc.0", "license": "SEE LICENSE IN LICENSE", "dependencies": { "graphql": "^16.8.1", diff --git a/package.json b/package.json index 96b3ff39..dda3a347 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "weaviate-client", - "version": "3.0.8", + "version": "3.1.0-rc.0", "description": "JS/TS client for Weaviate", "main": "dist/node/cjs/index.js", "type": "module", From 5a44b2d31bea50d667920d645ea9f3f4f78b83b1 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 10 Jul 2024 17:18:37 +0100 Subject: [PATCH 23/31] Undo RC release --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dda3a347..96b3ff39 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "weaviate-client", - "version": "3.1.0-rc.0", + "version": "3.0.8", "description": "JS/TS client for Weaviate", "main": "dist/node/cjs/index.js", "type": "module", From 5be92f54ad839974f55dbd36b822821be19d5572 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 11 Jul 2024 14:17:19 +0100 Subject: [PATCH 24/31] Update tenants naming to use latest API types in stable/v1.26 --- .github/workflows/main.yaml | 2 +- src/collections/deserialize/index.ts | 7 +++++- src/openapi/schema.ts | 14 +++++++++-- src/proto/v1/tenants.ts | 35 ++++++++++++++++++++++++---- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index fad869d1..6e2087d7 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.20 WEAVIATE_125: 1.25.7 - WEAVIATE_126: preview--30d0b3d + WEAVIATE_126: preview--8758c8d jobs: checks: diff --git a/src/collections/deserialize/index.ts b/src/collections/deserialize/index.ts index 00a9e45f..d01b7b1b 100644 --- a/src/collections/deserialize/index.ts +++ b/src/collections/deserialize/index.ts @@ -305,14 +305,19 @@ export class Deserialize { private static activityStatusGRPC(status: TenantActivityStatus): Tenant['activityStatus'] { switch (status) { case TenantActivityStatus.TENANT_ACTIVITY_STATUS_COLD: + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_INACTIVE: return 'INACTIVE'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_HOT: + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_ACTIVE: return 'ACTIVE'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FROZEN: + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_OFFLOADED: return 'OFFLOADED'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_OFFLOADING: return 'OFFLOADING'; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFREEZING: + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_ONLOADING: return 'ONLOADING'; default: throw new Error(`Unsupported tenant activity status: ${status}`); @@ -334,7 +339,7 @@ export class Deserialize { case undefined: return 'ACTIVE'; default: - throw new Error(`Unsupported tenant activity status: ${status}`); + return status; } } diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index 9ce33dd0..d8ddf03b 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -1289,10 +1289,20 @@ export interface definitions { /** @description name of the tenant */ name?: string; /** - * @description activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. For creation, allowed values are `HOT` - tenant is fully active and `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally. For updating, `HOT`, `COLD` and also `FROZEN` - as COLD, but files are stored on cloud storage. The following values are read-only and are set by the server for internal use: `FREEZING` - tenant is transitioning from HOT/COLD to FROZEN, `UNFREEZING` - tenant is transitioning from FROZEN to HOT/COLD + * @description activity status of the tenant's shard. Optional for creating tenant (implicit `ACTIVE`) and required for updating tenant. For creation, allowed values are `ACTIVE` - tenant is fully active and `INACTIVE` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally. For updating, `ACTIVE`, `INACTIVE` and also `OFFLOADED` - as INACTIVE, but files are stored on cloud storage. The following values are read-only and are set by the server for internal use: `OFFLOADING` - tenant is transitioning from ACTIVE/INACTIVE to OFFLOADED, `ONLOADING` - tenant is transitioning from OFFLOADED to ACTIVE/INACTIVE. We still accept deprecated names `HOT` (now `ACTIVE`), `COLD` (now `INACTIVE`), `FROZEN` (now `OFFLOADED`), `FREEZING` (now `OFFLOADING`), `UNFREEZING` (now `ONLOADING`). * @enum {string} */ - activityStatus?: 'HOT' | 'COLD' | 'FROZEN' | 'FREEZING' | 'UNFREEZING'; + activityStatus?: + | 'ACTIVE' + | 'INACTIVE' + | 'OFFLOADED' + | 'OFFLOADING' + | 'ONLOADING' + | 'HOT' + | 'COLD' + | 'FROZEN' + | 'FREEZING' + | 'UNFREEZING'; }; } diff --git a/src/proto/v1/tenants.ts b/src/proto/v1/tenants.ts index e35e570b..a369d0f9 100644 --- a/src/proto/v1/tenants.ts +++ b/src/proto/v1/tenants.ts @@ -16,7 +16,12 @@ export enum TenantActivityStatus { TENANT_ACTIVITY_STATUS_FROZEN = 4, TENANT_ACTIVITY_STATUS_UNFREEZING = 5, TENANT_ACTIVITY_STATUS_FREEZING = 6, - TENANT_ACTIVITY_STATUS_UNFROZEN = 7, + /** TENANT_ACTIVITY_STATUS_ACTIVE - not used yet - added to let the clients already add code to handle this in the future */ + TENANT_ACTIVITY_STATUS_ACTIVE = 7, + TENANT_ACTIVITY_STATUS_INACTIVE = 8, + TENANT_ACTIVITY_STATUS_OFFLOADED = 9, + TENANT_ACTIVITY_STATUS_OFFLOADING = 10, + TENANT_ACTIVITY_STATUS_ONLOADING = 11, UNRECOGNIZED = -1, } @@ -41,8 +46,20 @@ export function tenantActivityStatusFromJSON(object: any): TenantActivityStatus case "TENANT_ACTIVITY_STATUS_FREEZING": return TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING; case 7: - case "TENANT_ACTIVITY_STATUS_UNFROZEN": - return TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFROZEN; + case "TENANT_ACTIVITY_STATUS_ACTIVE": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_ACTIVE; + case 8: + case "TENANT_ACTIVITY_STATUS_INACTIVE": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_INACTIVE; + case 9: + case "TENANT_ACTIVITY_STATUS_OFFLOADED": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_OFFLOADED; + case 10: + case "TENANT_ACTIVITY_STATUS_OFFLOADING": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_OFFLOADING; + case 11: + case "TENANT_ACTIVITY_STATUS_ONLOADING": + return TenantActivityStatus.TENANT_ACTIVITY_STATUS_ONLOADING; case -1: case "UNRECOGNIZED": default: @@ -64,8 +81,16 @@ export function tenantActivityStatusToJSON(object: TenantActivityStatus): string return "TENANT_ACTIVITY_STATUS_UNFREEZING"; case TenantActivityStatus.TENANT_ACTIVITY_STATUS_FREEZING: return "TENANT_ACTIVITY_STATUS_FREEZING"; - case TenantActivityStatus.TENANT_ACTIVITY_STATUS_UNFROZEN: - return "TENANT_ACTIVITY_STATUS_UNFROZEN"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_ACTIVE: + return "TENANT_ACTIVITY_STATUS_ACTIVE"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_INACTIVE: + return "TENANT_ACTIVITY_STATUS_INACTIVE"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_OFFLOADED: + return "TENANT_ACTIVITY_STATUS_OFFLOADED"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_OFFLOADING: + return "TENANT_ACTIVITY_STATUS_OFFLOADING"; + case TenantActivityStatus.TENANT_ACTIVITY_STATUS_ONLOADING: + return "TENANT_ACTIVITY_STATUS_ONLOADING"; case TenantActivityStatus.UNRECOGNIZED: default: return "UNRECOGNIZED"; From b2afb99269f3db97a90a2432f7ad356f003935eb Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Fri, 12 Jul 2024 11:14:07 +0100 Subject: [PATCH 25/31] 3.1.0-rc.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f505042f..8ba356e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "weaviate-client", - "version": "3.0.9", + "version": "3.1.0-rc.0", "description": "JS/TS client for Weaviate", "main": "dist/node/cjs/index.js", "type": "module", From 6c8de30c10e5c3d02dfdaa74f4251f33f8671871 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 16 Jul 2024 14:04:58 +0100 Subject: [PATCH 26/31] Add missing `indexRangeFilters` to API with tests of functionality --- .github/workflows/main.yaml | 2 +- src/collections/config/integration.test.ts | 4 + src/collections/config/types/index.ts | 1 + src/collections/config/utils.ts | 1 + src/collections/configure/types/base.ts | 3 + src/collections/integration.test.ts | 108 ++++++++++++++++----- 6 files changed, 92 insertions(+), 27 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6e2087d7..21bb4fd0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.20 WEAVIATE_125: 1.25.7 - WEAVIATE_126: preview--8758c8d + WEAVIATE_126: preview--3aca6c5 jobs: checks: diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index c8cc7721..546be6df 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -40,6 +40,7 @@ describe('Testing of the collection.config namespace', () => { name: 'testProp', dataType: 'text', description: undefined, + indexRangeFilters: false, indexSearchable: true, indexFilterable: true, indexInverted: false, @@ -93,6 +94,7 @@ describe('Testing of the collection.config namespace', () => { name: 'testProp', dataType: 'text', description: undefined, + indexRangeFilters: false, indexSearchable: true, indexFilterable: true, indexInverted: false, @@ -342,6 +344,7 @@ describe('Testing of the collection.config namespace', () => { name: 'testProp', dataType: 'text', description: undefined, + indexRangeFilters: false, indexSearchable: true, indexFilterable: true, indexInverted: false, @@ -462,6 +465,7 @@ describe('Testing of the collection.config namespace', () => { name: 'testProp', dataType: 'text', description: undefined, + indexRangeFilters: false, indexSearchable: true, indexFilterable: true, indexInverted: false, diff --git a/src/collections/config/types/index.ts b/src/collections/config/types/index.ts index 75dc04cf..2af0f07e 100644 --- a/src/collections/config/types/index.ts +++ b/src/collections/config/types/index.ts @@ -59,6 +59,7 @@ export type PropertyConfig = { description?: string; indexInverted: boolean; indexFilterable: boolean; + indexRangeFilters: boolean; indexSearchable: boolean; nestedProperties?: PropertyConfig[]; tokenization: string; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index a5b791eb..79854a05 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -480,6 +480,7 @@ class ConfigMapping { description: prop.description, indexFilterable: prop.indexFilterable ? prop.indexFilterable : false, indexInverted: prop.indexInverted ? prop.indexInverted : false, + indexRangeFilters: prop.indexRangeFilters ? prop.indexRangeFilters : false, indexSearchable: prop.indexSearchable ? prop.indexSearchable : false, vectorizerConfig: prop.moduleConfig ? 'none' in prop.moduleConfig diff --git a/src/collections/configure/types/base.ts b/src/collections/configure/types/base.ts index 32b4d171..4e5d3bfd 100644 --- a/src/collections/configure/types/base.ts +++ b/src/collections/configure/types/base.ts @@ -61,6 +61,7 @@ export type PropertyConfigCreateBase = { description?: string; indexInverted?: boolean; indexFilterable?: boolean; + indexRangeFilters?: boolean; indexSearchable?: boolean; tokenization?: WeaviateProperty['tokenization']; skipVectorization?: boolean; @@ -71,6 +72,7 @@ export type NestedPropertyConfigCreateBase = { description?: string; indexInverted?: boolean; indexFilterable?: boolean; + indexRangeFilters?: boolean; indexSearchable?: boolean; tokenization?: WeaviateNestedProperty['tokenization']; }; @@ -83,6 +85,7 @@ export type PropertyConfigCreate = T extends undefined nestedProperties?: NestedPropertyConfigCreate[]; indexInverted?: boolean; indexFilterable?: boolean; + indexRangeFilters?: boolean; indexSearchable?: boolean; tokenization?: WeaviateProperty['tokenization']; skipVectorization?: boolean; diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index 308b2764..fa843e74 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -5,15 +5,12 @@ import { GeoCoordinate, PQConfig, PhoneNumber, + PropertyConfig, Text2VecContextionaryConfig, Text2VecOpenAIConfig, VectorIndexConfigHNSW, } from './types/index'; -const fail = (msg: string) => { - throw new Error(msg); -}; - describe('Testing of the collections.create method', () => { let cluster: WeaviateClient; let contextionary: WeaviateClient; @@ -34,6 +31,14 @@ describe('Testing of the collections.create method', () => { }); }); + afterAll(() => + Promise.all([ + cluster.collections.deleteAll(), + contextionary.collections.deleteAll(), + openai.collections.deleteAll(), + ]) + ); + it('should be able to create a simple collection with a generic', async () => { const collectionName = 'TestCollectionSimpleGeneric'; type TestCollectionSimple = { @@ -57,8 +62,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a simple collection without a generic', async () => { @@ -81,8 +84,79 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); + }); - await contextionary.collections.delete(collectionName); + it('should be able to create a collection with one fully customised text property', () => { + return contextionary.collections + .create({ + name: 'TestCollectionTextProperty', + properties: [ + { + name: 'text', + dataType: 'text', + indexFilterable: true, + indexSearchable: true, + skipVectorization: true, + tokenization: 'field', + vectorizePropertyName: true, + }, + ], + vectorizers: weaviate.configure.vectorizer.text2VecContextionary(), + }) + .then((collection) => collection.config.get()) + .then((config) => + expect(config.properties[0]).toEqual({ + name: 'text', + dataType: 'text', + indexFilterable: true, + indexInverted: false, + indexRangeFilters: false, + indexSearchable: true, + tokenization: 'field', + vectorizerConfig: { + 'text2vec-contextionary': { + skip: true, + vectorizePropertyName: true, + }, + }, + }) + ); + }); + + it('should be able to create a collection with one fully customised int property', () => { + return contextionary.collections + .create({ + name: 'TestCollectionIntProperty', + properties: [ + { + name: 'int', + dataType: 'int', + indexFilterable: true, + indexRangeFilters: true, + skipVectorization: true, + vectorizePropertyName: true, + }, + ], + vectorizers: weaviate.configure.vectorizer.text2VecContextionary(), + }) + .then((collection) => collection.config.get()) + .then(async (config) => + expect(config.properties[0]).toEqual({ + name: 'int', + dataType: 'int', + indexFilterable: true, + indexInverted: false, + indexRangeFilters: await contextionary.getWeaviateVersion().then((ver) => ver.isAtLeast(1, 26, 0)), + indexSearchable: false, + tokenization: 'none', + vectorizerConfig: { + 'text2vec-contextionary': { + skip: true, + vectorizePropertyName: true, + }, + }, + }) + ); }); it('should be able to create a simple collection without a generic and no properties', async () => { @@ -97,8 +171,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a simple collection without a generic using a schema var', async () => { @@ -123,8 +195,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a simple collection with a generic using a schema var with const', async () => { @@ -152,8 +222,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a simple collection with a generic using a schema var with type', async () => { @@ -181,8 +249,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a nested collection', async () => { @@ -219,8 +285,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexConfig).toBeDefined(); expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a collection with generic properties', () => { @@ -248,7 +312,7 @@ describe('Testing of the collections.create method', () => { phoneNumber: PhoneNumber; }; - cluster.collections.create({ + return cluster.collections.create({ name: collectionName, properties: [ { @@ -539,8 +603,6 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.indexType).toEqual('hnsw'); expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); - - await cluster.collections.delete(collectionName); }); it('should be able to create a collection with the contextionary vectorizer using configure.vectorizer', async () => { @@ -569,8 +631,6 @@ describe('Testing of the collections.create method', () => { expect( (response.vectorizers.default.vectorizer.config as Text2VecContextionaryConfig).vectorizeCollectionName ).toEqual(true); - - await contextionary.collections.delete(collectionName); }); it('should be able to create a collection with an openai vectorizer with configure.vectorizer', async () => { @@ -599,8 +659,6 @@ describe('Testing of the collections.create method', () => { expect( (response.vectorizers.default.vectorizer.config as Text2VecOpenAIConfig).vectorizeCollectionName ).toEqual(true); - - await openai.collections.delete(collectionName); }); it('should be able to create a collection with the openai generative with configure.Generative', async () => { @@ -626,7 +684,5 @@ describe('Testing of the collections.create method', () => { name: 'generative-openai', config: {}, }); - - await openai.collections.delete(collectionName); }); }); From 2c9854f9357dac57779f55006b64617cf905ee92 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 16 Jul 2024 14:14:56 +0100 Subject: [PATCH 27/31] Fix tests --- src/collections/integration.test.ts | 2 +- src/collections/journey.test.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index fa843e74..a1e4c325 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -160,7 +160,7 @@ describe('Testing of the collections.create method', () => { }); it('should be able to create a simple collection without a generic and no properties', async () => { - const collectionName = 'TestCollectionSimpleNonGeneric'; + const collectionName = 'TestCollectionSimpleNonGenericNoProperties'; const response = await contextionary.collections .create({ name: collectionName, diff --git a/src/collections/journey.test.ts b/src/collections/journey.test.ts index 44ae6cc4..5e50a070 100644 --- a/src/collections/journey.test.ts +++ b/src/collections/journey.test.ts @@ -86,6 +86,7 @@ describe('Journey testing of the client using a WCD cluster', () => { dataType: 'text', indexFilterable: true, indexInverted: false, + indexRangeFilters: false, indexSearchable: true, vectorizerConfig: { 'text2vec-cohere': { @@ -100,6 +101,7 @@ describe('Journey testing of the client using a WCD cluster', () => { dataType: 'int', indexFilterable: true, indexInverted: false, + indexRangeFilters: false, indexSearchable: false, vectorizerConfig: { 'text2vec-cohere': { @@ -114,6 +116,7 @@ describe('Journey testing of the client using a WCD cluster', () => { dataType: 'geoCoordinates', indexFilterable: true, indexInverted: false, + indexRangeFilters: false, indexSearchable: false, vectorizerConfig: { 'text2vec-cohere': { @@ -128,6 +131,7 @@ describe('Journey testing of the client using a WCD cluster', () => { dataType: 'date', indexFilterable: true, indexInverted: false, + indexRangeFilters: false, indexSearchable: false, vectorizerConfig: { 'text2vec-cohere': { From 181b0a381ea6886a9b35bdec072996389d2cecd7 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 18 Jul 2024 10:02:36 +0100 Subject: [PATCH 28/31] 3.1.0-rc.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 301d9746..657d21c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "weaviate-client", - "version": "3.1.0-rc.0", + "version": "3.1.0-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "weaviate-client", - "version": "3.1.0-rc.0", + "version": "3.1.0-rc.1", "license": "SEE LICENSE IN LICENSE", "dependencies": { "graphql": "^16.8.2", diff --git a/package.json b/package.json index 8ba356e1..70b35bc7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "weaviate-client", - "version": "3.1.0-rc.0", + "version": "3.1.0-rc.1", "description": "JS/TS client for Weaviate", "main": "dist/node/cjs/index.js", "type": "module", From b6f23a198ef706b5de2e7db8fc1127daa484d3d8 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 22 Jul 2024 18:58:57 +0100 Subject: [PATCH 29/31] Add batching of `tenants.create` and `tenants.update` with serial method invocations --- .github/workflows/main.yaml | 6 ++--- src/collections/serialize/index.ts | 22 ++++++++++++---- src/collections/tenants/index.ts | 28 +++++++++++++++------ src/collections/tenants/integration.test.ts | 18 +++++++++++++ 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 21bb4fd0..555db4f6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,9 +7,9 @@ on: pull_request: env: - WEAVIATE_124: 1.24.20 - WEAVIATE_125: 1.25.7 - WEAVIATE_126: preview--3aca6c5 + WEAVIATE_124: 1.24.21 + WEAVIATE_125: 1.25.8 + WEAVIATE_126: preview--caba86b jobs: checks: diff --git a/src/collections/serialize/index.ts b/src/collections/serialize/index.ts index ceb8939d..1d893dee 100644 --- a/src/collections/serialize/index.ts +++ b/src/collections/serialize/index.ts @@ -1329,7 +1329,19 @@ export class Serialize { }); }; - public static tenantsCreate(tenant: TenantBC | TenantCreate): { + public static tenants(tenants: T[], mapper: (tenant: T) => M): M[][] { + const mapped = []; + const batches = Math.ceil(tenants.length / 100); + for (let i = 0; i < batches; i++) { + const batch = tenants.slice(i * 100, (i + 1) * 100); + mapped.push(batch.map(mapper)); + } + return mapped; + } + + public static tenantCreate( + tenant: T + ): { name: string; activityStatus?: 'HOT' | 'COLD'; } { @@ -1361,9 +1373,9 @@ export class Serialize { }; } - public static tenantUpdate = ( - tenant: TenantBC | TenantUpdate - ): { name: string; activityStatus: 'HOT' | 'COLD' | 'FROZEN' } => { + public static tenantUpdate( + tenant: T + ): { name: string; activityStatus: 'HOT' | 'COLD' | 'FROZEN' } { let activityStatus: 'HOT' | 'COLD' | 'FROZEN'; switch (tenant.activityStatus) { case 'ACTIVE': @@ -1389,5 +1401,5 @@ export class Serialize { name: tenant.name, activityStatus, }; - }; + } } diff --git a/src/collections/tenants/index.ts b/src/collections/tenants/index.ts index 5390ac66..4c390a23 100644 --- a/src/collections/tenants/index.ts +++ b/src/collections/tenants/index.ts @@ -44,10 +44,16 @@ const tenants = ( return result; }); return { - create: (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => - new TenantsCreator(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantsCreate)) - .do() - .then((res) => res.map(parseTenantREST)), + create: async (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => { + const out: Tenant[] = []; + for await (const res of Serialize.tenants(parseValueOrValueArray(tenants), Serialize.tenantCreate).map( + (tenants) => + new TenantsCreator(connection, collection, tenants).do().then((res) => res.map(parseTenantREST)) + )) { + out.push(...res); + } + return out; + }, get: async function () { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); return check.supports ? getGRPC() : getREST(); @@ -63,10 +69,16 @@ const tenants = ( collection, parseValueOrValueArray(tenants).map(parseStringOrTenant) ).do(), - update: (tenants: TenantBC | TenantUpdate | (TenantBC | TenantUpdate)[]) => - new TenantsUpdater(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantUpdate)) - .do() - .then((res) => res.map(parseTenantREST)), + update: async (tenants: TenantBC | TenantUpdate | (TenantBC | TenantUpdate)[]) => { + const out: Tenant[] = []; + for await (const res of Serialize.tenants(parseValueOrValueArray(tenants), Serialize.tenantUpdate).map( + (tenants) => + new TenantsUpdater(connection, collection, tenants).do().then((res) => res.map(parseTenantREST)) + )) { + out.push(...res); + } + return out; + }, }; }; diff --git a/src/collections/tenants/integration.test.ts b/src/collections/tenants/integration.test.ts index aa623e5f..4cfb184e 100644 --- a/src/collections/tenants/integration.test.ts +++ b/src/collections/tenants/integration.test.ts @@ -160,4 +160,22 @@ describe('Testing of the collection.tenants methods', () => { expect(result).not.toHaveProperty('non-existing'); }); }); + + it('should be able to create and update 1000 tenants', async () => { + const howManyToAdd = 1000; + const howManyPreExisting = Object.entries(await collection.tenants.get()).length; + const howMany = howManyToAdd + howManyPreExisting; + const tenants = Array.from({ length: howManyToAdd }, (_, i) => ({ + name: `tenant-${i}`, + })); + await collection.tenants.create(tenants); + const getTenants = await collection.tenants.get(); + expect(Object.entries(getTenants).length).toBe(howMany); + expect(Object.values(getTenants).every((tenant) => tenant.activityStatus === 'ACTIVE')).toBe(true); + const updated = await collection.tenants.update( + Object.values(getTenants).map((tenant) => ({ name: tenant.name, activityStatus: 'INACTIVE' })) + ); + expect(Object.entries(updated).length).toBe(howMany); + expect(Object.values(updated).every((tenant) => tenant.activityStatus === 'INACTIVE')).toBe(true); + }); }); From bf4aec64cc9d06d1d352d6a986fe2b91d83718e8 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 23 Jul 2024 12:08:51 +0100 Subject: [PATCH 30/31] Update CI image to released 1.26 ver --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 555db4f6..1c56c3c6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.21 WEAVIATE_125: 1.25.8 - WEAVIATE_126: preview--caba86b + WEAVIATE_126: 1.26.0 jobs: checks: From 2987a27ea05386a6be9682baa73805a49cac1279 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 23 Jul 2024 17:41:16 +0100 Subject: [PATCH 31/31] Remove client-side batching of `tenants.create` endpoint --- .github/workflows/main.yaml | 2 +- src/collections/tenants/index.ts | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1c56c3c6..bc6b51c8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: env: WEAVIATE_124: 1.24.21 WEAVIATE_125: 1.25.8 - WEAVIATE_126: 1.26.0 + WEAVIATE_126: 1.26.1 jobs: checks: diff --git a/src/collections/tenants/index.ts b/src/collections/tenants/index.ts index 4c390a23..f72c6959 100644 --- a/src/collections/tenants/index.ts +++ b/src/collections/tenants/index.ts @@ -44,16 +44,10 @@ const tenants = ( return result; }); return { - create: async (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => { - const out: Tenant[] = []; - for await (const res of Serialize.tenants(parseValueOrValueArray(tenants), Serialize.tenantCreate).map( - (tenants) => - new TenantsCreator(connection, collection, tenants).do().then((res) => res.map(parseTenantREST)) - )) { - out.push(...res); - } - return out; - }, + create: (tenants: TenantBC | TenantCreate | (TenantBC | TenantCreate)[]) => + new TenantsCreator(connection, collection, parseValueOrValueArray(tenants).map(Serialize.tenantCreate)) + .do() + .then((res) => res.map(parseTenantREST)), get: async function () { const check = await dbVersionSupport.supportsTenantsGetGRPCMethod(); return check.supports ? getGRPC() : getREST();