Skip to content

Commit dcc1ede

Browse files
adhityanadhityan
authored
Merge pull request #192 from llm-tools/confluence_updates
Updates
2 parents 519c1c2 + db1432e commit dcc1ede

File tree

40 files changed

+580
-738
lines changed

40 files changed

+580
-738
lines changed

‎core/embedjs-interfaces/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-interfaces",
3-
"version": "0.1.25",
3+
"version": "0.1.26",
44
"description": "Interfaces for extending the embedjs ecosystem",
55
"dependencies": {
6-
"@langchain/core": "^0.3.25",
6+
"@langchain/core": "^0.3.26",
77
"debug": "^4.4.0",
88
"md5": "^2.3.0",
99
"uuid": "^11.0.3"

‎core/embedjs-utils/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-utils",
3-
"version": "0.1.25",
3+
"version": "0.1.26",
44
"description": "Useful util functions when extending the embedjs ecosystem",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.25"
6+
"@llm-tools/embedjs-interfaces": "0.1.26"
77
},
88
"type": "module",
99
"main": "./src/index.js",

‎core/embedjs-utils/src/util/strings.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ export function toTitleCase(str: string) {
4141
});
4242
}
4343

44-
export function isValidURL(url: string) {
44+
export function isValidURL(candidateUrl: string) {
4545
try {
46-
new URL(url);
47-
return true;
46+
const url = new URL(candidateUrl);
47+
return url.protocol === 'http:' || url.protocol === 'https:';
4848
} catch {
4949
return false;
5050
}

‎core/embedjs/package.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
{
22
"type": "module",
33
"name": "@llm-tools/embedjs",
4-
"version": "0.1.25",
4+
"version": "0.1.26",
55
"description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
66
"dependencies": {
77
"@langchain/textsplitters": "^0.1.0",
8-
"@llm-tools/embedjs-interfaces": "0.1.25",
9-
"@llm-tools/embedjs-utils": "0.1.25",
8+
"@llm-tools/embedjs-interfaces": "0.1.26",
9+
"@llm-tools/embedjs-utils": "0.1.26",
1010
"debug": "^4.4.0",
11-
"langchain": "^0.3.7",
11+
"langchain": "^0.3.8",
1212
"md5": "^2.3.0",
1313
"mime": "^4.0.6",
1414
"stream-mime-type": "^2.0.0"

‎core/embedjs/src/core/rag-application.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import createDebugMessages from 'debug';
22

3-
import { RAGEmbedding } from './rag-embedding.js';
43
import { RAGApplicationBuilder } from './rag-application-builder.js';
54
import {
65
AddLoaderReturn,
@@ -14,6 +13,7 @@ import {
1413
QueryResponse,
1514
SIMPLE_MODELS,
1615
DEFAULT_INSERT_BATCH_SIZE,
16+
BaseEmbeddings,
1717
} from '@llm-tools/embedjs-interfaces';
1818
import { cleanString, getUnique } from '@llm-tools/embedjs-utils';
1919

@@ -24,12 +24,14 @@ export class RAGApplication {
2424
private readonly searchResultCount: number;
2525
private readonly systemMessage: string;
2626
private readonly vectorDatabase: BaseVectorDatabase;
27+
private readonly embeddingModel: BaseEmbeddings;
2728
private readonly store: BaseStore;
2829
private loaders: BaseLoader[];
2930
private model: BaseModel;
3031

3132
constructor(llmBuilder: RAGApplicationBuilder) {
3233
if (!llmBuilder.getEmbeddingModel()) throw new Error('Embedding model must be set!');
34+
this.embeddingModel = llmBuilder.getEmbeddingModel();
3335

3436
this.storeConversationsToDefaultThread = llmBuilder.getParamStoreConversationsToDefaultThread();
3537
this.store = llmBuilder.getStore();
@@ -55,7 +57,7 @@ export class RAGApplication {
5557
* LLM based on the configuration provided
5658
*/
5759
public async init(llmBuilder: RAGApplicationBuilder) {
58-
await RAGEmbedding.init(llmBuilder.getEmbeddingModel());
60+
await this.embeddingModel.init();
5961

6062
this.model = await this.getModel(llmBuilder.getModel());
6163
if (!this.model) this.debug('No base model set; query function unavailable!');
@@ -68,7 +70,7 @@ export class RAGApplication {
6870
this.debug('Initialized LLM class');
6971
}
7072

71-
await this.vectorDatabase.init({ dimensions: await RAGEmbedding.getEmbedding().getDimensions() });
73+
await this.vectorDatabase.init({ dimensions: await this.embeddingModel.getDimensions() });
7274
this.debug('Initialized vector database');
7375

7476
if (this.store) {
@@ -117,7 +119,7 @@ export class RAGApplication {
117119
*/
118120
private async embedChunks(chunks: Pick<Chunk, 'pageContent'>[]) {
119121
const texts = chunks.map(({ pageContent }) => pageContent);
120-
return RAGEmbedding.getEmbedding().embedDocuments(texts);
122+
return this.embeddingModel.embedDocuments(texts);
121123
}
122124

123125
/**
@@ -352,7 +354,7 @@ export class RAGApplication {
352354
* only the number of results specified by the `searchResultCount` property.
353355
*/
354356
public async getEmbeddings(cleanQuery: string) {
355-
const queryEmbedded = await RAGEmbedding.getEmbedding().embedQuery(cleanQuery);
357+
const queryEmbedded = await this.embeddingModel.embedQuery(cleanQuery);
356358
const unfilteredResultSet = await this.vectorDatabase.similaritySearch(
357359
queryEmbedded,
358360
this.searchResultCount + 10,

‎core/embedjs/src/core/rag-embedding.ts

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎databases/embedjs-astra/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-astradb",
3-
"version": "0.1.25",
3+
"version": "0.1.26",
44
"description": "Add AstraDB support to embedjs",
55
"dependencies": {
66
"@datastax/astra-db-ts": "^1.5.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.25",
7+
"@llm-tools/embedjs-interfaces": "0.1.26",
88
"debug": "^4.4.0"
99
},
1010
"type": "module",

‎databases/embedjs-cosmos/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-cosmos",
3-
"version": "0.1.25",
3+
"version": "0.1.26",
44
"description": "Add CosmosDB support to embedjs",
55
"dependencies": {
66
"@azure/cosmos": "^4.2.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.25",
7+
"@llm-tools/embedjs-interfaces": "0.1.26",
88
"debug": "^4.4.0"
99
},
1010
"type": "module",

‎databases/embedjs-hnswlib/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-hnswlib",
3-
"version": "0.1.25",
3+
"version": "0.1.26",
44
"description": "Add HNSWLib support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.25",
6+
"@llm-tools/embedjs-interfaces": "0.1.26",
77
"debug": "^4.4.0",
88
"hnswlib-node": "^3.0.0"
99
},

‎databases/embedjs-lancedb/package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-lancedb",
3-
"version": "0.1.25",
3+
"version": "0.1.26",
44
"description": "Add LanceDb support to embedjs",
55
"dependencies": {
6-
"@lancedb/lancedb": "^0.14.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.25",
6+
"@lancedb/lancedb": "^0.14.1",
7+
"@llm-tools/embedjs-interfaces": "0.1.26",
88
"compute-cosine-similarity": "^1.1.0",
99
"debug": "^4.4.0"
1010
},

0 commit comments

Comments
 (0)