Merge pull request #907 from WolframResearch/bugfix/454044

Bugfix: Avoid an error message that occurs when giving whitespace strings to `SentenceBERTEmbedding`
WolframResearch · Nov 12, 2024 · ce2dd59 · ce2dd59
2 parents 512070d + 1980a25
commit ce2dd59
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 7 deletions.
diff --git a/Developer/VectorDatabases/VectorDatabaseBuilder.wl b/Developer/VectorDatabases/VectorDatabaseBuilder.wl
@@ -536,14 +536,36 @@ createEmbeddings[ string_String ] :=
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsection::Closed:: *)
 (*sentenceBERTEmbedding*)
-sentenceBERTEmbedding // beginDefinition;
+sentenceBERTEmbedding := getSentenceBERTEmbeddingFunction[ ];
 
-sentenceBERTEmbedding[ args___ ] := (
-    Needs[ "SemanticSearch`" -> None ];
-    SemanticSearch`SemanticSearch`Private`SentenceBERTEmbedding @ args
-);
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsubsection::Closed:: *)
+(*getSentenceBERTEmbeddingFunction*)
+getSentenceBERTEmbeddingFunction // beginDefinition;
+
+getSentenceBERTEmbeddingFunction[ ] := Enclose[
+    Module[ { name },
+
+        Needs[ "SemanticSearch`" -> None ];
+
+        name = ConfirmBy[
+            SelectFirst[
+                {
+                    "SemanticSearch`SentenceBERTEmbedding",
+                    "SemanticSearch`SemanticSearch`Private`SentenceBERTEmbedding"
+                },
+                NameQ @ # && ToExpression[ #, InputForm, System`Private`HasAnyEvaluationsQ ] &
+            ],
+            StringQ,
+            "SymbolName"
+        ];
+
+        getSentenceBERTEmbeddingFunction[ ] = Symbol @ name
+    ],
+    throwInternalFailure
+];
 
-sentenceBERTEmbedding // endDefinition;
+getSentenceBERTEmbeddingFunction // endDefinition;
 
 (* ::**************************************************************************************************************:: *)
 (* ::Subsection::Closed:: *)

diff --git a/Source/Chatbook/PromptGenerators/VectorDatabases.wl b/Source/Chatbook/PromptGenerators/VectorDatabases.wl
@@ -603,7 +603,10 @@ getAndCacheEmbeddings[ { } ] :=
 getAndCacheEmbeddings[ strings: { __String } ] /; $embeddingModel === "SentenceBERT" := Enclose[
     Module[ { vectors },
         vectors = ConfirmBy[
-            Developer`ToPackedArray @ sentenceBERTEmbedding @ strings,
+            If[ AllTrue[ strings, StringMatchQ[ WhitespaceCharacter... ] ],
+                Developer`ToPackedArray @ Rest @ sentenceBERTEmbedding @ Prepend[ strings, "hello" ],
+                Developer`ToPackedArray @ sentenceBERTEmbedding @ strings
+            ],
             Developer`PackedArrayQ,
             "PackedArray"
         ];