Skip to content

Commit

Permalink
update search index in bulk for performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@chubakueno
chubakueno committed Feb 3, 2025
1 parent c76c161 commit 7378460
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 34 deletions.
37 changes: 15 additions & 22 deletions src/DynamoCore/Models/DynamoModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1592,7 +1592,6 @@ private void InitializeIncludedNodes()

var cnbNode = new CodeBlockNodeSearchElement(cbnData, LibraryServices);
SearchModel?.Add(cnbNode);
LuceneUtility.AddNodeTypeToSearchIndex(cnbNode, iDoc);

var symbolSearchElement = new NodeModelSearchElement(symbolData)
{
Expand All @@ -1611,10 +1610,8 @@ private void InitializeIncludedNodes()
};

SearchModel?.Add(symbolSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(symbolSearchElement, iDoc);

SearchModel?.Add(outputSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc);
LuceneUtility.AddNodeTypeToSearchIndexBulk([cnbNode, symbolSearchElement, outputSearchElement], iDoc);

}

Expand Down Expand Up @@ -1755,6 +1752,7 @@ internal void LoadNodeLibrary(Assembly assem, bool suppressZeroTouchLibraryLoad
private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
{
var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
List<NodeSearchElement> nodeSearchElements = [];
foreach (var type in nodes)
{
// Protect ourselves from exceptions thrown by malformed third party nodes.
Expand All @@ -1769,14 +1767,15 @@ private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
// TODO: get search element some other way
if (ele != null)
{
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
nodeSearchElements.Add(ele);
}
}
catch (Exception e)
{
Logger.Log(e);
}
}
LuceneUtility.AddNodeTypeToSearchIndexBulk(nodeSearchElements, iDoc);
}

private void InitializePreferences()
Expand Down Expand Up @@ -3478,26 +3477,20 @@ internal void HideUnhideNamespace(bool hide, string library, string namespc)
internal void AddZeroTouchNodesToSearch(IEnumerable<FunctionGroup> functionGroups)
{
var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
List<NodeSearchElement> nodes = new();
foreach (var funcGroup in functionGroups)
AddZeroTouchNodeToSearch(funcGroup, iDoc);
}

private void AddZeroTouchNodeToSearch(FunctionGroup funcGroup, Document iDoc)
{
foreach (var functionDescriptor in funcGroup.Functions)
{
AddZeroTouchNodeToSearch(functionDescriptor, iDoc);
}
}

private void AddZeroTouchNodeToSearch(FunctionDescriptor functionDescriptor, Document iDoc)
{
if (functionDescriptor.IsVisibleInLibrary)
{
var ele = new ZeroTouchSearchElement(functionDescriptor);
SearchModel?.Add(ele);
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
foreach (var functionDescriptor in funcGroup.Functions)
{
if (functionDescriptor.IsVisibleInLibrary)
{
var ele = new ZeroTouchSearchElement(functionDescriptor);
SearchModel?.Add(ele);
nodes.Add(ele);
}
}
}
LuceneUtility.AddNodeTypeToSearchIndexBulk(nodes, iDoc);
}

/// <summary>
Expand Down
22 changes: 15 additions & 7 deletions src/DynamoCore/Utilities/LuceneSearchUtility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,12 +262,8 @@ internal void UpdateIndexedNodesInfo(List<NodeSearchElement> nodeList)
if(nodeList.Any())
{
writer.DeleteAll();
foreach(var node in nodeList)
{
var iDoc = InitializeIndexDocumentForNodes();
AddNodeTypeToSearchIndex(node, iDoc);
}
InitializeIndexSearcher();
var iDoc = InitializeIndexDocumentForNodes();
AddNodeTypeToSearchIndexBulk(nodeList, iDoc);
}
}

Expand Down Expand Up @@ -607,7 +603,7 @@ internal void CommitWriterChanges()
/// </summary>
/// <param name="node">node info that will be indexed</param>
/// <param name="doc">Lucene document in which the node info will be indexed</param>
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
internal void AddNodeTypeToSearchIndex_uninitialized(NodeSearchElement node, Document doc)
{
if (addedFields == null) return;
// During DynamoModel initialization, the index writer should still be valid here
Expand Down Expand Up @@ -645,6 +641,18 @@ internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);

writer?.AddDocument(doc);
}
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
{
AddNodeTypeToSearchIndex_uninitialized(node,doc);
InitializeIndexSearcher();
}
internal void AddNodeTypeToSearchIndexBulk(List<NodeSearchElement> nodes, Document doc)
{
foreach(var node in nodes)
{
AddNodeTypeToSearchIndex_uninitialized(node, doc);
}
InitializeIndexSearcher();
}
}
Expand Down
8 changes: 3 additions & 5 deletions src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,11 +733,9 @@ internal void SearchAutoCompleteCandidates(string input)
LuceneSearch.LuceneUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model, LuceneSearchUtility.DefaultStartConfig);

//Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search)
foreach (var node in searchElementsCache.Select(x => x.Model))
{
var doc = LuceneUtility.InitializeIndexDocumentForNodes();
LuceneUtility.AddNodeTypeToSearchIndex(node, doc);
}
var doc = LuceneUtility.InitializeIndexDocumentForNodes();
List<NodeSearchElement> nodeSearchElements = [.. searchElementsCache.Select(x => x.Model)];
LuceneUtility.AddNodeTypeToSearchIndexBulk(nodeSearchElements, doc);

//Write the Lucene documents to memory
LuceneUtility.CommitWriterChanges();
Expand Down

0 comments on commit 7378460

Please sign in to comment.