refactor for loops

FlowiseAI · Jun 10, 2023 · 0dccc6b · 0dccc6b
1 parent e9184ed
commit 0dccc6b
Show file tree

Hide file tree

Showing 20 changed files with 134 additions and 288 deletions.
diff --git a/packages/components/nodes/agents/BabyAGI/core.ts b/packages/components/nodes/agents/BabyAGI/core.ts
@@ -127,11 +127,7 @@ async function prioritizeTasks(
 
 export async function get_top_tasks(vectorStore: VectorStore, query: string, k: number): Promise<string[]> {
     const docs = await vectorStore.similaritySearch(query, k)
-    let returnDocs: string[] = []
-    for (const doc of docs) {
-        returnDocs.push(doc.metadata.task)
-    }
-    return returnDocs
+    return docs.map((doc) => doc.metadata.task);
 }
 
 async function executeTask(vectorStore: VectorStore, executionChain: LLMChain, objective: string, task: string, k = 5): Promise<string> {

diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
@@ -3,7 +3,7 @@ import { initializeAgentExecutorWithOptions, AgentExecutor, InitializeAgentExecu
 import { Tool } from 'langchain/tools'
 import { BaseChatMemory, ChatMessageHistory } from 'langchain/memory'
 import { getBaseClasses } from '../../../src/utils'
-import { AIChatMessage, HumanChatMessage } from 'langchain/schema'
+import { AIChatMessage, BaseChatMessage, HumanChatMessage } from 'langchain/schema'
 import { BaseLanguageModel } from 'langchain/base_language'
 import { flatten } from 'lodash'
 
@@ -94,17 +94,15 @@ class ConversationalAgent_Agents implements INode {
         const memory = nodeData.inputs?.memory as BaseChatMemory
 
         if (options && options.chatHistory) {
-            const chatHistory = []
-            const histories: IMessage[] = options.chatHistory
-
-            for (const message of histories) {
+            const messages: BaseChatMessage[] = options.chatHistory.map((message: IMessage) => {
                 if (message.type === 'apiMessage') {
-                    chatHistory.push(new AIChatMessage(message.message))
+                    return new AIChatMessage(message.message)
                 } else if (message.type === 'userMessage') {
-                    chatHistory.push(new HumanChatMessage(message.message))
+                    return new HumanChatMessage(message.message)
                 }
-            }
-            memory.chatHistory = new ChatMessageHistory(chatHistory)
+            })
+
+            memory.chatHistory = new ChatMessageHistory(messages)
             executor.memory = memory
         }
         const result = await executor.call({ input })

diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts
@@ -4,7 +4,7 @@ import { CustomChainHandler, getBaseClasses } from '../../../src/utils'
 import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from 'langchain/prompts'
 import { BufferMemory, ChatMessageHistory } from 'langchain/memory'
 import { BaseChatModel } from 'langchain/chat_models/base'
-import { AIChatMessage, HumanChatMessage } from 'langchain/schema'
+import { AIChatMessage, BaseChatMessage, HumanChatMessage } from 'langchain/schema'
 
 const systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
 
@@ -76,17 +76,15 @@ class ConversationChain_Chains implements INode {
         const memory = nodeData.inputs?.memory as BufferMemory
 
         if (options && options.chatHistory) {
-            const chatHistory = []
-            const histories: IMessage[] = options.chatHistory
-
-            for (const message of histories) {
+            const messages: BaseChatMessage[] = options.chatHistory.map((message: IMessage) => {
                 if (message.type === 'apiMessage') {
-                    chatHistory.push(new AIChatMessage(message.message))
+                    return new AIChatMessage(message.message)
                 } else if (message.type === 'userMessage') {
-                    chatHistory.push(new HumanChatMessage(message.message))
+                    return new HumanChatMessage(message.message)
                 }
-            }
-            memory.chatHistory = new ChatMessageHistory(chatHistory)
+            })
+
+            memory.chatHistory = new ChatMessageHistory(messages)
             chain.memory = memory
         }
 

diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts
@@ -111,14 +111,7 @@ const runPrediction = async (
             return res
         }
     } else if (inputVariables.length > 1) {
-        let seen: string[] = []
-
-        for (const variable of inputVariables) {
-            seen.push(variable)
-            if (promptValues[variable]) {
-                seen.pop()
-            }
-        }
+        const seen: string[] = inputVariables.filter((variable) => promptValues[variable]);
 
         if (seen.length === 0) {
             // All inputVariables have fixed values specified

diff --git a/packages/components/nodes/chains/MultiPromptChain/MultiPromptChain.ts b/packages/components/nodes/chains/MultiPromptChain/MultiPromptChain.ts
@@ -39,15 +39,9 @@ class MultiPromptChain_Chains implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const model = nodeData.inputs?.model as BaseLanguageModel
         const promptRetriever = nodeData.inputs?.promptRetriever as PromptRetriever[]
-        const promptNames = []
-        const promptDescriptions = []
-        const promptTemplates = []
-
-        for (const prompt of promptRetriever) {
-            promptNames.push(prompt.name)
-            promptDescriptions.push(prompt.description)
-            promptTemplates.push(prompt.systemMessage)
-        }
+        const promptNames = promptRetriever.map((p) => p.name);
+        const promptDescriptions = promptRetriever.map((p) => p.description);
+        const promptTemplates = promptRetriever.map((p) => p.systemMessage);
 
         const chain = MultiPromptChain.fromPrompts(model, promptNames, promptDescriptions, promptTemplates, undefined, {
             verbose: process.env.DEBUG === 'true' ? true : false

diff --git a/packages/components/nodes/chains/MultiRetrievalQAChain/MultiRetrievalQAChain.ts b/packages/components/nodes/chains/MultiRetrievalQAChain/MultiRetrievalQAChain.ts
@@ -39,15 +39,9 @@ class MultiRetrievalQAChain_Chains implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const model = nodeData.inputs?.model as BaseLanguageModel
         const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever as VectorStoreRetriever[]
-        const retrieverNames = []
-        const retrieverDescriptions = []
-        const retrievers = []
-
-        for (const vs of vectorStoreRetriever) {
-            retrieverNames.push(vs.name)
-            retrieverDescriptions.push(vs.description)
-            retrievers.push(vs.vectorStore.asRetriever((vs.vectorStore as any).k ?? 4))
-        }
+        const retrieverNames = vectorStoreRetriever.map((p) => p.name)
+        const retrieverDescriptions = vectorStoreRetriever.map((p) => p.description)
+        const retrievers = vectorStoreRetriever.map((p) => p.vectorStore.asRetriever((p.vectorStore as any).k ?? 4))
 
         const chain = MultiRetrievalQAChain.fromRetrievers(model, retrieverNames, retrieverDescriptions, retrievers, undefined, {
             verbose: process.env.DEBUG === 'true' ? true : false

diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
@@ -96,18 +96,15 @@ class Cheerio_DocumentLoaders implements INode {
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of docs) {
-                const newdoc = {
+            return docs.map((doc: any) => {
+                return {
                     ...doc,
                     metadata: {
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
 
         return docs

diff --git a/packages/components/nodes/documentloaders/Confluence/Confluence.ts b/packages/components/nodes/documentloaders/Confluence/Confluence.ts
@@ -97,21 +97,18 @@ class Confluence_DocumentLoaders implements INode {
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of docs) {
-                const newdoc = {
+            return docs.map((doc) => {
+                return {
                     ...doc,
                     metadata: {
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
 
-        return docs
+        return docs;
     }
 }
 

diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts
@@ -57,45 +57,30 @@ class Csv_DocumentLoaders implements INode {
         const columnName = nodeData.inputs?.columnName as string
         const metadata = nodeData.inputs?.metadata
 
-        let alldocs = []
-        let files: string[] = []
+        const files: string[] = (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) ? JSON.parse(csvFileBase64) : [csvFileBase64]
 
-        if (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) {
-            files = JSON.parse(csvFileBase64)
-        } else {
-            files = [csvFileBase64]
-        }
-
-        for (const file of files) {
+        const alldocs = await Promise.all(files.map(async (file) => {
             const splitDataURI = file.split(',')
             splitDataURI.pop()
             const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
             const blob = new Blob([bf])
             const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
 
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
-            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
-            }
-        }
+            return textSplitter ? await loader.loadAndSplit(textSplitter) : await loader.load()
+        }))
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of alldocs) {
-                const newdoc = {
+            return alldocs.map((doc) => {
+                return {
                     ...doc,
                     metadata: {
+                        // @ts-ignore-next-line
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
 
         return alldocs

diff --git a/packages/components/nodes/documentloaders/Docx/Docx.ts b/packages/components/nodes/documentloaders/Docx/Docx.ts
@@ -48,45 +48,30 @@ class Docx_DocumentLoaders implements INode {
         const docxFileBase64 = nodeData.inputs?.docxFile as string
         const metadata = nodeData.inputs?.metadata
 
-        let alldocs = []
-        let files: string[] = []
+        const files: string[] = (docxFileBase64.startsWith('[') && docxFileBase64.endsWith(']')) ? JSON.parse(docxFileBase64) : [docxFileBase64]
 
-        if (docxFileBase64.startsWith('[') && docxFileBase64.endsWith(']')) {
-            files = JSON.parse(docxFileBase64)
-        } else {
-            files = [docxFileBase64]
-        }
-
-        for (const file of files) {
+        const alldocs = await Promise.all(files.map((file) => {
             const splitDataURI = file.split(',')
             splitDataURI.pop()
             const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
             const blob = new Blob([bf])
             const loader = new DocxLoader(blob)
 
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
-            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
-            }
-        }
+            return (textSplitter) ? loader.loadAndSplit(textSplitter) : loader.load()
+        }))
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of alldocs) {
-                const newdoc = {
+            return alldocs.map((doc) => {
+                return {
                     ...doc,
                     metadata: {
+                        // @ts-ignore-next-line
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
 
         return alldocs

diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts
@@ -71,18 +71,15 @@ class Folder_DocumentLoaders implements INode {
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of docs) {
-                const newdoc = {
+            return docs.map((doc) => {
+                return {
                     ...doc,
                     metadata: {
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
 
         return docs

diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts
@@ -93,8 +93,6 @@ class Github_DocumentLoaders implements INode {
                 }
             })
         }
-
-        return docs
     }
 }
 

diff --git a/packages/components/nodes/documentloaders/Json/Json.ts b/packages/components/nodes/documentloaders/Json/Json.ts
@@ -63,45 +63,30 @@ class Json_DocumentLoaders implements INode {
             pointers = outputString.split(',').map((pointer) => '/' + pointer.trim())
         }
 
-        let alldocs = []
-        let files: string[] = []
+        const files: string[] = (jsonFileBase64.startsWith('[') && jsonFileBase64.endsWith(']')) ? JSON.parse(jsonFileBase64) : [jsonFileBase64]
 
-        if (jsonFileBase64.startsWith('[') && jsonFileBase64.endsWith(']')) {
-            files = JSON.parse(jsonFileBase64)
-        } else {
-            files = [jsonFileBase64]
-        }
-
-        for (const file of files) {
+        const alldocs = files.map((file) => {
             const splitDataURI = file.split(',')
             splitDataURI.pop()
             const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
             const blob = new Blob([bf])
             const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
 
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
-            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
-            }
-        }
+            return (textSplitter) ? loader.loadAndSplit(textSplitter) : loader.load()
+        })
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of alldocs) {
-                const newdoc = {
+            return alldocs.map((doc) => {
+                return {
                     ...doc,
                     metadata: {
+                        // @ts-ignore-next-line
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
 
         return alldocs

diff --git a/packages/components/nodes/documentloaders/NotionDB/NotionDB.ts b/packages/components/nodes/documentloaders/NotionDB/NotionDB.ts
@@ -80,21 +80,16 @@ class NotionDB_DocumentLoaders implements INode {
 
         if (metadata) {
             const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            let finaldocs = []
-            for (const doc of docs) {
-                const newdoc = {
+            return docs.map((doc) => {
+                return {
                     ...doc,
                     metadata: {
                         ...doc.metadata,
                         ...parsedMetadata
                     }
                 }
-                finaldocs.push(newdoc)
-            }
-            return finaldocs
+            })
         }
-
-        return docs
     }
 }