refactor: numerous improvements

.gitignore

@@ -1,8 +1,8 @@
 /tmp
-/functions.txt
-/functions.txt.test
-/functions.json
+functions.txt
+functions.json
 /bin
+/cache
 /tools/test.*
 /.env
 *.cmd

Argcfile.sh

@@ -148,15 +148,15 @@ test-tools() {
     )
 
     for test_case in "${test_cases[@]}"; do
-        IFS='#' read -r lang func data <<<"${test_case}"
+        IFS='#' read -r lang tool_name data <<<"${test_case}"
         cmd="$(_lang_to_cmd "$lang")"
-        cmd_path="$BIN_DIR/$func$ext"
+        cmd_path="$BIN_DIR/$tool_name$ext"
         if command -v "$cmd" &> /dev/null; then
             echo -n "Test $cmd_path: "
             "$cmd_path" "$data"
             if ! _is_win; then
-                echo -n "Test $cmd scripts/run-tool.$lang $func: "
-                "$cmd" "scripts/run-tool.$lang" "$func" "$data"
+                echo -n "Test $cmd scripts/run-tool.$lang $tool_name: "
+                "$cmd" "scripts/run-tool.$lang" "$tool_name" "$data"
             fi
         fi
     done
@@ -258,7 +258,7 @@ _choice_cmd() {
 }
 
 _die() {
-    echo "$*"
+    echo "$*" >&2
     exit 1
 }
 

scripts/build-declarations.js

@@ -1,11 +1,13 @@
 #!/usr/bin/env node
 
 const fs = require("fs");
+const path = require("path");
 
 const TOOL_ENTRY_FUNC = "run";
 
-function main(isTool = true) {
+function main() {
   const scriptfile = process.argv[2];
+  const isTool = path.dirname(scriptfile) == "tools";
   const contents = fs.readFileSync(process.argv[2], "utf8");
   const functions = extractFunctions(contents, isTool);
   let declarations = functions.map(({ funcName, jsdoc }) => {
@@ -56,13 +58,20 @@ function extractFunctions(contents, isTool) {
           });
         }
       } else {
-        const match = /function *([_A-Za-z]+)/.exec(line);
+        let match = /^export (async )?function ([A-Za-z0-9_]+)/.exec(line);
+        let funcName = null;
         if (match) {
-          const funcName = match[1];
-          if (!funcName.startsWith("_")) {
-            output.push({ funcName, jsdoc });
+          funcName = match[2];
+        }
+        if (!funcName) {
+          match = /^exports\.([A-Za-z0-9_]+) = (async )?function /.exec(line);
+          if (match) {
+            funcName = match[1];
           }
         }
+        if (funcName) {
+          output.push({ funcName, jsdoc });
+        }
       }
       jsdoc = "";
     }
@@ -165,21 +174,6 @@ function buildProperty(type, description) {
   return property;
 }
 
-/**
- * @param {string} filePath
- */
-function getBasename(filePath) {
-  const filenameWithExt = filePath.split(/[/\\]/).pop();
-
-  const lastDotIndex = filenameWithExt.lastIndexOf(".");
-
-  if (lastDotIndex === -1) {
-    return filenameWithExt;
-  }
-
-  return filenameWithExt.substring(0, lastDotIndex);
-}
-
 /**
  * @param {string} name
  * @param {string} description
@@ -204,4 +198,19 @@ function buildDeclaration(name, description, params) {
   return schema;
 }
 
+/**
+ * @param {string} filePath
+ */
+function getBasename(filePath) {
+  const filenameWithExt = filePath.split(/[/\\]/).pop();
+
+  const lastDotIndex = filenameWithExt.lastIndexOf(".");
+
+  if (lastDotIndex === -1) {
+    return filenameWithExt;
+  }
+
+  return filenameWithExt.substring(0, lastDotIndex);
+}
+
 main();

scripts/build-declarations.py

@@ -9,8 +9,11 @@
 
 TOOL_ENTRY_FUNC = "run"
 
-def main(is_tool = True):
+
+def main(is_tool=True):
     scriptfile = sys.argv[1]
+    is_tool = os.path.dirname(scriptfile) == "tools"
+
     with open(scriptfile, "r", encoding="utf-8") as f:
         contents = f.read()
 
@@ -92,8 +95,8 @@ def parse_docstring(docstring: str):
                 break
     params = {}
     for rawParam in rawParams:
-        name, type_, description = parse_param(rawParam)
-        params[name] = (type_, description)
+        name, type_, param_description = parse_param(rawParam)
+        params[name] = (type_, param_description)
     return (description.strip(), params)
 
 

scripts/build-declarations.sh

@@ -1,7 +1,23 @@
 #!/usr/bin/env bash
 
-argc --argc-export "$1" | \
-jq -r '
+main() {
+    scriptfile="$1"
+    is_tool=false
+    if [[ "$(dirname "$scriptfile")" == tools ]]; then
+        is_tool=true
+    fi
+    if [[ "$is_tool" == "true" ]]; then
+        expr='[.]' 
+    else
+        expr='.subcommands' 
+    fi
+    argc --argc-export "$scriptfile" | \
+    jq "$expr" | \
+    build_declarations
+}
+
+build_declarations() {
+    jq -r '
     def parse_description(flag_option):
         if flag_option.describe == "" then
             {}
@@ -36,8 +52,15 @@ jq -r '
             required: [flag_options[] | select(.required == true) | .id | sub("-"; "_"; "g")],
         };
 
-    [{
-        name: (.name | sub("-"; "_"; "g")),
-        description: .describe,
-        parameters: parse_parameter([.flag_options[] | select(.id != "help" and .id != "version")])
-    }]'
+    def parse_declaration:
+        {
+            name: (.name | sub("-"; "_"; "g")),
+            description: .describe,
+            parameters: parse_parameter([.flag_options[] | select(.id != "help" and .id != "version")])
+        };
+    [
+        .[] | parse_declaration
+    ]'
+}
+
+main "$@"

scripts/create-tool.sh

@@ -20,15 +20,15 @@ main() {
     ext="${argc_name##*.}"
     support_exts=('.sh' '.js' '.py')
     if [[ "$ext" == "$argc_name" ]]; then
-        _die "No extension name, pelease add one of ${support_exts[*]}" 
+        _die "error: no extension name, pelease add one of ${support_exts[*]}" 
     fi
     case $ext in
     sh) create_sh ;;
     js) create_js ;;
     py) create_py ;;
-    *) _die "Invalid extension name: $ext, must be one of ${support_exts[*]}" ;; 
+    *) _die "error: invalid extension name: $ext, must be one of ${support_exts[*]}" ;; 
     esac
-    _die "$output generated"
+    echo "$output generated"
 }
 
 create_sh() {
@@ -187,7 +187,7 @@ build_properties() {
 }
 
 _die() {
-    echo "$*"
+    echo "$*" >&2
     exit 1
 }
 

scripts/run-tool.js

@@ -4,34 +4,34 @@ const path = require("path");
 const fs = require("fs");
 
 function parseArgv() {
-  let funcName = process.argv[1];
-  let funcData = null;
+  let toolName = process.argv[1];
+  let toolData = null;
 
-  if (funcName.endsWith("run-tool.js")) {
-    funcName = process.argv[2];
-    funcData = process.argv[3];
+  if (toolName.endsWith("run-tool.js")) {
+    toolName = process.argv[2];
+    toolData = process.argv[3];
   } else {
-    funcName = path.basename(funcName);
-    funcData = process.argv[2];
+    toolName = path.basename(toolName);
+    toolData = process.argv[2];
   }
 
-  if (funcName.endsWith(".js")) {
-    funcName = funcName.slice(0, -3);
+  if (toolName.endsWith(".js")) {
+    toolName = toolName.slice(0, -3);
   }
 
-  return [funcName, funcData];
+  return [toolName, toolData];
 }
 
-function loadFunc(funcName) {
-  const funcFileName = `${funcName}.js`;
-  const funcPath = path.resolve(
-    process.env["LLM_FUNCTIONS_DIR"],
-    `tools/${funcFileName}`,
+function loadModule(toolName) {
+  const toolFileName = `${toolName}.js`;
+  const toolPath = path.resolve(
+    process.env["LLM_ROOT_DIR"],
+    `tools/${toolFileName}`,
   );
   try {
-    return require(funcPath);
+    return require(toolPath);
   } catch {
-    console.log(`Invalid function: ${funcFileName}`);
+    console.log(`Invalid tooltion: ${toolFileName}`);
     process.exit(1);
   }
 }
@@ -50,26 +50,32 @@ function loadEnv(filePath) {
   } catch {}
 }
 
-process.env["LLM_FUNCTIONS_DIR"] = path.resolve(__dirname, "..");
+const LLM_ROOT_DIR = path.resolve(__dirname, "..");
+process.env["LLM_ROOT_DIR"] = LLM_ROOT_DIR;
 
-loadEnv(path.resolve(process.env["LLM_FUNCTIONS_DIR"], ".env"));
+loadEnv(path.resolve(LLM_ROOT_DIR, ".env"));
 
-const [funcName, funcData] = parseArgv();
+const [toolName, toolData] = parseArgv();
 
-process.env["LLM_FUNCTION_NAME"] = funcName;
+process.env["LLM_TOOL_NAME"] = toolName;
+process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(
+  LLM_ROOT_DIR,
+  "cache",
+  toolName,
+);
 
-if (!funcData) {
+if (!toolData) {
   console.log("No json data");
   process.exit(1);
 }
 
-let args;
+let data = null;
 try {
-  args = JSON.parse(funcData);
+  data = JSON.parse(toolData);
 } catch {
   console.log("Invalid json data");
   process.exit(1);
 }
 
-const { run } = loadFunc(funcName);
-run(args);
+const { run } = loadModule(toolName);
+run(data);