refactor: search tools (#76)

Argcfile.sh

@@ -439,7 +439,7 @@ test-demo@agent() {
 # @cmd Clean tools
 # @alias tool:clean
 clean@tool() {
-    _choice_tool | sed 's/\.\([a-z]\+\)$//' |  xargs -I{} rm -rf "$BIN_DIR/{}"
+    _choice_tool | sed -E 's/\.([a-z]+)$//' |  xargs -I{} rm -rf "$BIN_DIR/{}"
     rm -rf functions.json
 }
 

tools/search_arxiv.sh

@@ -3,7 +3,7 @@ set -e
 
 # @describe Search arXiv for a query and return the top papers.
 
-# @env ARXIV_MAX_RESULTS=5 The max results to return.
+# @env ARXIV_MAX_RESULTS=3 The max results to return.
 # @option --query! The query to search for.
 
 main() {

tools/search_bing.sh

@@ -5,15 +5,15 @@ set -e
 # Use this when you need current information or feel a search could provide a better answer.
 
 # @env BING_API_KEY! The api key
-# @env BING_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
     encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
-    url="https://api.bing.microsoft.com/v7.0/search?q=$encoded_query&mkt=en-us&textdecorations=true&textformat=raw&count=$BING_MAX_RESULTS&offset=0"
+    url="https://api.bing.microsoft.com/v7.0/search?q=$encoded_query&mkt=en-us&textdecorations=true&textformat=raw&count=$SEARCH_MAX_RESULTS&offset=0"
     curl -fsSL "$url" \
         -H "Ocp-Apim-Subscription-Key: $BING_API_KEY" | \
-        jq '[.webPages.value[] | {name: .name, url: .url, snippet: .snippet}]' \
+        jq '[.webPages.value[] | {link: .url, title: .name, snippet: .snippet}]' \
         >> "$LLM_OUTPUT"
 }
 

tools/search_brave.sh

@@ -5,16 +5,16 @@ set -e
 # Use this when you need current information or feel a search could provide a better answer.
 
 # @env BRAVE_API_KEY! The api key
-# @env BRAVE_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
     encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
-    url="https://api.search.brave.com/res/v1/web/search?q=$encoded_query&count=$BRAVE_MAX_RESULTS"
+    url="https://api.search.brave.com/res/v1/web/search?q=$encoded_query&count=$SEARCH_MAX_RESULTS"
     curl -fsSL "$url" \
         -H "Accept: application/json" \
         -H "X-Subscription-Token: $BRAVE_API_KEY" | \
-        jq '[.web.results[] | {title: .title, url: .url, description: .description}]' \
+        jq '[.web.results[] | {link: .url, title: .title, snippet: .description}]' \
         >> "$LLM_OUTPUT"
 }
 

tools/search_duckduckgo.sh

@@ -4,12 +4,32 @@ set -e
 # @describe Perform a web search using DuckDuckGo API to get up-to-date information or additional context.
 # Use this when you need current information or feel a search could provide a better answer.
 
-# @meta require-tools ddgr
-# @env DDG_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
-    ddgr -n $DDG_MAX_RESULTS --json "$argc_query" >> "$LLM_OUTPUT"
+    encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
+    vqd="$(curl -fsSL -X POST https://duckduckgo.com -d "q=$encoded_query" | sed -En 's/.*vqd=([0-9-]+)&.*/\1/p')"
+    url="https://links.duckduckgo.com/d.js?q=$encoded_query&kl=wt-wt&l=wt-wt&p=&s=0&df=&vqd=$vqd&bing_market=wt-WT&ex=-1"
+    data="$(curl -fsSL "$url" | sed -En 's/.*DDG.pageLayout.load\(\x27d\x27,\[(.*)\]\);DDG.duckbar.load\(.*/\1/p')"
+    echo "[$data]" | jq '
+def strip_tags:
+  gsub("<[^>]*>"; "");
+
+def unescape_html_entities:
+  gsub("&amp;"; "&") |
+  gsub("&lt;"; "<") |
+  gsub("&gt;"; ">") |
+  gsub("&quot;"; "\"") | 
+  gsub("&apos;"; "'\''") |
+  gsub("&#x27;"; "'\''") |
+  gsub("&nbsp;"; " ");
+
+def normalize: strip_tags | unescape_html_entities;
+
+[.[:'"$SEARCH_MAX_RESULTS"'] | .[] | select(has("u")) | {link: .u, title: (.t | normalize), snippet: (.a | normalize)}]
+' >> "$LLM_OUTPUT"
+
 }
 
 eval "$(argc --argc-eval "$0" "$@")"

tools/search_exa.sh

@@ -5,7 +5,7 @@ set -e
 # Use this when you need current information or feel a search could provide a better answer.
 
 # @env EXA_API_KEY! The api key
-# @env EXA_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
@@ -15,15 +15,15 @@ main() {
         -d '
 {
     "query": "'"$argc_query"'",
-    "numResults": '"$EXA_MAX_RESULTS"',
+    "numResults": '"$SEARCH_MAX_RESULTS"',
     "type": "keyword",
     "contents": {
         "text": {
             "maxCharacters": 200
         }
     }
 }' | \
-        jq '[.results[] | {title: .title, url: .url, text: .text}]' \
+        jq '[.results[] | {link: .url, title: .title, snippet: .text}]' \
         >> "$LLM_OUTPUT"
 }
 

tools/search_google.sh

@@ -6,14 +6,14 @@ set -e
 
 # @env GOOGLE_API_KEY! The api key
 # @env GOOGLE_CSE_ID! The id of google search engine
-# @env GOOGLE_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
     encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
     url="https://www.googleapis.com/customsearch/v1?key=$GOOGLE_API_KEY&cx=$GOOGLE_CSE_ID&q=$encoded_query"
     curl -fsSL "$url" | \
-        jq '[.items[:'"$GOOGLE_MAX_RESULTS"'] | .[] | {title: .title, link: .link, snippet: .snippet}]' \
+        jq '[.items[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .link, title: .title, snippet: .snippet}]' \
         >> "$LLM_OUTPUT"
 }
 

tools/search_jina.sh

@@ -5,7 +5,7 @@ set -e
 # Use this when you need current information or feel a search could provide a better answer.
 
 # @env JINA_API_KEY The api key
-# @env JINA_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
@@ -15,7 +15,7 @@ main() {
     fi
     encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
     curl -fsSL "${curl_args[@]}" "https://s.jina.ai/$encoded_query" | \
-        jq '[.data[:'"$JINA_MAX_RESULTS"'] | .[] | {title: .title, url: .url, description: .description}]' \
+        jq '[.data[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .url, title: .title, snippet: .description}]' \
         >> "$LLM_OUTPUT"
 }
 

tools/search_searxng.sh

@@ -5,14 +5,14 @@ set -e
 # Use this when you need current information or feel a search could provide a better answer.
 
 # @env SEARXNG_API_BASE! The api url
-# @env SEARXNG_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
     encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
     url="$SEARXNG_API_BASE/search?q=$encoded_query&categories=general&language=en-US&format=json"
     curl -fsSL "$url" | \
-        jq '[.results[:'"$SEARXNG_MAX_RESULTS"'] | .[] | {url: .url, title: .title, content: .content}]' \
+        jq '[.results[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .url, title: .title, snippet: .content}]' \
         >> "$LLM_OUTPUT"
 
 }

tools/search_tavily.sh

@@ -5,7 +5,7 @@ set -e
 # Use this when you need current information or feel a search could provide a better answer.
 
 # @env TAVILY_API_KEY! The api key
-# @env TAVILY_MAX_RESULTS=5 The max results to return.
+# @env SEARCH_MAX_RESULTS=5 The max results to return.
 # @option --query! The query to search for.
 
 main() {
@@ -16,9 +16,9 @@ main() {
     "api_key": "'"$TAVILY_API_KEY"'",
     "query": "'"$argc_query"'",
     "search_depth": "advanced",
-    "max_results": "'"$TAVILY_MAX_RESULTS"'"
+    "max_results": "'"$SEARCH_MAX_RESULTS"'"
 }' | \
-        jq '[.results[] | {title: .title, url: .url, content: .content}]' \
+        jq '[.results[] | {link: .url, title: .title, snippet: .content}]' \
         >> "$LLM_OUTPUT"
 }