From d449cb8babd3ec3f7263a9a1dc039d4876ca1d4c Mon Sep 17 00:00:00 2001 From: sigoden Date: Wed, 17 Jul 2024 10:25:38 +0000 Subject: [PATCH] refactor: improve fetch_url_via_curl --- tools/fetch_url_via_curl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/fetch_url_via_curl.sh b/tools/fetch_url_via_curl.sh index 2f2efdc..540984e 100755 --- a/tools/fetch_url_via_curl.sh +++ b/tools/fetch_url_via_curl.sh @@ -8,8 +8,8 @@ set -e main() { # span and div tags are dropped from the HTML https://pandoc.org/MANUAL.html#raw-htmltex and sed removes any inline SVG images in image tags from the Markdown content. curl -fsSL "$argc_url" | \ - pandoc -f html-native_divs-native_spans -t gfm-raw_html | \ - sed -E 's/!\[.*?\]\((data:image\/svg\+xml[^)]+)\)//g' \ + pandoc -f html-native_divs-native_spans -t gfm-raw_html --wrap=none | \ + sed -E 's/!\[[^]]*\]\([^)]*\)//g' \ >> "$LLM_OUTPUT" }