Merge pull request #1576 from vespa-engine/thomasht86/exclude-non-pdf

(colpalidemo) do not download non pdf links
vespa-engine · Nov 12, 2024 · 71b5688 · 71b5688
2 parents 0b3cdfc + e434916
commit 71b5688
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/visual-retrieval-colpali/prepare_feed_deploy.py b/visual-retrieval-colpali/prepare_feed_deploy.py
@@ -178,8 +178,10 @@
     for a_tag in year_div.select("a.button.button--download-secondary[href]"):
         href = a_tag["href"]
         full_url = urljoin(url, href)
-        links.append(full_url)
-        url_to_year[full_url] = year
+        # exclude non-pdf links
+        if full_url.endswith(".pdf"):
+            links.append(full_url)
+            url_to_year[full_url] = year
 links, url_to_year
 # -