ignoring utf8 read errors

emcf · Sep 5, 2024 · 1c851da · 1c851da
1 parent 4fc33ec
commit 1c851da
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -10,7 +10,7 @@ def read_git_requirements(file):
 
 setup(
     name='thepipe_api',
-    version='1.2.7',
+    version='1.2.8',
     author='Emmett McFarlane',
     author_email='[email protected]',
     description='AI-native extractor, powered by multimodal LLMs.',

diff --git a/thepipe/scraper.py b/thepipe/scraper.py
@@ -126,7 +126,7 @@ def scrape_file(filepath: str, ai_extraction: bool = False, text_only: bool = Fa
     return scraped_chunks
 
 def scrape_plaintext(file_path: str) -> List[Chunk]:
-    with open(file_path, 'r', encoding='utf-8') as file:
+    with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:
         text = file.read()
     return [Chunk(path=file_path, texts=[text])]