diff --git a/pyproject.toml b/pyproject.toml index 67f68252..9c113ade 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "python-pptx", "pandas", "openpyxl", + "xlrd", "pdfminer.six", "puremagic", "pydub", diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.py index f8b97ee0..1656eaf4 100644 --- a/src/markitdown/_markitdown.py +++ b/src/markitdown/_markitdown.py @@ -173,7 +173,10 @@ def convert( # Only accept text files if content_type is None: return None - elif "text/" not in content_type.lower(): + elif all( + not content_type.lower().startswith(type_prefix) + for type_prefix in ["text/", "application/json"] + ): return None text_content = str(from_path(local_path).best()) @@ -760,7 +763,31 @@ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]: if extension.lower() != ".xlsx": return None - sheets = pd.read_excel(local_path, sheet_name=None) + sheets = pd.read_excel(local_path, sheet_name=None, engine="openpyxl") + md_content = "" + for s in sheets: + md_content += f"## {s}\n" + html_content = sheets[s].to_html(index=False) + md_content += self._convert(html_content).text_content.strip() + "\n\n" + + return DocumentConverterResult( + title=None, + text_content=md_content.strip(), + ) + + +class XlsConverter(HtmlConverter): + """ + Converts XLS files to Markdown, with each sheet presented as a separate Markdown table. + """ + + def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]: + # Bail if not a XLS + extension = kwargs.get("file_extension", "") + if extension.lower() != ".xls": + return None + + sheets = pd.read_excel(local_path, sheet_name=None, engine="xlrd") md_content = "" for s in sheets: md_content += f"## {s}\n" @@ -1387,6 +1414,7 @@ def __init__( self.register_page_converter(BingSerpConverter()) self.register_page_converter(DocxConverter()) self.register_page_converter(XlsxConverter()) + self.register_page_converter(XlsConverter()) self.register_page_converter(PptxConverter()) self.register_page_converter(WavConverter()) self.register_page_converter(Mp3Converter()) @@ -1603,6 +1631,25 @@ def _guess_ext_magic(self, path): # Use puremagic to guess try: guesses = puremagic.magic_file(path) + + # Fix for: https://github.com/microsoft/markitdown/issues/222 + # If there are no guesses, then try again after trimming leading ASCII whitespaces. + # ASCII whitespace characters are those byte values in the sequence b' \t\n\r\x0b\f' + # (space, tab, newline, carriage return, vertical tab, form feed). + if len(guesses) == 0: + with open(path, "rb") as file: + while True: + char = file.read(1) + if not char: # End of file + break + if not char.isspace(): + file.seek(file.tell() - 1) + break + try: + guesses = puremagic.magic_stream(file) + except puremagic.main.PureError: + pass + extensions = list() for g in guesses: ext = g.extension.strip() diff --git a/tests/test_files/test.json b/tests/test_files/test.json new file mode 100644 index 00000000..eba30594 --- /dev/null +++ b/tests/test_files/test.json @@ -0,0 +1,10 @@ +{ + "key1": "string_value", + "key2": 1234, + "key3": [ + "list_value1", + "list_value2" + ], + "5b64c88c-b3c3-4510-bcb8-da0b200602d8": "uuid_key", + "uuid_value": "9700dc99-6685-40b4-9a3a-5e406dcb37f3" +} diff --git a/tests/test_files/test.xls b/tests/test_files/test.xls new file mode 100644 index 00000000..de4f368c Binary files /dev/null and b/tests/test_files/test.xls differ diff --git a/tests/test_markitdown.py b/tests/test_markitdown.py index a0626d19..3333bcbc 100644 --- a/tests/test_markitdown.py +++ b/tests/test_markitdown.py @@ -54,6 +54,12 @@ "affc7dad-52dc-4b98-9b5d-51e65d8a8ad0", ] +XLS_TEST_STRINGS = [ + "## 09060124-b5e7-4717-9d07-3c046eb", + "6ff4173b-42a5-4784-9b19-f49caff4d93d", + "affc7dad-52dc-4b98-9b5d-51e65d8a8ad0", +] + DOCX_TEST_STRINGS = [ "314b0a30-5b04-470b-b9f7-eed2c2bec74a", "49e168b7-d2ae-407f-a055-2167576f39a1", @@ -139,6 +145,11 @@ "5bda1dd6", ] +JSON_TEST_STRINGS = [ + "5b64c88c-b3c3-4510-bcb8-da0b200602d8", + "9700dc99-6685-40b4-9a3a-5e406dcb37f3", +] + # --- Helper Functions --- def validate_strings(result, expected_strings, exclude_strings=None): @@ -185,6 +196,12 @@ def test_markitdown_local() -> None: result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsx")) validate_strings(result, XLSX_TEST_STRINGS) + # Test XLS processing + result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xls")) + for test_string in XLS_TEST_STRINGS: + text_content = result.text_content.replace("\\", "") + assert test_string in text_content + # Test DOCX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.docx")) validate_strings(result, DOCX_TEST_STRINGS) @@ -245,6 +262,15 @@ def test_markitdown_local() -> None: result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_outlook_msg.msg")) validate_strings(result, MSG_TEST_STRINGS) + # Test JSON processing + result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.json")) + validate_strings(result, JSON_TEST_STRINGS) + + # Test input with leading blank characters + input_data = b" \n\n\n

Test

" + result = markitdown.convert_stream(io.BytesIO(input_data)) + assert "# Test" in result.text_content + @pytest.mark.skipif( skip_exiftool,