From b900c9bac4dc83c26b3640ee70305cee1b48c3a2 Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Thu, 1 Oct 2026 21:42:59 -0700 Subject: [PATCH] Consolidate tests by format and strengthen regression coverage --- packages/markitdown/tests/README.md | 54 + .../markitdown/tests/test_charset_sample.py | 120 -- packages/markitdown/tests/test_cli_misc.py | 246 ++- packages/markitdown/tests/test_csv.py | 255 +++ .../markitdown/tests/test_csv_blank_runs.py | 35 - .../markitdown/tests/test_csv_line_endings.py | 53 - .../{test_cu_converter.py => test_cu.py} | 244 +-- ...test_docintel_html.py => test_docintel.py} | 8 +- packages/markitdown/tests/test_docx.py | 1489 ++++++++++++++++ packages/markitdown/tests/test_docx_images.py | 606 ------- packages/markitdown/tests/test_docx_math.py | 59 - .../tests/test_docx_math_accents.py | 80 - .../tests/test_docx_math_symbols.py | 102 -- packages/markitdown/tests/test_docx_omml.py | 48 - packages/markitdown/tests/test_docx_styles.py | 152 -- .../{test_epub_converter.py => test_epub.py} | 120 +- packages/markitdown/tests/test_file_paths.py | 138 -- packages/markitdown/tests/test_html.py | 349 ++++ .../markitdown/tests/test_html_converter.py | 153 -- ...{test_image_converter.py => test_image.py} | 122 +- packages/markitdown/tests/test_module_misc.py | 1587 +++-------------- .../markitdown/tests/test_module_vectors.py | 33 - ...utlook_msg_ansi.py => test_outlook_msg.py} | 19 +- .../tests/{test_pdf_tables.py => test_pdf.py} | 712 +++++--- .../markitdown/tests/test_pdf_masterformat.py | 171 -- packages/markitdown/tests/test_pdf_memory.py | 364 ---- .../{test_pptx_images.py => test_pptx.py} | 437 ++++- .../markitdown/tests/test_pptx_empty_title.py | 67 - .../markitdown/tests/test_pptx_none_text.py | 66 - packages/markitdown/tests/test_pptx_notes.py | 58 - packages/markitdown/tests/test_pptx_svg.py | 113 -- .../{test_rss_converter.py => test_rss.py} | 400 +++++ packages/markitdown/tests/test_rss_titles.py | 305 ---- .../tests/test_undetectable_charset.py | 74 - .../markitdown/tests/test_wikipedia_titles.py | 57 - .../{test_xlsx_images.py => test_xlsx.py} | 95 +- ...t_youtube_converter.py => test_youtube.py} | 86 +- .../tests/{test_zip_kwargs.py => test_zip.py} | 50 + 38 files changed, 4480 insertions(+), 4647 deletions(-) create mode 100644 packages/markitdown/tests/README.md delete mode 100644 packages/markitdown/tests/test_charset_sample.py create mode 100644 packages/markitdown/tests/test_csv.py delete mode 100644 packages/markitdown/tests/test_csv_blank_runs.py delete mode 100644 packages/markitdown/tests/test_csv_line_endings.py rename packages/markitdown/tests/{test_cu_converter.py => test_cu.py} (79%) rename packages/markitdown/tests/{test_docintel_html.py => test_docintel.py} (94%) create mode 100644 packages/markitdown/tests/test_docx.py delete mode 100644 packages/markitdown/tests/test_docx_images.py delete mode 100644 packages/markitdown/tests/test_docx_math.py delete mode 100644 packages/markitdown/tests/test_docx_math_accents.py delete mode 100644 packages/markitdown/tests/test_docx_math_symbols.py delete mode 100644 packages/markitdown/tests/test_docx_omml.py delete mode 100644 packages/markitdown/tests/test_docx_styles.py rename packages/markitdown/tests/{test_epub_converter.py => test_epub.py} (50%) delete mode 100644 packages/markitdown/tests/test_file_paths.py create mode 100644 packages/markitdown/tests/test_html.py delete mode 100644 packages/markitdown/tests/test_html_converter.py rename packages/markitdown/tests/{test_image_converter.py => test_image.py} (53%) rename packages/markitdown/tests/{test_outlook_msg_ansi.py => test_outlook_msg.py} (98%) rename packages/markitdown/tests/{test_pdf_tables.py => test_pdf.py} (66%) delete mode 100644 packages/markitdown/tests/test_pdf_masterformat.py delete mode 100644 packages/markitdown/tests/test_pdf_memory.py rename packages/markitdown/tests/{test_pptx_images.py => test_pptx.py} (55%) delete mode 100644 packages/markitdown/tests/test_pptx_empty_title.py delete mode 100644 packages/markitdown/tests/test_pptx_none_text.py delete mode 100644 packages/markitdown/tests/test_pptx_notes.py delete mode 100644 packages/markitdown/tests/test_pptx_svg.py rename packages/markitdown/tests/{test_rss_converter.py => test_rss.py} (64%) delete mode 100644 packages/markitdown/tests/test_rss_titles.py delete mode 100644 packages/markitdown/tests/test_undetectable_charset.py delete mode 100644 packages/markitdown/tests/test_wikipedia_titles.py rename packages/markitdown/tests/{test_xlsx_images.py => test_xlsx.py} (82%) rename packages/markitdown/tests/{test_youtube_converter.py => test_youtube.py} (65%) rename packages/markitdown/tests/{test_zip_kwargs.py => test_zip.py} (72%) diff --git a/packages/markitdown/tests/README.md b/packages/markitdown/tests/README.md new file mode 100644 index 0000000000..34c10c5ce3 --- /dev/null +++ b/packages/markitdown/tests/README.md @@ -0,0 +1,54 @@ +# Test organization + +Prefer the shared conversion matrix when a regression can be expressed as an +input file and expected or forbidden output strings: + +- Put input files in `test_files/` and define cases in `_test_vectors.py`. +- `test_cli_vectors.py` exercises the applicable vectors through the CLI. +- `test_module_vectors.py` exercises them through library file, stream, and URI + entry points. + +Tests that need custom setup, mocks, conversion options, or structural assertions +belong in the existing file for their format. Keep helpers and fixtures in that +file unless multiple suites actually need to share them. Add related tests to the +appropriate section instead of creating a file for each bug or feature. + +| Test file | Scope | +| --- | --- | +| `test_docx.py` | Word documents, math, styles, comments, and image hooks | +| `test_pptx.py` | PowerPoint slides, titles, notes, charts, SVGs, and image hooks | +| `test_xlsx.py` | Spreadsheet compatibility and image hooks, including XLS boundaries | +| `test_pdf.py` | PDF tables, numbering, page cleanup, and extraction fallback | +| `test_csv.py` | CSV escaping, blank rows, and line endings | +| `test_html.py` | HTML rendering, links, images, and Wikipedia pages | +| `test_rss.py` | RSS and Atom bodies, titles, metadata, and links | +| `test_epub.py` | EPUB archive paths, metadata, and embedded images | +| `test_outlook_msg.py` | Outlook MSG properties and encodings | +| `test_image.py` | Image captions and media metadata | +| `test_youtube.py` | YouTube URLs, extraction, transcripts, and fallback | +| `test_zip.py` | ZIP entries and nested conversion options | +| `test_docintel.py` | Document Intelligence routing and client options | +| `test_cu.py` | Content Understanding routing, conversion, and registration | +| `test_cli_misc.py` | CLI behavior outside the vector matrix, including service flags | +| `test_module_misc.py` | Library infrastructure, cross-format options, and formats without a dedicated suite | + +`test_module_misc.py` includes stream metadata, URI handling, charset detection, +cross-format LLM options, and the existing notebook and audio regressions. +Format-specific tests belong in their dedicated suite when one exists. + +Run the suite from `packages/markitdown`: + +```sh +hatch test +``` + +To run a single format: + +```sh +hatch test tests/test_docx.py +``` + +The existing remote, credential, platform, and optional-dependency skip conditions +apply. A consolidation should preserve test bodies, parametrization, fixtures, +and skip conditions; compare collected cases as well as test results to catch +accidentally dropped or overwritten tests. diff --git a/packages/markitdown/tests/test_charset_sample.py b/packages/markitdown/tests/test_charset_sample.py deleted file mode 100644 index 9b3036bd56..0000000000 --- a/packages/markitdown/tests/test_charset_sample.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Charset sampling must not mistake a split UTF-8 character for another encoding.""" - -import io - -import pytest - -from markitdown import MarkItDown, StreamInfo -from markitdown._markitdown import _read_charset_sample - - -_SAMPLE_SIZE = 65536 -_SPLIT_CHARACTERS = [ - (character, split) - for character in ("\u00e9", "\u65e5", "\U0001f600") - for split in range(1, len(character.encode("utf-8"))) -] - - -def _split_utf8_json(character: str, split: int) -> bytes: - prefix = '{"name":"r\u00e9sum\u00e9","notes":"'.encode("utf-8") - return ( - prefix - + b"a" * (_SAMPLE_SIZE - split - len(prefix)) - + (character + '"}').encode("utf-8") - ) - - -@pytest.fixture(scope="module") -def markitdown() -> MarkItDown: - return MarkItDown() - - -@pytest.mark.parametrize("character,split", _SPLIT_CHARACTERS) -def test_charset_sample_completes_only_the_split_character( - character: str, split: int -) -> None: - data = _split_utf8_json(character, split) - stream = io.BytesIO(data) - expected_size = _SAMPLE_SIZE + len(character.encode("utf-8")) - split - - sample = _read_charset_sample(stream) - - assert sample == data[:expected_size] - assert stream.tell() == expected_size - assert expected_size <= _SAMPLE_SIZE + 3 - sample.decode("utf-8") - - -@pytest.mark.parametrize("character,split", _SPLIT_CHARACTERS) -def test_split_utf8_json_preserves_content( - markitdown: MarkItDown, character: str, split: int -) -> None: - data = _split_utf8_json(character, split) - stream = io.BytesIO(data) - - guesses = markitdown._get_stream_info_guesses(stream, StreamInfo()) - assert guesses[0].charset == "utf-8" - assert stream.tell() == 0 - - result = markitdown.convert_stream(stream) - - assert result.markdown == data.decode("utf-8") - - -@pytest.mark.parametrize( - "sample,tail", - [ - (b"", b""), - (b"ordinary text", b""), - (b"short incomplete \xc3", b""), - (b"a" * _SAMPLE_SIZE, b"\xc3\xa9"), - (b"a" * (_SAMPLE_SIZE - 2) + b"\xc3\xa9", b"\xf0\x9f\x98\x80"), - (b"a" * (_SAMPLE_SIZE - 1) + b"\xc3", b""), - (b"a" * (_SAMPLE_SIZE - 1) + b"\xf0", b"\x9f"), - (b"a" * (_SAMPLE_SIZE - 1) + b"\xc3", b"x"), - (b"a" * (_SAMPLE_SIZE - 1) + b"\xe0", b"\x80\x80"), - (b"a" * (_SAMPLE_SIZE - 1) + b"\xed", b"\xa0\x80"), - (b"\xff" + b"a" * (_SAMPLE_SIZE - 2) + b"\xc3", b"\xa9"), - ], - ids=[ - "empty", - "short", - "short-incomplete", - "ascii", - "complete-utf8", - "incomplete-at-eof", - "incomplete-after-lookahead", - "invalid-continuation", - "overlong", - "surrogate", - "non-utf8-prefix", - ], -) -def test_other_charset_samples_are_unchanged(sample: bytes, tail: bytes) -> None: - stream = io.BytesIO(sample + tail) - - assert _read_charset_sample(stream) == sample - assert stream.tell() <= _SAMPLE_SIZE + 3 - - -def test_charset_guesses_restore_nonzero_stream_position( - markitdown: MarkItDown, -) -> None: - stream = io.BytesIO(b"prefix" + _split_utf8_json("\U0001f600", 1)) - stream.seek(len(b"prefix")) - - markitdown._get_stream_info_guesses(stream, StreamInfo()) - - assert stream.tell() == len(b"prefix") - - -def test_explicit_charset_still_takes_precedence(markitdown: MarkItDown) -> None: - data = _split_utf8_json("\u00e9", 1) - - result = markitdown.convert_stream( - io.BytesIO(data), - stream_info=StreamInfo(extension=".json", charset="cp1252"), - ) - - assert result.markdown == data.decode("cp1252") diff --git a/packages/markitdown/tests/test_cli_misc.py b/packages/markitdown/tests/test_cli_misc.py index c059d74477..7df1b0e43b 100644 --- a/packages/markitdown/tests/test_cli_misc.py +++ b/packages/markitdown/tests/test_cli_misc.py @@ -1,11 +1,20 @@ -#!/usr/bin/env python3 -m pytest +"""CLI behavior outside the shared file-vector matrix.""" + import io +import os import subprocess import sys from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest from markitdown import __version__ from markitdown.__main__ import main +from markitdown.converters._cu_converter import ContentUnderstandingFileType + + +# CLI behavior # This file contains CLI tests that are not directly tested by the FileTestVectors. # This includes things like help messages, version numbers, and invalid flags. @@ -55,8 +64,235 @@ def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: assert captured.out.strip() == "# Test HTML" +# Service flags and endpoints + +# --------------------------------------------------------------------------- +# CLI argument tests +# --------------------------------------------------------------------------- + + +class TestCLIArgs: + """Test CLI argument parsing for CU flags.""" + + def test_use_cu_without_endpoint_exits(self): + """--use-cu without --cu-endpoint should exit with error.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "markitdown", "--use-cu", "fake.pdf"], + capture_output=True, + text=True, + env={**os.environ, "MARKITDOWN_CU_ENDPOINT": ""}, + ) + assert result.returncode != 0 + assert ( + "cu-endpoint" in result.stderr.lower() + or "cu-endpoint" in (result.stdout or "").lower() + ) + + def test_use_cu_and_use_docintel_mutually_exclusive(self): + """--use-cu and --use-docintel cannot be used together.""" + import subprocess + + result = subprocess.run( + [ + sys.executable, + "-m", + "markitdown", + "--use-cu", + "--cu-endpoint", + "https://fake", + "--use-docintel", + "-e", + "https://fake-di", + "fake.pdf", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 2, result.stderr + assert ( + "argument -d/--use-docintel: not allowed with argument " + "--use-cu/--use-content-understanding" + ) in result.stderr, result.stderr + + def test_cu_file_types_parsing(self): + """--cu-file-types should parse comma-separated values into enum list.""" + from markitdown.converters import ContentUnderstandingFileType + + raw = "pdf,jpeg,mp4" + type_names = [t.strip().lower() for t in raw.split(",") if t.strip()] + cu_types = [ContentUnderstandingFileType(name) for name in type_names] + + assert cu_types == [ + ContentUnderstandingFileType.PDF, + ContentUnderstandingFileType.JPEG, + ContentUnderstandingFileType.MP4, + ] + + def test_cu_file_types_invalid_value(self): + """Unknown file type name should raise ValueError.""" + from markitdown.converters import ContentUnderstandingFileType + + with pytest.raises(ValueError): + ContentUnderstandingFileType("nonsense") + + def test_cu_file_types_single_value(self): + """Single file type (no comma) should parse correctly.""" + from markitdown.converters import ContentUnderstandingFileType + + cu_types = [ + ContentUnderstandingFileType(t.strip().lower()) + for t in "wav".split(",") + if t.strip() + ] + assert cu_types == [ContentUnderstandingFileType.WAV] + + def test_use_cu_wires_kwargs_to_markitdown(self, capsys): + """--use-cu should pass CU options through to MarkItDown.""" + import markitdown.__main__ as markitdown_cli + + markitdown_instance = MagicMock() + markitdown_instance.convert.return_value.markdown = "converted" + markitdown_cls = MagicMock(return_value=markitdown_instance) + + with patch.object( + sys, + "argv", + [ + "markitdown", + "--use-cu", + "--cu-endpoint", + "https://fake-cu", + "--cu-analyzer", + "custom-analyzer", + "--cu-file-types", + "pdf,jpeg,mp4", + "fake.pdf", + ], + ), patch.object(markitdown_cli, "MarkItDown", markitdown_cls): + markitdown_cli.main() + + markitdown_cls.assert_called_once_with( + enable_plugins=False, + cu_endpoint="https://fake-cu", + cu_analyzer_id="custom-analyzer", + cu_file_types=[ + ContentUnderstandingFileType.PDF, + ContentUnderstandingFileType.JPEG, + ContentUnderstandingFileType.MP4, + ], + ) + markitdown_instance.convert.assert_called_once_with( + "fake.pdf", stream_info=None, keep_data_uris=False + ) + assert capsys.readouterr().out == "converted\n" + + def test_use_cu_reads_from_stdin(self, capsys): + """--use-cu should preserve the CLI's filename-optional stdin mode.""" + import markitdown.__main__ as markitdown_cli + + input_buffer = io.BytesIO(b"fake pdf") + stdin = MagicMock(buffer=input_buffer) + markitdown_instance = MagicMock() + markitdown_instance.convert_stream.return_value.markdown = "converted" + markitdown_cls = MagicMock(return_value=markitdown_instance) + + with patch.object( + sys, + "argv", + [ + "markitdown", + "--use-cu", + "--cu-endpoint", + "https://fake-cu", + ], + ), patch.object(sys, "stdin", stdin), patch.object( + markitdown_cli, "MarkItDown", markitdown_cls + ): + markitdown_cli.main() + + markitdown_cls.assert_called_once_with( + enable_plugins=False, + cu_endpoint="https://fake-cu", + ) + + assert markitdown_instance.convert_stream.call_count == 1 + call_args, call_kwargs = markitdown_instance.convert_stream.call_args + assert call_args[0].read() == b"fake pdf" + assert call_kwargs == {"stream_info": None, "keep_data_uris": False} + assert capsys.readouterr().out == "converted\n" + + +# --------------------------------------------------------------------------- +# Endpoint environment variables (issue #2326) +# --------------------------------------------------------------------------- + + +class TestEndpointEnvVars: + """Endpoint flags fall back to operator-configured environment variables.""" + + @pytest.mark.parametrize( + "env_var, argv, kwarg", + [ + ( + "MARKITDOWN_CU_ENDPOINT", + ["markitdown", "--use-cu", "fake.pdf"], + "cu_endpoint", + ), + ( + "MARKITDOWN_DOCINTEL_ENDPOINT", + ["markitdown", "-d", "fake.pdf"], + "docintel_endpoint", + ), + ], + ) + def test_endpoint_read_from_environment(self, monkeypatch, env_var, argv, kwarg): + """With the env var set, the endpoint flag can be omitted entirely.""" + from markitdown.__main__ import main + + monkeypatch.setenv(env_var, "https://from-env") + monkeypatch.setattr(sys, "argv", argv) + + with patch("markitdown.__main__.MarkItDown") as mock_markitdown: + main() + + assert mock_markitdown.call_args.kwargs[kwarg] == "https://from-env" + + def test_flag_overrides_environment(self, monkeypatch): + """Fails if the env var is ever read after parsing instead of as an argparse default.""" + from markitdown.__main__ import main + + monkeypatch.setenv("MARKITDOWN_CU_ENDPOINT", "https://from-env") + monkeypatch.setattr( + sys, + "argv", + [ + "markitdown", + "--use-cu", + "--cu-endpoint", + "https://from-flag", + "fake.pdf", + ], + ) + + with patch("markitdown.__main__.MarkItDown") as mock_markitdown: + main() + + assert mock_markitdown.call_args.kwargs["cu_endpoint"] == "https://from-flag" + + def test_empty_environment_variable_is_treated_as_unset(self, monkeypatch, capsys): + """An empty env var must not pass as a valid endpoint.""" + from markitdown.__main__ import main + + monkeypatch.setenv("MARKITDOWN_CU_ENDPOINT", "") + monkeypatch.setattr(sys, "argv", ["markitdown", "--use-cu", "fake.pdf"]) + + with pytest.raises(SystemExit): + main() + + assert "MARKITDOWN_CU_ENDPOINT" in capsys.readouterr().out + + if __name__ == "__main__": - """Runs this file's tests from the command line.""" - test_version() - test_invalid_flag() - print("All tests passed!") + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_csv.py b/packages/markitdown/tests/test_csv.py new file mode 100644 index 0000000000..d21742bfbc --- /dev/null +++ b/packages/markitdown/tests/test_csv.py @@ -0,0 +1,255 @@ +"""CSV conversion, escaping, blank rows, and line endings.""" + +import io + +import pytest + +from markitdown import MarkItDown, StreamInfo + + +# Record separators + + +@pytest.fixture(scope="module") +def converter() -> MarkItDown: + return MarkItDown(enable_plugins=False) + + +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) +@pytest.mark.parametrize("final_newline", [False, True]) +def test_csv_record_separators_produce_a_table( + converter: MarkItDown, newline: str, final_newline: bool +) -> None: + content = newline.join(["name,age", "Alice,30"]) + if final_newline: + content += newline + + result = converter.convert_stream( + io.BytesIO(content.encode("utf-8")), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + + assert result.markdown == "| name | age |\n| --- | --- |\n| Alice | 30 |" + + +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) +@pytest.mark.parametrize( + "cell_newline", ["\n", "\r\n", "\r"], ids=["cell-LF", "cell-CRLF", "cell-CR"] +) +def test_csv_quoted_line_break_stays_inside_its_cell( + converter: MarkItDown, newline: str, cell_newline: str +) -> None: + content = newline.join( + ["name,notes", f'Alice,"first{cell_newline}second"', "Bob,plain"] + ) + + result = converter.convert_stream( + io.BytesIO(content.encode("utf-8")), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + + assert result.markdown == ( + "| name | notes |\n" + "| --- | --- |\n" + "| Alice | first second |\n" + "| Bob | plain |" + ) + + +# Long blank runs + + +@pytest.mark.parametrize("position", ["leading", "after_header", "trailing", "all"]) +def test_csv_long_blank_runs(position: str) -> None: + blank = b"\n" * 100_000 + header = b"name,value\n" + # An internal blank row and a wider data row must survive trimming. + data = b"Alice,1\n\nBob,2,extra\n" + content = { + "leading": blank + header + data, + "after_header": header + blank + data, + "trailing": header + data + blank, + "all": blank, + }[position] + + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(content), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + + expected = ( + "| name | value | |\n" + "| --- | --- | --- |\n" + "| Alice | 1 | |\n" + "| | | |\n" + "| Bob | 2 | extra |" + ) + assert result.markdown == ("" if position == "all" else expected) + + +# Conversion regressions + +############################################################################### +# CSV converter. +############################################################################### + + +def _convert_csv(data: bytes, charset: str | None = None) -> str: + stream_info = StreamInfo(extension=".csv", charset=charset) + return ( + MarkItDown() + .convert_stream(io.BytesIO(data), stream_info=stream_info) + .text_content + ) + + +def test_csv_utf8_bom_is_stripped_from_the_header() -> None: + result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n") + + assert "\ufeff" not in result + assert result.startswith("| name | age |") + assert "| Alice | 30 |" in result + + +def test_csv_utf8_bom_is_stripped_when_charset_is_known() -> None: + result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n", charset="utf-8") + + assert "\ufeff" not in result + assert result.startswith("| name | age |") + + +def test_csv_leading_blank_line_does_not_destroy_the_table() -> None: + result = _convert_csv(b"\nname,age\nAlice,30\n") + + assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |" + + +def test_csv_trailing_blank_lines_are_skipped() -> None: + result = _convert_csv(b"name,age\nAlice,30\n\n\n") + + assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |" + + +def test_csv_blank_lines_between_rows_are_kept() -> None: + result = _convert_csv(b"name,age\n\nAlice,30\n\nBob,40\n") + + assert ( + result == "| name | age |\n| --- | --- |\n| Alice | 30 |\n| | |\n| Bob | 40 |" + ) + + +def test_csv_all_blank_input_returns_empty_markdown() -> None: + result = _convert_csv(b"\n\n\n") + + assert result == "" + + +@pytest.mark.parametrize( + "data,expected", + [ + ( + b"banner\nname,age,city\nAlice,30,Seattle\n", + "| banner | | |\n| --- | --- | --- |\n" + "| name | age | city |\n| Alice | 30 | Seattle |", + ), + ( + b"name,age\nAlice\n\nBob,40,Seattle\nCarol,25\n", + "| name | age | |\n| --- | --- | --- |\n" + "| Alice | | |\n| | | |\n" + "| Bob | 40 | Seattle |\n| Carol | 25 | |", + ), + ( + b"name\nAlice,,\n", + "| name | | |\n| --- | --- | --- |\n| Alice | | |", + ), + ( + b"name,age,city\nAlice,30\nBob\n", + "| name | age | city |\n| --- | --- | --- |\n" + "| Alice | 30 | |\n| Bob | | |", + ), + ], + ids=["preamble", "widest-row-late", "trailing-empty-fields", "widest-header"], +) +def test_csv_table_matches_widest_row(data: bytes, expected: str) -> None: + assert _convert_csv(data) == expected + + +def test_csv_pipe_in_cell_is_escaped() -> None: + result = _convert_csv(b'name,description\nWidget,"cheap | fast"\n') + + # The pipe must be escaped rather than emitted raw, or it splits the row. + assert "| Widget | cheap \\| fast |" in result + + +def test_csv_pipe_preceded_by_backslash_is_still_escaped() -> None: + # `left\|right` must not become `left\\|right`: the doubled backslash is a + # literal backslash, which would leave the pipe acting as a delimiter and + # let a renderer drop `right`. The run of backslashes is doubled instead, + # so the escaping `\|` survives. + result = _convert_csv(b'name,description\nWidget,"left\\|right"\n') + + assert r"| Widget | left\\\|right |" in result + + +def test_csv_pipe_preceded_by_two_backslashes_is_still_escaped() -> None: + # An even-length run is just as dangerous once the naive `\|` is appended, + # so check a longer run as well. + result = _convert_csv(b'name,description\nWidget,"left\\\\|right"\n') + + assert r"| Widget | left\\\\\|right |" in result + + +def test_csv_newline_in_quoted_cell_does_not_split_the_row() -> None: + result = _convert_csv(b'name,notes\nWidget,"line one\nline two"\n') + + # The table must stay on one line per record; the embedded break collapses + # to a space. + assert len(result.splitlines()) == 3 + assert "| Widget | line one line two |" in result + + +def test_csv_carriage_returns_in_quoted_cell_collapse_to_spaces() -> None: + result = _convert_csv(b'name,notes\nWidget,"line one\r\nline two\rline three"\n') + + assert len(result.splitlines()) == 3 + assert "| Widget | line one line two line three |" in result + + +def test_csv_pipe_in_header_is_escaped() -> None: + result = _convert_csv(b'"a | b",c\n1,2\n') + + assert "| a \\| b | c |" in result + + +def test_csv_plain_values_are_unchanged() -> None: + # Guards against over-escaping ordinary content. + result = _convert_csv(b"name,description\nWidget,cheap and fast\n") + + assert "| Widget | cheap and fast |" in result + assert "\\" not in result + + +def test_csv_backslash_without_a_pipe_is_left_alone() -> None: + # Only backslashes that guard a pipe are doubled; a Windows path stays + # readable. + result = _convert_csv(b"name,path\nWidget,C:\\temp\\file.txt\n") + + assert r"| Widget | C:\temp\file.txt |" in result + + +@pytest.mark.parametrize( + "suffix,escaped_suffix", + [("", ""), ("|", r"\|"), ("x|", r"x\|")], +) +def test_csv_long_backslash_runs(suffix: str, escaped_suffix: str) -> None: + backslashes = "\\" * 65_536 + value = backslashes + suffix + expected = backslashes * (2 if suffix == "|" else 1) + escaped_suffix + + result = _convert_csv(f"{value}\n{value}\n".encode("utf-8"), charset="utf-8") + + assert result == f"| {expected} |\n| --- |\n| {expected} |" + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_csv_blank_runs.py b/packages/markitdown/tests/test_csv_blank_runs.py deleted file mode 100644 index 357af591e8..0000000000 --- a/packages/markitdown/tests/test_csv_blank_runs.py +++ /dev/null @@ -1,35 +0,0 @@ -"""CSV conversion preserves table contents when trimming long blank runs.""" - -import io - -import pytest - -from markitdown import MarkItDown, StreamInfo - - -@pytest.mark.parametrize("position", ["leading", "after_header", "trailing", "all"]) -def test_csv_long_blank_runs(position: str) -> None: - blank = b"\n" * 100_000 - header = b"name,value\n" - # An internal blank row and a wider data row must survive trimming. - data = b"Alice,1\n\nBob,2,extra\n" - content = { - "leading": blank + header + data, - "after_header": header + blank + data, - "trailing": header + data + blank, - "all": blank, - }[position] - - result = MarkItDown(enable_plugins=False).convert_stream( - io.BytesIO(content), - stream_info=StreamInfo(extension=".csv", charset="utf-8"), - ) - - expected = ( - "| name | value | |\n" - "| --- | --- | --- |\n" - "| Alice | 1 | |\n" - "| | | |\n" - "| Bob | 2 | extra |" - ) - assert result.markdown == ("" if position == "all" else expected) diff --git a/packages/markitdown/tests/test_csv_line_endings.py b/packages/markitdown/tests/test_csv_line_endings.py deleted file mode 100644 index 71d73b54d4..0000000000 --- a/packages/markitdown/tests/test_csv_line_endings.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Regression tests for CR-only CSV record separators (issue #2411).""" - -import io - -import pytest - -from markitdown import MarkItDown, StreamInfo - - -@pytest.fixture(scope="module") -def converter() -> MarkItDown: - return MarkItDown(enable_plugins=False) - - -@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) -@pytest.mark.parametrize("final_newline", [False, True]) -def test_csv_record_separators_produce_a_table( - converter: MarkItDown, newline: str, final_newline: bool -) -> None: - content = newline.join(["name,age", "Alice,30"]) - if final_newline: - content += newline - - result = converter.convert_stream( - io.BytesIO(content.encode("utf-8")), - stream_info=StreamInfo(extension=".csv", charset="utf-8"), - ) - - assert result.markdown == "| name | age |\n| --- | --- |\n| Alice | 30 |" - - -@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) -@pytest.mark.parametrize( - "cell_newline", ["\n", "\r\n", "\r"], ids=["cell-LF", "cell-CRLF", "cell-CR"] -) -def test_csv_quoted_line_break_stays_inside_its_cell( - converter: MarkItDown, newline: str, cell_newline: str -) -> None: - content = newline.join( - ["name,notes", f'Alice,"first{cell_newline}second"', "Bob,plain"] - ) - - result = converter.convert_stream( - io.BytesIO(content.encode("utf-8")), - stream_info=StreamInfo(extension=".csv", charset="utf-8"), - ) - - assert result.markdown == ( - "| name | notes |\n" - "| --- | --- |\n" - "| Alice | first second |\n" - "| Bob | plain |" - ) diff --git a/packages/markitdown/tests/test_cu_converter.py b/packages/markitdown/tests/test_cu.py similarity index 79% rename from packages/markitdown/tests/test_cu_converter.py rename to packages/markitdown/tests/test_cu.py index 487407bb78..2ab9e966e3 100644 --- a/packages/markitdown/tests/test_cu_converter.py +++ b/packages/markitdown/tests/test_cu.py @@ -1,27 +1,23 @@ -"""Tests for ContentUnderstandingConverter. - -Tests accepts() routing, smart routing modality logic, and convert() via mocks. -Follows the same pattern as test_docintel_html.py. -""" +"""Content Understanding routing, conversion, and registration.""" import io -import os -import sys from unittest.mock import MagicMock, patch import pytest +from markitdown import StreamInfo from markitdown.converters._cu_converter import ( ContentUnderstandingConverter, ContentUnderstandingFileType, - _resolve_analyzer_modality, - _get_modality, - _detect_file_type, _canonical_mime_type, _content_type_for, - _EXTENSION_MAP, + _detect_file_type, + _get_modality, + _resolve_analyzer_modality, ) -from markitdown._stream_info import StreamInfo + + +# Content Understanding # --------------------------------------------------------------------------- # Helper: create a converter with accepts() working but no SDK init @@ -786,160 +782,6 @@ def test_cu_registered_before_docintel(self): ), "CU should have higher priority (lower index) than Doc Intel" -# --------------------------------------------------------------------------- -# CLI argument tests -# --------------------------------------------------------------------------- - - -class TestCLIArgs: - """Test CLI argument parsing for CU flags.""" - - def test_use_cu_without_endpoint_exits(self): - """--use-cu without --cu-endpoint should exit with error.""" - import subprocess - - result = subprocess.run( - [sys.executable, "-m", "markitdown", "--use-cu", "fake.pdf"], - capture_output=True, - text=True, - env={**os.environ, "MARKITDOWN_CU_ENDPOINT": ""}, - ) - assert result.returncode != 0 - assert ( - "cu-endpoint" in result.stderr.lower() - or "cu-endpoint" in (result.stdout or "").lower() - ) - - def test_use_cu_and_use_docintel_mutually_exclusive(self): - """--use-cu and --use-docintel cannot be used together.""" - import subprocess - - result = subprocess.run( - [ - sys.executable, - "-m", - "markitdown", - "--use-cu", - "--cu-endpoint", - "https://fake", - "--use-docintel", - "-e", - "https://fake-di", - "fake.pdf", - ], - capture_output=True, - text=True, - ) - assert result.returncode != 0 - - def test_cu_file_types_parsing(self): - """--cu-file-types should parse comma-separated values into enum list.""" - from markitdown.converters import ContentUnderstandingFileType - - raw = "pdf,jpeg,mp4" - type_names = [t.strip().lower() for t in raw.split(",") if t.strip()] - cu_types = [ContentUnderstandingFileType(name) for name in type_names] - - assert cu_types == [ - ContentUnderstandingFileType.PDF, - ContentUnderstandingFileType.JPEG, - ContentUnderstandingFileType.MP4, - ] - - def test_cu_file_types_invalid_value(self): - """Unknown file type name should raise ValueError.""" - from markitdown.converters import ContentUnderstandingFileType - - with pytest.raises(ValueError): - ContentUnderstandingFileType("nonsense") - - def test_cu_file_types_single_value(self): - """Single file type (no comma) should parse correctly.""" - from markitdown.converters import ContentUnderstandingFileType - - cu_types = [ - ContentUnderstandingFileType(t.strip().lower()) - for t in "wav".split(",") - if t.strip() - ] - assert cu_types == [ContentUnderstandingFileType.WAV] - - def test_use_cu_wires_kwargs_to_markitdown(self, capsys): - """--use-cu should pass CU options through to MarkItDown.""" - import markitdown.__main__ as markitdown_cli - - markitdown_instance = MagicMock() - markitdown_instance.convert.return_value.markdown = "converted" - markitdown_cls = MagicMock(return_value=markitdown_instance) - - with patch.object( - sys, - "argv", - [ - "markitdown", - "--use-cu", - "--cu-endpoint", - "https://fake-cu", - "--cu-analyzer", - "custom-analyzer", - "--cu-file-types", - "pdf,jpeg,mp4", - "fake.pdf", - ], - ), patch.object(markitdown_cli, "MarkItDown", markitdown_cls): - markitdown_cli.main() - - markitdown_cls.assert_called_once_with( - enable_plugins=False, - cu_endpoint="https://fake-cu", - cu_analyzer_id="custom-analyzer", - cu_file_types=[ - ContentUnderstandingFileType.PDF, - ContentUnderstandingFileType.JPEG, - ContentUnderstandingFileType.MP4, - ], - ) - markitdown_instance.convert.assert_called_once_with( - "fake.pdf", stream_info=None, keep_data_uris=False - ) - assert capsys.readouterr().out == "converted\n" - - def test_use_cu_reads_from_stdin(self, capsys): - """--use-cu should preserve the CLI's filename-optional stdin mode.""" - import markitdown.__main__ as markitdown_cli - - input_buffer = io.BytesIO(b"fake pdf") - stdin = MagicMock(buffer=input_buffer) - markitdown_instance = MagicMock() - markitdown_instance.convert_stream.return_value.markdown = "converted" - markitdown_cls = MagicMock(return_value=markitdown_instance) - - with patch.object( - sys, - "argv", - [ - "markitdown", - "--use-cu", - "--cu-endpoint", - "https://fake-cu", - ], - ), patch.object(sys, "stdin", stdin), patch.object( - markitdown_cli, "MarkItDown", markitdown_cls - ): - markitdown_cli.main() - - markitdown_cls.assert_called_once_with( - enable_plugins=False, - cu_endpoint="https://fake-cu", - ) - - assert markitdown_instance.convert_stream.call_count == 1 - call_args, call_kwargs = markitdown_instance.convert_stream.call_args - assert call_args[0].read() == b"fake pdf" - assert call_kwargs == {"stream_info": None, "keep_data_uris": False} - assert capsys.readouterr().out == "converted\n" - - # --------------------------------------------------------------------------- # MissingDependencyException test # --------------------------------------------------------------------------- @@ -963,73 +805,3 @@ def test_missing_deps_message(self): assert "az-content-understanding" in str(exc_info.value) assert exc_info.value.__cause__ is import_error - - -# --------------------------------------------------------------------------- -# Endpoint environment variables (issue #2326) -# --------------------------------------------------------------------------- - - -class TestEndpointEnvVars: - """Endpoint flags fall back to operator-configured environment variables.""" - - @pytest.mark.parametrize( - "env_var, argv, kwarg", - [ - ( - "MARKITDOWN_CU_ENDPOINT", - ["markitdown", "--use-cu", "fake.pdf"], - "cu_endpoint", - ), - ( - "MARKITDOWN_DOCINTEL_ENDPOINT", - ["markitdown", "-d", "fake.pdf"], - "docintel_endpoint", - ), - ], - ) - def test_endpoint_read_from_environment(self, monkeypatch, env_var, argv, kwarg): - """With the env var set, the endpoint flag can be omitted entirely.""" - from markitdown.__main__ import main - - monkeypatch.setenv(env_var, "https://from-env") - monkeypatch.setattr(sys, "argv", argv) - - with patch("markitdown.__main__.MarkItDown") as mock_markitdown: - main() - - assert mock_markitdown.call_args.kwargs[kwarg] == "https://from-env" - - def test_flag_overrides_environment(self, monkeypatch): - """Fails if the env var is ever read after parsing instead of as an argparse default.""" - from markitdown.__main__ import main - - monkeypatch.setenv("MARKITDOWN_CU_ENDPOINT", "https://from-env") - monkeypatch.setattr( - sys, - "argv", - [ - "markitdown", - "--use-cu", - "--cu-endpoint", - "https://from-flag", - "fake.pdf", - ], - ) - - with patch("markitdown.__main__.MarkItDown") as mock_markitdown: - main() - - assert mock_markitdown.call_args.kwargs["cu_endpoint"] == "https://from-flag" - - def test_empty_environment_variable_is_treated_as_unset(self, monkeypatch, capsys): - """An empty env var must not pass as a valid endpoint.""" - from markitdown.__main__ import main - - monkeypatch.setenv("MARKITDOWN_CU_ENDPOINT", "") - monkeypatch.setattr(sys, "argv", ["markitdown", "--use-cu", "fake.pdf"]) - - with pytest.raises(SystemExit): - main() - - assert "MARKITDOWN_CU_ENDPOINT" in capsys.readouterr().out diff --git a/packages/markitdown/tests/test_docintel_html.py b/packages/markitdown/tests/test_docintel.py similarity index 94% rename from packages/markitdown/tests/test_docintel_html.py rename to packages/markitdown/tests/test_docintel.py index 89d454e48a..0e24265208 100644 --- a/packages/markitdown/tests/test_docintel_html.py +++ b/packages/markitdown/tests/test_docintel.py @@ -1,9 +1,15 @@ +"""Document Intelligence routing and client options.""" + import io + +from markitdown import StreamInfo from markitdown.converters._doc_intel_converter import ( DocumentIntelligenceConverter, DocumentIntelligenceFileType, ) -from markitdown._stream_info import StreamInfo + + +# Document Intelligence def _make_converter(file_types): diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py new file mode 100644 index 0000000000..4fa67684e7 --- /dev/null +++ b/packages/markitdown/tests/test_docx.py @@ -0,0 +1,1489 @@ +"""DOCX conversion, math, styles, and image hooks.""" + +import base64 +import inspect +import io +import os +import re +import zipfile +from pathlib import Path +from typing import Any, BinaryIO, Callable, Optional +from unittest.mock import Mock +from xml.etree import ElementTree as ET + +import mammoth +import pytest +from bs4 import BeautifulSoup +from lxml import etree +from mammoth.docx.files import InvalidFileReferenceError + +from markitdown import ( + FileConversionException, + MarkItDown, + MissingDependencyException, + StreamInfo, +) +from markitdown.converter_utils.docx.math.latex_dict import ( + CHR, + CHR_BO, + CHR_DEFAULT, + POS, + POS_DEFAULT, + T, +) +from markitdown.converter_utils.docx.math.omml import OMML_NS, load_string, oMath2Latex +from markitdown.converter_utils.docx.pre_process import _pre_process_styles +from markitdown.converters import DocxConverter, HtmlConverter, _docx_converter + + +# Math functions + + +def test_omml_known_function(): + omml_xml = """ + + + log + x + + + """ + results = list(load_string(omml_xml)) + assert len(results) == 1 + assert r"\log" in str(results[0]) + + +def test_omml_unknown_function_fallback(): + omml_xml = """ + + + customFunc + x + + + """ + results = list(load_string(omml_xml)) + assert len(results) == 1 + assert r"\operatorname{customFunc}" in str(results[0]) + + +def test_omml_function_name_split_across_runs(): + # Word can break a single function name into several runs (e.g. at a + # formatting boundary); the runs must be joined before the lookup. + omml_xml = """ + + + customFunc + x + + + """ + results = list(load_string(omml_xml)) + assert len(results) == 1 + assert str(results[0]) == r"\operatorname{customFunc}(x)" + + +def test_omml_known_function_split_across_runs(): + omml_xml = """ + + + sin + x + + + """ + results = list(load_string(omml_xml)) + assert len(results) == 1 + assert str(results[0]) == r"\sin(x)" + + +# Math accents + +# Tests for DOCX math accent templates. +# +# Every accent in ``latex_dict.CHR`` is a ``str.format`` template applied by +# ``oMath2Latex.do_acc`` as ``latex_s.format(c_dict["e"])``. A template whose +# braces are unbalanced raises ``ValueError`` at that call. +# +# That failure is not local to the equation. ``pre_process_docx`` runs +# ``_pre_process_math`` over the whole of ``word/document.xml`` inside a blanket +# ``except Exception`` and, on error, writes the *original* unprocessed XML back. +# Mammoth does not render OMML, so a single unformattable accent silently removes +# every equation in the document -- with no error and a zero exit code. +# +# ``test_accent_templates_are_formattable`` guards the whole table against that +# class of typo; ``test_caron_and_ring_accents`` covers the two entries that +# carried an extra closing brace. + +# Accent characters, as Word writes them into . +CARON = "\u030c" # COMBINING CARON -> \check + +RING_ABOVE = "\u030a" # COMBINING RING ABOVE -> \ocirc + +CIRCUMFLEX = "\u0302" # COMBINING CIRCUMFLEX -> \hat + +_OMML_DOC = ( + "' + "" + '' + "x" + "" +) + + +def _latex_for_accent(accent_char: str) -> str: + """Convert a single accented variable to LaTeX, as pre_process_docx would.""" + return next(load_string(_OMML_DOC.format(chr=accent_char))).latex + + +def test_accent_templates_are_formattable() -> None: + """Every {0} template must survive .format(); do_acc calls it unguarded.""" + unformattable = [] + for name, table in ( + ("CHR", CHR), + ("CHR_BO", CHR_BO), + ("CHR_DEFAULT", CHR_DEFAULT), + ("POS", POS), + ("POS_DEFAULT", POS_DEFAULT), + ): + for key, template in table.items(): + if not isinstance(template, str) or "{0}" not in template: + continue + try: + template.format("x") + except ValueError as exc: + unformattable.append(f"{name}[{key!r}] = {template!r}: {exc}") + + assert not unformattable, "unformattable accent templates: " + "; ".join( + unformattable + ) + + +def test_caron_and_ring_accents() -> None: + """U+030C and U+030A each carried an extra '}' and raised ValueError.""" + assert _latex_for_accent(CARON) == "\\check{x}" + assert _latex_for_accent(RING_ABOVE) == "\\ocirc{x}" + + +def test_unmodified_accent_still_converts() -> None: + """Control: a neighbouring accent that was never broken.""" + assert _latex_for_accent(CIRCUMFLEX) == "\\hat{x}" + + +# Math symbols + +# Tests for the OMML -> LaTeX symbol tables used by the DOCX converter. +# +# Two independent defects are covered here: +# +# ``CHR`` maps a combining accent to a LaTeX template. U+20EE COMBINING LEFT +# ARROW BELOW mapped to ``\underledtarrow``, which is not a LaTeX macro -- the +# name is a scrambled ``\underleftarrow``. Its two neighbours in the same table +# (U+20D6 -> ``\overleftarrow``, U+20EF -> ``\underrightarrow``) show the +# intent. The template still formats, so nothing raises: the document simply ends +# up with an undefined control sequence that no renderer can typeset. +# +# ``T`` normalizes the Mathematical Alphanumeric Symbols back to ASCII, so that +# an equation written with math italic letters yields ``h(x)`` rather than a +# string of astral-plane codepoints. U+1D455 -- the slot where math italic small +# h would sit -- is permanently reserved, because Unicode unifies that letter +# with U+210E PLANCK CONSTANT. The table followed the contiguous block and so +# skipped h, leaving it as the single letter of the alphabet that survived +# untranslated into the LaTeX output. + +# Math italic Latin: A-Z is contiguous, a-z has a hole at U+1D455 (small h), +# which Unicode unifies with U+210E. +ITALIC_UPPERCASE = { + chr(0x1D434 + i): c for i, c in enumerate("ABCDEFGHIJKLMNOPQRSTUVWXYZ") +} + +ITALIC_LOWERCASE = { + chr(0x1D44E + i): c for i, c in enumerate("abcdefghijklmnopqrstuvwxyz") +} + +ITALIC_LOWERCASE.pop(chr(0x1D455)) # reserved codepoint, never assigned + +ITALIC_LOWERCASE["ℎ"] = "h" + +_DOC = ( + "' + "{0}" +) + +_ACCENT = ( + "" + '' + "x" + "" +) + +_RUN = "{text}" + + +def _latex(fragment: str) -> str: + return next(load_string(_DOC.format(fragment))).latex + + +def _accent_latex(accent_char: str) -> str: + return _latex(_ACCENT.format(chr=accent_char)) + + +def _run_latex(text: str) -> str: + return _latex(_RUN.format(text=text)) + + +def test_arrow_accents_are_consistent() -> None: + """The three arrow accents share one naming scheme; none may be misspelled.""" + assert _accent_latex("⃖") == "\\overleftarrow{x}" # left arrow above + assert _accent_latex("⃯") == "\\underrightarrow{x}" # right arrow below + assert _accent_latex("⃮") == "\\underleftarrow{x}" # U+20EE, left arrow below + + +def test_math_italic_latin_alphabet_is_complete() -> None: + """Normalize the full alphabet, including U+210E (italic small h).""" + for char, expected in {**ITALIC_UPPERCASE, **ITALIC_LOWERCASE}.items(): + assert T.get(char) == expected, f"U+{ord(char):04X} is missing from T" + assert _run_latex(char) == expected + + +def test_math_italic_expression_is_fully_normalized() -> None: + """A whole expression must come out as plain ASCII LaTeX.""" + # h(x) = g(x), written entirely with math italic codepoints. + expression = "ℎ(\U0001d465)=\U0001d454(\U0001d465)" + assert _run_latex(expression) == "h(x)=g(x)" + + +# Malformed math runs + +# Regression test for a crash in the OMML -> LaTeX converter when a math run +# () has no text child (e.g. a run that only carries formatting +# properties). Previously `do_r()` called `elm.findtext(...)` directly and +# iterated over the result, which raised `TypeError: 'NoneType' object is not +# iterable` when the run had no text, aborting equation conversion for the +# entire document. + +MATH_NS_DECL = f'xmlns:m="{OMML_NS[1:-1]}"' + + +def _parse_omath(xml_fragment: str): + wrapped = f"{xml_fragment}" + return ET.fromstring(wrapped) + + +def test_run_without_text_child_does_not_crash(): + # with only , no child. + element = _parse_omath("") + # Should not raise TypeError: 'NoneType' object is not iterable + result = oMath2Latex(element) + assert result.latex == "" + + +def test_run_with_text_still_converts(): + element = _parse_omath("x") + result = oMath2Latex(element) + assert result.latex == "x" + + +def test_subscript_with_missing_text_run_does_not_crash(): + # Mirrors a real-world document: a subscript expression where one of the + # runs involved has no text (e.g. produced by some Word equation editors). + element = _parse_omath( + "" + "l" + "1" + "" + ) + result = oMath2Latex(element) + assert "l" in result.latex + assert "1" in result.latex + + +# Stylesheet repair + +WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + +W = f"{{{WORD_NAMESPACE}}}" + +TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx" + + +@pytest.mark.parametrize("double_strike", [False, True]) +@pytest.mark.parametrize("missing_type", [False, True]) +def test_docx_styles_with_redundant_default_namespace( + missing_type: bool, double_strike: bool +) -> None: + markitdown = MarkItDown() + expected = markitdown.convert(TEST_DOCX).markdown + fixture = io.BytesIO() + declaration = f'xmlns:w="{WORD_NAMESPACE}"'.encode("utf-8") + + with zipfile.ZipFile(TEST_DOCX) as source, zipfile.ZipFile(fixture, "w") as target: + for item in source.infolist(): + content = source.read(item) + if item.filename == "word/styles.xml": + if double_strike: + assert content.count(b"") == 1 + content = content.replace( + b"", + b'' + b'' + b'' + b"", + 1, + ) + assert content.count(declaration) == 1 + content = content.replace( + declaration, + declaration + f' xmlns="{WORD_NAMESPACE}"'.encode("utf-8"), + 1, + ) + if missing_type: + content, count = re.subn( + rb' bytes: + prefix = request.param + return f""" +<{prefix}:styles xmlns:{prefix}="{WORD_NAMESPACE}" xmlns="{WORD_NAMESPACE}" + xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:w14="http://schemas.microsoft.com/office/word/2010/wordml" + mc:Ignorable="w14"> + <{prefix}:style {prefix}:type="paragraph" {prefix}:styleId="Normal"> + <{prefix}:name {prefix}:val="Normal"/> + +""".encode( + "utf-8" + ) + + +def test_valid_styles_are_returned_unchanged(styles_xml: bytes) -> None: + assert _pre_process_styles(styles_xml) == styles_xml + + +def test_missing_type_repair_preserves_namespaces(styles_xml: bytes) -> None: + malformed, count = re.subn(rb'\s+(?:w|word):type="paragraph"', b"", styles_xml) + assert count == 1 + original = etree.fromstring(styles_xml) + + repaired = etree.fromstring(_pre_process_styles(malformed)) + + assert repaired.nsmap == original.nsmap + assert repaired.attrib == original.attrib + style = repaired.find(W + "style") + assert style is not None + assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} + name = style.find(W + "name") + assert name is not None + assert name.attrib == {W + "val": "Normal"} + + +def test_styles_without_ids_are_removed(styles_xml: bytes) -> None: + malformed, count = re.subn(rb'\s+(?:w|word):styleId="Normal"', b"", styles_xml) + assert count == 1 + + repaired = etree.fromstring(_pre_process_styles(malformed)) + + assert repaired.findall(W + "style") == [] + + +@pytest.mark.parametrize("value", [None, "0", "1"]) +@pytest.mark.parametrize("missing_type", [False, True]) +def test_style_strike_repair_preserves_namespaces( + styles_xml: bytes, value: str | None, missing_type: bool +) -> None: + original = etree.fromstring(styles_xml) + style = original.find(W + "style") + assert style is not None + if missing_type: + del style.attrib[W + "type"] + properties = etree.SubElement(style, W + "rPr") + strike = etree.SubElement(properties, W + "dstrike") + if value is not None: + strike.set(W + "val", value) + etree.SubElement(properties, W + "b") + + repaired = etree.fromstring(_pre_process_styles(etree.tostring(original))) + + assert repaired.nsmap == original.nsmap + assert repaired.attrib == original.attrib + style = repaired.find(W + "style") + assert style is not None + assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} + assert repaired.find(".//" + W + "dstrike") is None + strike = style.find(f"{W}rPr/{W}strike") + assert strike is not None + assert strike.attrib == ({} if value is None else {W + "val": value}) + assert style.find(f"{W}rPr/{W}b") is not None + + +def test_other_namespace_dstrike_is_unchanged(styles_xml: bytes) -> None: + original = etree.fromstring(styles_xml) + etree.SubElement(original, "{urn:extension}dstrike") + content = etree.tostring(original) + + assert _pre_process_styles(content) == content + + +# Image hooks + +_PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAM" + "BAQDJ/pLvAAAAAElFTkSuQmCC" +) + +_GIF = base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7") + +_INFO = StreamInfo(extension=".docx") + +_FILES = Path(__file__).parent / "test_files" + + +def _image(rid: str = "rIdPng", number: int = 1) -> str: + return f""" + + + + + + + + + + + +""" + + +def _text(text: str) -> str: + return f'{text}' + + +def _paragraph(content: str) -> str: + return f"{content}" + + +_BODY = _paragraph(_text("Before") + _image() + _text("After")) + + +def _docx(body: str = _BODY, *, embedded_style_map: Optional[str] = None) -> io.BytesIO: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr( + "[Content_Types].xml", + """ + + + + + +""", + ) + archive.writestr( + "_rels/.rels", + """ + +""", + ) + archive.writestr( + "word/_rels/document.xml.rels", + """ + + +""", + ) + archive.writestr( + "word/document.xml", + f""" + {body} +""", + ) + archive.writestr("word/media/image.png", _PNG) + archive.writestr("word/media/image.gif", _GIF) + if embedded_style_map is not None: + stream.seek(0) + mammoth.embed_style_map(stream, embedded_style_map) + stream.seek(0) + return stream + + +class _ImageConverter(DocxConverter): + def __init__(self, render: Callable[..., Optional[str]]): + super().__init__() + self.render = render + + def _image_to_html( + self, image_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any + ) -> Optional[str]: + return self.render(image_stream, stream_info, **kwargs) + + +def test_image_hook_does_not_change_public_conversion_signature() -> None: + assert list(inspect.signature(DocxConverter.convert).parameters) == [ + "self", + "file_stream", + "stream_info", + "kwargs", + ] + assert _ImageConverter.convert is DocxConverter.convert + assert list(inspect.signature(DocxConverter.__init__).parameters) == ["self"] + + +@pytest.mark.parametrize("via_dispatcher", [False, True]) +def test_inherited_hook_receives_images_and_options_in_order( + via_dispatcher: bool, +) -> None: + seen = [] + streams = [] + service = object() + options: dict[str, Any] = { + "ocr_service": service, + "heading_style": "underlined", + } + + def render(stream: BinaryIO, info: StreamInfo, **kwargs: Any) -> str: + assert stream.tell() == 0 + data = stream.read() + stream.seek(0) + assert stream.read() == data + seen.append((data, info, kwargs)) + streams.append(stream) + return f"image{len(seen)}" + + class InheritedImages(_ImageConverter): + pass + + converter = InheritedImages(render) + source = _docx( + _paragraph( + _image("rIdGif", 1) + + _text(" ") + + _image("rIdPng", 2) + + _text(" ") + + _image("rIdGif", 3) + ) + ) + original = source.getvalue() + info = StreamInfo( + extension=".docx", + filename="document.docx", + url="https://example.test/document.docx", + local_path="/document.docx", + ) + if via_dispatcher: + markitdown = MarkItDown() + markitdown.register_converter(converter, priority=-1) + result = markitdown.convert_stream(source, stream_info=info, **options) + else: + source.seek(7) + result = converter.convert(source, info, **options) + + assert [entry[:2] for entry in seen] == [ + (_GIF, StreamInfo(mimetype="image/gif", extension=".gif")), + (_PNG, StreamInfo(mimetype="image/png", extension=".png")), + (_GIF, StreamInfo(mimetype="image/gif", extension=".gif")), + ] + assert all(entry[2]["ocr_service"] is service for entry in seen) + assert all(entry[2]["heading_style"] == "underlined" for entry in seen) + assert result.markdown == "**image1** **image2** **image3**" + assert all(stream.closed for stream in streams) + assert not source.closed and source.getvalue() == original + + +@pytest.mark.parametrize("keep_data_uris", [False, True]) +@pytest.mark.parametrize("fallback", [None, "", " \n\t"]) +def test_declining_hook_preserves_native_output( + monkeypatch: pytest.MonkeyPatch, fallback: Optional[str], keep_data_uris: bool +) -> None: + expected_converter = DocxConverter() + expected_html = Mock(wraps=expected_converter._html_converter.convert_string) + monkeypatch.setattr( + expected_converter._html_converter, "convert_string", expected_html + ) + expected = expected_converter.convert(_docx(), _INFO, keep_data_uris=keep_data_uris) + converter = _ImageConverter(Mock(return_value=fallback)) + actual_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", actual_html) + + actual = converter.convert(_docx(), _INFO, keep_data_uris=keep_data_uris) + + assert actual_html.call_args == expected_html.call_args + assert actual.markdown == expected.markdown + assert actual.title == expected.title + + +def test_native_converter_does_not_add_image_processing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + convert_html = Mock(wraps=mammoth.convert_to_html) + monkeypatch.setattr(mammoth, "convert_to_html", convert_html) + + class UnmodifiedSubclass(DocxConverter): + pass + + for converter in (DocxConverter(), UnmodifiedSubclass()): + result = converter.convert(_docx(), _INFO) + assert "![Image 1](data:image/png;base64...)" in result.markdown + assert "convert_image" not in convert_html.call_args.kwargs + + +def test_override_can_delegate_to_super() -> None: + class NativeImages(DocxConverter): + def _image_to_html(self, image_stream, stream_info, **kwargs): + return super()._image_to_html(image_stream, stream_info, **kwargs) + + actual = NativeImages().convert(_docx(), _INFO) + assert actual.markdown == DocxConverter().convert(_docx(), _INFO).markdown + + +@pytest.mark.parametrize( + ("fragment", "expected"), + [ + ("text", "

BeforetextAfter

"), + ("one
two", "

Beforeone
twoAfter

"), + ("A & B < C", "

BeforeA & B < CAfter

"), + ( + "

one

two

", + "

Before

one

two

After

", + ), + ( + "

one

two

", + "

Before

one

two

After

", + ), + ( + "
  • one
  • two
", + "

Before

  • one
  • two

After

", + ), + ( + "
Key
Value
", + "

Before

" + "
Key
Value

After

", + ), + ( + "
Title

Body

", + "

Before

Title

Body

" + "

After

", + ), + ( + "lead

block

tail", + "

Beforelead

block

tailAfter

", + ), + ( + "
a < b\n  *literal*
", + "

Before

a < b\n  *literal*

After

", + ), + ], +) +def test_fragment_placement_precedes_shared_html_conversion( + monkeypatch: pytest.MonkeyPatch, fragment: str, expected: str +) -> None: + converter = _ImageConverter(Mock(return_value=fragment)) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + options = {"escape_asterisks": False, "custom_option": "forwarded"} + + result = converter.convert(_docx(), _INFO, **options) + + assert convert_html.call_args.args == (expected,) + assert convert_html.call_args.kwargs == options + assert ( + result.markdown + == HtmlConverter().convert_string(expected, escape_asterisks=False).markdown + ) + assert "data-markitdown-image" not in expected + + +@pytest.mark.parametrize( + ("style_map", "expected"), + [ + (None, "

one

two

"), + ("p => ul > li:fresh", "

  • one

    two

"), + ("p => blockquote > p:fresh", "

one

two

"), + ("p => h2:fresh > strong", "

one

two

"), + ], +) +def test_standalone_block_image_does_not_leave_empty_wrappers( + monkeypatch: pytest.MonkeyPatch, style_map: Optional[str], expected: str +) -> None: + converter = _ImageConverter(Mock(return_value="

one

two

")) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + converter.convert(_docx(_paragraph(_image())), _INFO, style_map=style_map) + + assert convert_html.call_args.args == (expected,) + + +def test_block_image_keeps_its_table_cell( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = ( + "" + "" + + _paragraph(_text("Item")) + + "" + + _paragraph(_text("Details")) + + "" + + _paragraph(_text("A")) + + "" + + _paragraph(_image()) + + "" + ) + converter = _ImageConverter( + Mock(return_value="

Serial: 12345

Status: active

") + ) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + result = converter.convert(_docx(body), _INFO) + + soup = BeautifulSoup(convert_html.call_args.args[0], "html.parser") + assert len(soup.find_all("tr")) == 2 + assert len(soup.find_all("td")) == 4 + assert ( + str(soup.find_all("td")[-1]) + == "

Serial: 12345

Status: active

" + ) + assert not soup.select("p p") + assert "| A | Serial: 12345 Status: active |" in result.markdown + + +def test_nested_run_formatting_is_preserved_around_blocks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + converter = _ImageConverter(Mock(return_value="

block

")) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + converter.convert(_docx(), _INFO, style_map="p => p:fresh > em > strong") + + assert convert_html.call_args.args == ( + "

Before

block

" + "

After

", + ) + + +def test_hook_inherits_preprocessing_and_style_precedence() -> None: + body = ( + _BODY + + _paragraph( + 'underlined' + ) + + _paragraph("deleted") + + _paragraph("x") + ) + + result = _ImageConverter(Mock(return_value="image")).convert( + _docx(body, embedded_style_map="u => em"), + _INFO, + style_map="u => strong", + ) + + assert result.markdown == "BeforeimageAfter\n\n**underlined**\n\n~~deleted~~\n\n$x$" + + +def test_multiple_images_and_native_fallback_keep_their_own_positions() -> None: + converter = _ImageConverter(Mock(side_effect=[None, "recognized", ""])) + body = _paragraph( + _image(number=1) + _text(" ") + _image(number=2) + _text(" ") + _image(number=3) + ) + + result = converter.convert(_docx(body), _INFO) + + assert result.markdown == ( + "![Image 1](data:image/png;base64...) *recognized* " + "![Image 3](data:image/png;base64...)" + ) + + +def test_many_images_have_independent_replacements() -> None: + converter = _ImageConverter( + Mock(side_effect=[f"

Image {number}

" for number in range(12)]) + ) + body = _paragraph("".join(_image(number=number) for number in range(12))) + + result = converter.convert(_docx(body), _INFO) + + assert result.markdown == "\n\n".join(f"Image {number}" for number in range(12)) + + +def test_returned_image_and_literal_text_use_normal_html_rendering() -> None: + render = Mock( + return_value=( + "

<literal> *not emphasis* & text

" + 'generated' + ) + ) + result = _ImageConverter(render).convert(_docx(_paragraph(_image())), _INFO) + + assert ( + result.markdown + == r" \*not emphasis\* & text" + "\n\n![generated](image.png)" + ) + render.assert_called_once() + + +@pytest.mark.parametrize("result", [False, 123, b"

not a string

"]) +def test_invalid_return_is_not_silent_native_fallback(result: Any) -> None: + with pytest.raises(TypeError, match="HTML string or None"): + _ImageConverter(Mock(return_value=result)).convert(_docx(), _INFO) + + +@pytest.mark.parametrize( + "fragment", + [ + "document", + "title", + "content", + "

content

", + ], +) +def test_full_document_is_not_a_valid_fragment(fragment: str) -> None: + with pytest.raises(ValueError, match="fragment, not a document"): + _ImageConverter(Mock(return_value=fragment)).convert(_docx(), _INFO) + + +@pytest.mark.parametrize("via_dispatcher", [False, True]) +def test_hook_errors_propagate_and_close_the_image_stream(via_dispatcher: bool) -> None: + streams = [] + error = RuntimeError("Image service failed") + + def render(stream, info, **kwargs): + streams.append(stream) + raise error + + converter = _ImageConverter(render) + if via_dispatcher: + markitdown = MarkItDown(enable_builtins=False) + markitdown.register_converter(converter, priority=-1) + with pytest.raises(FileConversionException) as caught: + markitdown.convert_stream(_docx(), stream_info=_INFO) + assert caught.value.attempts is not None + assert any( + attempt.exc_info and attempt.exc_info[1] is error + for attempt in caught.value.attempts + ) + else: + with pytest.raises(RuntimeError) as caught_direct: + converter.convert(_docx(), _INFO) + assert caught_direct.value is error + assert streams and all(stream.closed for stream in streams) + + +def test_dispatcher_can_fall_back_to_native_docx_after_hook_error() -> None: + render = Mock(side_effect=RuntimeError("Image service failed")) + markitdown = MarkItDown() + markitdown.register_converter(_ImageConverter(render), priority=-1) + + result = markitdown.convert_stream(_docx(), stream_info=_INFO) + + assert result.markdown == DocxConverter().convert(_docx(), _INFO).markdown + render.assert_called() + + +def test_mammoth_does_not_swallow_hook_file_reference_errors() -> None: + error = InvalidFileReferenceError("hook failed") + render = Mock(side_effect=error) + + with pytest.raises(RuntimeError, match="_image_to_html failed") as caught: + _ImageConverter(render).convert(_docx(), _INFO) + + assert caught.value.__cause__ is error + render.assert_called_once() + + +def test_per_call_options_and_fragments_do_not_leak_between_conversions() -> None: + converter = _ImageConverter(lambda stream, info, **kwargs: kwargs.get("image_text")) + first = converter.convert(_docx(), _INFO, image_text="

first

") + second = converter.convert(_docx(), _INFO, image_text="

second

") + third = converter.convert(_docx(), _INFO) + + assert first.markdown == "Before\n\nfirst\n\nAfter" + assert second.markdown == "Before\n\nsecond\n\nAfter" + assert third.markdown == DocxConverter().convert(_docx(), _INFO).markdown + + +def test_unreferenced_media_does_not_invoke_the_hook() -> None: + render = Mock(side_effect=AssertionError("unexpected image")) + + result = _ImageConverter(render).convert( + _docx(_paragraph(_text("Only text"))), _INFO + ) + + assert result.markdown == "Only text" + render.assert_not_called() + + +@pytest.mark.parametrize( + "fragment", + [ + 'new', + '

new

', + ], +) +def test_image_html_does_not_create_nested_links( + monkeypatch: pytest.MonkeyPatch, fragment: str +) -> None: + body = _paragraph( + '' + + _text("Before") + + _image() + + _text("After") + + "" + ) + converter = _ImageConverter(Mock(return_value=fragment)) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + result = converter.convert(_docx(body), _INFO) + + soup = BeautifulSoup(convert_html.call_args.args[0], "html.parser") + assert not soup.select("a a") + assert [link.get("href") for link in soup.find_all("a")] == [ + "#target", + "https://example.test/new", + "#target", + ] + assert "[Before](#target)" in result.markdown + assert "[new](https://example.test/new)" in result.markdown + assert "[After](#target)" in result.markdown + + +@pytest.mark.parametrize("keep_data_uris", [False, True]) +def test_declining_hook_preserves_a_real_document(keep_data_uris: bool) -> None: + content = (_FILES / "test.docx").read_bytes() + expected = DocxConverter().convert( + io.BytesIO(content), _INFO, keep_data_uris=keep_data_uris + ) + render = Mock(return_value=None) + + actual = _ImageConverter(render).convert( + io.BytesIO(content), _INFO, keep_data_uris=keep_data_uris + ) + + assert actual.markdown == expected.markdown + assert actual.title == expected.title + render.assert_called() + + +def test_hook_reads_images_from_the_preprocessed_archive() -> None: + import struct + + stream = _docx() + data = bytearray(stream.getvalue()) + with zipfile.ZipFile(stream) as archive: + offset = archive.getinfo("word/media/image.png").header_offset + length = struct.unpack_from("image" + + result = _ImageConverter(render).convert(io.BytesIO(data), _INFO) + + assert observed == [_PNG] + assert result.markdown == "BeforeimageAfter" + + +def test_missing_dependencies_fail_before_invoking_hook( + monkeypatch: pytest.MonkeyPatch, +) -> None: + error = ModuleNotFoundError("No module named 'mammoth'") + monkeypatch.setattr( + _docx_converter, "_dependency_exc_info", (ModuleNotFoundError, error, None) + ) + render = Mock() + + with pytest.raises(MissingDependencyException) as caught: + _ImageConverter(render).convert(_docx(), _INFO) + + assert caught.value.__cause__ is error + render.assert_not_called() + + +# Conversion regressions + +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") + +DOCX_COMMENT_TEST_STRINGS = [ + "314b0a30-5b04-470b-b9f7-eed2c2bec74a", + "49e168b7-d2ae-407f-a055-2167576f39a1", + "## d666f1f7-46cb-42bd-9a39-9a39cf2a509f", + "# Abstract", + "# Introduction", + "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", + "This is a test comment. 12df-321a", + "Yet another comment in the doc. 55yiyi-asd09", +] + + +# --- Helper Functions --- +def validate_strings(result, expected_strings, exclude_strings=None): + """Validate presence or absence of specific strings.""" + text_content = result.text_content.replace("\\", "") + for string in expected_strings: + assert string in text_content + if exclude_strings: + for string in exclude_strings: + assert string not in text_content + + +def test_docx_comments() -> None: + # Test DOCX processing, with comments and setting style_map on init + markitdown_with_style_map = MarkItDown(style_map="comment-reference => ") + result = markitdown_with_style_map.convert( + os.path.join(TEST_FILES_DIR, "test_with_comment.docx") + ) + validate_strings(result, DOCX_COMMENT_TEST_STRINGS) + + +def _write_underlined_docx( + path, + embedded_style_map: Optional[str] = None, + *, + paragraph_xml: str = ( + "plain " + 'underlined' + ), +) -> str: + """Write a minimal .docx holding one underlined run, and return its path.""" + document_xml = f""" + + + {paragraph_xml} + +""" + + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + "[Content_Types].xml", + """ + + + + +""", + ) + archive.writestr( + "_rels/.rels", + """ + + +""", + ) + archive.writestr( + "word/_rels/document.xml.rels", + """ +""", + ) + archive.writestr("word/document.xml", document_xml) + + if embedded_style_map is not None: + import mammoth + + with open(path, "r+b") as f: + mammoth.embed_style_map(f, embedded_style_map) + + return str(path) + + +def test_docx_underlined_text_is_preserved(tmp_path) -> None: + docx_file = _write_underlined_docx(tmp_path / "underlined.docx") + + result = MarkItDown().convert(docx_file) + + assert "plain underlined" in result.markdown + + +@pytest.mark.parametrize( + ("run_xml", "expected"), + [ + (' ', "First Last"), + ("", "First Last"), + (" ", "First\u00a0Last"), + ("", "First\nLast"), + ], +) +def test_docx_underlined_whitespace_is_preserved( + tmp_path, run_xml: str, expected: str +) -> None: + docx_file = _write_underlined_docx( + tmp_path / "underlined_whitespace.docx", + paragraph_xml=( + "First" + f'{run_xml}' + "Last" + ), + ) + + assert MarkItDown().convert(docx_file).markdown == expected + + +def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None: + # A style map embedded in the document takes precedence over the default + # "u => u" mapping that preserves underlines. + docx_file = _write_underlined_docx( + tmp_path / "embedded.docx", embedded_style_map="u => em" + ) + + result = MarkItDown().convert(docx_file) + + assert "plain *underlined*" in result.markdown + assert "" not in result.markdown + + +def test_docx_caller_style_map_overrides_embedded_style_map(tmp_path) -> None: + # ... and a caller-supplied style map still outranks the embedded one. + docx_file = _write_underlined_docx( + tmp_path / "embedded.docx", embedded_style_map="u => em" + ) + + result = MarkItDown(style_map="u => strong").convert(docx_file) + + assert "plain **underlined**" in result.markdown + + +def test_docx_equations() -> None: + markitdown = MarkItDown() + docx_file = os.path.join(TEST_FILES_DIR, "equations.docx") + result = markitdown.convert(docx_file) + + # Check for inline equation m=1 (wrapped with single $) is present + assert "$m=1$" in result.text_content, "Inline equation $m=1$ not found" + + # Find block equations wrapped with double $$ and check if they are present + block_equations = re.findall(r"\$\$(.+?)\$\$", result.text_content) + assert block_equations, "No block equations found in the document." + + +def test_docx_zip_filename_casing_mismatch() -> None: + """Test that DOCX files with inconsistent ZIP filename casing are handled. + + Some document generators produce .docx files where the central directory + records one casing (e.g. 'word/document.xml') but the local file headers + record another (e.g. 'Word/Document.XML'). Python's zipfile module raises + BadZipFile when reading such files. This test verifies that MarkItDown + handles this gracefully. + + See: https://github.com/microsoft/markitdown/issues/1812 + """ + import struct + + markitdown = MarkItDown() + docx_file = os.path.join(TEST_FILES_DIR, "test.docx") + + # Read the original docx and get its expected content + original_result = markitdown.convert(docx_file) + assert original_result.markdown.strip(), "Original DOCX should have content" + + # Read raw bytes and corrupt the local file header filenames + with open(docx_file, "rb") as f: + raw = bytearray(f.read()) + + # Find all local file headers and uppercase their filenames + corrupted = bytearray(raw) + offset = 0 + patched_count = 0 + while offset + 30 <= len(corrupted): + if corrupted[offset : offset + 4] != b"PK\x03\x04": + break + fname_len = struct.unpack_from(" len(corrupted): + break + # Uppercase the filename in the local header + old_name = corrupted[offset + 30 : offset + 30 + fname_len] + new_name = old_name.upper() + if old_name != new_name: + corrupted[offset + 30 : offset + 30 + fname_len] = new_name + patched_count += 1 + comp_size = struct.unpack_from(" 0, "Should have patched at least one local file header" + + # Verify the corrupted file would fail with plain zipfile + with pytest.raises(zipfile.BadZipFile): + with zipfile.ZipFile(io.BytesIO(bytes(corrupted)), "r") as zf: + for name in zf.namelist(): + zf.read(name) + + # Verify MarkItDown can still convert it + corrupted_result = markitdown.convert_stream( + io.BytesIO(bytes(corrupted)), + file_extension=".docx", + ) + assert ( + corrupted_result.markdown.strip() + ), "Corrupted DOCX should still produce content" + # Content should be equivalent to the original + assert ( + original_result.markdown.strip() == corrupted_result.markdown.strip() + ), "Corrupted DOCX should produce the same output as original" + + +def test_docx_zip_filename_non_casing_mismatch_still_rejected() -> None: + """Only case-only local/central filename disagreements are repaired. + + Equal encoded length does not imply two names differ only in case, so the + repair must not be used to wave through archives whose local and central + directory names genuinely disagree -- zipfile rejects those for good + reason, and honouring that keeps the workaround scoped to issue #1812. + """ + import struct + + from markitdown.converter_utils.docx.pre_process import ( + _fix_zip_filename_casing, + ) + + def build_zip(first_name: str) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr(first_name, b"payload-a") + zf.writestr("word/document.xml", b"payload-b") + return buf.getvalue() + + def replace_first_local_name(data: bytes, new_name: bytes) -> bytes: + """Rewrite only the first local file header's name, leaving the central + directory -- the authoritative copy -- untouched.""" + raw = bytearray(data) + fname_len = struct.unpack_from(" bool: + try: + with zipfile.ZipFile(io.BytesIO(data), "r") as zf: + for name in zf.namelist(): + zf.read(name) + except zipfile.BadZipFile: + return False + return True + + original = "[Content_Types].xml" + + # A case-only mismatch is repaired ... + case_only = replace_first_local_name(build_zip(original), original.upper().encode()) + assert not reads_cleanly(case_only) + repaired = _fix_zip_filename_casing(io.BytesIO(case_only)) + assert reads_cleanly(repaired.read()) + + # ... but an equal-length mismatch that is not case-only is left alone, + # so zipfile still refuses the archive. + for mutated_name in ( + b"ZBnoudou^Uxqdr\\/ylm", # unrelated, same length + b"[Content_Types].xmy", # a single differing character + b"[content_types]/xml", # differs only outside the cased characters + ): + data = replace_first_local_name(build_zip(original), mutated_name) + assert not reads_cleanly(data), f"{mutated_name!r} should not be readable" + result = _fix_zip_filename_casing(io.BytesIO(data)) + assert not reads_cleanly( + result.read() + ), f"{mutated_name!r} must not be silently repaired" + + +def test_docx_malformed_equations() -> None: + """Malformed equations should not crash the converter (issue #1979).""" + import zipfile + from io import BytesIO + + docx_xml = """ + + + + Normal text + + + + x+1 + + + + After good equation + + + + + + After empty oMathPara + + + + + + After empty inline oMath + + + + y+2 + + + After unnamespaced oMathPara + + z+3 + + After unnamespaced oMath + + +""" + + buf = BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("word/document.xml", docx_xml.encode("utf-8")) + z.writestr( + "[Content_Types].xml", + """ + + + + +""", + ) + z.writestr( + "word/_rels/document.xml.rels", + """ + +""", + ) + z.writestr( + "_rels/.rels", + """ + + +""", + ) + + buf.seek(0) + markitdown = MarkItDown() + result = markitdown.convert(buf) + assert "Normal text" in result.markdown + # A malformed equation must not abort the math pre-processing step, which + # would silently drop the LaTeX conversion of every other equation too + assert "$x+1$" in result.markdown or "$$x+1$$" in result.markdown + assert "After empty oMathPara" in result.markdown + assert "After empty inline oMath" in result.markdown + assert "After unnamespaced oMathPara" in result.markdown + assert "After unnamespaced oMath" in result.markdown + + +@pytest.mark.skipif( + os.name == "nt", + reason="The DOCX fixture embeds a POSIX file:///tmp/test_rlink.txt target.", +) +def test_doc_rlink() -> None: + # Test for: CVE-2025-11849 + markitdown = MarkItDown() + + # Document with rlink + docx_file = os.path.join(TEST_FILES_DIR, "rlink.docx") + + # Directory containing the target rlink file + rlink_tmp_dir = os.path.abspath(os.sep + "tmp") + + # Ensure the tmp directory exists + if not os.path.exists(rlink_tmp_dir): + pytest.skip(f"Skipping rlink test; {rlink_tmp_dir} directory does not exist.") + return + + rlink_file_path = os.path.join(rlink_tmp_dir, "test_rlink.txt") + rlink_content = "de658225-569e-4e3d-9ed2-cfb6abf927fc" + b64_prefix = ( + "ZGU2NTgyMjUtNTY5ZS00ZTNkLTllZDItY2ZiNmFiZjk" # base64 prefix of rlink_content + ) + + if os.path.exists(rlink_file_path): + with open(rlink_file_path, "r", encoding="utf-8") as f: + existing_content = f.read() + if existing_content != rlink_content: + raise ValueError( + f"Existing {rlink_file_path} content does not match expected content." + ) + else: + with open(rlink_file_path, "w", encoding="utf-8") as f: + f.write(rlink_content) + + try: + result = markitdown.convert(docx_file, keep_data_uris=True).text_content + assert ( + b64_prefix not in result + ) # Make sure the target file was NOT embedded in the output + finally: + os.remove(rlink_file_path) + + +# Styles missing a type + + +def test_convert_docx_with_style_missing_type(tmp_path): + """DOCX conversion should not fail when a style entry is missing w:type.""" + source_path = os.path.join(TEST_FILES_DIR, "test.docx") + malformed_path = tmp_path / "missing_style_type.docx" + + with zipfile.ZipFile(source_path, mode="r") as zip_input: + with zipfile.ZipFile(malformed_path, mode="w") as zip_output: + for item in zip_input.infolist(): + content = zip_input.read(item.filename) + if item.filename == "word/styles.xml": + styles_xml = content.decode("utf-8") + styles_xml, count = re.subn( + r' str: - return f""" - - - - - - - - - - - -""" - - -def _text(text: str) -> str: - return f'{text}' - - -def _paragraph(content: str) -> str: - return f"{content}" - - -_BODY = _paragraph(_text("Before") + _image() + _text("After")) - - -def _docx(body: str = _BODY, *, embedded_style_map: Optional[str] = None) -> io.BytesIO: - stream = io.BytesIO() - with zipfile.ZipFile(stream, "w") as archive: - archive.writestr( - "[Content_Types].xml", - """ - - - - - -""", - ) - archive.writestr( - "_rels/.rels", - """ - -""", - ) - archive.writestr( - "word/_rels/document.xml.rels", - """ - - -""", - ) - archive.writestr( - "word/document.xml", - f""" - {body} -""", - ) - archive.writestr("word/media/image.png", _PNG) - archive.writestr("word/media/image.gif", _GIF) - if embedded_style_map is not None: - stream.seek(0) - mammoth.embed_style_map(stream, embedded_style_map) - stream.seek(0) - return stream - - -class _ImageConverter(DocxConverter): - def __init__(self, render: Callable[..., Optional[str]]): - super().__init__() - self.render = render - - def _image_to_html( - self, image_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any - ) -> Optional[str]: - return self.render(image_stream, stream_info, **kwargs) - - -def test_image_hook_does_not_change_public_conversion_signature() -> None: - assert list(inspect.signature(DocxConverter.convert).parameters) == [ - "self", - "file_stream", - "stream_info", - "kwargs", - ] - assert _ImageConverter.convert is DocxConverter.convert - assert list(inspect.signature(DocxConverter.__init__).parameters) == ["self"] - - -@pytest.mark.parametrize("via_dispatcher", [False, True]) -def test_inherited_hook_receives_images_and_options_in_order( - via_dispatcher: bool, -) -> None: - seen = [] - streams = [] - service = object() - options: dict[str, Any] = { - "ocr_service": service, - "heading_style": "underlined", - } - - def render(stream: BinaryIO, info: StreamInfo, **kwargs: Any) -> str: - assert stream.tell() == 0 - data = stream.read() - stream.seek(0) - assert stream.read() == data - seen.append((data, info, kwargs)) - streams.append(stream) - return f"image{len(seen)}" - - class InheritedImages(_ImageConverter): - pass - - converter = InheritedImages(render) - source = _docx( - _paragraph( - _image("rIdGif", 1) - + _text(" ") - + _image("rIdPng", 2) - + _text(" ") - + _image("rIdGif", 3) - ) - ) - original = source.getvalue() - info = StreamInfo( - extension=".docx", - filename="document.docx", - url="https://example.test/document.docx", - local_path="/document.docx", - ) - if via_dispatcher: - markitdown = MarkItDown() - markitdown.register_converter(converter, priority=-1) - result = markitdown.convert_stream(source, stream_info=info, **options) - else: - source.seek(7) - result = converter.convert(source, info, **options) - - assert [entry[:2] for entry in seen] == [ - (_GIF, StreamInfo(mimetype="image/gif", extension=".gif")), - (_PNG, StreamInfo(mimetype="image/png", extension=".png")), - (_GIF, StreamInfo(mimetype="image/gif", extension=".gif")), - ] - assert all(entry[2]["ocr_service"] is service for entry in seen) - assert all(entry[2]["heading_style"] == "underlined" for entry in seen) - assert result.markdown == "**image1** **image2** **image3**" - assert all(stream.closed for stream in streams) - assert not source.closed and source.getvalue() == original - - -@pytest.mark.parametrize("keep_data_uris", [False, True]) -@pytest.mark.parametrize("fallback", [None, "", " \n\t"]) -def test_declining_hook_preserves_native_output( - monkeypatch: pytest.MonkeyPatch, fallback: Optional[str], keep_data_uris: bool -) -> None: - expected_converter = DocxConverter() - expected_html = Mock(wraps=expected_converter._html_converter.convert_string) - monkeypatch.setattr( - expected_converter._html_converter, "convert_string", expected_html - ) - expected = expected_converter.convert(_docx(), _INFO, keep_data_uris=keep_data_uris) - converter = _ImageConverter(Mock(return_value=fallback)) - actual_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", actual_html) - - actual = converter.convert(_docx(), _INFO, keep_data_uris=keep_data_uris) - - assert actual_html.call_args == expected_html.call_args - assert actual.markdown == expected.markdown - assert actual.title == expected.title - - -def test_native_converter_does_not_add_image_processing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - convert_html = Mock(wraps=mammoth.convert_to_html) - monkeypatch.setattr(mammoth, "convert_to_html", convert_html) - - class UnmodifiedSubclass(DocxConverter): - pass - - for converter in (DocxConverter(), UnmodifiedSubclass()): - result = converter.convert(_docx(), _INFO) - assert "![Image 1](data:image/png;base64...)" in result.markdown - assert "convert_image" not in convert_html.call_args.kwargs - - -def test_override_can_delegate_to_super() -> None: - class NativeImages(DocxConverter): - def _image_to_html(self, image_stream, stream_info, **kwargs): - return super()._image_to_html(image_stream, stream_info, **kwargs) - - actual = NativeImages().convert(_docx(), _INFO) - assert actual.markdown == DocxConverter().convert(_docx(), _INFO).markdown - - -@pytest.mark.parametrize( - ("fragment", "expected"), - [ - ("text", "

BeforetextAfter

"), - ("one
two", "

Beforeone
twoAfter

"), - ("A & B < C", "

BeforeA & B < CAfter

"), - ( - "

one

two

", - "

Before

one

two

After

", - ), - ( - "

one

two

", - "

Before

one

two

After

", - ), - ( - "
  • one
  • two
", - "

Before

  • one
  • two

After

", - ), - ( - "
Key
Value
", - "

Before

" - "
Key
Value

After

", - ), - ( - "
Title

Body

", - "

Before

Title

Body

" - "

After

", - ), - ( - "lead

block

tail", - "

Beforelead

block

tailAfter

", - ), - ( - "
a < b\n  *literal*
", - "

Before

a < b\n  *literal*

After

", - ), - ], -) -def test_fragment_placement_precedes_shared_html_conversion( - monkeypatch: pytest.MonkeyPatch, fragment: str, expected: str -) -> None: - converter = _ImageConverter(Mock(return_value=fragment)) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - options = {"escape_asterisks": False, "custom_option": "forwarded"} - - result = converter.convert(_docx(), _INFO, **options) - - assert convert_html.call_args.args == (expected,) - assert convert_html.call_args.kwargs == options - assert ( - result.markdown - == HtmlConverter().convert_string(expected, escape_asterisks=False).markdown - ) - assert "data-markitdown-image" not in expected - - -@pytest.mark.parametrize( - ("style_map", "expected"), - [ - (None, "

one

two

"), - ("p => ul > li:fresh", "

  • one

    two

"), - ("p => blockquote > p:fresh", "

one

two

"), - ("p => h2:fresh > strong", "

one

two

"), - ], -) -def test_standalone_block_image_does_not_leave_empty_wrappers( - monkeypatch: pytest.MonkeyPatch, style_map: Optional[str], expected: str -) -> None: - converter = _ImageConverter(Mock(return_value="

one

two

")) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - converter.convert(_docx(_paragraph(_image())), _INFO, style_map=style_map) - - assert convert_html.call_args.args == (expected,) - - -def test_block_image_keeps_its_table_cell( - monkeypatch: pytest.MonkeyPatch, -) -> None: - body = ( - "" - "" - + _paragraph(_text("Item")) - + "" - + _paragraph(_text("Details")) - + "" - + _paragraph(_text("A")) - + "" - + _paragraph(_image()) - + "" - ) - converter = _ImageConverter( - Mock(return_value="

Serial: 12345

Status: active

") - ) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - result = converter.convert(_docx(body), _INFO) - - soup = BeautifulSoup(convert_html.call_args.args[0], "html.parser") - assert len(soup.find_all("tr")) == 2 - assert len(soup.find_all("td")) == 4 - assert ( - str(soup.find_all("td")[-1]) - == "

Serial: 12345

Status: active

" - ) - assert not soup.select("p p") - assert "| A | Serial: 12345 Status: active |" in result.markdown - - -def test_nested_run_formatting_is_preserved_around_blocks( - monkeypatch: pytest.MonkeyPatch, -) -> None: - converter = _ImageConverter(Mock(return_value="

block

")) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - converter.convert(_docx(), _INFO, style_map="p => p:fresh > em > strong") - - assert convert_html.call_args.args == ( - "

Before

block

" - "

After

", - ) - - -def test_hook_inherits_preprocessing_and_style_precedence() -> None: - body = ( - _BODY - + _paragraph( - 'underlined' - ) - + _paragraph("deleted") - + _paragraph("x") - ) - - result = _ImageConverter(Mock(return_value="image")).convert( - _docx(body, embedded_style_map="u => em"), - _INFO, - style_map="u => strong", - ) - - assert result.markdown == "BeforeimageAfter\n\n**underlined**\n\n~~deleted~~\n\n$x$" - - -def test_multiple_images_and_native_fallback_keep_their_own_positions() -> None: - converter = _ImageConverter(Mock(side_effect=[None, "recognized", ""])) - body = _paragraph( - _image(number=1) + _text(" ") + _image(number=2) + _text(" ") + _image(number=3) - ) - - result = converter.convert(_docx(body), _INFO) - - assert result.markdown == ( - "![Image 1](data:image/png;base64...) *recognized* " - "![Image 3](data:image/png;base64...)" - ) - - -def test_many_images_have_independent_replacements() -> None: - converter = _ImageConverter( - Mock(side_effect=[f"

Image {number}

" for number in range(12)]) - ) - body = _paragraph("".join(_image(number=number) for number in range(12))) - - result = converter.convert(_docx(body), _INFO) - - assert result.markdown == "\n\n".join(f"Image {number}" for number in range(12)) - - -def test_returned_image_and_literal_text_use_normal_html_rendering() -> None: - render = Mock( - return_value=( - "

<literal> *not emphasis* & text

" - 'generated' - ) - ) - result = _ImageConverter(render).convert(_docx(_paragraph(_image())), _INFO) - - assert ( - result.markdown - == r" \*not emphasis\* & text" + "\n\n![generated](image.png)" - ) - render.assert_called_once() - - -@pytest.mark.parametrize("result", [False, 123, b"

not a string

"]) -def test_invalid_return_is_not_silent_native_fallback(result: Any) -> None: - with pytest.raises(TypeError, match="HTML string or None"): - _ImageConverter(Mock(return_value=result)).convert(_docx(), _INFO) - - -@pytest.mark.parametrize( - "fragment", - [ - "document", - "title", - "content", - "

content

", - ], -) -def test_full_document_is_not_a_valid_fragment(fragment: str) -> None: - with pytest.raises(ValueError, match="fragment, not a document"): - _ImageConverter(Mock(return_value=fragment)).convert(_docx(), _INFO) - - -@pytest.mark.parametrize("via_dispatcher", [False, True]) -def test_hook_errors_propagate_and_close_the_image_stream(via_dispatcher: bool) -> None: - streams = [] - error = RuntimeError("Image service failed") - - def render(stream, info, **kwargs): - streams.append(stream) - raise error - - converter = _ImageConverter(render) - if via_dispatcher: - markitdown = MarkItDown(enable_builtins=False) - markitdown.register_converter(converter, priority=-1) - with pytest.raises(FileConversionException) as caught: - markitdown.convert_stream(_docx(), stream_info=_INFO) - assert caught.value.attempts is not None - assert any( - attempt.exc_info and attempt.exc_info[1] is error - for attempt in caught.value.attempts - ) - else: - with pytest.raises(RuntimeError) as caught_direct: - converter.convert(_docx(), _INFO) - assert caught_direct.value is error - assert streams and all(stream.closed for stream in streams) - - -def test_dispatcher_can_fall_back_to_native_docx_after_hook_error() -> None: - render = Mock(side_effect=RuntimeError("Image service failed")) - markitdown = MarkItDown() - markitdown.register_converter(_ImageConverter(render), priority=-1) - - result = markitdown.convert_stream(_docx(), stream_info=_INFO) - - assert result.markdown == DocxConverter().convert(_docx(), _INFO).markdown - render.assert_called() - - -def test_mammoth_does_not_swallow_hook_file_reference_errors() -> None: - error = InvalidFileReferenceError("hook failed") - render = Mock(side_effect=error) - - with pytest.raises(RuntimeError, match="_image_to_html failed") as caught: - _ImageConverter(render).convert(_docx(), _INFO) - - assert caught.value.__cause__ is error - render.assert_called_once() - - -def test_per_call_options_and_fragments_do_not_leak_between_conversions() -> None: - converter = _ImageConverter(lambda stream, info, **kwargs: kwargs.get("image_text")) - first = converter.convert(_docx(), _INFO, image_text="

first

") - second = converter.convert(_docx(), _INFO, image_text="

second

") - third = converter.convert(_docx(), _INFO) - - assert first.markdown == "Before\n\nfirst\n\nAfter" - assert second.markdown == "Before\n\nsecond\n\nAfter" - assert third.markdown == DocxConverter().convert(_docx(), _INFO).markdown - - -def test_unreferenced_media_does_not_invoke_the_hook() -> None: - render = Mock(side_effect=AssertionError("unexpected image")) - - result = _ImageConverter(render).convert( - _docx(_paragraph(_text("Only text"))), _INFO - ) - - assert result.markdown == "Only text" - render.assert_not_called() - - -@pytest.mark.parametrize( - "fragment", - [ - 'new', - '

new

', - ], -) -def test_image_html_does_not_create_nested_links( - monkeypatch: pytest.MonkeyPatch, fragment: str -) -> None: - body = _paragraph( - '' - + _text("Before") - + _image() - + _text("After") - + "" - ) - converter = _ImageConverter(Mock(return_value=fragment)) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - result = converter.convert(_docx(body), _INFO) - - soup = BeautifulSoup(convert_html.call_args.args[0], "html.parser") - assert not soup.select("a a") - assert [link.get("href") for link in soup.find_all("a")] == [ - "#target", - "https://example.test/new", - "#target", - ] - assert "[Before](#target)" in result.markdown - assert "[new](https://example.test/new)" in result.markdown - assert "[After](#target)" in result.markdown - - -@pytest.mark.parametrize("keep_data_uris", [False, True]) -def test_declining_hook_preserves_a_real_document(keep_data_uris: bool) -> None: - content = (_FILES / "test.docx").read_bytes() - expected = DocxConverter().convert( - io.BytesIO(content), _INFO, keep_data_uris=keep_data_uris - ) - render = Mock(return_value=None) - - actual = _ImageConverter(render).convert( - io.BytesIO(content), _INFO, keep_data_uris=keep_data_uris - ) - - assert actual.markdown == expected.markdown - assert actual.title == expected.title - render.assert_called() - - -def test_hook_reads_images_from_the_preprocessed_archive() -> None: - import struct - - stream = _docx() - data = bytearray(stream.getvalue()) - with zipfile.ZipFile(stream) as archive: - offset = archive.getinfo("word/media/image.png").header_offset - length = struct.unpack_from("image" - - result = _ImageConverter(render).convert(io.BytesIO(data), _INFO) - - assert observed == [_PNG] - assert result.markdown == "BeforeimageAfter" - - -def test_missing_dependencies_fail_before_invoking_hook( - monkeypatch: pytest.MonkeyPatch, -) -> None: - error = ModuleNotFoundError("No module named 'mammoth'") - monkeypatch.setattr( - _docx_converter, "_dependency_exc_info", (ModuleNotFoundError, error, None) - ) - render = Mock() - - with pytest.raises(MissingDependencyException) as caught: - _ImageConverter(render).convert(_docx(), _INFO) - - assert caught.value.__cause__ is error - render.assert_not_called() diff --git a/packages/markitdown/tests/test_docx_math.py b/packages/markitdown/tests/test_docx_math.py deleted file mode 100644 index 07827becff..0000000000 --- a/packages/markitdown/tests/test_docx_math.py +++ /dev/null @@ -1,59 +0,0 @@ -from markitdown.converter_utils.docx.math.omml import load_string - - -def test_omml_known_function(): - omml_xml = """ - - - log - x - - - """ - results = list(load_string(omml_xml)) - assert len(results) == 1 - assert r"\log" in str(results[0]) - - -def test_omml_unknown_function_fallback(): - omml_xml = """ - - - customFunc - x - - - """ - results = list(load_string(omml_xml)) - assert len(results) == 1 - assert r"\operatorname{customFunc}" in str(results[0]) - - -def test_omml_function_name_split_across_runs(): - # Word can break a single function name into several runs (e.g. at a - # formatting boundary); the runs must be joined before the lookup. - omml_xml = """ - - - customFunc - x - - - """ - results = list(load_string(omml_xml)) - assert len(results) == 1 - assert str(results[0]) == r"\operatorname{customFunc}(x)" - - -def test_omml_known_function_split_across_runs(): - omml_xml = """ - - - sin - x - - - """ - results = list(load_string(omml_xml)) - assert len(results) == 1 - assert str(results[0]) == r"\sin(x)" diff --git a/packages/markitdown/tests/test_docx_math_accents.py b/packages/markitdown/tests/test_docx_math_accents.py deleted file mode 100644 index 821d057bc5..0000000000 --- a/packages/markitdown/tests/test_docx_math_accents.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for DOCX math accent templates. - -Every accent in ``latex_dict.CHR`` is a ``str.format`` template applied by -``oMath2Latex.do_acc`` as ``latex_s.format(c_dict["e"])``. A template whose -braces are unbalanced raises ``ValueError`` at that call. - -That failure is not local to the equation. ``pre_process_docx`` runs -``_pre_process_math`` over the whole of ``word/document.xml`` inside a blanket -``except Exception`` and, on error, writes the *original* unprocessed XML back. -Mammoth does not render OMML, so a single unformattable accent silently removes -every equation in the document -- with no error and a zero exit code. - -``test_accent_templates_are_formattable`` guards the whole table against that -class of typo; ``test_caron_and_ring_accents`` covers the two entries that -carried an extra closing brace. -""" - -from markitdown.converter_utils.docx.math.latex_dict import ( - CHR, - CHR_BO, - CHR_DEFAULT, - POS, - POS_DEFAULT, -) -from markitdown.converter_utils.docx.math.omml import load_string - -# Accent characters, as Word writes them into . -CARON = "\u030c" # COMBINING CARON -> \check -RING_ABOVE = "\u030a" # COMBINING RING ABOVE -> \ocirc -CIRCUMFLEX = "\u0302" # COMBINING CIRCUMFLEX -> \hat - -_OMML_DOC = ( - "' - "" - '' - "x" - "" -) - - -def _latex_for_accent(accent_char: str) -> str: - """Convert a single accented variable to LaTeX, as pre_process_docx would.""" - return next(load_string(_OMML_DOC.format(chr=accent_char))).latex - - -def test_accent_templates_are_formattable() -> None: - """Every {0} template must survive .format(); do_acc calls it unguarded.""" - unformattable = [] - for name, table in ( - ("CHR", CHR), - ("CHR_BO", CHR_BO), - ("CHR_DEFAULT", CHR_DEFAULT), - ("POS", POS), - ("POS_DEFAULT", POS_DEFAULT), - ): - for key, template in table.items(): - if not isinstance(template, str) or "{0}" not in template: - continue - try: - template.format("x") - except ValueError as exc: - unformattable.append(f"{name}[{key!r}] = {template!r}: {exc}") - - assert not unformattable, "unformattable accent templates: " + "; ".join( - unformattable - ) - - -def test_caron_and_ring_accents() -> None: - """U+030C and U+030A each carried an extra '}' and raised ValueError.""" - assert _latex_for_accent(CARON) == "\\check{x}" - assert _latex_for_accent(RING_ABOVE) == "\\ocirc{x}" - - -def test_unmodified_accent_still_converts() -> None: - """Control: a neighbouring accent that was never broken.""" - assert _latex_for_accent(CIRCUMFLEX) == "\\hat{x}" diff --git a/packages/markitdown/tests/test_docx_math_symbols.py b/packages/markitdown/tests/test_docx_math_symbols.py deleted file mode 100644 index babe976643..0000000000 --- a/packages/markitdown/tests/test_docx_math_symbols.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for the OMML -> LaTeX symbol tables used by the DOCX converter. - -Two independent defects are covered here: - -``CHR`` maps a combining accent to a LaTeX template. U+20EE COMBINING LEFT -ARROW BELOW mapped to ``\\underledtarrow``, which is not a LaTeX macro -- the -name is a scrambled ``\\underleftarrow``. Its two neighbours in the same table -(U+20D6 -> ``\\overleftarrow``, U+20EF -> ``\\underrightarrow``) show the -intent. The template still formats, so nothing raises: the document simply ends -up with an undefined control sequence that no renderer can typeset. - -``T`` normalizes the Mathematical Alphanumeric Symbols back to ASCII, so that -an equation written with math italic letters yields ``h(x)`` rather than a -string of astral-plane codepoints. U+1D455 -- the slot where math italic small -h would sit -- is permanently reserved, because Unicode unifies that letter -with U+210E PLANCK CONSTANT. The table followed the contiguous block and so -skipped h, leaving it as the single letter of the alphabet that survived -untranslated into the LaTeX output. -""" - -from markitdown.converter_utils.docx.math.latex_dict import T -from markitdown.converter_utils.docx.math.omml import load_string - -# Math italic Latin: A-Z is contiguous, a-z has a hole at U+1D455 (small h), -# which Unicode unifies with U+210E. -ITALIC_UPPERCASE = { - chr(0x1D434 + i): c for i, c in enumerate("ABCDEFGHIJKLMNOPQRSTUVWXYZ") -} -ITALIC_LOWERCASE = { - chr(0x1D44E + i): c for i, c in enumerate("abcdefghijklmnopqrstuvwxyz") -} -ITALIC_LOWERCASE.pop(chr(0x1D455)) # reserved codepoint, never assigned -ITALIC_LOWERCASE["ℎ"] = "h" - -_DOC = ( - "' - "{0}" -) - -_ACCENT = ( - "" - '' - "x" - "" -) - -_RUN = "{text}" - - -def _latex(fragment: str) -> str: - return next(load_string(_DOC.format(fragment))).latex - - -def _accent_latex(accent_char: str) -> str: - return _latex(_ACCENT.format(chr=accent_char)) - - -def _run_latex(text: str) -> str: - return _latex(_RUN.format(text=text)) - - -def test_combining_left_arrow_below() -> None: - """U+20EE must produce \\underleftarrow, not the misspelled macro.""" - assert _accent_latex("⃮") == "\\underleftarrow{x}" - - -def test_arrow_accents_are_consistent() -> None: - """The three arrow accents share one naming scheme; none may be misspelled.""" - assert _accent_latex("⃖") == "\\overleftarrow{x}" # left arrow above - assert _accent_latex("⃯") == "\\underrightarrow{x}" # right arrow below - assert _accent_latex("⃮") == "\\underleftarrow{x}" # left arrow below - - -def test_math_italic_small_h_is_normalized() -> None: - """U+210E is the math italic h; it must not survive into the LaTeX.""" - assert _run_latex("ℎ") == "h" - - -def test_math_italic_latin_alphabet_is_complete() -> None: - """No math italic Latin letter may leak through untranslated.""" - for char, expected in {**ITALIC_UPPERCASE, **ITALIC_LOWERCASE}.items(): - assert T.get(char) == expected, f"U+{ord(char):04X} is missing from T" - assert _run_latex(char) == expected - - -def test_math_italic_expression_is_fully_normalized() -> None: - """A whole expression must come out as plain ASCII LaTeX.""" - # h(x) = g(x), written entirely with math italic codepoints. - expression = "ℎ(\U0001d465)=\U0001d454(\U0001d465)" - assert _run_latex(expression) == "h(x)=g(x)" - - -if __name__ == "__main__": - test_combining_left_arrow_below() - test_arrow_accents_are_consistent() - test_math_italic_small_h_is_normalized() - test_math_italic_latin_alphabet_is_complete() - test_math_italic_expression_is_fully_normalized() - print("All tests passed!") diff --git a/packages/markitdown/tests/test_docx_omml.py b/packages/markitdown/tests/test_docx_omml.py deleted file mode 100644 index 98574108c4..0000000000 --- a/packages/markitdown/tests/test_docx_omml.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Regression test for a crash in the OMML -> LaTeX converter when a math run -() has no text child (e.g. a run that only carries formatting -properties). Previously `do_r()` called `elm.findtext(...)` directly and -iterated over the result, which raised `TypeError: 'NoneType' object is not -iterable` when the run had no text, aborting equation conversion for the -entire document. -""" - -from xml.etree import ElementTree as ET - -from markitdown.converter_utils.docx.math.omml import OMML_NS, oMath2Latex - -MATH_NS_DECL = f'xmlns:m="{OMML_NS[1:-1]}"' - - -def _parse_omath(xml_fragment: str): - wrapped = f"{xml_fragment}" - return ET.fromstring(wrapped) - - -def test_run_without_text_child_does_not_crash(): - # with only , no child. - element = _parse_omath("") - # Should not raise TypeError: 'NoneType' object is not iterable - result = oMath2Latex(element) - assert result.latex == "" - - -def test_run_with_text_still_converts(): - element = _parse_omath("x") - result = oMath2Latex(element) - assert result.latex == "x" - - -def test_subscript_with_missing_text_run_does_not_crash(): - # Mirrors a real-world document: a subscript expression where one of the - # runs involved has no text (e.g. produced by some Word equation editors). - element = _parse_omath( - "" - "l" - "1" - "" - ) - result = oMath2Latex(element) - assert "l" in result.latex - assert "1" in result.latex diff --git a/packages/markitdown/tests/test_docx_styles.py b/packages/markitdown/tests/test_docx_styles.py deleted file mode 100644 index 218fe03c5d..0000000000 --- a/packages/markitdown/tests/test_docx_styles.py +++ /dev/null @@ -1,152 +0,0 @@ -"""DOCX stylesheet repair must preserve namespace-qualified attributes.""" - -import io -from pathlib import Path -import re -import zipfile - -from lxml import etree -import pytest - -from markitdown import MarkItDown, StreamInfo -from markitdown.converter_utils.docx.pre_process import _pre_process_styles - - -WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" -W = f"{{{WORD_NAMESPACE}}}" -TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx" - - -@pytest.mark.parametrize("double_strike", [False, True]) -@pytest.mark.parametrize("missing_type", [False, True]) -def test_docx_styles_with_redundant_default_namespace( - missing_type: bool, double_strike: bool -) -> None: - markitdown = MarkItDown() - expected = markitdown.convert(TEST_DOCX).markdown - fixture = io.BytesIO() - declaration = f'xmlns:w="{WORD_NAMESPACE}"'.encode("utf-8") - - with zipfile.ZipFile(TEST_DOCX) as source, zipfile.ZipFile(fixture, "w") as target: - for item in source.infolist(): - content = source.read(item) - if item.filename == "word/styles.xml": - if double_strike: - assert content.count(b"") == 1 - content = content.replace( - b"", - b'' - b'' - b'' - b"", - 1, - ) - assert content.count(declaration) == 1 - content = content.replace( - declaration, - declaration + f' xmlns="{WORD_NAMESPACE}"'.encode("utf-8"), - 1, - ) - if missing_type: - content, count = re.subn( - rb' bytes: - prefix = request.param - return f""" -<{prefix}:styles xmlns:{prefix}="{WORD_NAMESPACE}" xmlns="{WORD_NAMESPACE}" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:w14="http://schemas.microsoft.com/office/word/2010/wordml" - mc:Ignorable="w14"> - <{prefix}:style {prefix}:type="paragraph" {prefix}:styleId="Normal"> - <{prefix}:name {prefix}:val="Normal"/> - -""".encode( - "utf-8" - ) - - -def test_valid_styles_are_returned_unchanged(styles_xml: bytes) -> None: - assert _pre_process_styles(styles_xml) == styles_xml - - -def test_missing_type_repair_preserves_namespaces(styles_xml: bytes) -> None: - malformed, count = re.subn(rb'\s+(?:w|word):type="paragraph"', b"", styles_xml) - assert count == 1 - original = etree.fromstring(styles_xml) - - repaired = etree.fromstring(_pre_process_styles(malformed)) - - assert repaired.nsmap == original.nsmap - assert repaired.attrib == original.attrib - style = repaired.find(W + "style") - assert style is not None - assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} - name = style.find(W + "name") - assert name is not None - assert name.attrib == {W + "val": "Normal"} - - -def test_styles_without_ids_are_removed(styles_xml: bytes) -> None: - malformed, count = re.subn(rb'\s+(?:w|word):styleId="Normal"', b"", styles_xml) - assert count == 1 - - repaired = etree.fromstring(_pre_process_styles(malformed)) - - assert repaired.findall(W + "style") == [] - - -@pytest.mark.parametrize("value", [None, "0", "1"]) -@pytest.mark.parametrize("missing_type", [False, True]) -def test_style_strike_repair_preserves_namespaces( - styles_xml: bytes, value: str | None, missing_type: bool -) -> None: - original = etree.fromstring(styles_xml) - style = original.find(W + "style") - assert style is not None - if missing_type: - del style.attrib[W + "type"] - properties = etree.SubElement(style, W + "rPr") - strike = etree.SubElement(properties, W + "dstrike") - if value is not None: - strike.set(W + "val", value) - etree.SubElement(properties, W + "b") - - repaired = etree.fromstring(_pre_process_styles(etree.tostring(original))) - - assert repaired.nsmap == original.nsmap - assert repaired.attrib == original.attrib - style = repaired.find(W + "style") - assert style is not None - assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} - assert repaired.find(".//" + W + "dstrike") is None - strike = style.find(f"{W}rPr/{W}strike") - assert strike is not None - assert strike.attrib == ({} if value is None else {W + "val": value}) - assert style.find(f"{W}rPr/{W}b") is not None - - -def test_other_namespace_dstrike_is_unchanged(styles_xml: bytes) -> None: - original = etree.fromstring(styles_xml) - etree.SubElement(original, "{urn:extension}dstrike") - content = etree.tostring(original) - - assert _pre_process_styles(content) == content diff --git a/packages/markitdown/tests/test_epub_converter.py b/packages/markitdown/tests/test_epub.py similarity index 50% rename from packages/markitdown/tests/test_epub_converter.py rename to packages/markitdown/tests/test_epub.py index ac8edd3202..f87f4d4437 100644 --- a/packages/markitdown/tests/test_epub_converter.py +++ b/packages/markitdown/tests/test_epub.py @@ -1,9 +1,16 @@ +"""EPUB conversion, archive paths, metadata, and embedded images.""" + import io import zipfile -from markitdown import StreamInfo +import pytest + +from markitdown import MarkItDown, StreamInfo from markitdown.converters import EpubConverter + +# Archive href resolution + CONTAINER_XML = """ @@ -107,9 +114,110 @@ def test_parent_relative_href_resolves() -> None: assert "SHARED_BODY" in _convert(stream) +# Conversion regressions + + +def test_epub_metadata_nodevalue(): + from defusedxml.minidom import parseString + from markitdown.converters._epub_converter import EpubConverter + + xml_data = ( + '' + "Structured Title" + "Author 1" + "Author 2" + "" + "" + "" + ) + dom = parseString(xml_data) + converter = EpubConverter() + + title = converter._get_text_from_node(dom, "dc:title") + assert title == "Structured Title" + + creators = converter._get_all_texts_from_nodes(dom, "dc:creator") + assert creators == ["Author 1", "Author 2"] + + publisher = converter._get_text_from_node(dom, "dc:publisher") + assert publisher is None + + missing = converter._get_text_from_node(dom, "dc:date") + assert missing is None + + +_EPUB_CONTAINER = ( + '' + '' + '' +) + +_EPUB_OPF = ( + '' + '' + '' + "Example book" + '' + '' +) + +_EPUB_CHAPTER = ( + '' + "

Chapter text.

" + 'diagram' + "" +) + + +def _build_epub_with_data_uri() -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("mimetype", "application/epub+zip") + zf.writestr("META-INF/container.xml", _EPUB_CONTAINER) + zf.writestr("OEBPS/content.opf", _EPUB_OPF) + zf.writestr("OEBPS/ch1.xhtml", _EPUB_CHAPTER) + return buf.getvalue() + + +def test_epub_honors_keep_data_uris() -> None: + """EPUB chapters must be converted with the options the caller passed.""" + result = MarkItDown().convert_stream( + io.BytesIO(_build_epub_with_data_uri()), + stream_info=StreamInfo(extension=".epub"), + keep_data_uris=True, + ) + + assert ( + "![diagram](data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==)" in result.markdown + ) + + +def test_epub_truncates_data_uris_by_default() -> None: + """Without the option, the default truncation must still apply.""" + result = MarkItDown().convert_stream( + io.BytesIO(_build_epub_with_data_uri()), + stream_info=StreamInfo(extension=".epub"), + ) + + assert "![diagram](data:image/png;base64...)" in result.markdown + assert "iVBORw0KGgo" not in result.markdown + + +def test_epub_metadata_and_text_are_unchanged() -> None: + """The rest of the conversion must not move.""" + result = MarkItDown().convert_stream( + io.BytesIO(_build_epub_with_data_uri()), + stream_info=StreamInfo(extension=".epub"), + ) + + assert result.title == "Example book" + assert "**Title:** Example book" in result.markdown + assert "Chapter text." in result.markdown + + if __name__ == "__main__": - test_percent_encoded_href_resolves_to_zip_entry() - test_non_ascii_percent_encoded_href_resolves() - test_literally_encoded_zip_entry_still_resolves() - test_parent_relative_href_resolves() - print("All tests passed") + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_file_paths.py b/packages/markitdown/tests/test_file_paths.py deleted file mode 100644 index 2930c2c522..0000000000 --- a/packages/markitdown/tests/test_file_paths.py +++ /dev/null @@ -1,138 +0,0 @@ -"""File URI validation must reject network paths before opening a file.""" - -import os -from pathlib import Path - -import pytest - -import markitdown._uri_utils as uri_utils -from markitdown import MarkItDown -from markitdown._uri_utils import _is_unc_or_device_path, file_uri_to_path - - -UNC_URI_PATHS = [ - "//server.example/share/x.txt", - "///server.example/share/x.txt", - "////server.example/share/x.txt", - r"/\server.example/share/x.txt", - r"\/server.example/share/x.txt", - r"\\server.example\share\x.txt", - "/%2Fserver.example/share/x.txt", - "/%2fserver.example/share/x.txt", - "/%5Cserver.example/share/x.txt", - "/%5cserver.example/share/x.txt", - "%2F%2Fserver.example/share/x.txt", - "%5C%5Cserver.example%5Cshare%5Cx.txt", - "/%5C%3F%5CUNC%5Cserver.example%5Cshare%5Cx.txt", - "/%5C.%5CUNC%5Cserver.example%5Cshare%5Cx.txt", - "/%5C%3F%5CC:%5CTemp%5Cx.txt", - "/%5C.%5CPhysicalDrive0", -] - -BLOCKED_URIS = ( - [ - f"file://{authority}{path}" - for authority in ("", "localhost") - for path in UNC_URI_PATHS - # An authority must be followed by a slash to introduce its path. - if path.startswith("/") - ] - + [f"file:{path}" for path in UNC_URI_PATHS if not path.startswith("/")] - + [ - "FILE:////server.example/share/x.txt", - "file:////C:/Temp/x.txt", - ] -) - - -@pytest.mark.parametrize("uri", BLOCKED_URIS) -def test_file_uri_rejects_unc_or_device_path(uri: str) -> None: - with pytest.raises(ValueError, match="UNC"): - file_uri_to_path(uri) - - -class _NoFileAccessMarkItDown(MarkItDown): - def convert_local(self, *args, **kwargs): - pytest.fail("Rejected file URIs must not reach convert_local()") - - -@pytest.fixture(scope="module") -def no_file_access_markitdown() -> MarkItDown: - return _NoFileAccessMarkItDown(enable_builtins=False) - - -@pytest.mark.parametrize("method", ["convert", "convert_uri"]) -@pytest.mark.parametrize("uri", BLOCKED_URIS + ["file://server.example/share/x.txt"]) -def test_convert_rejects_remote_file_uri_before_file_access( - no_file_access_markitdown: MarkItDown, method: str, uri: str -) -> None: - with pytest.raises(ValueError, match="Unsupported file URI"): - getattr(no_file_access_markitdown, method)(uri) - - -@pytest.mark.parametrize( - "path, blocked", - [ - (r"\\server\share\x.txt", True), - ("//server/share/x.txt", True), - (r"/\server/share/x.txt", True), - (r"\/server/share/x.txt", True), - (r"\\?\UNC\server\share\x.txt", True), - (r"\\?\C:\Temp\x.txt", True), - (r"\\.\PhysicalDrive0", True), - (r"C:\Temp\x.txt", False), - ("C:/Temp/x.txt", False), - ("/home/user/x.txt", False), - ("relative.txt", False), - ], -) -def test_resolved_path_validation(path: str, blocked: bool) -> None: - assert _is_unc_or_device_path(path) is blocked - - -def test_file_uri_rejects_unc_after_conversion(monkeypatch: pytest.MonkeyPatch) -> None: - # Check the final guard independently of the URI prefix check. Replace only - # the helper's converter binding, leaving the stdlib functions unchanged. - monkeypatch.setattr( - uri_utils, "url2pathname", lambda _: "//server.example/share/x.txt" - ) - - with pytest.raises(ValueError, match="UNC"): - file_uri_to_path("file:///local.txt") - - -@pytest.mark.parametrize("filename", ["notes.txt", "notes café.txt", "%5C%5Cx.txt"]) -@pytest.mark.parametrize("authority", ["", "localhost"]) -def test_local_file_uri_conversion( - tmp_path: Path, filename: str, authority: str -) -> None: - local_file = tmp_path / filename - local_file.write_text("Local file contents", encoding="utf-8") - uri = local_file.as_uri().replace("file://", f"file://{authority}", 1) - - netloc, path = file_uri_to_path(uri) - - assert netloc == (authority or None) - assert path == str(local_file) - markitdown = MarkItDown() - assert markitdown.convert(uri).markdown == "Local file contents" - assert markitdown.convert(str(local_file)).markdown == "Local file contents" - - -@pytest.mark.skipif(os.name != "nt", reason="Requires native Windows path conversion") -@pytest.mark.parametrize("authority", ["", "localhost"]) -@pytest.mark.parametrize( - "uri_path, expected", - [ - ("/C:/Temp/notes.txt", r"C:\Temp\notes.txt"), - ("/C%3A/Temp/notes.txt", r"C:\Temp\notes.txt"), - ("/C%3a/Temp/notes.txt", r"C:\Temp\notes.txt"), - ("/C:/Temp/notes%20caf%C3%A9.txt", "C:\\Temp\\notes café.txt"), - ("/C:/Temp/%255C%255Cx.txt", r"C:\Temp\%5C%5Cx.txt"), - ], -) -def test_windows_local_drive_uri(authority: str, uri_path: str, expected: str) -> None: - netloc, path = file_uri_to_path(f"file://{authority}{uri_path}") - - assert netloc == (authority or None) - assert path == expected diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py new file mode 100644 index 0000000000..94cece47b9 --- /dev/null +++ b/packages/markitdown/tests/test_html.py @@ -0,0 +1,349 @@ +"""HTML conversion, link handling, and Wikipedia pages.""" + +import io + +import pytest +from bs4 import BeautifulSoup + +from markitdown import MarkItDown, StreamInfo +from markitdown.converters import WikipediaConverter +from markitdown.converters._markdownify import _CustomMarkdownify + + +# HTML rendering + + +def _convert_html(html: str, **kwargs) -> str: + result = MarkItDown().convert_stream( + io.BytesIO(html.encode("utf-8")), + file_extension=".html", + **kwargs, + ) + return result.markdown + + +@pytest.mark.parametrize( + "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "] +) +def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None: + element = BeautifulSoup("", "html.parser").u + + assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + ("", "FirstLast"), + (" ", "First Last"), + ("\t", "First Last"), + (" ", "First\u00a0Last"), + ("
", "First\nLast"), + ("word", "FirstwordLast"), + (" word ", "First word Last"), + ], +) +def test_html_underlined_content_is_preserved(content: str, expected: str) -> None: + assert _convert_html(f"

First{content}Last

") == expected + + +def test_preserves_non_utf8_percent_encoded_href_path() -> None: + href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" + html = f'example' + + markdown = _convert_html(html) + + assert f"[example]({href})" in markdown + assert "%EF%BF%BD" not in markdown + + +def test_html_href_still_quotes_raw_unicode_and_spaces() -> None: + href = "https://example.com/a path/日本語" + expected_href = "https://example.com/a%20path/" "%E6%97%A5%E6%9C%AC%E8%AA%9E" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_quotes_literal_percent_sign() -> None: + href = "https://example.com/100% complete" + expected_href = "https://example.com/100%25%20complete" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_quotes_malformed_percent_escape() -> None: + href = "https://example.com/items/%ZZ/%2F" + expected_href = "https://example.com/items/%25ZZ/%2F" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_preserves_encoded_slash() -> None: + href = "https://example.com/items/a%2Fb" + + markdown = _convert_html(f'example') + + assert f"[example]({href})" in markdown + + +def test_html_href_does_not_quote_query_or_fragment() -> None: + href = "https://example.com/a path?query=a b%20c#fragment with spaces" + expected_href = "https://example.com/a%20path?query=a b%20c#fragment with spaces" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_img_prefers_data_src_over_placeholder_data_uri() -> None: + placeholder = ( + "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" + ) + real_src = "https://example.com/photo.jpg" + html = ( + f'A photo' + ) + + markdown = _convert_html(html) + + assert f"![A photo]({real_src})" in markdown + assert placeholder not in markdown + + +def test_img_uses_real_src_over_data_src_when_both_present() -> None: + real_src = "https://example.com/photo.jpg" + other_src = "https://example.com/photo-alt.jpg" + html = f'A photo' + + markdown = _convert_html(html) + + assert f"![A photo]({real_src})" in markdown + + +def test_img_falls_back_to_data_src_when_src_missing() -> None: + real_src = "https://example.com/photo.jpg" + html = f'A photo' + + markdown = _convert_html(html) + + assert f"![A photo]({real_src})" in markdown + + +def test_img_keeps_truncated_data_uri_when_no_data_src() -> None: + placeholder = ( + "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" + ) + html = f'A photo' + + markdown = _convert_html(html) + + assert "![A photo](data:image/gif;base64...)" in markdown + + +def test_img_keeps_embedded_data_uri_over_data_src_when_keeping_data_uris() -> None: + embedded = ( + "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" + ) + other_src = "https://example.com/photo.jpg" + html = f'A photo' + + markdown = _convert_html(html, keep_data_uris=True) + + assert f"![A photo]({embedded})" in markdown + assert other_src not in markdown + + +# Wikipedia article titles + +STREAM_INFO = StreamInfo( + url="https://en.wikipedia.org/wiki/Example", + mimetype="text/html", + extension=".html", +) + + +def _page(heading: str, document_title: str) -> io.BytesIO: + """Build a page shaped like the article HTML Wikipedia actually serves.""" + return io.BytesIO( + f"""{document_title} - Wikipedia +

{heading}

+

Body text.

+""".encode() + ) + + +def test_plain_title_is_read_from_the_title_span() -> None: + """A plain title keeps being read from mw-page-title-main.""" + page = _page( + 'Paris', + "Paris", + ) + + result = WikipediaConverter().convert(page, STREAM_INFO) + + assert result.title == "Paris" + assert result.markdown.lstrip().startswith("# Paris") + + +def test_fully_italicised_title_keeps_its_text() -> None: + """Wikipedia drops the title span for italicised titles, e.g. species names.""" + page = _page("Escherichia coli", "Escherichia coli") + + result = WikipediaConverter().convert(page, STREAM_INFO) + + # Without the first-heading fallback this became "Escherichia coli - Wikipedia". + assert result.title == "Escherichia coli" + assert result.markdown.lstrip().startswith("# Escherichia coli") + + +def test_title_mixing_markup_and_plain_text_keeps_both_parts() -> None: + """A disambiguated italic title spans several children, so .string is None.""" + page = _page("Titanic (1997 film)", "Titanic (1997 film)") + + result = WikipediaConverter().convert(page, STREAM_INFO) + + assert result.title == "Titanic (1997 film)" + assert result.markdown.lstrip().startswith("# Titanic (1997 film)") + + +# Conversion regressions + + +def test_wikipedia_converter_no_title() -> None: + """WikipediaConverter should not render '# None' when page has no title.""" + converter = WikipediaConverter() + html = b"

Hello

" + stream_info = StreamInfo( + mimetype="text/html", url="https://en.wikipedia.org/wiki/Test" + ) + result = converter.convert(io.BytesIO(html), stream_info) + assert "# None" not in result.markdown + assert "Hello" in result.markdown + assert result.markdown.strip() == "Hello" + + +def test_wikipedia_converter_blank_title() -> None: + """WikipediaConverter should not render an empty heading for a blank title.""" + converter = WikipediaConverter() + html = b"

Hello

" + stream_info = StreamInfo( + mimetype="text/html", url="https://en.wikipedia.org/wiki/Test" + ) + result = converter.convert(io.BytesIO(html), stream_info) + assert not result.markdown.lstrip().startswith("#") + assert result.title is None + assert result.text_content.strip() == "Hello" + + +def test_uppercase_data_image_uri_is_truncated_by_default() -> None: + markitdown = MarkItDown() + html = b'dot' + stream_info = StreamInfo(mimetype="text/html", extension=".html") + + result = markitdown.convert_stream(io.BytesIO(html), stream_info=stream_info) + assert result.markdown == "![dot](DATA:image/png;base64...)" + assert "AAAA" not in result.markdown + + result = markitdown.convert_stream( + io.BytesIO(html), stream_info=stream_info, keep_data_uris=True + ) + assert result.markdown == "![dot](DATA:image/png;base64,AAAA)" + + +def test_html_strikethrough_variants(tmp_path) -> None: + html = """ + +

Plain s element after.

+

Plain del element after.

+

Plain strike element after.

+

Spaces A B C.

+

Runs D E F.

+

Empty GH.

+

Newline IJ +KL.

+

Break MN
OP.

+ +""" + path = tmp_path / "strike.html" + path.write_text(html, encoding="utf-8") + markdown = MarkItDown().convert(str(path)).markdown + + assert markdown == "\n\n".join( + [ + # , and the obsolete all mean strikethrough + "Plain ~~s element~~ after.", + "Plain ~~del element~~ after.", + "Plain ~~strike element~~ after.", + # Surrounding whitespace stays outside of the markup ... + "Spaces A ~~B~~ C.", + # ... and runs of it collapse to a single space + "Runs D ~~E~~ F.", + # An empty element contributes nothing + "Empty GH.", + # A line break inside the element is kept, and the markup + # survives it because strikethrough may span a single newline + "Newline I~~J\nK~~L.", + "Break M~~N\nO~~P.", + ] + ) + + +def test_deeply_nested_html_fallback() -> None: + """Large, deeply nested HTML should fall back to plain-text extraction + instead of silently returning unconverted HTML (issue #1636). + + Note: This test uses sys.setrecursionlimit to guarantee a RecursionError + regardless of the host environment's default limit, making it deterministic + across different platforms and CI configurations. + """ + import sys + import warnings + + markitdown = MarkItDown() + + # Use a small recursion limit so the test is environment-independent. + # We restore the original limit in a finally block to avoid side-effects. + original_limit = sys.getrecursionlimit() + low_limit = 200 # well below markdownify's traversal depth for depth=500 + + # Build HTML with nesting deep enough to trigger RecursionError + depth = 500 + html = "" + for _ in range(depth): + html += '
' + html += "

Deep content with bold text

" + for _ in range(depth): + html += "
" + html += "" + + try: + sys.setrecursionlimit(low_limit) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = markitdown.convert_stream( + io.BytesIO(html.encode("utf-8")), + file_extension=".html", + ) + + # Should have emitted a warning about the fallback + recursion_warnings = [x for x in w if "deeply nested" in str(x.message)] + assert len(recursion_warnings) > 0 + + finally: + sys.setrecursionlimit(original_limit) + + # The output should contain the text content, not raw HTML + assert "Deep content" in result.markdown + assert "bold text" in result.markdown + assert "" not in result.markdown + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_html_converter.py b/packages/markitdown/tests/test_html_converter.py deleted file mode 100644 index 3c5d8976a1..0000000000 --- a/packages/markitdown/tests/test_html_converter.py +++ /dev/null @@ -1,153 +0,0 @@ -import io - -import pytest -from bs4 import BeautifulSoup - -from markitdown import MarkItDown -from markitdown.converters._markdownify import _CustomMarkdownify - - -def _convert_html(html: str, **kwargs) -> str: - result = MarkItDown().convert_stream( - io.BytesIO(html.encode("utf-8")), - file_extension=".html", - **kwargs, - ) - return result.markdown - - -@pytest.mark.parametrize( - "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "] -) -def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None: - element = BeautifulSoup("", "html.parser").u - - assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace - - -@pytest.mark.parametrize( - ("content", "expected"), - [ - ("", "FirstLast"), - (" ", "First Last"), - ("\t", "First Last"), - (" ", "First\u00a0Last"), - ("
", "First\nLast"), - ("word", "FirstwordLast"), - (" word ", "First word Last"), - ], -) -def test_html_underlined_content_is_preserved(content: str, expected: str) -> None: - assert _convert_html(f"

First{content}Last

") == expected - - -def test_preserves_non_utf8_percent_encoded_href_path() -> None: - href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" - html = f'example' - - markdown = _convert_html(html) - - assert f"[example]({href})" in markdown - assert "%EF%BF%BD" not in markdown - - -def test_html_href_still_quotes_raw_unicode_and_spaces() -> None: - href = "https://example.com/a path/日本語" - expected_href = "https://example.com/a%20path/" "%E6%97%A5%E6%9C%AC%E8%AA%9E" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_html_href_quotes_literal_percent_sign() -> None: - href = "https://example.com/100% complete" - expected_href = "https://example.com/100%25%20complete" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_html_href_quotes_malformed_percent_escape() -> None: - href = "https://example.com/items/%ZZ/%2F" - expected_href = "https://example.com/items/%25ZZ/%2F" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_html_href_preserves_encoded_slash() -> None: - href = "https://example.com/items/a%2Fb" - - markdown = _convert_html(f'example') - - assert f"[example]({href})" in markdown - - -def test_html_href_does_not_quote_query_or_fragment() -> None: - href = "https://example.com/a path?query=a b%20c#fragment with spaces" - expected_href = "https://example.com/a%20path?query=a b%20c#fragment with spaces" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_img_prefers_data_src_over_placeholder_data_uri() -> None: - placeholder = ( - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" - ) - real_src = "https://example.com/photo.jpg" - html = ( - f'A photo' - ) - - markdown = _convert_html(html) - - assert f"![A photo]({real_src})" in markdown - assert placeholder not in markdown - - -def test_img_uses_real_src_over_data_src_when_both_present() -> None: - real_src = "https://example.com/photo.jpg" - other_src = "https://example.com/photo-alt.jpg" - html = f'A photo' - - markdown = _convert_html(html) - - assert f"![A photo]({real_src})" in markdown - - -def test_img_falls_back_to_data_src_when_src_missing() -> None: - real_src = "https://example.com/photo.jpg" - html = f'A photo' - - markdown = _convert_html(html) - - assert f"![A photo]({real_src})" in markdown - - -def test_img_keeps_truncated_data_uri_when_no_data_src() -> None: - placeholder = ( - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" - ) - html = f'A photo' - - markdown = _convert_html(html) - - assert "![A photo](data:image/gif;base64...)" in markdown - - -def test_img_keeps_embedded_data_uri_over_data_src_when_keeping_data_uris() -> None: - embedded = ( - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" - ) - other_src = "https://example.com/photo.jpg" - html = f'A photo' - - markdown = _convert_html(html, keep_data_uris=True) - - assert f"![A photo]({embedded})" in markdown - assert other_src not in markdown diff --git a/packages/markitdown/tests/test_image_converter.py b/packages/markitdown/tests/test_image.py similarity index 53% rename from packages/markitdown/tests/test_image_converter.py rename to packages/markitdown/tests/test_image.py index ffb10102c1..2018f1b495 100644 --- a/packages/markitdown/tests/test_image_converter.py +++ b/packages/markitdown/tests/test_image.py @@ -1,10 +1,17 @@ +"""Image conversion, captions, and media metadata.""" + import io +import os +import re +import shutil from pathlib import Path +from types import SimpleNamespace from typing import Any, BinaryIO, Optional from unittest.mock import MagicMock import pytest +import markitdown.converters._image_converter as image_module from markitdown import ( DocumentConverterResult, FileConversionException, @@ -12,7 +19,9 @@ StreamInfo, ) from markitdown.converters import ImageConverter -import markitdown.converters._image_converter as image_module + + +# Image captions IMAGE_FILE = Path(__file__).parent / "test_files" / "test.jpg" @@ -135,3 +144,114 @@ def test_image_without_llm_is_unchanged( assert result.markdown == ("Title: Image title\n" if metadata else "") description.assert_not_called() + + +# Conversion regressions + +# Skip exiftool tests if not installed +skip_exiftool = shutil.which("exiftool") is None + +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") + +JPG_TEST_EXIFTOOL = { + "Author": "AutoGen Authors", + "Title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", + "Description": "AutoGen enables diverse LLM-based applications", + "ImageSize": "1615x1967", + "DateTimeOriginal": "2024:03:14 22:10:00", +} + +MP3_TEST_EXIFTOOL = { + "Title": "f67a499e-a7d0-4ca3-a49b-358bd934ae3e", + "Artist": "Artist Name Test String", + "Album": "Album Name Test String", + "SampleRate": "48000", +} + + +@pytest.mark.skipif( + skip_exiftool, + reason="do not run if exiftool is not installed", +) +def test_markitdown_exiftool() -> None: + which_exiftool = shutil.which("exiftool") + assert which_exiftool is not None + + # Test explicitly setting the location of exiftool + markitdown = MarkItDown(exiftool_path=which_exiftool) + result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg")) + for key in JPG_TEST_EXIFTOOL: + target = f"{key}: {JPG_TEST_EXIFTOOL[key]}" + assert target in result.text_content + + # Test setting the exiftool path through an environment variable + os.environ["EXIFTOOL_PATH"] = which_exiftool + markitdown = MarkItDown() + result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg")) + for key in JPG_TEST_EXIFTOOL: + target = f"{key}: {JPG_TEST_EXIFTOOL[key]}" + assert target in result.text_content + + # Test some other media types + result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.mp3")) + for key in MP3_TEST_EXIFTOOL: + target = f"{key}: {MP3_TEST_EXIFTOOL[key]}" + assert target in result.text_content + + +# --------------------------------------------------------------------------- +# Regression test for issue #1960: +# exiftool_path pointing to a nonexistent binary used to leak a raw +# FileNotFoundError. It should now be wrapped in RuntimeError with a +# message that includes the path. +# --------------------------------------------------------------------------- + + +def test_exiftool_metadata_with_nonexistent_binary(): + """#1960: nonexistent exiftool_path raises RuntimeError, not FileNotFoundError.""" + from markitdown.converters._exiftool import exiftool_metadata + + exiftool_path = "/this/does/not/exist/exiftool" + with pytest.raises( + RuntimeError, + match=re.escape(f"Failed to invoke exiftool at {exiftool_path}"), + ): + exiftool_metadata(io.BytesIO(b""), exiftool_path=exiftool_path) + + +def test_exiftool_metadata_with_no_path(): + """Sanity check: exiftool_path=None still returns {} (early return).""" + from markitdown.converters._exiftool import exiftool_metadata + + assert exiftool_metadata(io.BytesIO(b""), exiftool_path=None) == {} + + +def test_exiftool_metadata_invocation_oserror(): + """#1960: an OSError while running exiftool is wrapped, and the stream is restored.""" + from unittest.mock import patch + + from markitdown.converters._exiftool import exiftool_metadata + + def fake_run(args, **kwargs): + # The version check succeeds ... + if "-ver" in args: + return SimpleNamespace(stdout="12.24\n") + # ... but the actual metadata invocation fails. + raise OSError("exiftool vanished") + + exiftool_path = "/usr/bin/exiftool" + file_stream = io.BytesIO(b"0123456789") + file_stream.seek(4) + + with patch("markitdown.converters._exiftool.subprocess.run", side_effect=fake_run): + with pytest.raises( + RuntimeError, + match=re.escape(f"Failed to invoke exiftool at {exiftool_path}"), + ): + exiftool_metadata(file_stream, exiftool_path=exiftool_path) + + assert file_stream.tell() == 4 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 434cd3c5bf..716f5389b6 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -1,96 +1,50 @@ -#!/usr/bin/env python3 -m pytest +"""Library infrastructure, cross-format options, and remaining format regressions.""" + import io import json import ntpath import os -import re -import shutil -import zipfile -import pytest +from pathlib import Path from types import SimpleNamespace -from typing import Optional from unittest.mock import MagicMock -import markitdown._uri_utils as uri_utils -from markitdown._uri_utils import parse_data_uri, file_uri_to_path -from markitdown.converters._wikipedia_converter import WikipediaConverter -from markitdown._markitdown import _get_content_disposition_filename -from markitdown.converters import RssConverter +import pytest +from charset_normalizer import from_bytes +import markitdown._uri_utils as uri_utils from markitdown import ( - MarkItDown, - UnsupportedFormatException, FileConversionException, + MarkItDown, StreamInfo, + UnsupportedFormatException, +) +from markitdown._markitdown import ( + _get_content_disposition_filename, + _read_charset_sample, ) -from markitdown.converters import YouTubeConverter +from markitdown._uri_utils import ( + _is_unc_or_device_path, + file_uri_to_path, + parse_data_uri, +) + -# This file contains module tests that are not directly tested by the FileTestVectors. -# This includes things like helper functions and runtime conversion options -# (e.g., LLM clients, exiftool path, transcription services, etc.) +# Library behavior and remaining formats skip_remote = ( True if os.environ.get("GITHUB_ACTIONS") else False ) # Don't run these tests in CI - # Don't run the llm tests without a key and the client library skip_llm = False if os.environ.get("OPENAI_API_KEY") else True + try: import openai except ModuleNotFoundError: skip_llm = True -# Skip exiftool tests if not installed -skip_exiftool = shutil.which("exiftool") is None - TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") -JPG_TEST_EXIFTOOL = { - "Author": "AutoGen Authors", - "Title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", - "Description": "AutoGen enables diverse LLM-based applications", - "ImageSize": "1615x1967", - "DateTimeOriginal": "2024:03:14 22:10:00", -} - -MP3_TEST_EXIFTOOL = { - "Title": "f67a499e-a7d0-4ca3-a49b-358bd934ae3e", - "Artist": "Artist Name Test String", - "Album": "Album Name Test String", - "SampleRate": "48000", -} - -PDF_TEST_URL = "https://arxiv.org/pdf/2308.08155v2.pdf" -PDF_TEST_STRINGS = [ - "While there is contemporaneous exploration of multi-agent approaches" -] - -YOUTUBE_TEST_URL = "https://www.youtube.com/watch?v=V2qZ_lgxTzg" -YOUTUBE_TEST_STRINGS = [ - "## AutoGen FULL Tutorial with Python (Step-By-Step)", - "This is an intermediate tutorial for installing and using AutoGen locally", - "PT15M4S", - "the model we're going to be using today is GPT 3.5 turbo", # From the transcript -] - -DOCX_COMMENT_TEST_STRINGS = [ - "314b0a30-5b04-470b-b9f7-eed2c2bec74a", - "49e168b7-d2ae-407f-a055-2167576f39a1", - "## d666f1f7-46cb-42bd-9a39-9a39cf2a509f", - "# Abstract", - "# Introduction", - "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", - "This is a test comment. 12df-321a", - "Yet another comment in the doc. 55yiyi-asd09", -] - -BLOG_TEST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math" -BLOG_TEST_STRINGS = [ - "Large language models (LLMs) are powerful tools that can generate natural language texts for various applications, such as chatbots, summarization, translation, and more. GPT-4 is currently the state of the art LLM in the world. Is model selection irrelevant? What about inference parameters?", - "an example where high cost can easily prevent a generic complex", -] - LLM_TEST_STRINGS = [ "5bda1dd6", ] @@ -106,32 +60,6 @@ ] -def test_wikipedia_converter_no_title() -> None: - """WikipediaConverter should not render '# None' when page has no title.""" - converter = WikipediaConverter() - html = b"

Hello

" - stream_info = StreamInfo( - mimetype="text/html", url="https://en.wikipedia.org/wiki/Test" - ) - result = converter.convert(io.BytesIO(html), stream_info) - assert "# None" not in result.markdown - assert "Hello" in result.markdown - assert result.markdown.strip() == "Hello" - - -def test_wikipedia_converter_blank_title() -> None: - """WikipediaConverter should not render an empty heading for a blank title.""" - converter = WikipediaConverter() - html = b"

Hello

" - stream_info = StreamInfo( - mimetype="text/html", url="https://en.wikipedia.org/wiki/Test" - ) - result = converter.convert(io.BytesIO(html), stream_info) - assert not result.markdown.lstrip().startswith("#") - assert result.title is None - assert result.text_content.strip() == "Hello" - - # --- Helper Functions --- def validate_strings(result, expected_strings, exclude_strings=None): """Validate presence or absence of specific strings.""" @@ -215,38 +143,6 @@ def test_stream_info_operations() -> None: assert updated_stream_info.url == "url.1" -@pytest.mark.parametrize( - ("url", "video_id"), - [ - ("https://www.youtube.com/watch?v=V2qZ_lgxTzg", "V2qZ_lgxTzg"), - ("https://youtu.be/V2qZ_lgxTzg", "V2qZ_lgxTzg"), - ("https://www.youtube.com/shorts/V2qZ_lgxTzg", "V2qZ_lgxTzg"), - ("https://www.youtube.com/embed/V2qZ_lgxTzg", "V2qZ_lgxTzg"), - ], -) -def test_youtube_converter_extracts_supported_video_ids( - url: str, video_id: str -) -> None: - converter = YouTubeConverter() - assert converter._get_video_id(url) == video_id - - -@pytest.mark.parametrize( - "url", - [ - "https://www.youtube.com/watch?v=V2qZ_lgxTzg", - "https://youtu.be/V2qZ_lgxTzg", - "https://www.youtube.com/shorts/V2qZ_lgxTzg", - ], -) -def test_youtube_converter_accepts_supported_url_formats(url: str) -> None: - converter = YouTubeConverter() - assert converter.accepts( - io.BytesIO(b""), - StreamInfo(url=url, extension=".html"), - ) - - def test_data_uris() -> None: # Test basic parsing of data URIs data_uri = "data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==" @@ -295,21 +191,6 @@ def test_data_uris() -> None: assert data == b"Hello, World!" -def test_uppercase_data_image_uri_is_truncated_by_default() -> None: - markitdown = MarkItDown() - html = b'dot' - stream_info = StreamInfo(mimetype="text/html", extension=".html") - - result = markitdown.convert_stream(io.BytesIO(html), stream_info=stream_info) - assert result.markdown == "![dot](DATA:image/png;base64...)" - assert "AAAA" not in result.markdown - - result = markitdown.convert_stream( - io.BytesIO(html), stream_info=stream_info, keep_data_uris=True - ) - assert result.markdown == "![dot](DATA:image/png;base64,AAAA)" - - def test_file_uris() -> None: expected_path = os.path.abspath("/path/to/file.txt") @@ -378,460 +259,6 @@ def test_file_uri_with_percent_encoded_windows_drive( assert path == r"C:\Temp\example.md" -def test_docx_comments() -> None: - # Test DOCX processing, with comments and setting style_map on init - markitdown_with_style_map = MarkItDown(style_map="comment-reference => ") - result = markitdown_with_style_map.convert( - os.path.join(TEST_FILES_DIR, "test_with_comment.docx") - ) - validate_strings(result, DOCX_COMMENT_TEST_STRINGS) - - -def _write_underlined_docx( - path, - embedded_style_map: Optional[str] = None, - *, - paragraph_xml: str = ( - "plain " - 'underlined' - ), -) -> str: - """Write a minimal .docx holding one underlined run, and return its path.""" - document_xml = f""" - - - {paragraph_xml} - -""" - - with zipfile.ZipFile(path, "w") as archive: - archive.writestr( - "[Content_Types].xml", - """ - - - - -""", - ) - archive.writestr( - "_rels/.rels", - """ - - -""", - ) - archive.writestr( - "word/_rels/document.xml.rels", - """ -""", - ) - archive.writestr("word/document.xml", document_xml) - - if embedded_style_map is not None: - import mammoth - - with open(path, "r+b") as f: - mammoth.embed_style_map(f, embedded_style_map) - - return str(path) - - -def test_docx_underlined_text_is_preserved(tmp_path) -> None: - docx_file = _write_underlined_docx(tmp_path / "underlined.docx") - - result = MarkItDown().convert(docx_file) - - assert "plain underlined" in result.markdown - - -@pytest.mark.parametrize( - ("run_xml", "expected"), - [ - (' ', "First Last"), - ("", "First Last"), - (" ", "First\u00a0Last"), - ("", "First\nLast"), - ], -) -def test_docx_underlined_whitespace_is_preserved( - tmp_path, run_xml: str, expected: str -) -> None: - docx_file = _write_underlined_docx( - tmp_path / "underlined_whitespace.docx", - paragraph_xml=( - "First" - f'{run_xml}' - "Last" - ), - ) - - assert MarkItDown().convert(docx_file).markdown == expected - - -def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None: - # A style map embedded in the document takes precedence over the default - # "u => u" mapping that preserves underlines. - docx_file = _write_underlined_docx( - tmp_path / "embedded.docx", embedded_style_map="u => em" - ) - - result = MarkItDown().convert(docx_file) - - assert "plain *underlined*" in result.markdown - assert "" not in result.markdown - - -def test_docx_caller_style_map_overrides_embedded_style_map(tmp_path) -> None: - # ... and a caller-supplied style map still outranks the embedded one. - docx_file = _write_underlined_docx( - tmp_path / "embedded.docx", embedded_style_map="u => em" - ) - - result = MarkItDown(style_map="u => strong").convert(docx_file) - - assert "plain **underlined**" in result.markdown - - -def test_html_strikethrough_variants(tmp_path) -> None: - html = """ - -

Plain s element after.

-

Plain del element after.

-

Plain strike element after.

-

Spaces A B C.

-

Runs D E F.

-

Empty GH.

-

Newline IJ -KL.

-

Break MN
OP.

- -""" - path = tmp_path / "strike.html" - path.write_text(html, encoding="utf-8") - markdown = MarkItDown().convert(str(path)).markdown - - assert markdown == "\n\n".join( - [ - # , and the obsolete all mean strikethrough - "Plain ~~s element~~ after.", - "Plain ~~del element~~ after.", - "Plain ~~strike element~~ after.", - # Surrounding whitespace stays outside of the markup ... - "Spaces A ~~B~~ C.", - # ... and runs of it collapse to a single space - "Runs D ~~E~~ F.", - # An empty element contributes nothing - "Empty GH.", - # A line break inside the element is kept, and the markup - # survives it because strikethrough may span a single newline - "Newline I~~J\nK~~L.", - "Break M~~N\nO~~P.", - ] - ) - - -def test_docx_equations() -> None: - markitdown = MarkItDown() - docx_file = os.path.join(TEST_FILES_DIR, "equations.docx") - result = markitdown.convert(docx_file) - - # Check for inline equation m=1 (wrapped with single $) is present - assert "$m=1$" in result.text_content, "Inline equation $m=1$ not found" - - # Find block equations wrapped with double $$ and check if they are present - block_equations = re.findall(r"\$\$(.+?)\$\$", result.text_content) - assert block_equations, "No block equations found in the document." - - -def test_docx_zip_filename_casing_mismatch() -> None: - """Test that DOCX files with inconsistent ZIP filename casing are handled. - - Some document generators produce .docx files where the central directory - records one casing (e.g. 'word/document.xml') but the local file headers - record another (e.g. 'Word/Document.XML'). Python's zipfile module raises - BadZipFile when reading such files. This test verifies that MarkItDown - handles this gracefully. - - See: https://github.com/microsoft/markitdown/issues/1812 - """ - import struct - - markitdown = MarkItDown() - docx_file = os.path.join(TEST_FILES_DIR, "test.docx") - - # Read the original docx and get its expected content - original_result = markitdown.convert(docx_file) - assert original_result.markdown.strip(), "Original DOCX should have content" - - # Read raw bytes and corrupt the local file header filenames - with open(docx_file, "rb") as f: - raw = bytearray(f.read()) - - # Find all local file headers and uppercase their filenames - corrupted = bytearray(raw) - offset = 0 - patched_count = 0 - while offset + 30 <= len(corrupted): - if corrupted[offset : offset + 4] != b"PK\x03\x04": - break - fname_len = struct.unpack_from(" len(corrupted): - break - # Uppercase the filename in the local header - old_name = corrupted[offset + 30 : offset + 30 + fname_len] - new_name = old_name.upper() - if old_name != new_name: - corrupted[offset + 30 : offset + 30 + fname_len] = new_name - patched_count += 1 - comp_size = struct.unpack_from(" 0, "Should have patched at least one local file header" - - # Verify the corrupted file would fail with plain zipfile - with pytest.raises(zipfile.BadZipFile): - with zipfile.ZipFile(io.BytesIO(bytes(corrupted)), "r") as zf: - for name in zf.namelist(): - zf.read(name) - - # Verify MarkItDown can still convert it - corrupted_result = markitdown.convert_stream( - io.BytesIO(bytes(corrupted)), - file_extension=".docx", - ) - assert ( - corrupted_result.markdown.strip() - ), "Corrupted DOCX should still produce content" - # Content should be equivalent to the original - assert ( - original_result.markdown.strip() == corrupted_result.markdown.strip() - ), "Corrupted DOCX should produce the same output as original" - - -def test_docx_zip_filename_non_casing_mismatch_still_rejected() -> None: - """Only case-only local/central filename disagreements are repaired. - - Equal encoded length does not imply two names differ only in case, so the - repair must not be used to wave through archives whose local and central - directory names genuinely disagree -- zipfile rejects those for good - reason, and honouring that keeps the workaround scoped to issue #1812. - """ - import struct - - from markitdown.converter_utils.docx.pre_process import ( - _fix_zip_filename_casing, - ) - - def build_zip(first_name: str) -> bytes: - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as zf: - zf.writestr(first_name, b"payload-a") - zf.writestr("word/document.xml", b"payload-b") - return buf.getvalue() - - def replace_first_local_name(data: bytes, new_name: bytes) -> bytes: - """Rewrite only the first local file header's name, leaving the central - directory -- the authoritative copy -- untouched.""" - raw = bytearray(data) - fname_len = struct.unpack_from(" bool: - try: - with zipfile.ZipFile(io.BytesIO(data), "r") as zf: - for name in zf.namelist(): - zf.read(name) - except zipfile.BadZipFile: - return False - return True - - original = "[Content_Types].xml" - - # A case-only mismatch is repaired ... - case_only = replace_first_local_name(build_zip(original), original.upper().encode()) - assert not reads_cleanly(case_only) - repaired = _fix_zip_filename_casing(io.BytesIO(case_only)) - assert reads_cleanly(repaired.read()) - - # ... but an equal-length mismatch that is not case-only is left alone, - # so zipfile still refuses the archive. - for mutated_name in ( - b"ZBnoudou^Uxqdr\\/ylm", # unrelated, same length - b"[Content_Types].xmy", # a single differing character - b"[content_types]/xml", # differs only outside the cased characters - ): - data = replace_first_local_name(build_zip(original), mutated_name) - assert not reads_cleanly(data), f"{mutated_name!r} should not be readable" - result = _fix_zip_filename_casing(io.BytesIO(data)) - assert not reads_cleanly( - result.read() - ), f"{mutated_name!r} must not be silently repaired" - - -def test_docx_malformed_equations() -> None: - """Malformed equations should not crash the converter (issue #1979).""" - import zipfile - from io import BytesIO - - docx_xml = """ - - - - Normal text - - - - x+1 - - - - After good equation - - - - - - After empty oMathPara - - - - - - After empty inline oMath - - - - y+2 - - - After unnamespaced oMathPara - - z+3 - - After unnamespaced oMath - - -""" - - buf = BytesIO() - with zipfile.ZipFile(buf, "w") as z: - z.writestr("word/document.xml", docx_xml.encode("utf-8")) - z.writestr( - "[Content_Types].xml", - """ - - - - -""", - ) - z.writestr( - "word/_rels/document.xml.rels", - """ - -""", - ) - z.writestr( - "_rels/.rels", - """ - - -""", - ) - - buf.seek(0) - markitdown = MarkItDown() - result = markitdown.convert(buf) - assert "Normal text" in result.markdown - # A malformed equation must not abort the math pre-processing step, which - # would silently drop the LaTeX conversion of every other equation too - assert "$x+1$" in result.markdown or "$$x+1$$" in result.markdown - assert "After empty oMathPara" in result.markdown - assert "After empty inline oMath" in result.markdown - assert "After unnamespaced oMathPara" in result.markdown - assert "After unnamespaced oMath" in result.markdown - - -def test_xlsx_legacy_show_zeroes_sheetview(tmp_path) -> None: - from openpyxl import Workbook - - base_path = tmp_path / "base.xlsx" - xlsx_path = tmp_path / "legacy_show_zeroes.xlsx" - - workbook = Workbook() - sheet = workbook.active - sheet.title = "Data" - sheet["A1"] = "hello" - sheet["B1"] = "world" - # A cell whose text is byte-identical to the malformed attribute. openpyxl stores it - # as an inline string, so it lands in the very worksheet part that gets repaired. - sheet["A2"] = ' showZeroes="0" ' - workbook.save(base_path) - - with zipfile.ZipFile(base_path) as source: - with zipfile.ZipFile(xlsx_path, "w", zipfile.ZIP_DEFLATED) as target: - for item in source.infolist(): - data = source.read(item.filename) - if item.filename == "xl/worksheets/sheet1.xml": - data = data.replace( - b" None: - from markitdown.converters._xlsx_converter import _rename_show_zeroes_attribute - - worksheet_xml = ( - b"" - b'' - b'' - b"" - # openpyxl ignores custom sheet views entirely, so they need no repair - b'' - b'' - b' showZeroes="0" ' - b"" - b'IF(A1=" showZeroes=","x","y")y' - b"" - ) - - repaired = _rename_show_zeroes_attribute(worksheet_xml) - - # Every start tag is repaired, whatever separates its attributes ... - assert repaired.count(b"showZeros=") == 2 - assert b'' in repaired - assert b'' in repaired - - # ... and nothing outside those start tags is rewritten. - assert b' showZeroes="0" ' in repaired - assert b'IF(A1=" showZeroes=","x","y")' in repaired - assert b'' in repaired - - def test_input_as_strings() -> None: markitdown = MarkItDown() @@ -895,263 +322,10 @@ def test_get_content_disposition_filename_decodes_rfc5987() -> None: ) -def test_pptx_chart_multi_series_conversion() -> None: - """Charts with multiple series and many categories must convert correctly. - - Regression test for the slow path in PptxConverter._convert_chart_to_markdown, - where ``series.values[idx]`` was evaluated inside the (category x series) loop. - In python-pptx each ``series.values`` access rescans the cached points via - XPath (O(n) per lookup), so the old code was O(n^2) per series (and rebuilt - the whole tuple for every category), making large charts extremely slow. - - The values are now materialized once per series. This test builds a chart - with enough categories that the regressed code path would be pathologically - slow, and verifies the resulting Markdown table is correct across series. - """ - pptx = pytest.importorskip("pptx") - from pptx.util import Inches - from pptx.chart.data import CategoryChartData - from pptx.enum.chart import XL_CHART_TYPE - - n_categories = 200 - categories = [f"C{i}" for i in range(n_categories)] - series_a = [float(i) for i in range(n_categories)] - series_b = [float(i * 2) for i in range(n_categories)] - - presentation = pptx.Presentation() - slide = presentation.slides.add_slide(presentation.slide_layouts[5]) - chart_data = CategoryChartData() - chart_data.categories = categories - chart_data.add_series("Series A", series_a) - chart_data.add_series("Series B", series_b) - slide.shapes.add_chart( - XL_CHART_TYPE.COLUMN_CLUSTERED, - Inches(1), - Inches(1), - Inches(8), - Inches(5), - chart_data, - ) - - buffer = io.BytesIO() - presentation.save(buffer) - buffer.seek(0) - - result = MarkItDown().convert_stream(buffer, file_extension=".pptx") - md = result.markdown - - # Both series headers are present - assert "Series A" in md - assert "Series B" in md - # First and last categories are present (nothing truncated) - assert "| C0 |" in md - assert f"| C{n_categories - 1} |" in md - # A representative row carries the correct value for each series - assert "| C10 | 10.0 | 20.0 |" in md - - -def test_deeply_nested_html_fallback() -> None: - """Large, deeply nested HTML should fall back to plain-text extraction - instead of silently returning unconverted HTML (issue #1636). - - Note: This test uses sys.setrecursionlimit to guarantee a RecursionError - regardless of the host environment's default limit, making it deterministic - across different platforms and CI configurations. - """ - import sys - import warnings - - markitdown = MarkItDown() - - # Use a small recursion limit so the test is environment-independent. - # We restore the original limit in a finally block to avoid side-effects. - original_limit = sys.getrecursionlimit() - low_limit = 200 # well below markdownify's traversal depth for depth=500 - - # Build HTML with nesting deep enough to trigger RecursionError - depth = 500 - html = "" - for _ in range(depth): - html += '
' - html += "

Deep content with bold text

" - for _ in range(depth): - html += "
" - html += "" - - try: - sys.setrecursionlimit(low_limit) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - result = markitdown.convert_stream( - io.BytesIO(html.encode("utf-8")), - file_extension=".html", - ) - - # Should have emitted a warning about the fallback - recursion_warnings = [x for x in w if "deeply nested" in str(x.message)] - assert len(recursion_warnings) > 0 - - finally: - sys.setrecursionlimit(original_limit) - - # The output should contain the text content, not raw HTML - assert "Deep content" in result.markdown - assert "bold text" in result.markdown - assert "" not in result.markdown - - -def test_deeply_nested_rss_item_fallback() -> None: - """Deeply nested HTML inside an RSS item should fall back to plain-text - extraction instead of silently embedding raw unconverted HTML in the - markdown output (same failure class as the HTML converter fix in #1644). - - Note: This test uses sys.setrecursionlimit to guarantee a RecursionError - regardless of the host environment's default limit, making it deterministic - across different platforms and CI configurations. - """ - import sys - import warnings - - markitdown = MarkItDown() - - # Use a small recursion limit so the test is environment-independent. - # We restore the original limit in a finally block to avoid side-effects. - original_limit = sys.getrecursionlimit() - low_limit = 200 # well below markdownify's traversal depth for depth=500 - - # Build an RSS item whose content is deeply nested HTML - depth = 500 - item_html = "" - for _ in range(depth): - item_html += '
' - item_html += "

Deep feed content with bold text

" - for _ in range(depth): - item_html += "
" - - rss = ( - '' - '' - "" - "Test Feed" - "A test feed" - "" - "Deep Item" - f"" - "" - "" - "" - ) - atom = ( - '' - '' - "Test Feed" - "" - "Deep Entry" - f'' - "" - "" - ) - - try: - sys.setrecursionlimit(low_limit) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - result = markitdown.convert_stream( - io.BytesIO(rss.encode("utf-8")), - file_extension=".rss", - ) - - # Should have emitted a warning about the fallback - recursion_warnings = [x for x in w if "deeply nested" in str(x.message)] - assert len(recursion_warnings) > 0 - - # strict=True should expose the conversion failure rather than applying - # the plain-text fallback. - with pytest.raises(RecursionError): - RssConverter().convert( - io.BytesIO(rss.encode("utf-8")), - StreamInfo(extension=".rss"), - strict=True, - ) - with pytest.raises(RecursionError): - RssConverter().convert( - io.BytesIO(atom.encode("utf-8")), - StreamInfo(extension=".atom"), - strict=True, - ) - finally: - sys.setrecursionlimit(original_limit) - - # The output should contain the text content, not raw HTML - assert "Deep feed content" in result.markdown - assert "bold text" in result.markdown - assert "" not in result.markdown - - -@pytest.mark.skipif( - os.name == "nt", - reason="The DOCX fixture embeds a POSIX file:///tmp/test_rlink.txt target.", -) -def test_doc_rlink() -> None: - # Test for: CVE-2025-11849 - markitdown = MarkItDown() - - # Document with rlink - docx_file = os.path.join(TEST_FILES_DIR, "rlink.docx") - - # Directory containing the target rlink file - rlink_tmp_dir = os.path.abspath(os.sep + "tmp") - - # Ensure the tmp directory exists - if not os.path.exists(rlink_tmp_dir): - pytest.skip(f"Skipping rlink test; {rlink_tmp_dir} directory does not exist.") - return - - rlink_file_path = os.path.join(rlink_tmp_dir, "test_rlink.txt") - rlink_content = "de658225-569e-4e3d-9ed2-cfb6abf927fc" - b64_prefix = ( - "ZGU2NTgyMjUtNTY5ZS00ZTNkLTllZDItY2ZiNmFiZjk" # base64 prefix of rlink_content - ) - - if os.path.exists(rlink_file_path): - with open(rlink_file_path, "r", encoding="utf-8") as f: - existing_content = f.read() - if existing_content != rlink_content: - raise ValueError( - f"Existing {rlink_file_path} content does not match expected content." - ) - else: - with open(rlink_file_path, "w", encoding="utf-8") as f: - f.write(rlink_content) - - try: - result = markitdown.convert(docx_file, keep_data_uris=True).text_content - assert ( - b64_prefix not in result - ) # Make sure the target file was NOT embedded in the output - finally: - os.remove(rlink_file_path) - - -@pytest.mark.skipif( - skip_remote, - reason="do not run tests that query external urls", -) -def test_markitdown_remote() -> None: - markitdown = MarkItDown() - - # By URL - result = markitdown.convert(PDF_TEST_URL) - for test_string in PDF_TEST_STRINGS: - assert test_string in result.text_content - - # Youtube - # result = markitdown.convert(YOUTUBE_TEST_URL) - # for test_string in YOUTUBE_TEST_STRINGS: - # assert test_string in result.text_content +# Youtube +# result = markitdown.convert(YOUTUBE_TEST_URL) +# for test_string in YOUTUBE_TEST_STRINGS: +# assert test_string in result.text_content @pytest.mark.skipif( @@ -1189,61 +363,6 @@ def test_exceptions() -> None: assert type(exc_info.value.attempts[0].converter).__name__ == "PptxConverter" -def test_pptx_converter_treats_none_llm_caption_as_empty(monkeypatch) -> None: - from markitdown.converters import _pptx_converter - - calls = 0 - - def none_caption(*args, **kwargs): - nonlocal calls - calls += 1 - return None - - monkeypatch.setattr(_pptx_converter, "llm_caption", none_caption) - - result = MarkItDown().convert( - os.path.join(TEST_FILES_DIR, "test.pptx"), - llm_client=MagicMock(), - llm_model="test-model", - ) - - assert calls > 0 - assert ( - "![This phrase of the caption is Human-written.](Picture4.jpg)" - in result.markdown - ) - - -@pytest.mark.skipif( - skip_exiftool, - reason="do not run if exiftool is not installed", -) -def test_markitdown_exiftool() -> None: - which_exiftool = shutil.which("exiftool") - assert which_exiftool is not None - - # Test explicitly setting the location of exiftool - markitdown = MarkItDown(exiftool_path=which_exiftool) - result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg")) - for key in JPG_TEST_EXIFTOOL: - target = f"{key}: {JPG_TEST_EXIFTOOL[key]}" - assert target in result.text_content - - # Test setting the exiftool path through an environment variable - os.environ["EXIFTOOL_PATH"] = which_exiftool - markitdown = MarkItDown() - result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg")) - for key in JPG_TEST_EXIFTOOL: - target = f"{key}: {JPG_TEST_EXIFTOOL[key]}" - assert target in result.text_content - - # Test some other media types - result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.mp3")) - for key in MP3_TEST_EXIFTOOL: - target = f"{key}: {MP3_TEST_EXIFTOOL[key]}" - assert target in result.text_content - - def test_markitdown_llm_parameters() -> None: """Test that LLM parameters are correctly passed to the client.""" mock_client = MagicMock() @@ -1314,137 +433,6 @@ def test_markitdown_llm() -> None: validate_strings(result, PPTX_TEST_STRINGS) -def test_pptx_chart_no_title_text_frame() -> None: - from markitdown.converters._pptx_converter import PptxConverter - - # python-pptx's ChartTitle.text_frame is destructive -- it creates a text - # frame if one isn't already present, so it never returns None. - # has_text_frame is the property that actually reflects presence/absence, - # which is what the has_title=True-but-no-text-frame case looks like - # against the real library. - mock_chart = MagicMock() - mock_chart.has_title = True - mock_chart.chart_title.has_text_frame = False - - mock_category = MagicMock() - mock_category.label = "Cat 1" - mock_chart.plots = [MagicMock(categories=[mock_category])] - - mock_series = MagicMock() - mock_series.name = "Series 1" - mock_series.values = [10.0] - mock_chart.series = [mock_series] - - converter = PptxConverter() - result = converter._convert_chart_to_markdown(mock_chart) - - assert "### Chart" in result - assert "Cat 1" in result - assert "Series 1" in result - assert ":" not in result - - -def test_pptx_chart_with_title_text_frame() -> None: - from markitdown.converters._pptx_converter import PptxConverter - - mock_chart = MagicMock() - mock_chart.has_title = True - mock_chart.chart_title.has_text_frame = True - mock_chart.chart_title.text_frame.text = "Revenue" - - mock_category = MagicMock() - mock_category.label = "Cat 1" - mock_chart.plots = [MagicMock(categories=[mock_category])] - - mock_series = MagicMock() - mock_series.name = "Series 1" - mock_series.values = [10.0] - mock_chart.series = [mock_series] - - converter = PptxConverter() - result = converter._convert_chart_to_markdown(mock_chart) - - assert "### Chart: Revenue" in result - - -def test_youtube_converter_missing_title_metadata() -> None: - """Missing titles fall back to HTML when no video content is extracted.""" - from unittest.mock import patch - from markitdown.converters._youtube_converter import YouTubeConverter - - converter = YouTubeConverter() - stream_info = StreamInfo( - mimetype="text/html", - extension=".html", - url="https://www.youtube.com/watch?v=12345", - ) - - with patch( - "markitdown.converters._youtube_converter.IS_YOUTUBE_TRANSCRIPT_CAPABLE", - False, - ): - # Case 1: Stream with no title metadata or title tag - html_content_no_title = b"Video Content" - stream_no_title = io.BytesIO(html_content_no_title) - result_no_title = converter.convert(stream_no_title, stream_info) - assert result_no_title.title is None - assert result_no_title.markdown == "Video Content" - - # Case 2: Stream with an empty tag - html_content_empty_title = ( - b"<html><head><title>Video Content" - ) - stream_empty_title = io.BytesIO(html_content_empty_title) - result_empty_title = converter.convert(stream_empty_title, stream_info) - assert result_empty_title.title is None - assert result_empty_title.markdown == "Video Content" - - # Case 3: Stream whose title is only available from the tag - html_content_title_tag = b"<html><head><title>Fallback TitleVideo Content" - stream_title_tag = io.BytesIO(html_content_title_tag) - result_title_tag = converter.convert(stream_title_tag, stream_info) - assert result_title_tag.title == "Fallback Title" - assert result_title_tag.markdown == "# YouTube\n\n## Fallback Title\n" - - -def test_zip_duplicate_filenames_preserve_each_entry() -> None: - """Same-named ZIP entries must retain their own content and archive order.""" - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as archive: - archive.writestr("notes.txt", "First archived entry.") - with pytest.warns(UserWarning, match="Duplicate name"): - archive.writestr("notes.txt", "Second archived entry.") - buf.seek(0) - - result = MarkItDown().convert_stream( - buf, stream_info=StreamInfo(extension=".zip", filename="duplicate.zip") - ) - - assert result.markdown == ( - "Content from the zip file `duplicate.zip`:\n\n" - "## File: notes.txt\n\nFirst archived entry.\n\n" - "## File: notes.txt\n\nSecond archived entry." - ) - - -def test_zip_stream_no_filename_header() -> None: - """Regression test: ZipConverter must not render the literal string 'None' - in the output header when the stream has no associated URL, local path, or - filename (e.g. when called via convert_stream() without stream_info).""" - markitdown = MarkItDown() - - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as zf: - zf.writestr("hello.txt", "Hello world") - buf.seek(0) - - result = markitdown.convert_stream( - buf, stream_info=StreamInfo(mimetype="application/zip") - ) - assert result.markdown.startswith("Content from the zip file `(unknown)`:\n\n") - assert "Hello world" in result.markdown - - def test_ipynb_heading_title_preserves_leading_hash() -> None: """Heading marker removal must not eat a leading '#' from the title text. @@ -1563,367 +551,320 @@ def test_ipynb_accepts_non_ascii() -> None: assert result is True -def test_epub_metadata_nodevalue(): - from defusedxml.minidom import parseString - from markitdown.converters._epub_converter import EpubConverter - - xml_data = ( - '' - "Structured Title" - "Author 1" - "Author 2" - "" - "" - "" - ) - dom = parseString(xml_data) - converter = EpubConverter() +# File URI validation + +UNC_URI_PATHS = [ + "//server.example/share/x.txt", + "///server.example/share/x.txt", + "////server.example/share/x.txt", + r"/\server.example/share/x.txt", + r"\/server.example/share/x.txt", + r"\\server.example\share\x.txt", + "/%2Fserver.example/share/x.txt", + "/%2fserver.example/share/x.txt", + "/%5Cserver.example/share/x.txt", + "/%5cserver.example/share/x.txt", + "%2F%2Fserver.example/share/x.txt", + "%5C%5Cserver.example%5Cshare%5Cx.txt", + "/%5C%3F%5CUNC%5Cserver.example%5Cshare%5Cx.txt", + "/%5C.%5CUNC%5Cserver.example%5Cshare%5Cx.txt", + "/%5C%3F%5CC:%5CTemp%5Cx.txt", + "/%5C.%5CPhysicalDrive0", +] - title = converter._get_text_from_node(dom, "dc:title") - assert title == "Structured Title" +BLOCKED_URIS = ( + [ + f"file://{authority}{path}" + for authority in ("", "localhost") + for path in UNC_URI_PATHS + # An authority must be followed by a slash to introduce its path. + if path.startswith("/") + ] + + [f"file:{path}" for path in UNC_URI_PATHS if not path.startswith("/")] + + [ + "FILE:////server.example/share/x.txt", + "file:////C:/Temp/x.txt", + ] +) - creators = converter._get_all_texts_from_nodes(dom, "dc:creator") - assert creators == ["Author 1", "Author 2"] - publisher = converter._get_text_from_node(dom, "dc:publisher") - assert publisher is None +@pytest.mark.parametrize("uri", BLOCKED_URIS) +def test_file_uri_rejects_unc_or_device_path(uri: str) -> None: + with pytest.raises(ValueError, match="UNC"): + file_uri_to_path(uri) - missing = converter._get_text_from_node(dom, "dc:date") - assert missing is None +class _NoFileAccessMarkItDown(MarkItDown): + def convert_local(self, *args, **kwargs): + pytest.fail("Rejected file URIs must not reach convert_local()") -_EPUB_CONTAINER = ( - '' - '' - '' -) -_EPUB_OPF = ( - '' - '' - '' - "Example book" - '' - '' -) +@pytest.fixture(scope="module") +def no_file_access_markitdown() -> MarkItDown: + return _NoFileAccessMarkItDown(enable_builtins=False) -_EPUB_CHAPTER = ( - '' - "

Chapter text.

" - 'diagram' - "" -) +@pytest.mark.parametrize("method", ["convert", "convert_uri"]) +@pytest.mark.parametrize("uri", BLOCKED_URIS + ["file://server.example/share/x.txt"]) +def test_convert_rejects_remote_file_uri_before_file_access( + no_file_access_markitdown: MarkItDown, method: str, uri: str +) -> None: + with pytest.raises(ValueError, match="Unsupported file URI"): + getattr(no_file_access_markitdown, method)(uri) -def _build_epub() -> bytes: - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as zf: - zf.writestr("mimetype", "application/epub+zip") - zf.writestr("META-INF/container.xml", _EPUB_CONTAINER) - zf.writestr("OEBPS/content.opf", _EPUB_OPF) - zf.writestr("OEBPS/ch1.xhtml", _EPUB_CHAPTER) - return buf.getvalue() +@pytest.mark.parametrize( + "path, blocked", + [ + (r"\\server\share\x.txt", True), + ("//server/share/x.txt", True), + (r"/\server/share/x.txt", True), + (r"\/server/share/x.txt", True), + (r"\\?\UNC\server\share\x.txt", True), + (r"\\?\C:\Temp\x.txt", True), + (r"\\.\PhysicalDrive0", True), + (r"C:\Temp\x.txt", False), + ("C:/Temp/x.txt", False), + ("/home/user/x.txt", False), + ("relative.txt", False), + ], +) +def test_resolved_path_validation(path: str, blocked: bool) -> None: + assert _is_unc_or_device_path(path) is blocked -def test_epub_honors_keep_data_uris() -> None: - """EPUB chapters must be converted with the options the caller passed.""" - result = MarkItDown().convert_stream( - io.BytesIO(_build_epub()), - stream_info=StreamInfo(extension=".epub"), - keep_data_uris=True, - ) - assert ( - "![diagram](data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==)" in result.markdown +def test_file_uri_rejects_unc_after_conversion(monkeypatch: pytest.MonkeyPatch) -> None: + # Check the final guard independently of the URI prefix check. Replace only + # the helper's converter binding, leaving the stdlib functions unchanged. + monkeypatch.setattr( + uri_utils, "url2pathname", lambda _: "//server.example/share/x.txt" ) + with pytest.raises(ValueError, match="UNC"): + file_uri_to_path("file:///local.txt") -def test_epub_truncates_data_uris_by_default() -> None: - """Without the option, the default truncation must still apply.""" - result = MarkItDown().convert_stream( - io.BytesIO(_build_epub()), stream_info=StreamInfo(extension=".epub") - ) - assert "![diagram](data:image/png;base64...)" in result.markdown - assert "iVBORw0KGgo" not in result.markdown +@pytest.mark.parametrize("filename", ["notes.txt", "notes café.txt", "%5C%5Cx.txt"]) +@pytest.mark.parametrize("authority", ["", "localhost"]) +def test_local_file_uri_conversion( + tmp_path: Path, filename: str, authority: str +) -> None: + local_file = tmp_path / filename + local_file.write_text("Local file contents", encoding="utf-8") + uri = local_file.as_uri().replace("file://", f"file://{authority}", 1) + netloc, path = file_uri_to_path(uri) -def test_epub_metadata_and_text_are_unchanged() -> None: - """The rest of the conversion must not move.""" - result = MarkItDown().convert_stream( - io.BytesIO(_build_epub()), stream_info=StreamInfo(extension=".epub") - ) + assert netloc == (authority or None) + assert path == str(local_file) + markitdown = MarkItDown() + assert markitdown.convert(uri).markdown == "Local file contents" + assert markitdown.convert(str(local_file)).markdown == "Local file contents" - assert result.title == "Example book" - assert "**Title:** Example book" in result.markdown - assert "Chapter text." in result.markdown +@pytest.mark.skipif(os.name != "nt", reason="Requires native Windows path conversion") +@pytest.mark.parametrize("authority", ["", "localhost"]) +@pytest.mark.parametrize( + "uri_path, expected", + [ + ("/C:/Temp/notes.txt", r"C:\Temp\notes.txt"), + ("/C%3A/Temp/notes.txt", r"C:\Temp\notes.txt"), + ("/C%3a/Temp/notes.txt", r"C:\Temp\notes.txt"), + ("/C:/Temp/notes%20caf%C3%A9.txt", "C:\\Temp\\notes café.txt"), + ("/C:/Temp/%255C%255Cx.txt", r"C:\Temp\%5C%5Cx.txt"), + ], +) +def test_windows_local_drive_uri(authority: str, uri_path: str, expected: str) -> None: + netloc, path = file_uri_to_path(f"file://{authority}{uri_path}") -def test_json_with_late_non_ascii_character(tmp_path) -> None: - payload = { - "record": { - "title": "Example record", - "abstract": "This is sample test. " * 500, - }, - "notes": "non-ASCII character: è", - } - json_path = tmp_path / "input.json" - json_path.write_text( - json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" - ) + assert netloc == (authority or None) + assert path == expected - result = MarkItDown().convert(str(json_path)) - assert "non-ASCII character: è" in result.text_content +# Charset sampling +_SAMPLE_SIZE = 65536 -############################################################################### -# CSV converter. -############################################################################### +_SPLIT_CHARACTERS = [ + (character, split) + for character in ("\u00e9", "\u65e5", "\U0001f600") + for split in range(1, len(character.encode("utf-8"))) +] -def _convert_csv(data: bytes, charset: str | None = None) -> str: - stream_info = StreamInfo(extension=".csv", charset=charset) +def _split_utf8_json(character: str, split: int) -> bytes: + prefix = '{"name":"r\u00e9sum\u00e9","notes":"'.encode("utf-8") return ( - MarkItDown() - .convert_stream(io.BytesIO(data), stream_info=stream_info) - .text_content + prefix + + b"a" * (_SAMPLE_SIZE - split - len(prefix)) + + (character + '"}').encode("utf-8") ) -def test_csv_utf8_bom_is_stripped_from_the_header() -> None: - result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n") - - assert "\ufeff" not in result - assert result.startswith("| name | age |") - assert "| Alice | 30 |" in result - - -def test_csv_utf8_bom_is_stripped_when_charset_is_known() -> None: - result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n", charset="utf-8") - - assert "\ufeff" not in result - assert result.startswith("| name | age |") - - -def test_csv_leading_blank_line_does_not_destroy_the_table() -> None: - result = _convert_csv(b"\nname,age\nAlice,30\n") - - assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |" +@pytest.fixture(scope="module") +def markitdown() -> MarkItDown: + return MarkItDown() -def test_csv_trailing_blank_lines_are_skipped() -> None: - result = _convert_csv(b"name,age\nAlice,30\n\n\n") +@pytest.mark.parametrize("character,split", _SPLIT_CHARACTERS) +def test_charset_sample_completes_only_the_split_character( + character: str, split: int +) -> None: + data = _split_utf8_json(character, split) + stream = io.BytesIO(data) + expected_size = _SAMPLE_SIZE + len(character.encode("utf-8")) - split - assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |" + sample = _read_charset_sample(stream) + assert sample == data[:expected_size] + assert stream.tell() == expected_size + assert expected_size <= _SAMPLE_SIZE + 3 + sample.decode("utf-8") -def test_csv_blank_lines_between_rows_are_kept() -> None: - result = _convert_csv(b"name,age\n\nAlice,30\n\nBob,40\n") - assert ( - result == "| name | age |\n| --- | --- |\n| Alice | 30 |\n| | |\n| Bob | 40 |" - ) +@pytest.mark.parametrize("character,split", _SPLIT_CHARACTERS) +def test_split_utf8_json_preserves_content( + markitdown: MarkItDown, character: str, split: int +) -> None: + data = _split_utf8_json(character, split) + stream = io.BytesIO(data) + guesses = markitdown._get_stream_info_guesses(stream, StreamInfo()) + assert guesses[0].charset == "utf-8" + assert stream.tell() == 0 -def test_csv_all_blank_input_returns_empty_markdown() -> None: - result = _convert_csv(b"\n\n\n") + result = markitdown.convert_stream(stream) - assert result == "" + assert result.markdown == data.decode("utf-8") @pytest.mark.parametrize( - "data,expected", + "sample,tail", [ - ( - b"banner\nname,age,city\nAlice,30,Seattle\n", - "| banner | | |\n| --- | --- | --- |\n" - "| name | age | city |\n| Alice | 30 | Seattle |", - ), - ( - b"name,age\nAlice\n\nBob,40,Seattle\nCarol,25\n", - "| name | age | |\n| --- | --- | --- |\n" - "| Alice | | |\n| | | |\n" - "| Bob | 40 | Seattle |\n| Carol | 25 | |", - ), - ( - b"name\nAlice,,\n", - "| name | | |\n| --- | --- | --- |\n| Alice | | |", - ), - ( - b"name,age,city\nAlice,30\nBob\n", - "| name | age | city |\n| --- | --- | --- |\n" - "| Alice | 30 | |\n| Bob | | |", - ), + (b"", b""), + (b"ordinary text", b""), + (b"short incomplete \xc3", b""), + (b"a" * _SAMPLE_SIZE, b"\xc3\xa9"), + (b"a" * (_SAMPLE_SIZE - 2) + b"\xc3\xa9", b"\xf0\x9f\x98\x80"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xc3", b""), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xf0", b"\x9f"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xc3", b"x"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xe0", b"\x80\x80"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xed", b"\xa0\x80"), + (b"\xff" + b"a" * (_SAMPLE_SIZE - 2) + b"\xc3", b"\xa9"), + ], + ids=[ + "empty", + "short", + "short-incomplete", + "ascii", + "complete-utf8", + "incomplete-at-eof", + "incomplete-after-lookahead", + "invalid-continuation", + "overlong", + "surrogate", + "non-utf8-prefix", ], - ids=["preamble", "widest-row-late", "trailing-empty-fields", "widest-header"], ) -def test_csv_table_matches_widest_row(data: bytes, expected: str) -> None: - assert _convert_csv(data) == expected - - -def test_csv_pipe_in_cell_is_escaped() -> None: - result = _convert_csv(b'name,description\nWidget,"cheap | fast"\n') - - # The pipe must be escaped rather than emitted raw, or it splits the row. - assert "| Widget | cheap \\| fast |" in result - - -def test_csv_pipe_preceded_by_backslash_is_still_escaped() -> None: - # `left\|right` must not become `left\\|right`: the doubled backslash is a - # literal backslash, which would leave the pipe acting as a delimiter and - # let a renderer drop `right`. The run of backslashes is doubled instead, - # so the escaping `\|` survives. - result = _convert_csv(b'name,description\nWidget,"left\\|right"\n') - - assert r"| Widget | left\\\|right |" in result - - -def test_csv_pipe_preceded_by_two_backslashes_is_still_escaped() -> None: - # An even-length run is just as dangerous once the naive `\|` is appended, - # so check a longer run as well. - result = _convert_csv(b'name,description\nWidget,"left\\\\|right"\n') - - assert r"| Widget | left\\\\\|right |" in result - - -def test_csv_newline_in_quoted_cell_does_not_split_the_row() -> None: - result = _convert_csv(b'name,notes\nWidget,"line one\nline two"\n') - - # The table must stay on one line per record; the embedded break collapses - # to a space. - assert len(result.splitlines()) == 3 - assert "| Widget | line one line two |" in result - - -def test_csv_carriage_returns_in_quoted_cell_collapse_to_spaces() -> None: - result = _convert_csv(b'name,notes\nWidget,"line one\r\nline two\rline three"\n') +def test_other_charset_samples_are_unchanged(sample: bytes, tail: bytes) -> None: + stream = io.BytesIO(sample + tail) - assert len(result.splitlines()) == 3 - assert "| Widget | line one line two line three |" in result + assert _read_charset_sample(stream) == sample + assert stream.tell() <= _SAMPLE_SIZE + 3 -def test_csv_pipe_in_header_is_escaped() -> None: - result = _convert_csv(b'"a | b",c\n1,2\n') +def test_charset_guesses_restore_nonzero_stream_position( + markitdown: MarkItDown, +) -> None: + stream = io.BytesIO(b"prefix" + _split_utf8_json("\U0001f600", 1)) + stream.seek(len(b"prefix")) - assert "| a \\| b | c |" in result + markitdown._get_stream_info_guesses(stream, StreamInfo()) + assert stream.tell() == len(b"prefix") -def test_csv_plain_values_are_unchanged() -> None: - # Guards against over-escaping ordinary content. - result = _convert_csv(b"name,description\nWidget,cheap and fast\n") - assert "| Widget | cheap and fast |" in result - assert "\\" not in result +def test_explicit_charset_still_takes_precedence(markitdown: MarkItDown) -> None: + data = _split_utf8_json("\u00e9", 1) + result = markitdown.convert_stream( + io.BytesIO(data), + stream_info=StreamInfo(extension=".json", charset="cp1252"), + ) -def test_csv_backslash_without_a_pipe_is_left_alone() -> None: - # Only backslashes that guard a pipe are doubled; a Windows path stays - # readable. - result = _convert_csv(b"name,path\nWidget,C:\\temp\\file.txt\n") + assert result.markdown == data.decode("cp1252") - assert r"| Widget | C:\temp\file.txt |" in result +# Undetectable charsets -@pytest.mark.parametrize( - "suffix,escaped_suffix", - [("", ""), ("|", r"\|"), ("x|", r"x\|")], -) -def test_csv_long_backslash_runs(suffix: str, escaped_suffix: str) -> None: - backslashes = "\\" * 65_536 - value = backslashes + suffix - expected = backslashes * (2 if suffix == "|" else 1) + escaped_suffix +# Tests for bytes no charset decodes. +# +# ``charset_normalizer.from_bytes(data).best()`` returns ``None`` when nothing +# decodes the bytes, and ``str(None)`` is the four-character string ``"None"``, +# so a binary file handed over with a text extension became a document whose +# entire content was the word "None" -- four characters that were never in the +# file. - result = _convert_csv(f"{value}\n{value}\n".encode("utf-8"), charset="utf-8") +# Random bytes: no charset claims them, which is what makes `best()` answer None. +# Fixed rather than generated, so the test does not depend on chance. +UNDECODABLE = bytes((7 * i * i + 251 * i + 193) % 256 for i in range(4096)) - assert result == f"| {expected} |\n| --- |\n| {expected} |" +def test_the_fixture_really_is_undecodable() -> None: + """Guards the guard: if some charset claimed these bytes the rest would pass + for the wrong reason.""" + assert from_bytes(UNDECODABLE).best() is None -# --------------------------------------------------------------------------- -# Regression test for issue #1960: -# exiftool_path pointing to a nonexistent binary used to leak a raw -# FileNotFoundError. It should now be wrapped in RuntimeError with a -# message that includes the path. -# --------------------------------------------------------------------------- - -def test_exiftool_metadata_with_nonexistent_binary(): - """#1960: nonexistent exiftool_path raises RuntimeError, not FileNotFoundError.""" - from markitdown.converters._exiftool import exiftool_metadata - - exiftool_path = "/this/does/not/exist/exiftool" - with pytest.raises( - RuntimeError, - match=re.escape(f"Failed to invoke exiftool at {exiftool_path}"), - ): - exiftool_metadata(io.BytesIO(b""), exiftool_path=exiftool_path) +def test_binary_with_a_text_extension_is_not_the_word_none() -> None: + markitdown = MarkItDown() + for extension in (".txt", ".md"): + result = markitdown.convert_stream( + io.BytesIO(UNDECODABLE), file_extension=extension + ) + assert result.markdown != "None" -def test_exiftool_metadata_with_no_path(): - """Sanity check: exiftool_path=None still returns {} (early return).""" - from markitdown.converters._exiftool import exiftool_metadata +def test_binary_with_a_csv_extension_is_not_a_table_of_none() -> None: + markitdown = MarkItDown() + result = markitdown.convert_stream(io.BytesIO(UNDECODABLE), file_extension=".csv") + assert result.markdown != "| None |\n| --- |" - assert exiftool_metadata(io.BytesIO(b""), exiftool_path=None) == {} +def test_ordinary_text_is_unchanged() -> None: + """The fallback only runs when detection fails; everything else is as it was.""" + markitdown = MarkItDown() -def test_exiftool_metadata_invocation_oserror(): - """#1960: an OSError while running exiftool is wrapped, and the stream is restored.""" - from unittest.mock import patch + text = markitdown.convert_stream( + io.BytesIO(b"hello, world\n"), file_extension=".txt" + ) + assert text.markdown == "hello, world\n" - from markitdown.converters._exiftool import exiftool_metadata + table = markitdown.convert_stream(io.BytesIO(b"a,b\n1,2\n"), file_extension=".csv") + assert table.markdown == "| a | b |\n| --- | --- |\n| 1 | 2 |" - def fake_run(args, **kwargs): - # The version check succeeds ... - if "-ver" in args: - return SimpleNamespace(stdout="12.24\n") - # ... but the actual metadata invocation fails. - raise OSError("exiftool vanished") - exiftool_path = "/usr/bin/exiftool" - file_stream = io.BytesIO(b"0123456789") - file_stream.seek(4) +def test_text_containing_the_word_none_survives() -> None: + """The bug was a whole document reading "None", not the word appearing in + one.""" + markitdown = MarkItDown() + result = markitdown.convert_stream( + io.BytesIO(b"value,note\nNone,missing\n"), file_extension=".csv" + ) + assert result.markdown == "| value | note |\n| --- | --- |\n| None | missing |" - with patch("markitdown.converters._exiftool.subprocess.run", side_effect=fake_run): - with pytest.raises( - RuntimeError, - match=re.escape(f"Failed to invoke exiftool at {exiftool_path}"), - ): - exiftool_metadata(file_stream, exiftool_path=exiftool_path) - assert file_stream.tell() == 4 +def test_a_declared_charset_still_wins() -> None: + """`stream_info.charset` short-circuits detection and is untouched.""" + markitdown = MarkItDown() + body = "名称,代码\n浦发银行,600000\n".encode("gb18030") + result = markitdown.convert_stream( + io.BytesIO(body), + stream_info=StreamInfo(extension=".csv", charset="gb18030"), + ) + assert "浦发银行" in result.markdown if __name__ == "__main__": - """Runs this file's tests from the command line.""" - for test in [ - test_stream_info_operations, - test_data_uris, - test_file_uris, - test_docx_comments, - test_docx_zip_filename_casing_mismatch, - test_docx_zip_filename_non_casing_mismatch_still_rejected, - test_zip_duplicate_filenames_preserve_each_entry, - test_input_as_strings, - test_markitdown_remote, - test_speech_transcription, - test_exceptions, - test_doc_rlink, - test_markitdown_exiftool, - test_markitdown_llm_parameters, - test_markitdown_llm, - test_pptx_chart_no_title_text_frame, - test_pptx_chart_with_title_text_frame, - test_ipynb_accepts_non_ascii, - test_epub_metadata_nodevalue, - test_exiftool_metadata_with_nonexistent_binary, - test_exiftool_metadata_with_no_path, - test_exiftool_metadata_invocation_oserror, - ]: - print(f"Running {test.__name__}...", end="") - test() - print("OK") - print("All tests passed!") + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_module_vectors.py b/packages/markitdown/tests/test_module_vectors.py index 3f51550a83..d6d9d680c6 100644 --- a/packages/markitdown/tests/test_module_vectors.py +++ b/packages/markitdown/tests/test_module_vectors.py @@ -3,8 +3,6 @@ import time import pytest import base64 -import re -import zipfile from pathlib import Path @@ -201,37 +199,6 @@ def test_convert_stream_keep_data_uris(test_vector): assert string not in result.markdown -def test_convert_docx_with_style_missing_type(tmp_path): - """DOCX conversion should not fail when a style entry is missing w:type.""" - source_path = os.path.join(TEST_FILES_DIR, "test.docx") - malformed_path = tmp_path / "missing_style_type.docx" - - with zipfile.ZipFile(source_path, mode="r") as zip_input: - with zipfile.ZipFile(malformed_path, mode="w") as zip_output: - for item in zip_input.infolist(): - content = zip_input.read(item.filename) - if item.filename == "word/styles.xml": - styles_xml = content.decode("utf-8") - styles_xml, count = re.subn( - r' None: if __name__ == "__main__": - pytest.main([__file__, "-v"]) + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_pdf_tables.py b/packages/markitdown/tests/test_pdf.py similarity index 66% rename from packages/markitdown/tests/test_pdf_tables.py rename to packages/markitdown/tests/test_pdf.py index d26de0cb98..8ae2ec464a 100644 --- a/packages/markitdown/tests/test_pdf_tables.py +++ b/packages/markitdown/tests/test_pdf.py @@ -1,11 +1,20 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for PDF table extraction functionality.""" +"""PDF conversion, table extraction, numbering, and page cleanup.""" +import io import os import re +from unittest.mock import MagicMock, patch + import pytest from markitdown import MarkItDown +from markitdown.converters._pdf_converter import ( + PARTIAL_NUMBERING_PATTERN, + _merge_partial_numbering_lines, +) + + +# Table extraction TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") @@ -51,8 +60,8 @@ def extract_markdown_tables(text_content): for line in lines: line = line.strip() if line.startswith("|") and line.endswith("|"): - # Skip separator rows (contain only dashes and pipes) - if re.match(r"^\|[\s\-|]+\|$", line): + # A separator needs dashes in every cell; retain blank data rows. + if re.match(r"^\|(?:\s*-+\s*\|)+$", line): continue # Parse cells from the row cells = [cell.strip() for cell in line.split("|")[1:-1]] @@ -271,7 +280,7 @@ def test_borderless_table_extraction(self, markitdown): ), "Second table should come before Recommendations" def test_borderless_table_no_duplication(self, markitdown): - """Test that borderless table content is not duplicated excessively.""" + """Each SKU reference appears once, without missing or duplicated content.""" pdf_path = os.path.join( TEST_FILES_DIR, "SPARSE-2024-INV-1234_borderless_table.pdf" ) @@ -282,13 +291,9 @@ def test_borderless_table_no_duplication(self, markitdown): result = markitdown.convert(pdf_path) text_content = result.text_content - # Count occurrences of unique table data - should not be excessively duplicated - # SKU-8847 appears in both tables, plus possibly once in summary text + # SKU-8847 appears once in each of two tables and once in the analysis. sku_count = text_content.count("SKU-8847") - # Should appear at most 4 times (2 tables + minor text references), not more - assert ( - sku_count <= 4 - ), f"SKU-8847 appears too many times ({sku_count}), suggests duplication issue" + assert sku_count == 3, f"Expected 3 SKU-8847 references, found {sku_count}" def test_borderless_table_correct_position(self, markitdown): """Test that tables appear in correct positions relative to text.""" @@ -626,31 +631,6 @@ def test_academic_pdf_extraction(self, markitdown): "multiagentconversations" not in text_content.lower() ), "Text should have proper spacing between words" - def test_scanned_pdf_handling(self, markitdown): - """Test handling of scanned/image-based PDF (no text layer). - - Expected output: Empty - scanned PDFs without OCR have no text layer. - """ - pdf_path = os.path.join( - TEST_FILES_DIR, "MEDRPT-2024-PAT-3847_medical_report_scan.pdf" - ) - - if not os.path.exists(pdf_path): - pytest.skip(f"Test file not found: {pdf_path}") - - result = markitdown.convert(pdf_path) - - # Scanned PDFs without OCR have no text layer, so extraction should be empty - assert ( - result is not None - ), "Converter should return a result even for scanned PDFs" - assert result.text_content is not None, "text_content should not be None" - - # Verify extraction is empty (no text layer in scanned PDF) - assert ( - result.text_content.strip() == "" - ), f"Scanned PDF should have empty extraction, got: '{result.text_content[:100]}...'" - def test_movie_theater_booking_pdf_extraction(self, markitdown): """Test extraction of movie theater booking PDF with complex tables. @@ -746,35 +726,7 @@ def test_movie_theater_full_output(self, markitdown): with open(expected_path, "r", encoding="utf-8") as f: expected_output = f.read() - # Compare outputs - actual_lines = [line.rstrip() for line in actual_output.split("\n")] - expected_lines = [line.rstrip() for line in expected_output.split("\n")] - - # Check line count - assert abs(len(actual_lines) - len(expected_lines)) <= 2, ( - f"Line count mismatch: actual={len(actual_lines)}, " - f"expected={len(expected_lines)}" - ) - - # Check structural elements - assert actual_output.count("|") > 80, "Should have many pipe separators" - assert actual_output.count("---") > 8, "Should have table separators" - - # Validate critical sections - for section in [ - "BOOKING ORDER", - "STARLIGHT CINEMAS", - "2024-12-5678", - "Holiday Spectacular", - "$12,500.00", - ]: - assert section in actual_output, f"Missing section: {section}" - - # Check table structure - table_rows = [line for line in actual_lines if line.startswith("|")] - assert ( - len(table_rows) > 15 - ), f"Should have >15 table rows, got {len(table_rows)}" + assert actual_output == expected_output def test_sparse_borderless_table_full_output(self, markitdown): """Test complete output for SPARSE borderless table PDF.""" @@ -799,28 +751,7 @@ def test_sparse_borderless_table_full_output(self, markitdown): with open(expected_path, "r", encoding="utf-8") as f: expected_output = f.read() - # Compare outputs - actual_lines = [line.rstrip() for line in actual_output.split("\n")] - expected_lines = [line.rstrip() for line in expected_output.split("\n")] - - # Check line count is close - assert abs(len(actual_lines) - len(expected_lines)) <= 2, ( - f"Line count mismatch: actual={len(actual_lines)}, " - f"expected={len(expected_lines)}" - ) - - # Check structural elements - assert actual_output.count("|") > 50, "Should have many pipe separators" - - # Validate critical sections - for section in [ - "INVENTORY RECONCILIATION REPORT", - "SPARSE-2024-INV-1234", - "SKU-8847", - "SKU-9201", - "Variance Analysis", - ]: - assert section in actual_output, f"Missing section: {section}" + assert actual_output == expected_output def test_repair_multipage_full_output(self, markitdown): """Test complete output for REPAIR multipage invoice PDF.""" @@ -841,28 +772,7 @@ def test_repair_multipage_full_output(self, markitdown): with open(expected_path, "r", encoding="utf-8") as f: expected_output = f.read() - # Compare outputs - actual_lines = [line.rstrip() for line in actual_output.split("\n")] - expected_lines = [line.rstrip() for line in expected_output.split("\n")] - - # Check line count is close - assert abs(len(actual_lines) - len(expected_lines)) <= 2, ( - f"Line count mismatch: actual={len(actual_lines)}, " - f"expected={len(expected_lines)}" - ) - - # Check structural elements - assert actual_output.count("|") > 40, "Should have many pipe separators" - - # Validate critical sections - for section in [ - "ZAVA AUTO REPAIR", - "Gabriel Diaz", - "Jeep", - "Grand Cherokee", - "GRAND TOTAL", - ]: - assert section in actual_output, f"Missing section: {section}" + assert actual_output == expected_output def test_receipt_full_output(self, markitdown): """Test complete output for RECEIPT retail purchase PDF.""" @@ -887,25 +797,7 @@ def test_receipt_full_output(self, markitdown): with open(expected_path, "r", encoding="utf-8") as f: expected_output = f.read() - # Compare outputs - actual_lines = [line.rstrip() for line in actual_output.split("\n")] - expected_lines = [line.rstrip() for line in expected_output.split("\n")] - - # Check line count is close - assert abs(len(actual_lines) - len(expected_lines)) <= 2, ( - f"Line count mismatch: actual={len(actual_lines)}, " - f"expected={len(expected_lines)}" - ) - - # Validate critical sections - for section in [ - "TECHMART ELECTRONICS", - "TXN-98765-2024", - "Sarah Mitchell", - "$821.14", - "RETURN POLICY", - ]: - assert section in actual_output, f"Missing section: {section}" + assert actual_output == expected_output def test_academic_paper_full_output(self, markitdown): """Test complete output for academic paper PDF.""" @@ -924,32 +816,10 @@ def test_academic_paper_full_output(self, markitdown): with open(expected_path, "r", encoding="utf-8") as f: expected_output = f.read() - # Compare outputs - actual_lines = [line.rstrip() for line in actual_output.split("\n")] - expected_lines = [line.rstrip() for line in expected_output.split("\n")] - - # Check line count is close - assert abs(len(actual_lines) - len(expected_lines)) <= 2, ( - f"Line count mismatch: actual={len(actual_lines)}, " - f"expected={len(expected_lines)}" - ) - - # Academic paper should not have pipe separators - assert ( - actual_output.count("|") == 0 - ), "Academic paper should not have pipe separators" - - # Validate critical sections - for section in [ - "Introduction", - "Large language models", - "agents", - "multi-agent", - ]: - assert section in actual_output, f"Missing section: {section}" + assert actual_output == expected_output def test_medical_scan_full_output(self, markitdown): - """Test complete output for medical report scan PDF (empty, no text layer).""" + """A scanned PDF without OCR produces empty output and no Markdown tables.""" pdf_path = os.path.join( TEST_FILES_DIR, "MEDRPT-2024-PAT-3847_medical_report_scan.pdf" ) @@ -976,6 +846,9 @@ def test_medical_scan_full_output(self, markitdown): assert ( expected_output.strip() == "" ), "Expected output should be empty for scanned PDF" + assert ( + extract_markdown_tables(actual_output) == [] + ), "Scanned PDF should have no extracted tables" class TestPdfTableMarkdownFormat: @@ -1112,64 +985,38 @@ def test_receipt_has_no_tables(self, markitdown): total_table_rows < 5 ), f"Receipt should not have significant tables, found {total_table_rows} rows" - def test_scanned_pdf_no_tables(self, markitdown): - """Test that scanned PDF has empty extraction and no tables.""" - pdf_path = os.path.join( - TEST_FILES_DIR, "MEDRPT-2024-PAT-3847_medical_report_scan.pdf" - ) - - if not os.path.exists(pdf_path): - pytest.skip(f"Test file not found: {pdf_path}") - - result = markitdown.convert(pdf_path) - - # Scanned PDF with no text layer should have empty extraction - assert ( - result.text_content.strip() == "" - ), "Scanned PDF should have empty extraction" - - tables = extract_markdown_tables(result.text_content) - - # Scanned PDF with no text layer should have no tables - assert len(tables) == 0, "Scanned PDF should have no extracted tables" - - def test_all_pdfs_table_rows_consistent(self, markitdown): + @pytest.mark.parametrize( + "pdf_file, expected_table_count", + [ + ("SPARSE-2024-INV-1234_borderless_table.pdf", 2), + ("REPAIR-2022-INV-001_multipage.pdf", 6), + ("RECEIPT-2024-TXN-98765_retail_purchase.pdf", 0), + ("test.pdf", 0), + ], + ) + def test_all_pdfs_table_rows_consistent( + self, markitdown, pdf_file, expected_table_count + ): """Test that all PDF tables have rows with pipe-separated content. Note: With gap-based column detection, rows may have different column counts depending on how content is spaced in the PDF. What's important is that each row has pipe separators and the content is readable. """ - pdf_files = [ - "SPARSE-2024-INV-1234_borderless_table.pdf", - "REPAIR-2022-INV-001_multipage.pdf", - "RECEIPT-2024-TXN-98765_retail_purchase.pdf", - "test.pdf", - ] - - for pdf_file in pdf_files: - pdf_path = os.path.join(TEST_FILES_DIR, pdf_file) - if not os.path.exists(pdf_path): - continue - - result = markitdown.convert(pdf_path) - tables = extract_markdown_tables(result.text_content) - - for table_idx, table in enumerate(tables): - if not table: - continue + pdf_path = os.path.join(TEST_FILES_DIR, pdf_file) + result = markitdown.convert(pdf_path) + assert result.text_content.strip(), f"{pdf_file}: No text extracted" - # Verify each row has at least one column (pipe-separated content) - for row_idx, row in enumerate(table): - assert ( - len(row) >= 1 - ), f"{pdf_file}: Table {table_idx}, row {row_idx} has no columns" + tables = extract_markdown_tables(result.text_content) + assert ( + len(tables) == expected_table_count + ), f"{pdf_file}: Expected {expected_table_count} tables, found {len(tables)}" - # Verify the row has non-empty content - row_content = " ".join(cell.strip() for cell in row) - assert ( - len(row_content.strip()) > 0 - ), f"{pdf_file}: Table {table_idx}, row {row_idx} is empty" + for table_idx, table in enumerate(tables): + for row_idx, row in enumerate(table): + assert any( + cell.strip() for cell in row + ), f"{pdf_file}: Table {table_idx}, row {row_idx} is empty" def test_borderless_table_data_integrity(self, markitdown): """Test that borderless table extraction preserves data integrity.""" @@ -1196,3 +1043,464 @@ def test_borderless_table_data_integrity(self, markitdown): table_text = str(second_table) assert "Electronics" in table_text, "Second table should contain Electronics" assert "Hardware" in table_text, "Second table should contain Hardware" + + +# MasterFormat numbering + + +class TestMasterFormatPartialNumbering: + """Test handling of MasterFormat-style partial numbering (.1, .2, etc.).""" + + def test_partial_numbering_pattern_regex(self): + """Test that the partial numbering regex pattern correctly matches.""" + + # Should match partial numbering patterns + assert PARTIAL_NUMBERING_PATTERN.match(".1") is not None + assert PARTIAL_NUMBERING_PATTERN.match(".2") is not None + assert PARTIAL_NUMBERING_PATTERN.match(".10") is not None + assert PARTIAL_NUMBERING_PATTERN.match(".99") is not None + + # Should NOT match other patterns + assert PARTIAL_NUMBERING_PATTERN.match("1.") is None + assert PARTIAL_NUMBERING_PATTERN.match("1.2") is None + assert PARTIAL_NUMBERING_PATTERN.match(".1.2") is None + assert PARTIAL_NUMBERING_PATTERN.match("text") is None + assert PARTIAL_NUMBERING_PATTERN.match(".a") is None + assert PARTIAL_NUMBERING_PATTERN.match("") is None + + def test_masterformat_partial_numbering_not_split(self): + """Test that MasterFormat partial numbering stays with associated text. + + MasterFormat documents use partial numbering like: + .1 The intent of this Request for Proposal... + .2 Available information relative to... + + These should NOT be split into separate table columns, but kept + as coherent text lines with the number followed by its description. + """ + pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") + + markitdown = MarkItDown() + result = markitdown.convert(pdf_path) + text_content = result.text_content + + # Partial numberings should NOT appear isolated on their own lines + # If they're isolated, it means the parser incorrectly split them from their text + lines = text_content.split("\n") + isolated_numberings = [] + for line in lines: + stripped = line.strip() + # Check if line contains ONLY a partial numbering (with possible whitespace/pipes) + cleaned = stripped.replace("|", "").strip() + if cleaned in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".10"]: + isolated_numberings.append(stripped) + + assert len(isolated_numberings) == 0, ( + f"Partial numberings should not be isolated from their text. " + f"Found isolated: {isolated_numberings}" + ) + + # Verify that partial numberings appear WITH following text on the same line + # Look for patterns like ".1 The intent" or ".1 Some text" + partial_with_text = re.findall(r"\.\d+\s+\w+", text_content) + assert ( + len(partial_with_text) > 0 + ), "Expected to find partial numberings followed by text on the same line" + + def test_masterformat_content_preserved(self): + """Test that MasterFormat document content is fully preserved.""" + pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") + + markitdown = MarkItDown() + result = markitdown.convert(pdf_path) + text_content = result.text_content + + # Verify key content from the MasterFormat document is preserved + expected_content = [ + "RFP for Construction Management Services", + "Section 00 00 43", + "Instructions to Respondents", + "Ken Sargent House", + "INTENT", + "Request for Proposal", + "KEN SARGENT HOUSE", + "GRANDE PRAIRIE, ALBERTA", + "Section 00 00 45", + ] + + for content in expected_content: + assert ( + content in text_content + ), f"Expected content '{content}' not found in extracted text" + + # Verify partial numbering is followed by text on the same line + # .1 should be followed by "The intent" on the same line + assert re.search( + r"\.1\s+The intent", text_content + ), "Partial numbering .1 should be followed by 'The intent' text" + + # .2 should be followed by "Available information" on the same line + assert re.search( + r"\.2\s+Available information", text_content + ), "Partial numbering .2 should be followed by 'Available information' text" + + # Ensure text content is not empty and has reasonable length + assert ( + len(text_content.strip()) > 100 + ), "MasterFormat document should have substantial text content" + + @pytest.mark.parametrize( + "blank_lines", + ["", "\n\n", " \n\t"], + ids=["one-blank-line", "multiple-blank-lines", "whitespace-only-lines"], + ) + def test_merge_partial_numbering_with_empty_lines_between(self, blank_lines): + """Merge across blank lines while preserving surrounding text and paragraphs.""" + text = ( + "Heading\n\n" + f".1\n{blank_lines}\nThe intent of this Request...\n" + "Continuation.\n\n" + f".10\n{blank_lines}\nAvailable information." + ) + expected = ( + "Heading\n\n" + ".1 The intent of this Request...\n" + "Continuation.\n\n" + ".10 Available information." + ) + + assert _merge_partial_numbering_lines(text) == expected + + def test_multiple_partial_numberings_all_merged(self): + """Test that all partial numberings in a document are properly merged.""" + pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") + + markitdown = MarkItDown() + result = markitdown.convert(pdf_path) + text_content = result.text_content + + # Count occurrences of merged partial numberings (number followed by text) + merged_count = len(re.findall(r"\.\d+\s+[A-Za-z]", text_content)) + + # Count isolated partial numberings (number alone on a line) + isolated_count = 0 + for line in text_content.split("\n"): + stripped = line.strip() + if re.match(r"^\.\d+$", stripped): + isolated_count += 1 + + assert ( + merged_count >= 2 + ), f"Expected at least 2 merged partial numberings, found {merged_count}" + assert ( + isolated_count == 0 + ), f"Found {isolated_count} isolated partial numberings that weren't merged" + + +# Page cleanup and extraction fallback + +# Tests for PDF converter memory optimization. +# +# Verifies that: +# - page.close() is called after processing each page (frees cached data) +# - Plain-text PDFs fall back to pdfminer when no form pages are found +# - Mixed PDFs use form extraction only on form-style pages + + +def _make_form_page(): + """Create a mock page with 3-column table-like word positions.""" + page = MagicMock() + page.width = 612 + page.close = MagicMock() + page.extract_words.return_value = [ + {"text": "Name", "x0": 50, "x1": 100, "top": 10, "bottom": 20}, + {"text": "Value", "x0": 250, "x1": 300, "top": 10, "bottom": 20}, + {"text": "Unit", "x0": 450, "x1": 500, "top": 10, "bottom": 20}, + {"text": "Alpha", "x0": 50, "x1": 100, "top": 30, "bottom": 40}, + {"text": "100", "x0": 250, "x1": 280, "top": 30, "bottom": 40}, + {"text": "kg", "x0": 450, "x1": 470, "top": 30, "bottom": 40}, + {"text": "Beta", "x0": 50, "x1": 100, "top": 50, "bottom": 60}, + {"text": "200", "x0": 250, "x1": 280, "top": 50, "bottom": 60}, + {"text": "lb", "x0": 450, "x1": 470, "top": 50, "bottom": 60}, + ] + return page + + +def _make_plain_page(): + """Create a mock page with single-line paragraph (no table structure).""" + page = MagicMock() + page.width = 612 + page.close = MagicMock() + page.extract_words.return_value = [ + { + "text": "This is a long paragraph of plain text.", + "x0": 50, + "x1": 550, + "top": 10, + "bottom": 20, + }, + ] + page.extract_text.return_value = "This is a long paragraph of plain text." + return page + + +def _mock_pdfplumber_open(pages): + """Return a mock pdfplumber.open that yields the given pages.""" + + def mock_open(stream): + mock_pdf = MagicMock() + mock_pdf.pages = pages + mock_pdf.__enter__ = MagicMock(return_value=mock_pdf) + mock_pdf.__exit__ = MagicMock(return_value=False) + return mock_pdf + + return mock_open + + +class TestPdfMemoryOptimization: + """Test that PDF conversion cleans up per-page caches to limit memory.""" + + def test_page_close_called_on_every_page(self): + """Verify page.close() is called on every page during conversion. + + This ensures cached word/layout data is freed after each page, + preventing O(n) memory growth with page count. + """ + num_pages = 20 + pages = [_make_form_page() for _ in range(num_pages)] + + with patch( + "markitdown.converters._pdf_converter.pdfplumber" + ) as mock_pdfplumber: + mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) + + md = MarkItDown() + buf = io.BytesIO(b"fake pdf content") + from markitdown import StreamInfo + + md.convert_stream( + buf, + stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), + ) + + # page.close() must be called on ALL pages + for i, page in enumerate(pages): + assert page.close.called, ( + f"page.close() was NOT called on page {i} — " + "this would cause memory to accumulate" + ) + + def test_plain_text_pdf_falls_back_to_pdfminer(self): + """Verify all-plain-text PDFs fall back to pdfminer. + + When no page has form-style content, the converter should discard + pdfplumber results and use pdfminer for the whole document (better + text spacing for prose). + """ + num_pages = 50 + pages = [_make_plain_page() for _ in range(num_pages)] + + with patch( + "markitdown.converters._pdf_converter.pdfplumber" + ) as mock_pdfplumber, patch( + "markitdown.converters._pdf_converter.pdfminer" + ) as mock_pdfminer: + mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) + mock_pdfminer.high_level.extract_text.return_value = "Plain text content" + + md = MarkItDown() + buf = io.BytesIO(b"fake pdf content") + from markitdown import StreamInfo + + result = md.convert_stream( + buf, + stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), + ) + + # pdfminer should be used for the final text extraction + assert mock_pdfminer.high_level.extract_text.called, ( + "pdfminer.high_level.extract_text was not called — " + "plain-text PDFs should fall back to pdfminer" + ) + assert result.text_content is not None + + def test_plain_text_pdf_still_closes_all_pages(self): + """Even for plain-text PDFs, page.close() must be called on every page.""" + num_pages = 30 + pages = [_make_plain_page() for _ in range(num_pages)] + + with patch( + "markitdown.converters._pdf_converter.pdfplumber" + ) as mock_pdfplumber, patch( + "markitdown.converters._pdf_converter.pdfminer" + ) as mock_pdfminer: + mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) + mock_pdfminer.high_level.extract_text.return_value = "text" + + md = MarkItDown() + buf = io.BytesIO(b"fake pdf content") + from markitdown import StreamInfo + + md.convert_stream( + buf, + stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), + ) + + for i, page in enumerate(pages): + assert ( + page.close.called + ), f"page.close() was NOT called on plain-text page {i}" + + def test_mixed_pdf_uses_form_extraction_per_page(self): + """In a mixed PDF, form pages get table extraction while plain pages don't. + + Ensures we don't miss form-style pages and don't waste work + running form extraction on plain-text pages. + """ + # Pages 0,2,4 are form-style; pages 1,3 are plain text + pages = [ + _make_form_page(), # 0 - form + _make_plain_page(), # 1 - plain + _make_form_page(), # 2 - form + _make_plain_page(), # 3 - plain + _make_form_page(), # 4 - form + ] + + with patch( + "markitdown.converters._pdf_converter.pdfplumber" + ) as mock_pdfplumber: + mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) + + md = MarkItDown() + buf = io.BytesIO(b"fake pdf content") + from markitdown import StreamInfo + + result = md.convert_stream( + buf, + stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), + ) + + # All pages should have close() called + for i, page in enumerate(pages): + assert page.close.called, f"page.close() not called on page {i}" + + # Form pages (0,2,4) should have extract_words called + for i in [0, 2, 4]: + assert pages[ + i + ].extract_words.called, f"extract_words not called on form page {i}" + + # Result should contain table content from form pages + assert result.text_content is not None + assert ( + "|" in result.text_content + ), "Expected markdown table pipes in output from form-style pages" + + def test_only_one_pdfplumber_open_call(self): + """Verify pdfplumber.open is called exactly once (single pass).""" + pages = [_make_form_page() for _ in range(10)] + + with patch( + "markitdown.converters._pdf_converter.pdfplumber" + ) as mock_pdfplumber: + mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) + + md = MarkItDown() + buf = io.BytesIO(b"fake pdf content") + from markitdown import StreamInfo + + md.convert_stream( + buf, + stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), + ) + + assert mock_pdfplumber.open.call_count == 1, ( + f"Expected 1 pdfplumber.open call (single pass), " + f"got {mock_pdfplumber.open.call_count}" + ) + + @pytest.mark.skipif( + not os.path.exists( + os.path.join(TEST_FILES_DIR, "REPAIR-2022-INV-001_multipage.pdf") + ), + reason="Multipage invoice PDF not available", + ) + def test_real_pdf_page_cleanup(self): + """Close each real page before the next page or document cleanup begins.""" + import pdfplumber + + events = [] + page_numbers = [] + original_extract_words = pdfplumber.page.Page.extract_words + original_page_close = pdfplumber.page.Page.close + original_pdf_close = pdfplumber.PDF.close + + def tracking_extract_words(page, *args, **kwargs): + events.append(("extract", page.page_number)) + return original_extract_words(page, *args, **kwargs) + + def tracking_page_close(page): + events.append(("close", page.page_number)) + return original_page_close(page) + + def tracking_pdf_close(pdf): + page_numbers.extend(page.page_number for page in pdf.pages) + events.append(("document_close", None)) + return original_pdf_close(pdf) + + with patch.object( + pdfplumber.page.Page, "extract_words", tracking_extract_words + ), patch.object( + pdfplumber.page.Page, "close", tracking_page_close + ), patch.object( + pdfplumber.PDF, "close", tracking_pdf_close + ): + pdf_path = os.path.join(TEST_FILES_DIR, "REPAIR-2022-INV-001_multipage.pdf") + MarkItDown().convert(pdf_path) + + # Assert outside the spies: conversion catches pdfplumber exceptions and + # falls back to pdfminer, which could otherwise hide assertion failures. + assert len(page_numbers) > 1, "The fixture must exercise multiple pages" + document_close = events.index(("document_close", None)) + for index, page_number in enumerate(page_numbers): + extracted = events.index(("extract", page_number)) + closed = events.index(("close", page_number)) + next_extraction = ( + events.index(("extract", page_numbers[index + 1])) + if index + 1 < len(page_numbers) + else document_close + ) + assert extracted < closed < next_extraction, ( + f"Page {page_number} must be closed after extraction and before " + f"the next page or document cleanup: {events}" + ) + + +# Conversion regressions + +skip_remote = ( + True if os.environ.get("GITHUB_ACTIONS") else False +) # Don't run these tests in CI + +PDF_TEST_URL = "https://arxiv.org/pdf/2308.08155v2.pdf" + +PDF_TEST_STRINGS = [ + "While there is contemporaneous exploration of multi-agent approaches" +] + + +@pytest.mark.skipif( + skip_remote, + reason="do not run tests that query external urls", +) +def test_markitdown_remote() -> None: + markitdown = MarkItDown() + + # By URL + result = markitdown.convert(PDF_TEST_URL) + for test_string in PDF_TEST_STRINGS: + assert test_string in result.text_content + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_pdf_masterformat.py b/packages/markitdown/tests/test_pdf_masterformat.py deleted file mode 100644 index 8d3eb07398..0000000000 --- a/packages/markitdown/tests/test_pdf_masterformat.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for MasterFormat-style partial numbering in PDF conversion.""" - -import os -import re -import pytest - -from markitdown import MarkItDown -from markitdown.converters._pdf_converter import PARTIAL_NUMBERING_PATTERN - -TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") - - -class TestMasterFormatPartialNumbering: - """Test handling of MasterFormat-style partial numbering (.1, .2, etc.).""" - - def test_partial_numbering_pattern_regex(self): - """Test that the partial numbering regex pattern correctly matches.""" - - # Should match partial numbering patterns - assert PARTIAL_NUMBERING_PATTERN.match(".1") is not None - assert PARTIAL_NUMBERING_PATTERN.match(".2") is not None - assert PARTIAL_NUMBERING_PATTERN.match(".10") is not None - assert PARTIAL_NUMBERING_PATTERN.match(".99") is not None - - # Should NOT match other patterns - assert PARTIAL_NUMBERING_PATTERN.match("1.") is None - assert PARTIAL_NUMBERING_PATTERN.match("1.2") is None - assert PARTIAL_NUMBERING_PATTERN.match(".1.2") is None - assert PARTIAL_NUMBERING_PATTERN.match("text") is None - assert PARTIAL_NUMBERING_PATTERN.match(".a") is None - assert PARTIAL_NUMBERING_PATTERN.match("") is None - - def test_masterformat_partial_numbering_not_split(self): - """Test that MasterFormat partial numbering stays with associated text. - - MasterFormat documents use partial numbering like: - .1 The intent of this Request for Proposal... - .2 Available information relative to... - - These should NOT be split into separate table columns, but kept - as coherent text lines with the number followed by its description. - """ - pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") - - markitdown = MarkItDown() - result = markitdown.convert(pdf_path) - text_content = result.text_content - - # Partial numberings should NOT appear isolated on their own lines - # If they're isolated, it means the parser incorrectly split them from their text - lines = text_content.split("\n") - isolated_numberings = [] - for line in lines: - stripped = line.strip() - # Check if line contains ONLY a partial numbering (with possible whitespace/pipes) - cleaned = stripped.replace("|", "").strip() - if cleaned in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".10"]: - isolated_numberings.append(stripped) - - assert len(isolated_numberings) == 0, ( - f"Partial numberings should not be isolated from their text. " - f"Found isolated: {isolated_numberings}" - ) - - # Verify that partial numberings appear WITH following text on the same line - # Look for patterns like ".1 The intent" or ".1 Some text" - partial_with_text = re.findall(r"\.\d+\s+\w+", text_content) - assert ( - len(partial_with_text) > 0 - ), "Expected to find partial numberings followed by text on the same line" - - def test_masterformat_content_preserved(self): - """Test that MasterFormat document content is fully preserved.""" - pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") - - markitdown = MarkItDown() - result = markitdown.convert(pdf_path) - text_content = result.text_content - - # Verify key content from the MasterFormat document is preserved - expected_content = [ - "RFP for Construction Management Services", - "Section 00 00 43", - "Instructions to Respondents", - "Ken Sargent House", - "INTENT", - "Request for Proposal", - "KEN SARGENT HOUSE", - "GRANDE PRAIRIE, ALBERTA", - "Section 00 00 45", - ] - - for content in expected_content: - assert ( - content in text_content - ), f"Expected content '{content}' not found in extracted text" - - # Verify partial numbering is followed by text on the same line - # .1 should be followed by "The intent" on the same line - assert re.search( - r"\.1\s+The intent", text_content - ), "Partial numbering .1 should be followed by 'The intent' text" - - # .2 should be followed by "Available information" on the same line - assert re.search( - r"\.2\s+Available information", text_content - ), "Partial numbering .2 should be followed by 'Available information' text" - - # Ensure text content is not empty and has reasonable length - assert ( - len(text_content.strip()) > 100 - ), "MasterFormat document should have substantial text content" - - def test_merge_partial_numbering_with_empty_lines_between(self): - """Test that partial numberings merge correctly even with empty lines between. - - When PDF extractors produce output like: - .1 - - The intent of this Request... - - The merge logic should still combine them properly. - """ - pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") - - markitdown = MarkItDown() - result = markitdown.convert(pdf_path) - text_content = result.text_content - - # The merged result should have .1 and .2 followed by text - # Check that we don't have patterns like ".1\n\nThe intent" (unmerged) - lines = text_content.split("\n") - - for i, line in enumerate(lines): - stripped = line.strip() - # If we find an isolated partial numbering, the merge failed - if stripped in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8"]: - # Check if next non-empty line exists and wasn't merged - for j in range(i + 1, min(i + 3, len(lines))): - if lines[j].strip(): - pytest.fail( - f"Partial numbering '{stripped}' on line {i} was not " - f"merged with following text '{lines[j].strip()[:30]}...'" - ) - break - - def test_multiple_partial_numberings_all_merged(self): - """Test that all partial numberings in a document are properly merged.""" - pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf") - - markitdown = MarkItDown() - result = markitdown.convert(pdf_path) - text_content = result.text_content - - # Count occurrences of merged partial numberings (number followed by text) - merged_count = len(re.findall(r"\.\d+\s+[A-Za-z]", text_content)) - - # Count isolated partial numberings (number alone on a line) - isolated_count = 0 - for line in text_content.split("\n"): - stripped = line.strip() - if re.match(r"^\.\d+$", stripped): - isolated_count += 1 - - assert ( - merged_count >= 2 - ), f"Expected at least 2 merged partial numberings, found {merged_count}" - assert ( - isolated_count == 0 - ), f"Found {isolated_count} isolated partial numberings that weren't merged" diff --git a/packages/markitdown/tests/test_pdf_memory.py b/packages/markitdown/tests/test_pdf_memory.py deleted file mode 100644 index 1731dd63e6..0000000000 --- a/packages/markitdown/tests/test_pdf_memory.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for PDF converter memory optimization. - -Verifies that: -- page.close() is called after processing each page (frees cached data) -- Plain-text PDFs fall back to pdfminer when no form pages are found -- Mixed PDFs use form extraction only on form-style pages -- Memory stays constant regardless of page count -""" - -import gc -import io -import os -import tracemalloc - -import pytest -from unittest.mock import patch, MagicMock - -from markitdown import MarkItDown - -TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") - - -def _has_fpdf2() -> bool: - try: - import fpdf # noqa: F401 - - return True - except ImportError: - return False - - -def _make_form_page(): - """Create a mock page with 3-column table-like word positions.""" - page = MagicMock() - page.width = 612 - page.close = MagicMock() - page.extract_words.return_value = [ - {"text": "Name", "x0": 50, "x1": 100, "top": 10, "bottom": 20}, - {"text": "Value", "x0": 250, "x1": 300, "top": 10, "bottom": 20}, - {"text": "Unit", "x0": 450, "x1": 500, "top": 10, "bottom": 20}, - {"text": "Alpha", "x0": 50, "x1": 100, "top": 30, "bottom": 40}, - {"text": "100", "x0": 250, "x1": 280, "top": 30, "bottom": 40}, - {"text": "kg", "x0": 450, "x1": 470, "top": 30, "bottom": 40}, - {"text": "Beta", "x0": 50, "x1": 100, "top": 50, "bottom": 60}, - {"text": "200", "x0": 250, "x1": 280, "top": 50, "bottom": 60}, - {"text": "lb", "x0": 450, "x1": 470, "top": 50, "bottom": 60}, - ] - return page - - -def _make_plain_page(): - """Create a mock page with single-line paragraph (no table structure).""" - page = MagicMock() - page.width = 612 - page.close = MagicMock() - page.extract_words.return_value = [ - { - "text": "This is a long paragraph of plain text.", - "x0": 50, - "x1": 550, - "top": 10, - "bottom": 20, - }, - ] - page.extract_text.return_value = "This is a long paragraph of plain text." - return page - - -def _mock_pdfplumber_open(pages): - """Return a mock pdfplumber.open that yields the given pages.""" - - def mock_open(stream): - mock_pdf = MagicMock() - mock_pdf.pages = pages - mock_pdf.__enter__ = MagicMock(return_value=mock_pdf) - mock_pdf.__exit__ = MagicMock(return_value=False) - return mock_pdf - - return mock_open - - -class TestPdfMemoryOptimization: - """Test that PDF conversion cleans up per-page caches to limit memory.""" - - def test_page_close_called_on_every_page(self): - """Verify page.close() is called on every page during conversion. - - This ensures cached word/layout data is freed after each page, - preventing O(n) memory growth with page count. - """ - num_pages = 20 - pages = [_make_form_page() for _ in range(num_pages)] - - with patch( - "markitdown.converters._pdf_converter.pdfplumber" - ) as mock_pdfplumber: - mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) - - md = MarkItDown() - buf = io.BytesIO(b"fake pdf content") - from markitdown import StreamInfo - - md.convert_stream( - buf, - stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), - ) - - # page.close() must be called on ALL pages - for i, page in enumerate(pages): - assert page.close.called, ( - f"page.close() was NOT called on page {i} — " - "this would cause memory to accumulate" - ) - - def test_plain_text_pdf_falls_back_to_pdfminer(self): - """Verify all-plain-text PDFs fall back to pdfminer. - - When no page has form-style content, the converter should discard - pdfplumber results and use pdfminer for the whole document (better - text spacing for prose). - """ - num_pages = 50 - pages = [_make_plain_page() for _ in range(num_pages)] - - with patch( - "markitdown.converters._pdf_converter.pdfplumber" - ) as mock_pdfplumber, patch( - "markitdown.converters._pdf_converter.pdfminer" - ) as mock_pdfminer: - mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) - mock_pdfminer.high_level.extract_text.return_value = "Plain text content" - - md = MarkItDown() - buf = io.BytesIO(b"fake pdf content") - from markitdown import StreamInfo - - result = md.convert_stream( - buf, - stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), - ) - - # pdfminer should be used for the final text extraction - assert mock_pdfminer.high_level.extract_text.called, ( - "pdfminer.high_level.extract_text was not called — " - "plain-text PDFs should fall back to pdfminer" - ) - assert result.text_content is not None - - def test_plain_text_pdf_still_closes_all_pages(self): - """Even for plain-text PDFs, page.close() must be called on every page.""" - num_pages = 30 - pages = [_make_plain_page() for _ in range(num_pages)] - - with patch( - "markitdown.converters._pdf_converter.pdfplumber" - ) as mock_pdfplumber, patch( - "markitdown.converters._pdf_converter.pdfminer" - ) as mock_pdfminer: - mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) - mock_pdfminer.high_level.extract_text.return_value = "text" - - md = MarkItDown() - buf = io.BytesIO(b"fake pdf content") - from markitdown import StreamInfo - - md.convert_stream( - buf, - stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), - ) - - for i, page in enumerate(pages): - assert ( - page.close.called - ), f"page.close() was NOT called on plain-text page {i}" - - def test_mixed_pdf_uses_form_extraction_per_page(self): - """In a mixed PDF, form pages get table extraction while plain pages don't. - - Ensures we don't miss form-style pages and don't waste work - running form extraction on plain-text pages. - """ - # Pages 0,2,4 are form-style; pages 1,3 are plain text - pages = [ - _make_form_page(), # 0 - form - _make_plain_page(), # 1 - plain - _make_form_page(), # 2 - form - _make_plain_page(), # 3 - plain - _make_form_page(), # 4 - form - ] - - with patch( - "markitdown.converters._pdf_converter.pdfplumber" - ) as mock_pdfplumber: - mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) - - md = MarkItDown() - buf = io.BytesIO(b"fake pdf content") - from markitdown import StreamInfo - - result = md.convert_stream( - buf, - stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), - ) - - # All pages should have close() called - for i, page in enumerate(pages): - assert page.close.called, f"page.close() not called on page {i}" - - # Form pages (0,2,4) should have extract_words called - for i in [0, 2, 4]: - assert pages[ - i - ].extract_words.called, f"extract_words not called on form page {i}" - - # Result should contain table content from form pages - assert result.text_content is not None - assert ( - "|" in result.text_content - ), "Expected markdown table pipes in output from form-style pages" - - def test_only_one_pdfplumber_open_call(self): - """Verify pdfplumber.open is called exactly once (single pass).""" - pages = [_make_form_page() for _ in range(10)] - - with patch( - "markitdown.converters._pdf_converter.pdfplumber" - ) as mock_pdfplumber: - mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages) - - md = MarkItDown() - buf = io.BytesIO(b"fake pdf content") - from markitdown import StreamInfo - - md.convert_stream( - buf, - stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"), - ) - - assert mock_pdfplumber.open.call_count == 1, ( - f"Expected 1 pdfplumber.open call (single pass), " - f"got {mock_pdfplumber.open.call_count}" - ) - - @pytest.mark.skipif( - not os.path.exists(os.path.join(TEST_FILES_DIR, "test.pdf")), - reason="test.pdf not available", - ) - def test_real_pdf_page_cleanup(self): - """Integration test: verify page.close() is called with a real PDF.""" - import pdfplumber - - close_call_count = 0 - original_close = pdfplumber.page.Page.close - - def tracking_close(self): - nonlocal close_call_count - close_call_count += 1 - original_close(self) - - with patch.object(pdfplumber.page.Page, "close", tracking_close): - md = MarkItDown() - pdf_path = os.path.join(TEST_FILES_DIR, "test.pdf") - md.convert(pdf_path) - - assert ( - close_call_count > 0 - ), "page.close() was never called during PDF conversion" - - -def _generate_table_pdf(num_pages: int) -> bytes: - """Generate a PDF with table-like content on every page.""" - from fpdf import FPDF - - pdf = FPDF() - pdf.set_auto_page_break(auto=False) - for page_num in range(num_pages): - pdf.add_page() - pdf.set_font("Helvetica", size=10) - pdf.set_xy(10, 10) - pdf.cell(60, 8, "Parameter", border=1) - pdf.cell(60, 8, "Value", border=1) - pdf.cell(60, 8, "Unit", border=1) - pdf.ln() - for row in range(20): - y = 18 + row * 8 - if y > 270: - break - pdf.set_xy(10, y) - pdf.cell(60, 8, f"Param_{page_num}_{row}", border=1) - pdf.cell(60, 8, f"{(page_num * 100 + row) * 1.23:.2f}", border=1) - pdf.cell(60, 8, "kg/m2", border=1) - return pdf.output() - - -@pytest.mark.skipif( - not _has_fpdf2(), - reason="fpdf2 not installed", -) -class TestPdfMemoryBenchmark: - """Benchmark: verify memory stays constant with page.close() fix.""" - - def test_memory_does_not_grow_linearly(self): - """Peak memory for 200 pages should be far less than without the fix. - - Without page.close(), 200 pages uses ~225 MiB (linear growth). - With the fix, peak memory should stay under 30 MiB. - """ - from markitdown import StreamInfo - - num_pages = 200 - pdf_bytes = _generate_table_pdf(num_pages) - - gc.collect() - tracemalloc.start() - - md = MarkItDown() - buf = io.BytesIO(pdf_bytes) - md.convert_stream(buf, stream_info=StreamInfo(extension=".pdf")) - - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - - peak_mib = peak / 1024 / 1024 - # Without the fix this would be ~225 MiB. With the fix it should - # be well under 30 MiB. Use a generous threshold to avoid flaky - # failures on different machines. - assert peak_mib < 30, ( - f"Peak memory {peak_mib:.1f} MiB for {num_pages} pages is too high. " - f"Expected < 30 MiB with page.close() fix." - ) - - def test_memory_constant_across_page_counts(self): - """Peak memory should not scale linearly with page count. - - Converts 50-page and 200-page PDFs and asserts the peak memory - ratio is much less than the 4x page count ratio. - """ - from markitdown import StreamInfo - - results = {} - for num_pages in [50, 200]: - pdf_bytes = _generate_table_pdf(num_pages) - - gc.collect() - tracemalloc.start() - - md = MarkItDown() - buf = io.BytesIO(pdf_bytes) - md.convert_stream(buf, stream_info=StreamInfo(extension=".pdf")) - - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - results[num_pages] = peak - - ratio = results[200] / results[50] - # With O(n) memory growth the ratio would be ~4x. - # With the fix it should be close to 1x (well under 2x). - assert ratio < 2.0, ( - f"Memory ratio 200p/50p = {ratio:.2f}x — " - f"expected < 2.0x (constant memory). " - f"50p={results[50] / 1024 / 1024:.1f} MiB, " - f"200p={results[200] / 1024 / 1024:.1f} MiB" - ) diff --git a/packages/markitdown/tests/test_pptx_images.py b/packages/markitdown/tests/test_pptx.py similarity index 55% rename from packages/markitdown/tests/test_pptx_images.py rename to packages/markitdown/tests/test_pptx.py index fa6d579aef..8d2b9eb19c 100644 --- a/packages/markitdown/tests/test_pptx_images.py +++ b/packages/markitdown/tests/test_pptx.py @@ -1,12 +1,17 @@ -"""PPTX image hooks share native slide traversal and HTML rendering.""" +"""PowerPoint conversion, titles, notes, charts, and image hooks.""" import base64 import inspect import io +import os from pathlib import Path from typing import Any, BinaryIO, Callable, Optional -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock +import pptx +import pptx.shapes.autoshape +import pptx.slide +import pytest from bs4 import BeautifulSoup from lxml import etree from pptx import Presentation @@ -14,7 +19,6 @@ from pptx.enum.chart import XL_CHART_TYPE from pptx.enum.shapes import MSO_SHAPE_TYPE from pptx.util import Inches -import pytest from markitdown import ( FileConversionException, @@ -22,16 +26,299 @@ MissingDependencyException, StreamInfo, ) -from markitdown.converters import HtmlConverter, PptxConverter -from markitdown.converters import _pptx_converter +from markitdown.converters import HtmlConverter, PptxConverter, _pptx_converter + + +# Slide titles + +BODY_TEXT = "Some body text on the slide." + + +def _build_pptx_with_title(title: str | None) -> io.BytesIO: + """Build a one-slide "Title and Content" deck, optionally filling the title. + + Passing None leaves the title placeholder as PowerPoint creates it: present + on the slide, showing "Click to add title", and carrying no text. + """ + prs = pptx.Presentation() + slide = prs.slides.add_slide(prs.slide_layouts[1]) + if title is not None: + slide.shapes.title.text = title + slide.placeholders[1].text = BODY_TEXT + buf = io.BytesIO() + prs.save(buf) + buf.seek(0) + return buf + + +def _convert_title_slide(title: str | None) -> str: + return ( + MarkItDown() + .convert_stream( + _build_pptx_with_title(title), stream_info=StreamInfo(extension=".pptx") + ) + .markdown + ) + + +def _heading_lines(markdown: str) -> list[str]: + return [line for line in markdown.splitlines() if line.startswith("#")] + + +def test_untouched_title_placeholder_produces_no_heading() -> None: + markdown = _convert_title_slide(None) + + assert BODY_TEXT in markdown + assert _heading_lines(markdown) == [] + + +def test_empty_title_produces_no_heading() -> None: + markdown = _convert_title_slide("") + + assert BODY_TEXT in markdown + assert _heading_lines(markdown) == [] + + +def test_whitespace_only_title_produces_no_heading() -> None: + markdown = _convert_title_slide(" ") + + assert BODY_TEXT in markdown + assert _heading_lines(markdown) == [] + + +def test_title_with_text_is_still_emitted() -> None: + markdown = _convert_title_slide("Quarterly Results") + + assert "# Quarterly Results" in markdown + assert BODY_TEXT in markdown + + +# Speaker notes + + +def _build_pptx_with_optional_notes(notes: str | None) -> io.BytesIO: + """Build a one-slide deck in memory, optionally attaching a notes slide.""" + prs = pptx.Presentation() + slide = prs.slides.add_slide(prs.slide_layouts[5]) + slide.shapes.title.text = "Slide title" + if notes is not None: + # Touching notes_slide creates the part, which is what PowerPoint does + # for a deck whose notes pane has been opened. + slide.notes_slide.notes_text_frame.text = notes + buf = io.BytesIO() + prs.save(buf) + buf.seek(0) + return buf + + +def _convert_notes_slide(notes: str | None) -> str: + return ( + MarkItDown() + .convert_stream( + _build_pptx_with_optional_notes(notes), + stream_info=StreamInfo(extension=".pptx"), + ) + .markdown + ) + + +def test_empty_notes_slide_produces_no_notes_heading() -> None: + markdown = _convert_notes_slide("") + + assert "Slide title" in markdown + assert "### Notes:" not in markdown + + +def test_whitespace_only_notes_produce_no_notes_heading() -> None: + markdown = _convert_notes_slide(" \n\n ") + + assert "Slide title" in markdown + assert "### Notes:" not in markdown + + +def test_slide_without_a_notes_slide_produces_no_notes_heading() -> None: + markdown = _convert_notes_slide(None) + + assert "Slide title" in markdown + assert "### Notes:" not in markdown + + +def test_notes_with_text_are_still_emitted() -> None: + markdown = _convert_notes_slide("Remember to mention the budget.") + + assert "### Notes:\nRemember to mention the budget." in markdown + + +# Missing text + +# Regression test for #1808: PptxConverter must not crash when ``shape.text`` +# or ``notes_text_frame.text`` returns ``None`` (e.g. an ```` run with no +# ```` child, or certain third-party-generated decks). + +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") + +PPTX_FIXTURE = os.path.join(TEST_FILES_DIR, "test.pptx") + + +def _build_pptx_with_notes() -> io.BytesIO: + """Build a minimal pptx with a notes slide attached, in memory.""" + prs = pptx.Presentation() + blank = prs.slide_layouts[6] + slide = prs.slides.add_slide(blank) + slide.notes_slide.notes_text_frame.text = "some notes" + buf = io.BytesIO() + prs.save(buf) + buf.seek(0) + return buf + + +def test_pptx_converter_handles_none_shape_text(monkeypatch): + """``shape.text`` returning None must not blow up the title / body paths.""" + monkeypatch.setattr( + pptx.shapes.autoshape.Shape, + "text", + property(lambda self: None), + raising=True, + ) + + result = MarkItDown().convert(PPTX_FIXTURE) + assert result.markdown is not None + + +def test_pptx_converter_handles_none_notes_text(monkeypatch): + """``notes_text_frame.text`` returning None must not blow up the notes path.""" + real_notes = pptx.slide.NotesSlide.notes_text_frame + + class _NoneText: + text = None + + def fake_notes(self): + frame = real_notes.fget(self) + if frame is None: + return None + return _NoneText() + + monkeypatch.setattr( + pptx.slide.NotesSlide, + "notes_text_frame", + property(fake_notes), + raising=True, + ) + + buf = _build_pptx_with_notes() + result = MarkItDown().convert_stream(buf, file_extension=".pptx") + assert result.markdown is not None + + +# SVG pictures + +# Tests for PPTX SVG images that lack a rasterized fallback. +# +# PowerPoint stores an SVG picture as an ```` whose ``r:embed`` points to +# a rasterized PNG fallback, plus an ```` extension that points to +# the SVG. When a picture has no raster fallback the ```` has no +# ``r:embed`` at all, so python-pptx's ``shape.image`` raises +# ``ValueError("no embedded image")``. The converter must handle this gracefully +# (resolving the SVG blip directly) instead of failing the whole conversion. +# A tiny synthetic PPTX whose only picture is an SVG without a rasterized +# fallback: the has no r:embed, only an extension +# pointing to an embedded SVG. Its alt text is "Red square SVG". +SVG_NO_FALLBACK_PPTX = os.path.join(TEST_FILES_DIR, "test_svg_no_fallback.pptx") + +_SVG_NS = "http://schemas.microsoft.com/office/drawing/2016/SVG/main" + + +def _first_picture_shape(pptx_path): + presentation = Presentation(pptx_path) + converter = PptxConverter() + for slide in presentation.slides: + for shape in slide.shapes: + if converter._is_picture(shape): + return converter, shape + raise AssertionError(f"No picture shape found in {pptx_path}") + + +def test_pptx_svg_without_raster_fallback() -> None: + md = MarkItDown() + + # Default conversion should not raise and should emit the image alt text. + result = md.convert(SVG_NO_FALLBACK_PPTX) + assert "Red square SVG" in result.markdown + + # keep_data_uris used to crash with ValueError("no embedded image"). It + # should now embed the SVG as a data URI. + result = md.convert(SVG_NO_FALLBACK_PPTX, keep_data_uris=True) + assert "data:image/svg+xml;base64," in result.markdown + + +def test_get_image_info_resolves_svg_blip_without_fallback() -> None: + # Unit-level check: _get_image_info must resolve the raw SVG blob directly + # from the extension when shape.image raises because there is + # no rasterized fallback (no r:embed on the ). + converter, shape = _first_picture_shape(SVG_NO_FALLBACK_PPTX) + + blob, content_type, _filename = converter._get_image_info(shape) + + assert blob is not None and len(blob) > 0 + assert content_type == "image/svg+xml" + assert b"' + "" % _SVG_NS + ) + self._element = etree.fromstring(xml) + self.part = _FakePart() + + @property + def image(self): + raise ValueError("no embedded image") + + +def test_is_picture_true_for_svg_placeholder() -> None: + # _is_picture must not let shape.image's ValueError propagate for SVG + # placeholders lacking a raster fallback; it should still report a picture + # because an embedded SVG blip is present. + converter = PptxConverter() + assert converter._is_picture(_FakeSvgPlaceholderShape()) is True + + +# Image hooks _PNG = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAM" "BAQDJ/pLvAAAAAElFTkSuQmCC" ) + _GIF = base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7") + _INFO = StreamInfo(extension=".pptx") + _FILES = Path(__file__).parent / "test_files" @@ -500,3 +787,143 @@ def test_missing_optional_dependencies_fail_before_hook( _ImageConverter(render).convert(io.BytesIO(b""), _INFO) assert caught.value.__cause__ is error render.assert_not_called() + + +# Conversion regressions + + +def test_pptx_chart_multi_series_conversion() -> None: + """Charts with multiple series and many categories must convert correctly. + + Regression test for the slow path in PptxConverter._convert_chart_to_markdown, + where ``series.values[idx]`` was evaluated inside the (category x series) loop. + In python-pptx each ``series.values`` access rescans the cached points via + XPath (O(n) per lookup), so the old code was O(n^2) per series (and rebuilt + the whole tuple for every category), making large charts extremely slow. + + The values are now materialized once per series. This test builds a chart + with enough categories that the regressed code path would be pathologically + slow, and verifies the resulting Markdown table is correct across series. + """ + pptx = pytest.importorskip("pptx") + from pptx.util import Inches + from pptx.chart.data import CategoryChartData + from pptx.enum.chart import XL_CHART_TYPE + + n_categories = 200 + categories = [f"C{i}" for i in range(n_categories)] + series_a = [float(i) for i in range(n_categories)] + series_b = [float(i * 2) for i in range(n_categories)] + + presentation = pptx.Presentation() + slide = presentation.slides.add_slide(presentation.slide_layouts[5]) + chart_data = CategoryChartData() + chart_data.categories = categories + chart_data.add_series("Series A", series_a) + chart_data.add_series("Series B", series_b) + slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1), + Inches(1), + Inches(8), + Inches(5), + chart_data, + ) + + buffer = io.BytesIO() + presentation.save(buffer) + buffer.seek(0) + + result = MarkItDown().convert_stream(buffer, file_extension=".pptx") + md = result.markdown + + # Both series headers are present + assert "Series A" in md + assert "Series B" in md + # First and last categories are present (nothing truncated) + assert "| C0 |" in md + assert f"| C{n_categories - 1} |" in md + # A representative row carries the correct value for each series + assert "| C10 | 10.0 | 20.0 |" in md + + +def test_pptx_converter_treats_none_llm_caption_as_empty(monkeypatch) -> None: + from markitdown.converters import _pptx_converter + + calls = 0 + + def none_caption(*args, **kwargs): + nonlocal calls + calls += 1 + return None + + monkeypatch.setattr(_pptx_converter, "llm_caption", none_caption) + + result = MarkItDown().convert( + os.path.join(TEST_FILES_DIR, "test.pptx"), + llm_client=MagicMock(), + llm_model="test-model", + ) + + assert calls > 0 + assert ( + "![This phrase of the caption is Human-written.](Picture4.jpg)" + in result.markdown + ) + + +def test_pptx_chart_no_title_text_frame() -> None: + from markitdown.converters._pptx_converter import PptxConverter + + # python-pptx's ChartTitle.text_frame is destructive -- it creates a text + # frame if one isn't already present, so it never returns None. + # has_text_frame is the property that actually reflects presence/absence, + # which is what the has_title=True-but-no-text-frame case looks like + # against the real library. + mock_chart = MagicMock() + mock_chart.has_title = True + mock_chart.chart_title.has_text_frame = False + + mock_category = MagicMock() + mock_category.label = "Cat 1" + mock_chart.plots = [MagicMock(categories=[mock_category])] + + mock_series = MagicMock() + mock_series.name = "Series 1" + mock_series.values = [10.0] + mock_chart.series = [mock_series] + + converter = PptxConverter() + result = converter._convert_chart_to_markdown(mock_chart) + + assert "### Chart" in result + assert "Cat 1" in result + assert "Series 1" in result + assert ":" not in result + + +def test_pptx_chart_with_title_text_frame() -> None: + from markitdown.converters._pptx_converter import PptxConverter + + mock_chart = MagicMock() + mock_chart.has_title = True + mock_chart.chart_title.has_text_frame = True + mock_chart.chart_title.text_frame.text = "Revenue" + + mock_category = MagicMock() + mock_category.label = "Cat 1" + mock_chart.plots = [MagicMock(categories=[mock_category])] + + mock_series = MagicMock() + mock_series.name = "Series 1" + mock_series.values = [10.0] + mock_chart.series = [mock_series] + + converter = PptxConverter() + result = converter._convert_chart_to_markdown(mock_chart) + + assert "### Chart: Revenue" in result + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_pptx_empty_title.py b/packages/markitdown/tests/test_pptx_empty_title.py deleted file mode 100644 index b66aaf2cb3..0000000000 --- a/packages/markitdown/tests/test_pptx_empty_title.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""A slide whose title placeholder carries no text must not get a bare heading.""" - -import io - -import pptx - -from markitdown import MarkItDown, StreamInfo - -BODY_TEXT = "Some body text on the slide." - - -def _build_pptx(title: str | None) -> io.BytesIO: - """Build a one-slide "Title and Content" deck, optionally filling the title. - - Passing None leaves the title placeholder as PowerPoint creates it: present - on the slide, showing "Click to add title", and carrying no text. - """ - prs = pptx.Presentation() - slide = prs.slides.add_slide(prs.slide_layouts[1]) - if title is not None: - slide.shapes.title.text = title - slide.placeholders[1].text = BODY_TEXT - buf = io.BytesIO() - prs.save(buf) - buf.seek(0) - return buf - - -def _convert(title: str | None) -> str: - return ( - MarkItDown() - .convert_stream(_build_pptx(title), stream_info=StreamInfo(extension=".pptx")) - .markdown - ) - - -def _heading_lines(markdown: str) -> list[str]: - return [line for line in markdown.splitlines() if line.startswith("#")] - - -def test_untouched_title_placeholder_produces_no_heading() -> None: - markdown = _convert(None) - - assert BODY_TEXT in markdown - assert _heading_lines(markdown) == [] - - -def test_empty_title_produces_no_heading() -> None: - markdown = _convert("") - - assert BODY_TEXT in markdown - assert _heading_lines(markdown) == [] - - -def test_whitespace_only_title_produces_no_heading() -> None: - markdown = _convert(" ") - - assert BODY_TEXT in markdown - assert _heading_lines(markdown) == [] - - -def test_title_with_text_is_still_emitted() -> None: - markdown = _convert("Quarterly Results") - - assert "# Quarterly Results" in markdown - assert BODY_TEXT in markdown diff --git a/packages/markitdown/tests/test_pptx_none_text.py b/packages/markitdown/tests/test_pptx_none_text.py deleted file mode 100644 index be5d5cbe9a..0000000000 --- a/packages/markitdown/tests/test_pptx_none_text.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Regression test for #1808: PptxConverter must not crash when ``shape.text`` -or ``notes_text_frame.text`` returns ``None`` (e.g. an ```` run with no -```` child, or certain third-party-generated decks).""" - -import io -import os - -import pptx -import pptx.shapes.autoshape -import pptx.slide - -from markitdown import MarkItDown - -TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") -PPTX_FIXTURE = os.path.join(TEST_FILES_DIR, "test.pptx") - - -def _build_pptx_with_notes() -> io.BytesIO: - """Build a minimal pptx with a notes slide attached, in memory.""" - prs = pptx.Presentation() - blank = prs.slide_layouts[6] - slide = prs.slides.add_slide(blank) - slide.notes_slide.notes_text_frame.text = "some notes" - buf = io.BytesIO() - prs.save(buf) - buf.seek(0) - return buf - - -def test_pptx_converter_handles_none_shape_text(monkeypatch): - """``shape.text`` returning None must not blow up the title / body paths.""" - monkeypatch.setattr( - pptx.shapes.autoshape.Shape, - "text", - property(lambda self: None), - raising=True, - ) - - result = MarkItDown().convert(PPTX_FIXTURE) - assert result.markdown is not None - - -def test_pptx_converter_handles_none_notes_text(monkeypatch): - """``notes_text_frame.text`` returning None must not blow up the notes path.""" - real_notes = pptx.slide.NotesSlide.notes_text_frame - - class _NoneText: - text = None - - def fake_notes(self): - frame = real_notes.fget(self) - if frame is None: - return None - return _NoneText() - - monkeypatch.setattr( - pptx.slide.NotesSlide, - "notes_text_frame", - property(fake_notes), - raising=True, - ) - - buf = _build_pptx_with_notes() - result = MarkItDown().convert_stream(buf, file_extension=".pptx") - assert result.markdown is not None diff --git a/packages/markitdown/tests/test_pptx_notes.py b/packages/markitdown/tests/test_pptx_notes.py deleted file mode 100644 index 7289ffa798..0000000000 --- a/packages/markitdown/tests/test_pptx_notes.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""A slide whose notes slide carries no text must not get a "Notes:" heading.""" - -import io - -import pptx - -from markitdown import MarkItDown, StreamInfo - - -def _build_pptx(notes: str | None) -> io.BytesIO: - """Build a one-slide deck in memory, optionally attaching a notes slide.""" - prs = pptx.Presentation() - slide = prs.slides.add_slide(prs.slide_layouts[5]) - slide.shapes.title.text = "Slide title" - if notes is not None: - # Touching notes_slide creates the part, which is what PowerPoint does - # for a deck whose notes pane has been opened. - slide.notes_slide.notes_text_frame.text = notes - buf = io.BytesIO() - prs.save(buf) - buf.seek(0) - return buf - - -def _convert(notes: str | None) -> str: - return ( - MarkItDown() - .convert_stream(_build_pptx(notes), stream_info=StreamInfo(extension=".pptx")) - .markdown - ) - - -def test_empty_notes_slide_produces_no_notes_heading() -> None: - markdown = _convert("") - - assert "Slide title" in markdown - assert "### Notes:" not in markdown - - -def test_whitespace_only_notes_produce_no_notes_heading() -> None: - markdown = _convert(" \n\n ") - - assert "Slide title" in markdown - assert "### Notes:" not in markdown - - -def test_slide_without_a_notes_slide_produces_no_notes_heading() -> None: - markdown = _convert(None) - - assert "Slide title" in markdown - assert "### Notes:" not in markdown - - -def test_notes_with_text_are_still_emitted() -> None: - markdown = _convert("Remember to mention the budget.") - - assert "### Notes:\nRemember to mention the budget." in markdown diff --git a/packages/markitdown/tests/test_pptx_svg.py b/packages/markitdown/tests/test_pptx_svg.py deleted file mode 100644 index 8bb0d16330..0000000000 --- a/packages/markitdown/tests/test_pptx_svg.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for PPTX SVG images that lack a rasterized fallback. - -PowerPoint stores an SVG picture as an ```` whose ``r:embed`` points to -a rasterized PNG fallback, plus an ```` extension that points to -the SVG. When a picture has no raster fallback the ```` has no -``r:embed`` at all, so python-pptx's ``shape.image`` raises -``ValueError("no embedded image")``. The converter must handle this gracefully -(resolving the SVG blip directly) instead of failing the whole conversion. -""" -import os - -from lxml import etree -from pptx import Presentation -from pptx.enum.shapes import MSO_SHAPE_TYPE - -from markitdown import MarkItDown -from markitdown.converters._pptx_converter import PptxConverter - -TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") - -# A tiny synthetic PPTX whose only picture is an SVG without a rasterized -# fallback: the has no r:embed, only an extension -# pointing to an embedded SVG. Its alt text is "Red square SVG". -SVG_NO_FALLBACK_PPTX = os.path.join(TEST_FILES_DIR, "test_svg_no_fallback.pptx") - -_SVG_NS = "http://schemas.microsoft.com/office/drawing/2016/SVG/main" - - -def _first_picture_shape(pptx_path): - presentation = Presentation(pptx_path) - converter = PptxConverter() - for slide in presentation.slides: - for shape in slide.shapes: - if converter._is_picture(shape): - return converter, shape - raise AssertionError(f"No picture shape found in {pptx_path}") - - -def test_pptx_svg_without_raster_fallback() -> None: - md = MarkItDown() - - # Default conversion should not raise and should emit the image alt text. - result = md.convert(SVG_NO_FALLBACK_PPTX) - assert "Red square SVG" in result.markdown - - # keep_data_uris used to crash with ValueError("no embedded image"). It - # should now embed the SVG as a data URI. - result = md.convert(SVG_NO_FALLBACK_PPTX, keep_data_uris=True) - assert "data:image/svg+xml;base64," in result.markdown - - -def test_get_image_info_resolves_svg_blip_without_fallback() -> None: - # Unit-level check: _get_image_info must resolve the raw SVG blob directly - # from the extension when shape.image raises because there is - # no rasterized fallback (no r:embed on the ). - converter, shape = _first_picture_shape(SVG_NO_FALLBACK_PPTX) - - blob, content_type, _filename = converter._get_image_info(shape) - - assert blob is not None and len(blob) > 0 - assert content_type == "image/svg+xml" - assert b"' - "" % _SVG_NS - ) - self._element = etree.fromstring(xml) - self.part = _FakePart() - - @property - def image(self): - raise ValueError("no embedded image") - - -def test_is_picture_true_for_svg_placeholder() -> None: - # _is_picture must not let shape.image's ValueError propagate for SVG - # placeholders lacking a raster fallback; it should still report a picture - # because an embedded SVG blip is present. - converter = PptxConverter() - assert converter._is_picture(_FakeSvgPlaceholderShape()) is True - - -if __name__ == "__main__": - test_pptx_svg_without_raster_fallback() - test_get_image_info_resolves_svg_blip_without_fallback() - test_is_picture_true_for_svg_placeholder() - print("All tests passed!") diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss.py similarity index 64% rename from packages/markitdown/tests/test_rss_converter.py rename to packages/markitdown/tests/test_rss.py index 6aaec65a75..89d2cbb796 100644 --- a/packages/markitdown/tests/test_rss_converter.py +++ b/packages/markitdown/tests/test_rss.py @@ -1,3 +1,5 @@ +"""RSS and Atom conversion, metadata, and link resolution.""" + import io import sys @@ -7,6 +9,9 @@ from markitdown.converters import RssConverter +# Feed bodies and links + + @pytest.mark.parametrize( "root_prefix, child_prefix", [("", ""), ("a:", "a:"), ("a:", "b:"), ("a:", ""), ("", "a:")], @@ -688,3 +693,398 @@ def test_document_base_does_not_leak_between_conversions() -> None: assert first.markdown.endswith("[Read](https://example.com/story)") assert second.markdown.endswith("[Read](story)") + + +# Feed and entry titles + + +def test_rss_pretty_printed_titles_still_make_headings() -> None: + """A title written on its own line must not end the heading before its text.""" + feed = b""" + + + Example feed + + + Example feed description + + + + A story about things + + + Mon, 01 Jan 2024 00:00:00 GMT + + Body text. + + +""" + + result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss")) + + assert result.title == "Example feed" + # The channel description retains its original whitespace. + assert result.markdown.splitlines() == [ + "# Example feed", + "", + " Example feed description", + " ", + "", + "## A story about things", + "Published on: Mon, 01 Jan 2024 00:00:00 GMT", + "Body text.", + ] + + +def test_atom_pretty_printed_titles_still_make_headings() -> None: + feed = b""" + + + Example feed + + + Example subtitle + + + + Example entry + + + 2024-01-01T00:00:00Z + + Body text. + + +""" + + result = RssConverter().convert( + io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml") + ) + + assert result.title == "Example feed" + assert result.markdown.splitlines() == [ + "# Example feed", + "Example subtitle", + "", + "## Example entry", + "Updated on: 2024-01-01T00:00:00Z", + "Body text.", + ] + + +def test_rss_whitespace_only_title_makes_no_heading() -> None: + """An empty title must be absent, not an empty '#' line.""" + feed = b""" + + + Example feed description + + Example item + Body text. + + +""" + + result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss")) + + assert result.title is None + assert not result.markdown.startswith("#\n") + assert result.markdown.splitlines()[0] == "Example feed description" + + +def test_atom_feed_without_any_title_makes_no_heading() -> None: + """A missing title must not render the word 'None' as the document heading.""" + feed = b""" + + urn:example:feed + + urn:example:entry + Body text. + + +""" + + result = RssConverter().convert( + io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml") + ) + + assert "None" not in result.markdown + assert "Body text." in result.markdown + + +def test_rss_single_line_titles_are_unchanged() -> None: + """The ordinary layout must produce exactly the same markdown as before.""" + feed = b""" + + Example feed + Example feed description + + Example item + Mon, 01 Jan 2024 00:00:00 GMT + Body text. + + +""" + + result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss")) + + assert result.title == "Example feed" + assert result.markdown == ( + "# Example feed\n" + "Example feed description\n" + "\n" + "## Example item\n" + "Published on: Mon, 01 Jan 2024 00:00:00 GMT\n" + "Body text." + ) + + +@pytest.mark.parametrize("extension", [".rss", ".atom"]) +@pytest.mark.parametrize( + "payload, expected", + [ + ("Quarterly report", "Quarterly R&D report"), + (" report", "Quarterly R&D report"), + ("\n \n", "Example"), + ("nested trailing", "nested trailing"), + ("leading nested trailing", "leading nested trailing"), + ("leading nested", "leading nested"), + ("nested", "nested"), + ( + " \nnested trailing", + "nested trailing", + ), + ("cooperate", "cooperate"), + ("

First

Second

", "First Second"), + ( + "Use <slot> and &lt;literal&gt;", + "Use and <literal>", + ), + ("", None), + (" \n", None), + ("", None), + ("", None), + ], +) +def test_complete_feed_and_entry_titles( + extension: str, payload: str, expected: str | None +) -> None: + if extension == ".rss": + xml = ( + f"{payload}" + f"{payload}Body." + "" + ) + else: + xml = ( + f'{payload}' + f"{payload}Body." + ) + result = RssConverter().convert( + io.BytesIO(xml.encode()), StreamInfo(extension=extension) + ) + + assert result.title == expected + assert result.markdown == ( + f"# {expected}\n\n## {expected}\nBody." if expected else "Body." + ) + + +@pytest.mark.parametrize("prefix", ["", "a:"]) +@pytest.mark.parametrize( + "content_type, payload, expected", + [ + ("html", "<b>nested</b> trailing", "nested trailing"), + ("html", "nes]]>]]> trailing", "nested trailing"), + ("html", "nested trailing", "nested trailing"), + ("html", "<p>First</p><p>Second</p>", "First Second"), + ("html", "Use &lt;slot&gt;", "Use "), + ( + "xhtml", + 'cooperate' + " <literal>]]>", + "cooperate <literal>", + ), + ("text/plain", "Use <slot>", "Use "), + ], +) +def test_atom_typed_titles_and_subtitles( + prefix: str, content_type: str, payload: str, expected: str +) -> None: + xml = ( + f'<{prefix}feed xmlns="http://www.w3.org/2005/Atom" ' + 'xmlns:a="http://www.w3.org/2005/Atom">' + f'<{prefix}title type="{content_type}">{payload}' + f'<{prefix}subtitle type="{content_type}">{payload}' + f'<{prefix}entry><{prefix}title type="{content_type}">{payload}' + f"<{prefix}content>Body." + ) + result = RssConverter().convert( + io.BytesIO(xml.encode()), StreamInfo(extension=".atom") + ) + + assert result.title == expected + assert result.markdown == f"# {expected}\n{expected}\n\n## {expected}\nBody." + + +@pytest.mark.parametrize("extension", [".rss", ".atom"]) +def test_feed_does_not_borrow_nested_metadata(extension: str) -> None: + if extension == ".rss": + xml = ( + "Wrong feed title" + "EntryBody." + "Wrong entry title" + "Second body." + ) + else: + xml = ( + '' + "EntryBody." + "Wrong entry titleWrong body." + "Second body." + ) + result = RssConverter().convert( + io.BytesIO(xml.encode()), StreamInfo(extension=extension) + ) + + assert result.title is None + assert result.markdown.count("## Entry") == 1 + assert "Wrong" not in result.markdown + assert not result.markdown.startswith("# ") + assert "Body.\n\nSecond body." in result.markdown + + +@pytest.mark.parametrize("content_type", ["rss", "text", "html", "xhtml"]) +def test_deeply_nested_title_is_read_without_recursion(content_type: str) -> None: + payload = "" * 500 + "Deep title" + "" * 500 + if content_type == "rss": + xml = f"{payload}".encode() + extension = ".rss" + else: + if content_type == "html": + payload = f"" + elif content_type == "xhtml": + payload = f'
{payload}
' + xml = ( + f'' + f'{payload}' + ).encode() + extension = ".atom" + original_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(200) + result = RssConverter().convert( + io.BytesIO(xml), StreamInfo(extension=extension) + ) + finally: + sys.setrecursionlimit(original_limit) + + assert result.title == "Deep title" + + +def test_rss_channel_description_and_date_preserve_complete_text() -> None: + xml = b"""Feed + About and research. + EntryMon, 00:00:00 GMT + """ + result = RssConverter().convert(io.BytesIO(xml), StreamInfo(extension=".rss")) + + assert "About R&D and research." in result.markdown + assert "Published on: Mon, 01 Jan 2024 00:00:00 GMT" in result.markdown + + +# Conversion regressions + + +def test_deeply_nested_rss_item_fallback() -> None: + """Deeply nested HTML inside an RSS item should fall back to plain-text + extraction instead of silently embedding raw unconverted HTML in the + markdown output (same failure class as the HTML converter fix in #1644). + + Note: This test uses sys.setrecursionlimit to guarantee a RecursionError + regardless of the host environment's default limit, making it deterministic + across different platforms and CI configurations. + """ + import sys + import warnings + + markitdown = MarkItDown() + + # Use a small recursion limit so the test is environment-independent. + # We restore the original limit in a finally block to avoid side-effects. + original_limit = sys.getrecursionlimit() + low_limit = 200 # well below markdownify's traversal depth for depth=500 + + # Build an RSS item whose content is deeply nested HTML + depth = 500 + item_html = "" + for _ in range(depth): + item_html += '
' + item_html += "

Deep feed content with bold text

" + for _ in range(depth): + item_html += "
" + + rss = ( + '' + '' + "" + "Test Feed" + "A test feed" + "" + "Deep Item" + f"" + "" + "" + "" + ) + atom = ( + '' + '' + "Test Feed" + "" + "Deep Entry" + f'' + "" + "" + ) + + try: + sys.setrecursionlimit(low_limit) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = markitdown.convert_stream( + io.BytesIO(rss.encode("utf-8")), + file_extension=".rss", + ) + + # Should have emitted a warning about the fallback + recursion_warnings = [x for x in w if "deeply nested" in str(x.message)] + assert len(recursion_warnings) > 0 + + # strict=True should expose the conversion failure rather than applying + # the plain-text fallback. + with pytest.raises(RecursionError): + RssConverter().convert( + io.BytesIO(rss.encode("utf-8")), + StreamInfo(extension=".rss"), + strict=True, + ) + with pytest.raises(RecursionError): + RssConverter().convert( + io.BytesIO(atom.encode("utf-8")), + StreamInfo(extension=".atom"), + strict=True, + ) + finally: + sys.setrecursionlimit(original_limit) + + # The output should contain the text content, not raw HTML + assert "Deep feed content" in result.markdown + assert "bold text" in result.markdown + assert "" not in result.markdown + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_rss_titles.py b/packages/markitdown/tests/test_rss_titles.py deleted file mode 100644 index ce21b3a61f..0000000000 --- a/packages/markitdown/tests/test_rss_titles.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Feed and entry titles must survive a pretty-printed feed.""" - -import io -import sys - -import pytest - -from markitdown import StreamInfo -from markitdown.converters import RssConverter - - -def test_rss_pretty_printed_titles_still_make_headings() -> None: - """A title written on its own line must not end the heading before its text.""" - feed = b""" - - - Example feed - - - Example feed description - - - - A story about things - - - Mon, 01 Jan 2024 00:00:00 GMT - - Body text. - - -""" - - result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss")) - - assert result.title == "Example feed" - # The channel description retains its original whitespace. - assert result.markdown.splitlines() == [ - "# Example feed", - "", - " Example feed description", - " ", - "", - "## A story about things", - "Published on: Mon, 01 Jan 2024 00:00:00 GMT", - "Body text.", - ] - - -def test_atom_pretty_printed_titles_still_make_headings() -> None: - feed = b""" - - - Example feed - - - Example subtitle - - - - Example entry - - - 2024-01-01T00:00:00Z - - Body text. - - -""" - - result = RssConverter().convert( - io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml") - ) - - assert result.title == "Example feed" - assert result.markdown.splitlines() == [ - "# Example feed", - "Example subtitle", - "", - "## Example entry", - "Updated on: 2024-01-01T00:00:00Z", - "Body text.", - ] - - -def test_rss_whitespace_only_title_makes_no_heading() -> None: - """An empty title must be absent, not an empty '#' line.""" - feed = b""" - - - Example feed description - - Example item - Body text. - - -""" - - result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss")) - - assert result.title is None - assert not result.markdown.startswith("#\n") - assert result.markdown.splitlines()[0] == "Example feed description" - - -def test_atom_feed_without_any_title_makes_no_heading() -> None: - """A missing title must not render the word 'None' as the document heading.""" - feed = b""" - - urn:example:feed - - urn:example:entry - Body text. - - -""" - - result = RssConverter().convert( - io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml") - ) - - assert "None" not in result.markdown - assert "Body text." in result.markdown - - -def test_rss_single_line_titles_are_unchanged() -> None: - """The ordinary layout must produce exactly the same markdown as before.""" - feed = b""" - - Example feed - Example feed description - - Example item - Mon, 01 Jan 2024 00:00:00 GMT - Body text. - - -""" - - result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss")) - - assert result.title == "Example feed" - assert result.markdown == ( - "# Example feed\n" - "Example feed description\n" - "\n" - "## Example item\n" - "Published on: Mon, 01 Jan 2024 00:00:00 GMT\n" - "Body text." - ) - - -@pytest.mark.parametrize("extension", [".rss", ".atom"]) -@pytest.mark.parametrize( - "payload, expected", - [ - ("Quarterly report", "Quarterly R&D report"), - (" report", "Quarterly R&D report"), - ("\n \n", "Example"), - ("nested trailing", "nested trailing"), - ("leading nested trailing", "leading nested trailing"), - ("leading nested", "leading nested"), - ("nested", "nested"), - ( - " \nnested trailing", - "nested trailing", - ), - ("cooperate", "cooperate"), - ("

First

Second

", "First Second"), - ( - "Use <slot> and &lt;literal&gt;", - "Use and <literal>", - ), - ("", None), - (" \n", None), - ("", None), - ("", None), - ], -) -def test_complete_feed_and_entry_titles( - extension: str, payload: str, expected: str | None -) -> None: - if extension == ".rss": - xml = ( - f"{payload}" - f"{payload}Body." - "" - ) - else: - xml = ( - f'{payload}' - f"{payload}Body." - ) - result = RssConverter().convert( - io.BytesIO(xml.encode()), StreamInfo(extension=extension) - ) - - assert result.title == expected - assert result.markdown == ( - f"# {expected}\n\n## {expected}\nBody." if expected else "Body." - ) - - -@pytest.mark.parametrize("prefix", ["", "a:"]) -@pytest.mark.parametrize( - "content_type, payload, expected", - [ - ("html", "<b>nested</b> trailing", "nested trailing"), - ("html", "nes]]>]]> trailing", "nested trailing"), - ("html", "nested trailing", "nested trailing"), - ("html", "<p>First</p><p>Second</p>", "First Second"), - ("html", "Use &lt;slot&gt;", "Use "), - ( - "xhtml", - 'cooperate' - " <literal>]]>", - "cooperate <literal>", - ), - ("text/plain", "Use <slot>", "Use "), - ], -) -def test_atom_typed_titles_and_subtitles( - prefix: str, content_type: str, payload: str, expected: str -) -> None: - xml = ( - f'<{prefix}feed xmlns="http://www.w3.org/2005/Atom" ' - 'xmlns:a="http://www.w3.org/2005/Atom">' - f'<{prefix}title type="{content_type}">{payload}' - f'<{prefix}subtitle type="{content_type}">{payload}' - f'<{prefix}entry><{prefix}title type="{content_type}">{payload}' - f"<{prefix}content>Body." - ) - result = RssConverter().convert( - io.BytesIO(xml.encode()), StreamInfo(extension=".atom") - ) - - assert result.title == expected - assert result.markdown == f"# {expected}\n{expected}\n\n## {expected}\nBody." - - -@pytest.mark.parametrize("extension", [".rss", ".atom"]) -def test_feed_does_not_borrow_nested_metadata(extension: str) -> None: - if extension == ".rss": - xml = ( - "Wrong feed title" - "EntryBody." - "Wrong entry title" - "Second body." - ) - else: - xml = ( - '' - "EntryBody." - "Wrong entry titleWrong body." - "Second body." - ) - result = RssConverter().convert( - io.BytesIO(xml.encode()), StreamInfo(extension=extension) - ) - - assert result.title is None - assert result.markdown.count("## Entry") == 1 - assert "Wrong" not in result.markdown - assert not result.markdown.startswith("# ") - assert "Body.\n\nSecond body." in result.markdown - - -@pytest.mark.parametrize("content_type", ["rss", "text", "html", "xhtml"]) -def test_deeply_nested_title_is_read_without_recursion(content_type: str) -> None: - payload = "" * 500 + "Deep title" + "" * 500 - if content_type == "rss": - xml = f"{payload}".encode() - extension = ".rss" - else: - if content_type == "html": - payload = f"" - elif content_type == "xhtml": - payload = f'
{payload}
' - xml = ( - f'' - f'{payload}' - ).encode() - extension = ".atom" - original_limit = sys.getrecursionlimit() - try: - sys.setrecursionlimit(200) - result = RssConverter().convert( - io.BytesIO(xml), StreamInfo(extension=extension) - ) - finally: - sys.setrecursionlimit(original_limit) - - assert result.title == "Deep title" - - -def test_rss_channel_description_and_date_preserve_complete_text() -> None: - xml = b"""Feed - About and research. - EntryMon, 00:00:00 GMT - """ - result = RssConverter().convert(io.BytesIO(xml), StreamInfo(extension=".rss")) - - assert "About R&D and research." in result.markdown - assert "Published on: Mon, 01 Jan 2024 00:00:00 GMT" in result.markdown diff --git a/packages/markitdown/tests/test_undetectable_charset.py b/packages/markitdown/tests/test_undetectable_charset.py deleted file mode 100644 index eb87a3cf63..0000000000 --- a/packages/markitdown/tests/test_undetectable_charset.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Tests for bytes no charset decodes. - -``charset_normalizer.from_bytes(data).best()`` returns ``None`` when nothing -decodes the bytes, and ``str(None)`` is the four-character string ``"None"``, -so a binary file handed over with a text extension became a document whose -entire content was the word "None" -- four characters that were never in the -file. -""" - -import io - -from charset_normalizer import from_bytes - -from markitdown import MarkItDown, StreamInfo - -# Random bytes: no charset claims them, which is what makes `best()` answer None. -# Fixed rather than generated, so the test does not depend on chance. -UNDECODABLE = bytes((7 * i * i + 251 * i + 193) % 256 for i in range(4096)) - - -def test_the_fixture_really_is_undecodable() -> None: - """Guards the guard: if some charset claimed these bytes the rest would pass - for the wrong reason.""" - assert from_bytes(UNDECODABLE).best() is None - - -def test_binary_with_a_text_extension_is_not_the_word_none() -> None: - markitdown = MarkItDown() - for extension in (".txt", ".md"): - result = markitdown.convert_stream( - io.BytesIO(UNDECODABLE), file_extension=extension - ) - assert result.markdown != "None" - - -def test_binary_with_a_csv_extension_is_not_a_table_of_none() -> None: - markitdown = MarkItDown() - result = markitdown.convert_stream(io.BytesIO(UNDECODABLE), file_extension=".csv") - assert result.markdown != "| None |\n| --- |" - - -def test_ordinary_text_is_unchanged() -> None: - """The fallback only runs when detection fails; everything else is as it was.""" - markitdown = MarkItDown() - - text = markitdown.convert_stream( - io.BytesIO(b"hello, world\n"), file_extension=".txt" - ) - assert text.markdown == "hello, world\n" - - table = markitdown.convert_stream(io.BytesIO(b"a,b\n1,2\n"), file_extension=".csv") - assert table.markdown == "| a | b |\n| --- | --- |\n| 1 | 2 |" - - -def test_text_containing_the_word_none_survives() -> None: - """The bug was a whole document reading "None", not the word appearing in - one.""" - markitdown = MarkItDown() - result = markitdown.convert_stream( - io.BytesIO(b"value,note\nNone,missing\n"), file_extension=".csv" - ) - assert result.markdown == "| value | note |\n| --- | --- |\n| None | missing |" - - -def test_a_declared_charset_still_wins() -> None: - """`stream_info.charset` short-circuits detection and is untouched.""" - markitdown = MarkItDown() - body = "名称,代码\n浦发银行,600000\n".encode("gb18030") - result = markitdown.convert_stream( - io.BytesIO(body), - stream_info=StreamInfo(extension=".csv", charset="gb18030"), - ) - assert "浦发银行" in result.markdown diff --git a/packages/markitdown/tests/test_wikipedia_titles.py b/packages/markitdown/tests/test_wikipedia_titles.py deleted file mode 100644 index 47708e466b..0000000000 --- a/packages/markitdown/tests/test_wikipedia_titles.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -m pytest -"""Article titles that carry markup must survive conversion.""" - -import io - -from markitdown import StreamInfo -from markitdown.converters import WikipediaConverter - -STREAM_INFO = StreamInfo( - url="https://en.wikipedia.org/wiki/Example", - mimetype="text/html", - extension=".html", -) - - -def _page(heading: str, document_title: str) -> io.BytesIO: - """Build a page shaped like the article HTML Wikipedia actually serves.""" - return io.BytesIO( - f"""{document_title} - Wikipedia -

{heading}

-

Body text.

-""".encode() - ) - - -def test_plain_title_is_read_from_the_title_span() -> None: - """A plain title keeps being read from mw-page-title-main.""" - page = _page( - 'Paris', - "Paris", - ) - - result = WikipediaConverter().convert(page, STREAM_INFO) - - assert result.title == "Paris" - assert result.markdown.lstrip().startswith("# Paris") - - -def test_fully_italicised_title_keeps_its_text() -> None: - """Wikipedia drops the title span for italicised titles, e.g. species names.""" - page = _page("Escherichia coli", "Escherichia coli") - - result = WikipediaConverter().convert(page, STREAM_INFO) - - # Without the first-heading fallback this became "Escherichia coli - Wikipedia". - assert result.title == "Escherichia coli" - assert result.markdown.lstrip().startswith("# Escherichia coli") - - -def test_title_mixing_markup_and_plain_text_keeps_both_parts() -> None: - """A disambiguated italic title spans several children, so .string is None.""" - page = _page("Titanic (1997 film)", "Titanic (1997 film)") - - result = WikipediaConverter().convert(page, STREAM_INFO) - - assert result.title == "Titanic (1997 film)" - assert result.markdown.lstrip().startswith("# Titanic (1997 film)") diff --git a/packages/markitdown/tests/test_xlsx_images.py b/packages/markitdown/tests/test_xlsx.py similarity index 82% rename from packages/markitdown/tests/test_xlsx_images.py rename to packages/markitdown/tests/test_xlsx.py index 3dce30bfce..bb071c868f 100644 --- a/packages/markitdown/tests/test_xlsx_images.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -1,16 +1,18 @@ -"""Spreadsheet image hooks retain the native table pipeline and package bytes.""" +"""Spreadsheet conversion, compatibility repairs, and image hooks.""" import inspect import io +import zipfile from pathlib import Path from typing import Any, BinaryIO, Callable, Optional from unittest.mock import Mock -import zipfile from xml.etree import ElementTree as ET +import openpyxl +import pytest +from PIL import Image from bs4 import BeautifulSoup from defusedxml.common import EntitiesForbidden -import openpyxl from openpyxl.drawing.image import Image as SheetImage from openpyxl.drawing.spreadsheet_drawing import ( AbsoluteAnchor, @@ -18,14 +20,13 @@ OneCellAnchor, TwoCellAnchor, ) -from PIL import Image -import pytest -from markitdown import MissingDependencyException, StreamInfo -from markitdown.converters import XlsConverter, XlsxConverter -from markitdown.converters import _xlsx_converter +from markitdown import MarkItDown, MissingDependencyException, StreamInfo from markitdown.converter_utils import _xlsx_images +from markitdown.converters import XlsConverter, XlsxConverter, _xlsx_converter + +# Image hooks _INFO = StreamInfo(extension=".xlsx") @@ -464,3 +465,81 @@ def test_missing_dependencies_and_legacy_xls_stay_separate( legacy.convert(io.BytesIO(data), StreamInfo(extension=".xls")).markdown == expected ) + + +# Conversion regressions + + +def test_xlsx_legacy_show_zeroes_sheetview(tmp_path) -> None: + from openpyxl import Workbook + + base_path = tmp_path / "base.xlsx" + xlsx_path = tmp_path / "legacy_show_zeroes.xlsx" + + workbook = Workbook() + sheet = workbook.active + sheet.title = "Data" + sheet["A1"] = "hello" + sheet["B1"] = "world" + # A cell whose text is byte-identical to the malformed attribute. openpyxl stores it + # as an inline string, so it lands in the very worksheet part that gets repaired. + sheet["A2"] = ' showZeroes="0" ' + workbook.save(base_path) + + with zipfile.ZipFile(base_path) as source: + with zipfile.ZipFile(xlsx_path, "w", zipfile.ZIP_DEFLATED) as target: + for item in source.infolist(): + data = source.read(item.filename) + if item.filename == "xl/worksheets/sheet1.xml": + data = data.replace( + b" None: + from markitdown.converters._xlsx_converter import _rename_show_zeroes_attribute + + worksheet_xml = ( + b"" + b'' + b'' + b"" + # openpyxl ignores custom sheet views entirely, so they need no repair + b'' + b'' + b' showZeroes="0" ' + b"" + b'IF(A1=" showZeroes=","x","y")y' + b"" + ) + + repaired = _rename_show_zeroes_attribute(worksheet_xml) + + # Every start tag is repaired, whatever separates its attributes ... + assert repaired.count(b"showZeros=") == 2 + assert b'' in repaired + assert b'' in repaired + + # ... and nothing outside those start tags is rewritten. + assert b' showZeroes="0" ' in repaired + assert b'IF(A1=" showZeroes=","x","y")' in repaired + assert b'' in repaired + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_youtube_converter.py b/packages/markitdown/tests/test_youtube.py similarity index 65% rename from packages/markitdown/tests/test_youtube_converter.py rename to packages/markitdown/tests/test_youtube.py index 4293f620f2..1ec139ded7 100644 --- a/packages/markitdown/tests/test_youtube_converter.py +++ b/packages/markitdown/tests/test_youtube.py @@ -1,12 +1,17 @@ +"""YouTube URL handling, metadata, transcripts, and HTML fallback.""" + import io from types import SimpleNamespace from unittest.mock import MagicMock import pytest +import markitdown.converters._youtube_converter as youtube_module from markitdown import MarkItDown, StreamInfo from markitdown.converters import HtmlConverter, YouTubeConverter -import markitdown.converters._youtube_converter as youtube_module + + +# Extraction and fallback @pytest.fixture @@ -166,3 +171,82 @@ def test_youtube_preserves_library_transcript( ) transcript_api.return_value.list.assert_called_once_with("12345") transcript_api.return_value.fetch.assert_called_once_with("12345", languages=["fr"]) + + +# Conversion regressions + + +@pytest.mark.parametrize( + ("url", "video_id"), + [ + ("https://www.youtube.com/watch?v=V2qZ_lgxTzg", "V2qZ_lgxTzg"), + ("https://youtu.be/V2qZ_lgxTzg", "V2qZ_lgxTzg"), + ("https://www.youtube.com/shorts/V2qZ_lgxTzg", "V2qZ_lgxTzg"), + ("https://www.youtube.com/embed/V2qZ_lgxTzg", "V2qZ_lgxTzg"), + ], +) +def test_youtube_converter_extracts_supported_video_ids( + url: str, video_id: str +) -> None: + converter = YouTubeConverter() + assert converter._get_video_id(url) == video_id + + +@pytest.mark.parametrize( + "url", + [ + "https://www.youtube.com/watch?v=V2qZ_lgxTzg", + "https://youtu.be/V2qZ_lgxTzg", + "https://www.youtube.com/shorts/V2qZ_lgxTzg", + ], +) +def test_youtube_converter_accepts_supported_url_formats(url: str) -> None: + converter = YouTubeConverter() + assert converter.accepts( + io.BytesIO(b""), + StreamInfo(url=url, extension=".html"), + ) + + +def test_youtube_converter_missing_title_metadata() -> None: + """Missing titles fall back to HTML when no video content is extracted.""" + from unittest.mock import patch + from markitdown.converters._youtube_converter import YouTubeConverter + + converter = YouTubeConverter() + stream_info = StreamInfo( + mimetype="text/html", + extension=".html", + url="https://www.youtube.com/watch?v=12345", + ) + + with patch( + "markitdown.converters._youtube_converter.IS_YOUTUBE_TRANSCRIPT_CAPABLE", + False, + ): + # Case 1: Stream with no title metadata or title tag + html_content_no_title = b"Video Content" + stream_no_title = io.BytesIO(html_content_no_title) + result_no_title = converter.convert(stream_no_title, stream_info) + assert result_no_title.title is None + assert result_no_title.markdown == "Video Content" + + # Case 2: Stream with an empty tag + html_content_empty_title = ( + b"<html><head><title>Video Content" + ) + stream_empty_title = io.BytesIO(html_content_empty_title) + result_empty_title = converter.convert(stream_empty_title, stream_info) + assert result_empty_title.title is None + assert result_empty_title.markdown == "Video Content" + + # Case 3: Stream whose title is only available from the tag + html_content_title_tag = b"<html><head><title>Fallback TitleVideo Content" + stream_title_tag = io.BytesIO(html_content_title_tag) + result_title_tag = converter.convert(stream_title_tag, stream_info) + assert result_title_tag.title == "Fallback Title" + assert result_title_tag.markdown == "# YouTube\n\n## Fallback Title\n" + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_zip_kwargs.py b/packages/markitdown/tests/test_zip.py similarity index 72% rename from packages/markitdown/tests/test_zip_kwargs.py rename to packages/markitdown/tests/test_zip.py index a1470b92fd..aa253f915f 100644 --- a/packages/markitdown/tests/test_zip_kwargs.py +++ b/packages/markitdown/tests/test_zip.py @@ -1,3 +1,5 @@ +"""ZIP conversion and forwarding options to nested converters.""" + import io import os import zipfile @@ -13,6 +15,9 @@ ) from markitdown.converters._zip_converter import ZipConverter + +# Nested conversion + TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") @@ -134,3 +139,48 @@ def test_zip_member_docx_converts_as_docx(archive_url: Optional[str]) -> None: # ... and it was not unpacked as a nested archive assert "word/document.xml" not in result assert "[Content_Types].xml" not in result + + +# Conversion regressions + + +def test_zip_duplicate_filenames_preserve_each_entry() -> None: + """Same-named ZIP entries must retain their own content and archive order.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as archive: + archive.writestr("notes.txt", "First archived entry.") + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("notes.txt", "Second archived entry.") + buf.seek(0) + + result = MarkItDown().convert_stream( + buf, stream_info=StreamInfo(extension=".zip", filename="duplicate.zip") + ) + + assert result.markdown == ( + "Content from the zip file `duplicate.zip`:\n\n" + "## File: notes.txt\n\nFirst archived entry.\n\n" + "## File: notes.txt\n\nSecond archived entry." + ) + + +def test_zip_stream_no_filename_header() -> None: + """Regression test: ZipConverter must not render the literal string 'None' + in the output header when the stream has no associated URL, local path, or + filename (e.g. when called via convert_stream() without stream_info).""" + markitdown = MarkItDown() + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("hello.txt", "Hello world") + buf.seek(0) + + result = markitdown.convert_stream( + buf, stream_info=StreamInfo(mimetype="application/zip") + ) + assert result.markdown.startswith("Content from the zip file `(unknown)`:\n\n") + assert "Hello world" in result.markdown + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__]))