diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index fc359d6c52..1093cc695d 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -28,7 +28,7 @@ TEST_DATA_DIR = Path(__file__).parent / "ocr_test_data" _MOCK_TEXT = "MOCK_OCR_TEXT_12345" -_MOCK_BLOCK = "*[Image OCR] \nMOCK\\_OCR\\_TEXT\\_12345 \n[End OCR]*" +_MOCK_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*" class MockOCRService: diff --git a/packages/markitdown-ocr/tests/test_docx_inheritance.py b/packages/markitdown-ocr/tests/test_docx_inheritance.py index 7cf02d705d..a2cea38a22 100644 --- a/packages/markitdown-ocr/tests/test_docx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_docx_inheritance.py @@ -104,7 +104,7 @@ def test_plugin_registration_full_trip(monkeypatch: pytest.MonkeyPatch) -> None: assert result.markdown == ( "# Heading\n\nBefore\n\n" - "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n" + "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n" "After\n\n**Native content**" ) client.chat.completions.create.assert_called_once() @@ -145,8 +145,8 @@ def test_recognition_cache_is_local_and_service_override_is_preserved() -> None: overridden = _convert(converter, data, ocr_service=second) again = _convert(converter, data) - assert default.count("*[Image OCR] \nfirst \n[End OCR]*") == 12 - assert overridden.count("*[Image OCR] \nsecond \n[End OCR]*") == 12 + assert default.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 12 + assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 12 assert again == default assert first.extract_text.call_count == 2 second.extract_text.assert_called_once() @@ -189,7 +189,7 @@ def test_ocr_text_is_escaped_and_line_endings_are_preserved_as_html_breaks( "
final
[End OCR]

" ) in convert_html.call_args.args[0] assert ( - "*[Image OCR] \nA\\_B \\*literal\\* \n" " & value \nfinal \n[End OCR]*" + "*[Image OCR]\\\nA\\_B \\*literal\\*\\\n" " & value\\\nfinal\\\n[End OCR]*" ) in result assert "data-markitdown-image-" not in convert_html.call_args.args[0] diff --git a/packages/markitdown-ocr/tests/test_pptx_converter.py b/packages/markitdown-ocr/tests/test_pptx_converter.py index 683a688511..5a01ca9c91 100644 --- a/packages/markitdown-ocr/tests/test_pptx_converter.py +++ b/packages/markitdown-ocr/tests/test_pptx_converter.py @@ -34,7 +34,7 @@ TEST_DATA_DIR = Path(__file__).parent / "ocr_test_data" _MOCK_TEXT = "MOCK_OCR_TEXT_12345" -_OCR_BLOCK = "*[Image OCR] \nMOCK\\_OCR\\_TEXT\\_12345 \n[End OCR]*" +_OCR_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*" class MockOCRService: diff --git a/packages/markitdown-ocr/tests/test_pptx_inheritance.py b/packages/markitdown-ocr/tests/test_pptx_inheritance.py index 6ee3b89d22..0fe26753aa 100644 --- a/packages/markitdown-ocr/tests/test_pptx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_pptx_inheritance.py @@ -103,8 +103,8 @@ def test_ocr_cache_is_document_local_and_honors_service_override() -> None: overridden = _convert(converter, data, ocr_service=second) again = _convert(converter, data) - assert original.count("*[Image OCR] \nfirst \n[End OCR]*") == 12 - assert overridden.count("*[Image OCR] \nsecond \n[End OCR]*") == 12 + assert original.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 12 + assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 12 assert again == original assert first.extract_text.call_count == 2 second.extract_text.assert_called_once() @@ -148,7 +148,7 @@ def test_ocr_text_is_escaped_with_normalized_html_breaks_and_html_options( "
final
[End OCR]

", ) assert ( - "*[Image OCR] \nA\\_B \\*literal\\* \n" " & value \nfinal \n[End OCR]*" + "*[Image OCR]\\\nA\\_B \\*literal\\*\\\n" " & value\\\nfinal\\\n[End OCR]*" ) in result assert "*literal*" in _convert( converter, data, escape_asterisks=False, escape_underscores=False @@ -213,7 +213,7 @@ def recognize(stream): ) assert len(seen) == 1 and b" None: result = _convert(PptxConverterWithOCR(service), data) - assert "*[Image OCR] \nSVG text \n[End OCR]*" in result + assert "*[Image OCR]\\\nSVG text\\\n[End OCR]*" in result client.chat.completions.create.assert_called_once() request = client.chat.completions.create.call_args.kwargs assert request["model"] == "vision-model" @@ -271,7 +271,7 @@ def test_plugin_full_trip_with_mocked_model_only( expected_image = ( "![Recognized_text](Picture1.jpg)" if caption_succeeds - else "*[Image OCR]\nRecognized\\_text\n[End OCR]*" + else "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*" ) assert result.markdown == ( "\n# Heading\n\n" diff --git a/packages/markitdown-ocr/tests/test_xlsx_converter.py b/packages/markitdown-ocr/tests/test_xlsx_converter.py index 113960dddb..2c81671ef8 100644 --- a/packages/markitdown-ocr/tests/test_xlsx_converter.py +++ b/packages/markitdown-ocr/tests/test_xlsx_converter.py @@ -32,7 +32,7 @@ _MOCK_TEXT = "MOCK_OCR_TEXT_12345" -_OCR_BLOCK = "*[Image OCR] \nMOCK\\_OCR\\_TEXT\\_12345 \n[End OCR]*" +_OCR_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*" class MockOCRService: diff --git a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py index c94c8fac5f..1f227239d4 100644 --- a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py @@ -107,8 +107,8 @@ def test_plugin_full_trip_uses_native_cells_and_original_image( "| --- | --- |\n" "| & \\*value\\* | 12 |\n\n" "### Images in this sheet:\n\n" - "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n" - "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n" + "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n" + "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n" "## Other\n| Last |\n| --- |\n| Final |" ) client.chat.completions.create.assert_called_once() @@ -149,8 +149,8 @@ def test_cache_is_document_local_and_per_call_service_overrides_work() -> None: overridden = _convert(converter, data, ocr_service=second) again = _convert(converter, data) - assert default.count("*[Image OCR] \nfirst \n[End OCR]*") == 4 - assert overridden.count("*[Image OCR] \nsecond \n[End OCR]*") == 4 + assert default.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 4 + assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 4 assert again == default assert first.extract_text.call_count == 2 second.extract_text.assert_called_once() @@ -172,7 +172,7 @@ def recognize(stream): assert calls == [_RED, _BLUE] assert result.count("[Image OCR]") == 2 assert "Image at " not in result - assert result.count("*[Image OCR] \nblue \n[End OCR]*") == 2 + assert result.count("*[Image OCR]\\\nblue\\\n[End OCR]*") == 2 def test_mixed_anchor_recognition_keeps_legacy_openpyxl_order() -> None: @@ -222,7 +222,7 @@ def recognize(stream): data, ) assert calls == legacy_order - blocks = [f"*[Image OCR] \n{label} \n[End OCR]*" for label in legacy_order] + blocks = [f"*[Image OCR]\\\n{label}\\\n[End OCR]*" for label in legacy_order] assert "\n\n".join(blocks) in result @@ -242,7 +242,7 @@ def test_ocr_html_is_escaped_and_existing_html_options_are_forwarded( "
final
[End OCR]

" ) in convert_html.call_args_list[1].args[0] assert ( - "*[Image OCR] \nA_B \\*literal\\* \n & value \nfinal \n[End OCR]*" + "*[Image OCR]\\\nA_B \\*literal\\*\\\n & value\\\nfinal\\\n[End OCR]*" in result ) assert "| Header_one | Unnamed: 1 |" in result diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py index ed34144863..c5c85cb2d8 100644 --- a/packages/markitdown/src/markitdown/converters/_markdownify.py +++ b/packages/markitdown/src/markitdown/converters/_markdownify.py @@ -1,6 +1,7 @@ import re import markdownify +from bs4 import Comment from typing import Any, Optional from urllib.parse import quote, urlparse, urlunparse @@ -22,11 +23,25 @@ def _quote_path_preserving_percent_encoded_octets(path: str) -> str: return "".join(parts) +def _shows_content(nodes: Any) -> bool: + """Whether any of these sibling nodes shows text or an image.""" + for node in nodes: + if isinstance(node, Comment): + continue + if isinstance(node, str): + if node.strip(): + return True + elif node.name == "img" or node.get_text(strip=True) or node.find("img"): + return True + return False + + class _CustomMarkdownify(markdownify.MarkdownConverter): """ A custom version of markdownify's MarkdownConverter. Changes include: - Altering the default heading style to use '#', '##', etc. + - Writing hard line breaks as backslashes, which survive whitespace normalization. - Removing javascript hyperlinks. - Truncating images with large data:uri sources. - Ensuring URIs are properly escaped, and do not conflict with Markdown syntax @@ -143,6 +158,30 @@ def convert_img( return "![%s](%s%s)" % (alt, src, title_part) + def convert_br( + self, + el: Any, + text: str, + parent_tags: Any = None, + **kwargs, + ) -> str: + """Write a hard line break as a backslash rather than two trailing spaces. + + MarkItDown strips trailing whitespace from every output line, which removes + markdownify's two-space hard breaks. A break with text or an image on both sides + of it in its parent element is written as a backslash, which survives. Other + breaks, and those in table cells, headings, and preformatted text, keep + markdownify's handling. + """ + if ( + isinstance(parent_tags, (set, frozenset)) + and not parent_tags & {"_inline", "_noformat"} + and _shows_content(el.previous_siblings) + and _shows_content(el.next_siblings) + ): + return "\\\n" + text + return super().convert_br(el, text, parent_tags, **kwargs) # type: ignore + def convert_input( self, el: Any, diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py index 4fa67684e7..9c991a5a8d 100644 --- a/packages/markitdown/tests/test_docx.py +++ b/packages/markitdown/tests/test_docx.py @@ -1485,5 +1485,13 @@ def test_convert_docx_with_style_missing_type(tmp_path): assert "# Abstract" in result.markdown +def test_word_line_breaks_are_kept() -> None: + body = _paragraph(_text("Line one") + "" + _text("Line two")) + + result = MarkItDown().convert_stream(_docx(body), stream_info=_INFO) + + assert result.markdown == "Line one\\\nLine two" + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py index 94cece47b9..d99956c7c6 100644 --- a/packages/markitdown/tests/test_html.py +++ b/packages/markitdown/tests/test_html.py @@ -101,6 +101,50 @@ def test_html_href_does_not_quote_query_or_fragment() -> None: assert f"[example]({expected_href})" in markdown +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

Line one
Line two

", "Line one\\\nLine two"), + ("

Line one

Line two

", "Line one\\\n\\\nLine two"), + ("

Bold
text

", "**Bold\\\ntext**"), + ("
  • Item
    continued
", "* Item\\\n continued"), + ("

Before
x

", "Before\\\n![x](a.png)"), + ], +) +def test_html_line_breaks_survive_whitespace_normalization( + html: str, expected: str +) -> None: + assert expected in _convert_html(html) + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

Trailing

", "Trailing"), + ("


Leading

", "Leading"), + ("

First
Last

", "First\nLast"), + ("

Text

", "Text"), + ], +) +def test_html_line_breaks_at_element_edges_are_unchanged( + html: str, expected: str +) -> None: + assert _convert_html(html) == expected + + +def test_html_line_breaks_in_cells_headings_and_code_are_unchanged() -> None: + markdown = _convert_html( + "
a
b
" + "

Head
ing

" + "
one
two
" + ) + + assert "| a b |" in markdown + assert "## Head ing" in markdown + assert "```\none\ntwo\n```" in markdown + assert "\\" not in markdown + + def test_img_prefers_data_src_over_placeholder_data_uri() -> None: placeholder = ( "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" @@ -287,9 +331,10 @@ def test_html_strikethrough_variants(tmp_path) -> None: # An empty element contributes nothing "Empty GH.", # A line break inside the element is kept, and the markup - # survives it because strikethrough may span a single newline + # survives it because strikethrough may span a single newline; + # a
stays a hard line break "Newline I~~J\nK~~L.", - "Break M~~N\nO~~P.", + "Break M~~N\\\nO~~P.", ] ) diff --git a/packages/markitdown/tests/test_rss.py b/packages/markitdown/tests/test_rss.py index 89d2cbb796..3006e7ecfd 100644 --- a/packages/markitdown/tests/test_rss.py +++ b/packages/markitdown/tests/test_rss.py @@ -384,7 +384,7 @@ def _feed_with_body(field: str, payload: str, *, atom_type: str = "html") -> byt ("nested trailing", "**nested** trailing"), ("leading nested trailing", "leading **nested** trailing"), ("leading nested", "leading **nested**"), - ("First
Second", "First \nSecond"), + ("First
Second", "First\\\nSecond"), ("
line 1\n  line 2
", "```\nline 1\n line 2\n```"), ( "\n nested trailing", diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index bb071c868f..cd115d3ca4 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -196,7 +196,7 @@ class InheritedImages(_ImageConverter): [ ("{}", "**one**\n\n**two**\n\n**three**"), ("{}", "one\n\ntwo\n\nthree"), - ("{}
line", "one \nline\n\ntwo \nline\n\nthree \nline"), + ("{}
line", "one\\\nline\n\ntwo\\\nline\n\nthree\\\nline"), ( "

{}

paragraph

", "one\n\nparagraph\n\ntwo\n\nparagraph\n\nthree\n\nparagraph",