From 7c88cb21a7f7ff53e1beadbe177766f0f8118e51 Mon Sep 17 00:00:00 2001 From: Patrick Creighton Date: Tue, 6 Oct 2026 10:19:14 -0700 Subject: [PATCH] Keep hard line breaks that the output normalization strips markdownify writes
as two trailing spaces, which MarkItDown.convert() then strips from every output line, so Word line breaks and HTML
became soft breaks that render as a space. A break with text or an image on both sides of it in its parent element is now written as a backslash hard break, which survives the normalization. Other breaks, and those in table cells, headings, and preformatted text, keep markdownify's handling. Tests that recorded the old converter output, including the OCR plugin's, now expect the backslash. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tests/test_docx_converter.py | 2 +- .../tests/test_docx_inheritance.py | 8 +-- .../tests/test_pptx_converter.py | 2 +- .../tests/test_pptx_inheritance.py | 12 ++--- .../tests/test_xlsx_converter.py | 2 +- .../tests/test_xlsx_inheritance.py | 14 +++--- .../src/markitdown/converters/_markdownify.py | 39 +++++++++++++++ packages/markitdown/tests/test_docx.py | 8 +++ packages/markitdown/tests/test_html.py | 49 ++++++++++++++++++- packages/markitdown/tests/test_rss.py | 2 +- packages/markitdown/tests/test_xlsx.py | 2 +- 11 files changed, 116 insertions(+), 24 deletions(-) diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index fc359d6c52..1093cc695d 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -28,7 +28,7 @@ TEST_DATA_DIR = Path(__file__).parent / "ocr_test_data" _MOCK_TEXT = "MOCK_OCR_TEXT_12345" -_MOCK_BLOCK = "*[Image OCR] \nMOCK\\_OCR\\_TEXT\\_12345 \n[End OCR]*" +_MOCK_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*" class MockOCRService: diff --git a/packages/markitdown-ocr/tests/test_docx_inheritance.py b/packages/markitdown-ocr/tests/test_docx_inheritance.py index 7cf02d705d..a2cea38a22 100644 --- a/packages/markitdown-ocr/tests/test_docx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_docx_inheritance.py @@ -104,7 +104,7 @@ def test_plugin_registration_full_trip(monkeypatch: pytest.MonkeyPatch) -> None: assert result.markdown == ( "# Heading\n\nBefore\n\n" - "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n" + "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n" "After\n\n**Native content**" ) client.chat.completions.create.assert_called_once() @@ -145,8 +145,8 @@ def test_recognition_cache_is_local_and_service_override_is_preserved() -> None: overridden = _convert(converter, data, ocr_service=second) again = _convert(converter, data) - assert default.count("*[Image OCR] \nfirst \n[End OCR]*") == 12 - assert overridden.count("*[Image OCR] \nsecond \n[End OCR]*") == 12 + assert default.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 12 + assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 12 assert again == default assert first.extract_text.call_count == 2 second.extract_text.assert_called_once() @@ -189,7 +189,7 @@ def test_ocr_text_is_escaped_and_line_endings_are_preserved_as_html_breaks( "
final
[End OCR]

" ) in convert_html.call_args.args[0] assert ( - "*[Image OCR] \nA\\_B \\*literal\\* \n" " & value \nfinal \n[End OCR]*" + "*[Image OCR]\\\nA\\_B \\*literal\\*\\\n" " & value\\\nfinal\\\n[End OCR]*" ) in result assert "data-markitdown-image-" not in convert_html.call_args.args[0] diff --git a/packages/markitdown-ocr/tests/test_pptx_converter.py b/packages/markitdown-ocr/tests/test_pptx_converter.py index 683a688511..5a01ca9c91 100644 --- a/packages/markitdown-ocr/tests/test_pptx_converter.py +++ b/packages/markitdown-ocr/tests/test_pptx_converter.py @@ -34,7 +34,7 @@ TEST_DATA_DIR = Path(__file__).parent / "ocr_test_data" _MOCK_TEXT = "MOCK_OCR_TEXT_12345" -_OCR_BLOCK = "*[Image OCR] \nMOCK\\_OCR\\_TEXT\\_12345 \n[End OCR]*" +_OCR_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*" class MockOCRService: diff --git a/packages/markitdown-ocr/tests/test_pptx_inheritance.py b/packages/markitdown-ocr/tests/test_pptx_inheritance.py index 6ee3b89d22..0fe26753aa 100644 --- a/packages/markitdown-ocr/tests/test_pptx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_pptx_inheritance.py @@ -103,8 +103,8 @@ def test_ocr_cache_is_document_local_and_honors_service_override() -> None: overridden = _convert(converter, data, ocr_service=second) again = _convert(converter, data) - assert original.count("*[Image OCR] \nfirst \n[End OCR]*") == 12 - assert overridden.count("*[Image OCR] \nsecond \n[End OCR]*") == 12 + assert original.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 12 + assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 12 assert again == original assert first.extract_text.call_count == 2 second.extract_text.assert_called_once() @@ -148,7 +148,7 @@ def test_ocr_text_is_escaped_with_normalized_html_breaks_and_html_options( "
final
[End OCR]

", ) assert ( - "*[Image OCR] \nA\\_B \\*literal\\* \n" " & value \nfinal \n[End OCR]*" + "*[Image OCR]\\\nA\\_B \\*literal\\*\\\n" " & value\\\nfinal\\\n[End OCR]*" ) in result assert "*literal*" in _convert( converter, data, escape_asterisks=False, escape_underscores=False @@ -213,7 +213,7 @@ def recognize(stream): ) assert len(seen) == 1 and b" None: result = _convert(PptxConverterWithOCR(service), data) - assert "*[Image OCR] \nSVG text \n[End OCR]*" in result + assert "*[Image OCR]\\\nSVG text\\\n[End OCR]*" in result client.chat.completions.create.assert_called_once() request = client.chat.completions.create.call_args.kwargs assert request["model"] == "vision-model" @@ -271,7 +271,7 @@ def test_plugin_full_trip_with_mocked_model_only( expected_image = ( "![Recognized_text](Picture1.jpg)" if caption_succeeds - else "*[Image OCR]\nRecognized\\_text\n[End OCR]*" + else "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*" ) assert result.markdown == ( "\n# Heading\n\n" diff --git a/packages/markitdown-ocr/tests/test_xlsx_converter.py b/packages/markitdown-ocr/tests/test_xlsx_converter.py index 113960dddb..2c81671ef8 100644 --- a/packages/markitdown-ocr/tests/test_xlsx_converter.py +++ b/packages/markitdown-ocr/tests/test_xlsx_converter.py @@ -32,7 +32,7 @@ _MOCK_TEXT = "MOCK_OCR_TEXT_12345" -_OCR_BLOCK = "*[Image OCR] \nMOCK\\_OCR\\_TEXT\\_12345 \n[End OCR]*" +_OCR_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*" class MockOCRService: diff --git a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py index c94c8fac5f..1f227239d4 100644 --- a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py @@ -107,8 +107,8 @@ def test_plugin_full_trip_uses_native_cells_and_original_image( "| --- | --- |\n" "| & \\*value\\* | 12 |\n\n" "### Images in this sheet:\n\n" - "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n" - "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n" + "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n" + "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n" "## Other\n| Last |\n| --- |\n| Final |" ) client.chat.completions.create.assert_called_once() @@ -149,8 +149,8 @@ def test_cache_is_document_local_and_per_call_service_overrides_work() -> None: overridden = _convert(converter, data, ocr_service=second) again = _convert(converter, data) - assert default.count("*[Image OCR] \nfirst \n[End OCR]*") == 4 - assert overridden.count("*[Image OCR] \nsecond \n[End OCR]*") == 4 + assert default.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 4 + assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 4 assert again == default assert first.extract_text.call_count == 2 second.extract_text.assert_called_once() @@ -172,7 +172,7 @@ def recognize(stream): assert calls == [_RED, _BLUE] assert result.count("[Image OCR]") == 2 assert "Image at " not in result - assert result.count("*[Image OCR] \nblue \n[End OCR]*") == 2 + assert result.count("*[Image OCR]\\\nblue\\\n[End OCR]*") == 2 def test_mixed_anchor_recognition_keeps_legacy_openpyxl_order() -> None: @@ -222,7 +222,7 @@ def recognize(stream): data, ) assert calls == legacy_order - blocks = [f"*[Image OCR] \n{label} \n[End OCR]*" for label in legacy_order] + blocks = [f"*[Image OCR]\\\n{label}\\\n[End OCR]*" for label in legacy_order] assert "\n\n".join(blocks) in result @@ -242,7 +242,7 @@ def test_ocr_html_is_escaped_and_existing_html_options_are_forwarded( "
final
[End OCR]

" ) in convert_html.call_args_list[1].args[0] assert ( - "*[Image OCR] \nA_B \\*literal\\* \n & value \nfinal \n[End OCR]*" + "*[Image OCR]\\\nA_B \\*literal\\*\\\n & value\\\nfinal\\\n[End OCR]*" in result ) assert "| Header_one | Unnamed: 1 |" in result diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py index ed34144863..c5c85cb2d8 100644 --- a/packages/markitdown/src/markitdown/converters/_markdownify.py +++ b/packages/markitdown/src/markitdown/converters/_markdownify.py @@ -1,6 +1,7 @@ import re import markdownify +from bs4 import Comment from typing import Any, Optional from urllib.parse import quote, urlparse, urlunparse @@ -22,11 +23,25 @@ def _quote_path_preserving_percent_encoded_octets(path: str) -> str: return "".join(parts) +def _shows_content(nodes: Any) -> bool: + """Whether any of these sibling nodes shows text or an image.""" + for node in nodes: + if isinstance(node, Comment): + continue + if isinstance(node, str): + if node.strip(): + return True + elif node.name == "img" or node.get_text(strip=True) or node.find("img"): + return True + return False + + class _CustomMarkdownify(markdownify.MarkdownConverter): """ A custom version of markdownify's MarkdownConverter. Changes include: - Altering the default heading style to use '#', '##', etc. + - Writing hard line breaks as backslashes, which survive whitespace normalization. - Removing javascript hyperlinks. - Truncating images with large data:uri sources. - Ensuring URIs are properly escaped, and do not conflict with Markdown syntax @@ -143,6 +158,30 @@ def convert_img( return "![%s](%s%s)" % (alt, src, title_part) + def convert_br( + self, + el: Any, + text: str, + parent_tags: Any = None, + **kwargs, + ) -> str: + """Write a hard line break as a backslash rather than two trailing spaces. + + MarkItDown strips trailing whitespace from every output line, which removes + markdownify's two-space hard breaks. A break with text or an image on both sides + of it in its parent element is written as a backslash, which survives. Other + breaks, and those in table cells, headings, and preformatted text, keep + markdownify's handling. + """ + if ( + isinstance(parent_tags, (set, frozenset)) + and not parent_tags & {"_inline", "_noformat"} + and _shows_content(el.previous_siblings) + and _shows_content(el.next_siblings) + ): + return "\\\n" + text + return super().convert_br(el, text, parent_tags, **kwargs) # type: ignore + def convert_input( self, el: Any, diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py index 4fa67684e7..9c991a5a8d 100644 --- a/packages/markitdown/tests/test_docx.py +++ b/packages/markitdown/tests/test_docx.py @@ -1485,5 +1485,13 @@ def test_convert_docx_with_style_missing_type(tmp_path): assert "# Abstract" in result.markdown +def test_word_line_breaks_are_kept() -> None: + body = _paragraph(_text("Line one") + "" + _text("Line two")) + + result = MarkItDown().convert_stream(_docx(body), stream_info=_INFO) + + assert result.markdown == "Line one\\\nLine two" + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py index 94cece47b9..d99956c7c6 100644 --- a/packages/markitdown/tests/test_html.py +++ b/packages/markitdown/tests/test_html.py @@ -101,6 +101,50 @@ def test_html_href_does_not_quote_query_or_fragment() -> None: assert f"[example]({expected_href})" in markdown +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

Line one
Line two

", "Line one\\\nLine two"), + ("

Line one

Line two

", "Line one\\\n\\\nLine two"), + ("

Bold
text

", "**Bold\\\ntext**"), + ("
  • Item
    continued
", "* Item\\\n continued"), + ("

Before
x

", "Before\\\n![x](a.png)"), + ], +) +def test_html_line_breaks_survive_whitespace_normalization( + html: str, expected: str +) -> None: + assert expected in _convert_html(html) + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

Trailing

", "Trailing"), + ("


Leading

", "Leading"), + ("

First
Last

", "First\nLast"), + ("

Text

", "Text"), + ], +) +def test_html_line_breaks_at_element_edges_are_unchanged( + html: str, expected: str +) -> None: + assert _convert_html(html) == expected + + +def test_html_line_breaks_in_cells_headings_and_code_are_unchanged() -> None: + markdown = _convert_html( + "
a
b
" + "

Head
ing

" + "
one
two
" + ) + + assert "| a b |" in markdown + assert "## Head ing" in markdown + assert "```\none\ntwo\n```" in markdown + assert "\\" not in markdown + + def test_img_prefers_data_src_over_placeholder_data_uri() -> None: placeholder = ( "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" @@ -287,9 +331,10 @@ def test_html_strikethrough_variants(tmp_path) -> None: # An empty element contributes nothing "Empty GH.", # A line break inside the element is kept, and the markup - # survives it because strikethrough may span a single newline + # survives it because strikethrough may span a single newline; + # a
stays a hard line break "Newline I~~J\nK~~L.", - "Break M~~N\nO~~P.", + "Break M~~N\\\nO~~P.", ] ) diff --git a/packages/markitdown/tests/test_rss.py b/packages/markitdown/tests/test_rss.py index 89d2cbb796..3006e7ecfd 100644 --- a/packages/markitdown/tests/test_rss.py +++ b/packages/markitdown/tests/test_rss.py @@ -384,7 +384,7 @@ def _feed_with_body(field: str, payload: str, *, atom_type: str = "html") -> byt ("nested trailing", "**nested** trailing"), ("leading nested trailing", "leading **nested** trailing"), ("leading nested", "leading **nested**"), - ("First
Second", "First \nSecond"), + ("First
Second", "First\\\nSecond"), ("
line 1\n  line 2
", "```\nline 1\n line 2\n```"), ( "\n nested trailing", diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index bb071c868f..cd115d3ca4 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -196,7 +196,7 @@ class InheritedImages(_ImageConverter): [ ("{}", "**one**\n\n**two**\n\n**three**"), ("{}", "one\n\ntwo\n\nthree"), - ("{}
line", "one \nline\n\ntwo \nline\n\nthree \nline"), + ("{}
line", "one\\\nline\n\ntwo\\\nline\n\nthree\\\nline"), ( "

{}

paragraph

", "one\n\nparagraph\n\ntwo\n\nparagraph\n\nthree\n\nparagraph",