Keep hard line breaks that the output normalization strips

packages/markitdown-ocr/tests/test_docx_converter.py

@@ -28,7 +28,7 @@
 TEST_DATA_DIR = Path(__file__).parent / "ocr_test_data"
 
 _MOCK_TEXT = "MOCK_OCR_TEXT_12345"
-_MOCK_BLOCK = "*[Image OCR]  \nMOCK\\_OCR\\_TEXT\\_12345  \n[End OCR]*"
+_MOCK_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*"
 
 
 class MockOCRService:

packages/markitdown-ocr/tests/test_docx_inheritance.py

@@ -104,7 +104,7 @@ def test_plugin_registration_full_trip(monkeypatch: pytest.MonkeyPatch) -> None:
 
     assert result.markdown == (
         "# Heading\n\nBefore\n\n"
-        "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n"
+        "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n"
         "After\n\n**Native content**"
     )
     client.chat.completions.create.assert_called_once()
@@ -145,8 +145,8 @@ def test_recognition_cache_is_local_and_service_override_is_preserved() -> None:
     overridden = _convert(converter, data, ocr_service=second)
     again = _convert(converter, data)
 
-    assert default.count("*[Image OCR]  \nfirst  \n[End OCR]*") == 12
-    assert overridden.count("*[Image OCR]  \nsecond  \n[End OCR]*") == 12
+    assert default.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 12
+    assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 12
     assert again == default
     assert first.extract_text.call_count == 2
     second.extract_text.assert_called_once()
@@ -189,7 +189,7 @@ def test_ocr_text_is_escaped_and_line_endings_are_preserved_as_html_breaks(
         "<br/>final<br/>[End OCR]</em></p>"
     ) in convert_html.call_args.args[0]
     assert (
-        "*[Image OCR]  \nA\\_B \\*literal\\*  \n" "<tag> & value  \nfinal  \n[End OCR]*"
+        "*[Image OCR]\\\nA\\_B \\*literal\\*\\\n" "<tag> & value\\\nfinal\\\n[End OCR]*"
     ) in result
     assert "data-markitdown-image-" not in convert_html.call_args.args[0]
 

packages/markitdown-ocr/tests/test_pptx_converter.py

@@ -34,7 +34,7 @@
 TEST_DATA_DIR = Path(__file__).parent / "ocr_test_data"
 
 _MOCK_TEXT = "MOCK_OCR_TEXT_12345"
-_OCR_BLOCK = "*[Image OCR]  \nMOCK\\_OCR\\_TEXT\\_12345  \n[End OCR]*"
+_OCR_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*"
 
 
 class MockOCRService:

packages/markitdown-ocr/tests/test_pptx_inheritance.py

@@ -103,8 +103,8 @@ def test_ocr_cache_is_document_local_and_honors_service_override() -> None:
     overridden = _convert(converter, data, ocr_service=second)
     again = _convert(converter, data)
 
-    assert original.count("*[Image OCR]  \nfirst  \n[End OCR]*") == 12
-    assert overridden.count("*[Image OCR]  \nsecond  \n[End OCR]*") == 12
+    assert original.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 12
+    assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 12
     assert again == original
     assert first.extract_text.call_count == 2
     second.extract_text.assert_called_once()
@@ -148,7 +148,7 @@ def test_ocr_text_is_escaped_with_normalized_html_breaks_and_html_options(
         "<br/>final<br/>[End OCR]</em></p>",
     )
     assert (
-        "*[Image OCR]  \nA\\_B \\*literal\\*  \n" "<tag> & value  \nfinal  \n[End OCR]*"
+        "*[Image OCR]\\\nA\\_B \\*literal\\*\\\n" "<tag> & value\\\nfinal\\\n[End OCR]*"
     ) in result
     assert "*literal*" in _convert(
         converter, data, escape_asterisks=False, escape_underscores=False
@@ -213,7 +213,7 @@ def recognize(stream):
     )
 
     assert len(seen) == 1 and b"<svg" in seen[0]
-    assert "*[Image OCR]  \nSVG text  \n[End OCR]*" in result
+    assert "*[Image OCR]\\\nSVG text\\\n[End OCR]*" in result
     assert "data:image" not in result
 
 
@@ -231,7 +231,7 @@ def test_svg_metadata_reaches_bundled_vision_request() -> None:
 
     result = _convert(PptxConverterWithOCR(service), data)
 
-    assert "*[Image OCR]  \nSVG text  \n[End OCR]*" in result
+    assert "*[Image OCR]\\\nSVG text\\\n[End OCR]*" in result
     client.chat.completions.create.assert_called_once()
     request = client.chat.completions.create.call_args.kwargs
     assert request["model"] == "vision-model"
@@ -271,7 +271,7 @@ def test_plugin_full_trip_with_mocked_model_only(
     expected_image = (
         "![Recognized_text](Picture1.jpg)"
         if caption_succeeds
-        else "*[Image OCR]\nRecognized\\_text\n[End OCR]*"
+        else "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*"
     )
     assert result.markdown == (
         "<!-- Slide number: 1 -->\n# Heading\n\n"

packages/markitdown-ocr/tests/test_xlsx_converter.py

@@ -32,7 +32,7 @@
 _MOCK_TEXT = "MOCK_OCR_TEXT_12345"
 
 
-_OCR_BLOCK = "*[Image OCR]  \nMOCK\\_OCR\\_TEXT\\_12345  \n[End OCR]*"
+_OCR_BLOCK = "*[Image OCR]\\\nMOCK\\_OCR\\_TEXT\\_12345\\\n[End OCR]*"
 
 
 class MockOCRService:

packages/markitdown-ocr/tests/test_xlsx_inheritance.py

@@ -107,8 +107,8 @@ def test_plugin_full_trip_uses_native_cells_and_original_image(
         "| --- | --- |\n"
         "| <native> & \\*value\\* | 12 |\n\n"
         "### Images in this sheet:\n\n"
-        "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n"
-        "*[Image OCR]\nRecognized\\_text\n[End OCR]*\n\n"
+        "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n"
+        "*[Image OCR]\\\nRecognized\\_text\\\n[End OCR]*\n\n"
         "## Other\n| Last |\n| --- |\n| Final |"
     )
     client.chat.completions.create.assert_called_once()
@@ -149,8 +149,8 @@ def test_cache_is_document_local_and_per_call_service_overrides_work() -> None:
     overridden = _convert(converter, data, ocr_service=second)
     again = _convert(converter, data)
 
-    assert default.count("*[Image OCR]  \nfirst  \n[End OCR]*") == 4
-    assert overridden.count("*[Image OCR]  \nsecond  \n[End OCR]*") == 4
+    assert default.count("*[Image OCR]\\\nfirst\\\n[End OCR]*") == 4
+    assert overridden.count("*[Image OCR]\\\nsecond\\\n[End OCR]*") == 4
     assert again == default
     assert first.extract_text.call_count == 2
     second.extract_text.assert_called_once()
@@ -172,7 +172,7 @@ def recognize(stream):
     assert calls == [_RED, _BLUE]
     assert result.count("[Image OCR]") == 2
     assert "Image at " not in result
-    assert result.count("*[Image OCR]  \nblue  \n[End OCR]*") == 2
+    assert result.count("*[Image OCR]\\\nblue\\\n[End OCR]*") == 2
 
 
 def test_mixed_anchor_recognition_keeps_legacy_openpyxl_order() -> None:
@@ -222,7 +222,7 @@ def recognize(stream):
         data,
     )
     assert calls == legacy_order
-    blocks = [f"*[Image OCR]  \n{label}  \n[End OCR]*" for label in legacy_order]
+    blocks = [f"*[Image OCR]\\\n{label}\\\n[End OCR]*" for label in legacy_order]
     assert "\n\n".join(blocks) in result
 
 
@@ -242,7 +242,7 @@ def test_ocr_html_is_escaped_and_existing_html_options_are_forwarded(
         "<br/>final<br/>[End OCR]</em></p>"
     ) in convert_html.call_args_list[1].args[0]
     assert (
-        "*[Image OCR]  \nA_B \\*literal\\*  \n<tag> & value  \nfinal  \n[End OCR]*"
+        "*[Image OCR]\\\nA_B \\*literal\\*\\\n<tag> & value\\\nfinal\\\n[End OCR]*"
         in result
     )
     assert "| Header_one | Unnamed: 1 |" in result

packages/markitdown/src/markitdown/converters/_markdownify.py

@@ -1,6 +1,7 @@
 import re
 import markdownify
 
+from bs4 import Comment
 from typing import Any, Optional
 from urllib.parse import quote, urlparse, urlunparse
 
@@ -22,11 +23,25 @@ def _quote_path_preserving_percent_encoded_octets(path: str) -> str:
     return "".join(parts)
 
 
+def _shows_content(nodes: Any) -> bool:
+    """Whether any of these sibling nodes shows text or an image."""
+    for node in nodes:
+        if isinstance(node, Comment):
+            continue
+        if isinstance(node, str):
+            if node.strip():
+                return True
+        elif node.name == "img" or node.get_text(strip=True) or node.find("img"):
+            return True
+    return False
+
+
 class _CustomMarkdownify(markdownify.MarkdownConverter):
     """
     A custom version of markdownify's MarkdownConverter. Changes include:
 
     - Altering the default heading style to use '#', '##', etc.
+    - Writing hard line breaks as backslashes, which survive whitespace normalization.
     - Removing javascript hyperlinks.
     - Truncating images with large data:uri sources.
     - Ensuring URIs are properly escaped, and do not conflict with Markdown syntax
@@ -143,6 +158,30 @@ def convert_img(
 
         return "![%s](%s%s)" % (alt, src, title_part)
 
+    def convert_br(
+        self,
+        el: Any,
+        text: str,
+        parent_tags: Any = None,
+        **kwargs,
+    ) -> str:
+        """Write a hard line break as a backslash rather than two trailing spaces.
+
+        MarkItDown strips trailing whitespace from every output line, which removes
+        markdownify's two-space hard breaks. A break with text or an image on both sides
+        of it in its parent element is written as a backslash, which survives. Other
+        breaks, and those in table cells, headings, and preformatted text, keep
+        markdownify's handling.
+        """
+        if (
+            isinstance(parent_tags, (set, frozenset))
+            and not parent_tags & {"_inline", "_noformat"}
+            and _shows_content(el.previous_siblings)
+            and _shows_content(el.next_siblings)
+        ):
+            return "\\\n" + text
+        return super().convert_br(el, text, parent_tags, **kwargs)  # type: ignore
+
     def convert_input(
         self,
         el: Any,

packages/markitdown/tests/test_docx.py

@@ -1485,5 +1485,13 @@ def test_convert_docx_with_style_missing_type(tmp_path):
     assert "# Abstract" in result.markdown
 
 
+def test_word_line_breaks_are_kept() -> None:
+    body = _paragraph(_text("Line one") + "<w:r><w:br/></w:r>" + _text("Line two"))
+
+    result = MarkItDown().convert_stream(_docx(body), stream_info=_INFO)
+
+    assert result.markdown == "Line one\\\nLine two"
+
+
 if __name__ == "__main__":
     raise SystemExit(pytest.main([__file__]))

packages/markitdown/tests/test_html.py

@@ -101,6 +101,50 @@ def test_html_href_does_not_quote_query_or_fragment() -> None:
     assert f"[example]({expected_href})" in markdown
 
 
+@pytest.mark.parametrize(
+    ("html", "expected"),
+    [
+        ("<p>Line one<br>Line two</p>", "Line one\\\nLine two"),
+        ("<p>Line one<br><br>Line two</p>", "Line one\\\n\\\nLine two"),
+        ("<p><strong>Bold<br>text</strong></p>", "**Bold\\\ntext**"),
+        ("<ul><li>Item<br>continued</li></ul>", "* Item\\\n  continued"),
+        ("<p>Before<br><img src='a.png' alt='x'></p>", "Before\\\n![x](a.png)"),
+    ],
+)
+def test_html_line_breaks_survive_whitespace_normalization(
+    html: str, expected: str
+) -> None:
+    assert expected in _convert_html(html)
+
+
+@pytest.mark.parametrize(
+    ("html", "expected"),
+    [
+        ("<p>Trailing<br></p>", "Trailing"),
+        ("<p><br>Leading</p>", "Leading"),
+        ("<p>First<u><br></u>Last</p>", "First\nLast"),
+        ("<p>Text<br><!-- note --></p>", "Text"),
+    ],
+)
+def test_html_line_breaks_at_element_edges_are_unchanged(
+    html: str, expected: str
+) -> None:
+    assert _convert_html(html) == expected
+
+
+def test_html_line_breaks_in_cells_headings_and_code_are_unchanged() -> None:
+    markdown = _convert_html(
+        "<table><tr><td>a<br>b</td></tr></table>"
+        "<h2>Head<br>ing</h2>"
+        "<pre>one<br>two</pre>"
+    )
+
+    assert "| a b |" in markdown
+    assert "## Head ing" in markdown
+    assert "```\none\ntwo\n```" in markdown
+    assert "\\" not in markdown
+
+
 def test_img_prefers_data_src_over_placeholder_data_uri() -> None:
     placeholder = (
         "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7"
@@ -287,9 +331,10 @@ def test_html_strikethrough_variants(tmp_path) -> None:
             # An empty element contributes nothing
             "Empty GH.",
             # A line break inside the element is kept, and the markup
-            # survives it because strikethrough may span a single newline
+            # survives it because strikethrough may span a single newline;
+            # a <br> stays a hard line break
             "Newline I~~J\nK~~L.",
-            "Break M~~N\nO~~P.",
+            "Break M~~N\\\nO~~P.",
         ]
     )
 

packages/markitdown/tests/test_rss.py

@@ -384,7 +384,7 @@ def _feed_with_body(field: str, payload: str, *, atom_type: str = "html") -> byt
         ("<b>nested</b> trailing", "**nested** trailing"),
         ("leading <b>nested</b> trailing", "leading **nested** trailing"),
         ("leading <b>nested</b>", "leading **nested**"),
-        ("First<br/>Second", "First  \nSecond"),
+        ("First<br/>Second", "First\\\nSecond"),
         ("<pre>line 1\n  line 2</pre>", "```\nline 1\n  line 2\n```"),
         (
             "\n <!--ignore--><?instruction ignore?><b>nested<!--ignore--></b> trailing",

packages/markitdown/tests/test_xlsx.py

@@ -196,7 +196,7 @@ class InheritedImages(_ImageConverter):
     [
         ("<strong>{}</strong>", "**one**\n\n**two**\n\n**three**"),
         ("{}", "one\n\ntwo\n\nthree"),
-        ("{}<br>line", "one  \nline\n\ntwo  \nline\n\nthree  \nline"),
+        ("{}<br>line", "one\\\nline\n\ntwo\\\nline\n\nthree\\\nline"),
         (
             "<p>{}</p><p>paragraph</p>",
             "one\n\nparagraph\n\ntwo\n\nparagraph\n\nthree\n\nparagraph",