fix: preserve Markdown hard line breaks

packages/markitdown/src/markitdown/_markitdown.py

@@ -98,6 +98,13 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes:
     return sample
 
 
+def _rstrip_preserving_hard_break(line: str) -> str:
+    """Retain Markdown input's two-space break syntax on nonblank lines."""
+    if line.endswith("  ") and line.strip():
+        return line
+    return line.rstrip()
+
+
 # Lower priority values are tried first.
 PRIORITY_SPECIFIC_FILE_FORMAT = (
     0.0  # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia
@@ -678,9 +685,21 @@ def _convert(
                         file_stream.seek(cur_pos)
 
                 if res is not None:
+                    preserve_hard_breaks = isinstance(
+                        converter, PlainTextConverter
+                    ) and (
+                        (stream_info.extension or "").lower() in (".md", ".markdown")
+                        or (stream_info.mimetype or "").split(";", 1)[0].strip().lower()
+                        in ("text/markdown", "application/markdown")
+                    )
                     # Normalize the content
                     res.text_content = "\n".join(
-                        [line.rstrip() for line in re.split(r"\r?\n", res.text_content)]
+                        [
+                            _rstrip_preserving_hard_break(line)
+                            if preserve_hard_breaks
+                            else line.rstrip()
+                            for line in re.split(r"\r?\n", res.text_content)
+                        ]
                     )
                     res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
                     return res

packages/markitdown/tests/test_cli_misc.py

@@ -20,6 +20,24 @@
 # This includes things like help messages, version numbers, and invalid flags.
 
 
+@pytest.mark.parametrize(
+    "extension,expected", [(".md", "First  \nSecond\n"), (".txt", "First\nSecond\n")]
+)
+def test_cli_preserves_hard_breaks_only_for_markdown(
+    tmp_path, extension, expected
+) -> None:
+    path = tmp_path / ("notes" + extension)
+    path.write_text("First  \nSecond", encoding="utf-8")
+    result = subprocess.run(
+        [sys.executable, "-m", "markitdown", str(path)],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert result.stdout == expected
+
+
 def test_version() -> None:
     result = subprocess.run(
         [sys.executable, "-m", "markitdown", "--version"],

packages/markitdown/tests/test_docx.py

@@ -534,6 +534,18 @@ def _docx(body: str = _BODY, *, embedded_style_map: Optional[str] = None) -> io.
     return stream
 
 
+@pytest.mark.parametrize(
+    "breaks,expected", [(1, "First\nSecond"), (2, "First\n\nSecond")]
+)
+def test_docx_keeps_upstream_break_normalization(breaks, expected) -> None:
+    body = _paragraph(_text("First") + "<w:r><w:br/></w:r>" * breaks + _text("Second"))
+    result = MarkItDown(enable_plugins=False).convert_stream(
+        _docx(body), stream_info=StreamInfo(extension=".docx")
+    )
+
+    assert result.markdown == expected
+
+
 class _ImageConverter(DocxConverter):
     def __init__(self, render: Callable[..., Optional[str]]):
         super().__init__()

packages/markitdown/tests/test_html.py

@@ -22,6 +22,28 @@ def _convert_html(html: str, **kwargs) -> str:
     return result.markdown
 
 
+@pytest.mark.parametrize(
+    "html,expected",
+    [
+        ("<p>First<br>Second</p>", "First\nSecond"),
+        ("<p>First<br><br>Second</p>", "First\n\nSecond"),
+        ("<p>First<u><br></u>Second</p>", "First\nSecond"),
+        ("<pre>First<br>Second</pre>", "```\nFirst\nSecond\n```"),
+    ],
+)
+def test_html_keeps_upstream_break_normalization(html, expected) -> None:
+    assert _convert_html(html) == expected
+
+
+def test_html_with_markdown_extension_keeps_upstream_normalization() -> None:
+    result = MarkItDown(enable_plugins=False).convert_stream(
+        io.BytesIO(b"<p>First<br>Second</p>"),
+        stream_info=StreamInfo(extension=".md", mimetype="text/html", charset="utf-8"),
+    )
+
+    assert result.markdown == "First\nSecond"
+
+
 @pytest.mark.parametrize(
     "whitespace", ["", " ", "  ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "]
 )

packages/markitdown/tests/test_module_misc.py

@@ -14,6 +14,7 @@
 
 import markitdown._uri_utils as uri_utils
 from markitdown import (
+    DocumentConverterResult,
     FileConversionException,
     MarkItDown,
     StreamInfo,
@@ -72,6 +73,100 @@ def validate_strings(result, expected_strings, exclude_strings=None):
             assert string not in text_content
 
 
+@pytest.mark.parametrize(
+    "stream_info",
+    [
+        StreamInfo(extension=".md"),
+        StreamInfo(extension=".markdown"),
+        StreamInfo(extension=".MD"),
+        StreamInfo(mimetype="text/markdown"),
+        StreamInfo(mimetype="application/markdown"),
+        StreamInfo(mimetype="Text/Markdown; charset=utf-8"),
+        StreamInfo(extension=".md", mimetype="text/plain"),
+    ],
+)
+def test_markdown_input_keeps_hard_breaks(stream_info: StreamInfo) -> None:
+    source = "First  \nSecond   \r\nThird\n"
+    result = MarkItDown(enable_plugins=False).convert_stream(
+        io.BytesIO(source.encode()),
+        stream_info=stream_info.copy_and_update(charset="utf-8"),
+    )
+
+    assert result.markdown == "First  \nSecond   \nThird\n"
+
+
+@pytest.mark.parametrize(
+    "extension,mimetype",
+    [(".txt", "text/plain"), (".text", "text/plain"), (None, "text/plain")],
+)
+def test_non_markdown_text_keeps_upstream_normalization(extension, mimetype) -> None:
+    result = MarkItDown(enable_plugins=False).convert_stream(
+        io.BytesIO(b"First  \nSecond   \r\nThird \t\n\n\nLast\n"),
+        stream_info=StreamInfo(extension=extension, mimetype=mimetype, charset="utf-8"),
+    )
+
+    assert result.markdown == "First\nSecond\nThird\n\nLast\n"
+
+
+@pytest.mark.parametrize(
+    "source,expected",
+    [
+        ("First\\\nSecond\n", "First\\\nSecond\n"),
+        ("First \nSecond\n", "First\nSecond\n"),
+        ("First\t\nSecond\n", "First\nSecond\n"),
+        ("First  \t\nSecond\n", "First\nSecond\n"),
+        ("First\n   \n\n\nSecond\n", "First\n\nSecond\n"),
+        ("", ""),
+    ],
+)
+def test_markdown_other_normalization_is_unchanged(source, expected) -> None:
+    result = MarkItDown(enable_plugins=False).convert_stream(
+        io.BytesIO(source.encode()),
+        stream_info=StreamInfo(extension=".md", charset="utf-8"),
+    )
+
+    assert result.markdown == expected
+
+
+@pytest.mark.parametrize("entry_point", ["local", "file_uri", "data_uri", "response"])
+def test_markdown_hard_breaks_across_entry_points(tmp_path, entry_point) -> None:
+    source = b"First  \nSecond\n"
+    markitdown = MarkItDown(enable_plugins=False)
+    path = tmp_path / "notes.md"
+    path.write_bytes(source)
+
+    if entry_point == "local":
+        result = markitdown.convert(path)
+    elif entry_point == "file_uri":
+        result = markitdown.convert_uri(path.as_uri())
+    elif entry_point == "data_uri":
+        result = markitdown.convert_uri("data:text/markdown,First%20%20%0ASecond%0A")
+    else:
+        response = requests.Response()
+        response.status_code = 200
+        response.url = "https://example.com/notes"
+        response.headers["Content-Type"] = "text/markdown; charset=utf-8"
+        response.raw = io.BytesIO(source)
+        result = markitdown.convert_response(response)
+
+    assert result.markdown == source.decode()
+
+
+def test_custom_converter_keeps_upstream_normalization_for_markdown_input() -> None:
+    converter = MagicMock()
+    converter.accepts.return_value = True
+    converter.convert.return_value = DocumentConverterResult(markdown="First  \nSecond")
+    markitdown = MarkItDown(enable_builtins=False, enable_plugins=False)
+    markitdown.register_converter(converter)
+
+    result = markitdown.convert_stream(
+        io.BytesIO(b"First  \nSecond"),
+        stream_info=StreamInfo(extension=".md", charset="utf-8"),
+    )
+
+    assert result.markdown == "First\nSecond"
+
+
 def test_stream_info_operations() -> None:
     """Test operations performed on StreamInfo objects."""