diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py
index 8e2fe6a37c..8e5f13ee79 100644
--- a/packages/markitdown/src/markitdown/_markitdown.py
+++ b/packages/markitdown/src/markitdown/_markitdown.py
@@ -98,6 +98,13 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes:
return sample
+def _rstrip_preserving_hard_break(line: str) -> str:
+ """Retain Markdown input's two-space break syntax on nonblank lines."""
+ if line.endswith(" ") and line.strip():
+ return line
+ return line.rstrip()
+
+
# Lower priority values are tried first.
PRIORITY_SPECIFIC_FILE_FORMAT = (
0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia
@@ -678,9 +685,21 @@ def _convert(
file_stream.seek(cur_pos)
if res is not None:
+ preserve_hard_breaks = isinstance(
+ converter, PlainTextConverter
+ ) and (
+ (stream_info.extension or "").lower() in (".md", ".markdown")
+ or (stream_info.mimetype or "").split(";", 1)[0].strip().lower()
+ in ("text/markdown", "application/markdown")
+ )
# Normalize the content
res.text_content = "\n".join(
- [line.rstrip() for line in re.split(r"\r?\n", res.text_content)]
+ [
+ _rstrip_preserving_hard_break(line)
+ if preserve_hard_breaks
+ else line.rstrip()
+ for line in re.split(r"\r?\n", res.text_content)
+ ]
)
res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
return res
diff --git a/packages/markitdown/tests/test_cli_misc.py b/packages/markitdown/tests/test_cli_misc.py
index 5e42b16b7c..e4c2dc3fcf 100644
--- a/packages/markitdown/tests/test_cli_misc.py
+++ b/packages/markitdown/tests/test_cli_misc.py
@@ -20,6 +20,24 @@
# This includes things like help messages, version numbers, and invalid flags.
+@pytest.mark.parametrize(
+ "extension,expected", [(".md", "First \nSecond\n"), (".txt", "First\nSecond\n")]
+)
+def test_cli_preserves_hard_breaks_only_for_markdown(
+ tmp_path, extension, expected
+) -> None:
+ path = tmp_path / ("notes" + extension)
+ path.write_text("First \nSecond", encoding="utf-8")
+ result = subprocess.run(
+ [sys.executable, "-m", "markitdown", str(path)],
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert result.stdout == expected
+
+
def test_version() -> None:
result = subprocess.run(
[sys.executable, "-m", "markitdown", "--version"],
diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py
index 4fa67684e7..57b40b54b9 100644
--- a/packages/markitdown/tests/test_docx.py
+++ b/packages/markitdown/tests/test_docx.py
@@ -534,6 +534,18 @@ def _docx(body: str = _BODY, *, embedded_style_map: Optional[str] = None) -> io.
return stream
+@pytest.mark.parametrize(
+ "breaks,expected", [(1, "First\nSecond"), (2, "First\n\nSecond")]
+)
+def test_docx_keeps_upstream_break_normalization(breaks, expected) -> None:
+ body = _paragraph(_text("First") + "" * breaks + _text("Second"))
+ result = MarkItDown(enable_plugins=False).convert_stream(
+ _docx(body), stream_info=StreamInfo(extension=".docx")
+ )
+
+ assert result.markdown == expected
+
+
class _ImageConverter(DocxConverter):
def __init__(self, render: Callable[..., Optional[str]]):
super().__init__()
diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py
index 94cece47b9..123fca8ef1 100644
--- a/packages/markitdown/tests/test_html.py
+++ b/packages/markitdown/tests/test_html.py
@@ -22,6 +22,28 @@ def _convert_html(html: str, **kwargs) -> str:
return result.markdown
+@pytest.mark.parametrize(
+ "html,expected",
+ [
+ ("
First
Second
", "First\nSecond"),
+ ("First
Second
", "First\n\nSecond"),
+ ("First
Second
", "First\nSecond"),
+ ("First
Second
", "```\nFirst\nSecond\n```"),
+ ],
+)
+def test_html_keeps_upstream_break_normalization(html, expected) -> None:
+ assert _convert_html(html) == expected
+
+
+def test_html_with_markdown_extension_keeps_upstream_normalization() -> None:
+ result = MarkItDown(enable_plugins=False).convert_stream(
+ io.BytesIO(b"First
Second
"),
+ stream_info=StreamInfo(extension=".md", mimetype="text/html", charset="utf-8"),
+ )
+
+ assert result.markdown == "First\nSecond"
+
+
@pytest.mark.parametrize(
"whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "]
)
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index 4d6c9be59b..500dfabac9 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -14,6 +14,7 @@
import markitdown._uri_utils as uri_utils
from markitdown import (
+ DocumentConverterResult,
FileConversionException,
MarkItDown,
StreamInfo,
@@ -72,6 +73,100 @@ def validate_strings(result, expected_strings, exclude_strings=None):
assert string not in text_content
+@pytest.mark.parametrize(
+ "stream_info",
+ [
+ StreamInfo(extension=".md"),
+ StreamInfo(extension=".markdown"),
+ StreamInfo(extension=".MD"),
+ StreamInfo(mimetype="text/markdown"),
+ StreamInfo(mimetype="application/markdown"),
+ StreamInfo(mimetype="Text/Markdown; charset=utf-8"),
+ StreamInfo(extension=".md", mimetype="text/plain"),
+ ],
+)
+def test_markdown_input_keeps_hard_breaks(stream_info: StreamInfo) -> None:
+ source = "First \nSecond \r\nThird\n"
+ result = MarkItDown(enable_plugins=False).convert_stream(
+ io.BytesIO(source.encode()),
+ stream_info=stream_info.copy_and_update(charset="utf-8"),
+ )
+
+ assert result.markdown == "First \nSecond \nThird\n"
+
+
+@pytest.mark.parametrize(
+ "extension,mimetype",
+ [(".txt", "text/plain"), (".text", "text/plain"), (None, "text/plain")],
+)
+def test_non_markdown_text_keeps_upstream_normalization(extension, mimetype) -> None:
+ result = MarkItDown(enable_plugins=False).convert_stream(
+ io.BytesIO(b"First \nSecond \r\nThird \t\n\n\nLast\n"),
+ stream_info=StreamInfo(extension=extension, mimetype=mimetype, charset="utf-8"),
+ )
+
+ assert result.markdown == "First\nSecond\nThird\n\nLast\n"
+
+
+@pytest.mark.parametrize(
+ "source,expected",
+ [
+ ("First\\\nSecond\n", "First\\\nSecond\n"),
+ ("First \nSecond\n", "First\nSecond\n"),
+ ("First\t\nSecond\n", "First\nSecond\n"),
+ ("First \t\nSecond\n", "First\nSecond\n"),
+ ("First\n \n\n\nSecond\n", "First\n\nSecond\n"),
+ ("", ""),
+ ],
+)
+def test_markdown_other_normalization_is_unchanged(source, expected) -> None:
+ result = MarkItDown(enable_plugins=False).convert_stream(
+ io.BytesIO(source.encode()),
+ stream_info=StreamInfo(extension=".md", charset="utf-8"),
+ )
+
+ assert result.markdown == expected
+
+
+@pytest.mark.parametrize("entry_point", ["local", "file_uri", "data_uri", "response"])
+def test_markdown_hard_breaks_across_entry_points(tmp_path, entry_point) -> None:
+ source = b"First \nSecond\n"
+ markitdown = MarkItDown(enable_plugins=False)
+ path = tmp_path / "notes.md"
+ path.write_bytes(source)
+
+ if entry_point == "local":
+ result = markitdown.convert(path)
+ elif entry_point == "file_uri":
+ result = markitdown.convert_uri(path.as_uri())
+ elif entry_point == "data_uri":
+ result = markitdown.convert_uri("data:text/markdown,First%20%20%0ASecond%0A")
+ else:
+ response = requests.Response()
+ response.status_code = 200
+ response.url = "https://example.com/notes"
+ response.headers["Content-Type"] = "text/markdown; charset=utf-8"
+ response.raw = io.BytesIO(source)
+ result = markitdown.convert_response(response)
+
+ assert result.markdown == source.decode()
+
+
+def test_custom_converter_keeps_upstream_normalization_for_markdown_input() -> None:
+ converter = MagicMock()
+ converter.accepts.return_value = True
+ converter.convert.return_value = DocumentConverterResult(markdown="First \nSecond")
+ markitdown = MarkItDown(enable_builtins=False, enable_plugins=False)
+ markitdown.register_converter(converter)
+
+ result = markitdown.convert_stream(
+ io.BytesIO(b"First \nSecond"),
+ stream_info=StreamInfo(extension=".md", charset="utf-8"),
+ )
+
+ assert result.markdown == "First\nSecond"
+
+
def test_stream_info_operations() -> None:
"""Test operations performed on StreamInfo objects."""