From 3faa1f01fe85619d69e8b53c043cf6831383a3fe Mon Sep 17 00:00:00 2001 From: MaciejZet Date: Thu, 8 Oct 2026 22:21:25 +0200 Subject: [PATCH 1/2] fix: preserve Markdown hard line breaks --- .../markitdown/src/markitdown/_markitdown.py | 17 +- packages/markitdown/tests/test_docx.py | 2 +- packages/markitdown/tests/test_html.py | 4 +- .../tests/test_markdown_whitespace.py | 200 ++++++++++++++++++ 4 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 packages/markitdown/tests/test_markdown_whitespace.py diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 8e2fe6a37c..1726e6882b 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -98,6 +98,18 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes: return sample +def _rstrip_preserving_hard_break(line: str) -> str: + """Drop insignificant trailing whitespace, but keep a CommonMark hard break. + + Two or more trailing spaces before a newline are a hard line break. A line + that contains only whitespace is not one. Collapsing runs of blank lines is + unchanged. + """ + if line.endswith(" ") and line.strip(): + return line + return line.rstrip() + + # Lower priority values are tried first. PRIORITY_SPECIFIC_FILE_FORMAT = ( 0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia @@ -680,7 +692,10 @@ def _convert( if res is not None: # Normalize the content res.text_content = "\n".join( - [line.rstrip() for line in re.split(r"\r?\n", res.text_content)] + [ + _rstrip_preserving_hard_break(line) + for line in re.split(r"\r?\n", res.text_content) + ] ) res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content) return res diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py index 4fa67684e7..bea4de59e5 100644 --- a/packages/markitdown/tests/test_docx.py +++ b/packages/markitdown/tests/test_docx.py @@ -1134,7 +1134,7 @@ def test_docx_underlined_text_is_preserved(tmp_path) -> None: (' ', "First Last"), ("", "First Last"), (" ", "First\u00a0Last"), - ("", "First\nLast"), + ("", "First \nLast"), ], ) def test_docx_underlined_whitespace_is_preserved( diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py index 94cece47b9..c5b3d90a96 100644 --- a/packages/markitdown/tests/test_html.py +++ b/packages/markitdown/tests/test_html.py @@ -38,7 +38,7 @@ def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None: (" ", "First Last"), ("\t", "First Last"), (" ", "First\u00a0Last"), - ("
", "First\nLast"), + ("
", "First \nLast"), ("word", "FirstwordLast"), (" word ", "First word Last"), ], @@ -289,7 +289,7 @@ def test_html_strikethrough_variants(tmp_path) -> None: # A line break inside the element is kept, and the markup # survives it because strikethrough may span a single newline "Newline I~~J\nK~~L.", - "Break M~~N\nO~~P.", + "Break M~~N \nO~~P.", ] ) diff --git a/packages/markitdown/tests/test_markdown_whitespace.py b/packages/markitdown/tests/test_markdown_whitespace.py new file mode 100644 index 0000000000..45351e3a89 --- /dev/null +++ b/packages/markitdown/tests/test_markdown_whitespace.py @@ -0,0 +1,200 @@ +"""Trailing spaces that encode a hard break must survive normalization. + +``MarkItDown._convert`` strips trailing whitespace from every converter's +Markdown. Two or more trailing spaces are a CommonMark hard line break, so +those are kept. A single trailing space, a trailing tab, and a line that is +only whitespace are still removed. Runs of blank lines are still collapsed. +""" + +import io +from typing import Any, BinaryIO + +import pytest + +from markitdown import ( + DocumentConverter, + DocumentConverterResult, + MarkItDown, + StreamInfo, +) +from markitdown._markitdown import _rstrip_preserving_hard_break + + +class _PassthroughConverter(DocumentConverter): + def accepts( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, + ) -> bool: + return (stream_info.extension or "").lower() == ".fixture" + + def convert( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, + ) -> DocumentConverterResult: + return DocumentConverterResult(markdown=file_stream.read().decode("utf-8")) + + +def _convert(source: str) -> str: + markitdown = MarkItDown(enable_builtins=False, enable_plugins=False) + markitdown.register_converter(_PassthroughConverter(), priority=-1.0) + return markitdown.convert_stream( + io.BytesIO(source.encode("utf-8")), + stream_info=StreamInfo(extension=".fixture", charset="utf-8"), + ).markdown + + +def _convert_builtin(source: str, *, extension: str, mimetype: str) -> str: + return ( + MarkItDown(enable_plugins=False) + .convert_stream( + io.BytesIO(source.encode("utf-8")), + stream_info=StreamInfo( + extension=extension, mimetype=mimetype, charset="utf-8" + ), + ) + .markdown + ) + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + ("First line ", "First line "), + ("First line ", "First line "), + ("First line ", "First line"), + ("First line\t", "First line"), + ("First line \t", "First line"), + ("First line \t", "First line"), + ("First line\t ", "First line\t "), + (" ", ""), + ("", ""), + ("First line\u00a0\u00a0", "First line"), + ], +) +def test_rstrip_keeps_only_a_hard_break(line: str, expected: str) -> None: + assert _rstrip_preserving_hard_break(line) == expected + + +def test_hard_line_breaks_survive_normalization() -> None: + source = "First line \nSecond line\n" + + assert _convert(source) == source + + +def test_single_trailing_space_is_still_trimmed() -> None: + source = "First line \nSecond line\n" + + assert _convert(source) == "First line\nSecond line\n" + + +def test_hard_line_breaks_with_more_than_two_spaces_are_preserved() -> None: + source = "First line \nSecond line\n" + + assert _convert(source) == source + + +def test_several_hard_breaks_in_one_document_are_preserved() -> None: + source = "One \nTwo \nThree\n" + + assert _convert(source) == source + + +def test_crlf_hard_break_becomes_lf_and_keeps_the_spaces() -> None: + source = "First line \r\nSecond line\r\n" + + assert _convert(source) == "First line \nSecond line\n" + + +def test_backslash_hard_break_is_unchanged() -> None: + source = "First line\\\nSecond line\n" + + assert _convert(source) == source + + +def test_trailing_tab_is_still_trimmed() -> None: + source = "First line\t\nSecond line\n" + + assert _convert(source) == "First line\nSecond line\n" + + +def test_spaces_followed_by_a_tab_are_not_a_hard_break() -> None: + source = "First line \t\nSecond line\n" + + assert _convert(source) == "First line\nSecond line\n" + + +def test_whitespace_only_line_is_still_trimmed() -> None: + source = "First line\n \nSecond line\n" + + assert _convert(source) == "First line\n\nSecond line\n" + + +def test_blank_line_runs_are_still_collapsed() -> None: + source = "First line\n\n\n\nSecond line\n" + + assert _convert(source) == "First line\n\nSecond line\n" + + +def test_hard_break_survives_beside_a_collapsed_blank_run() -> None: + source = "First line \n\n\n\nSecond line \nThird\n" + + assert _convert(source) == "First line \n\nSecond line \nThird\n" + + +def test_two_trailing_spaces_inside_a_fence_are_kept() -> None: + source = "```\ncode \n```\n" + + assert _convert(source) == source + + +def test_one_trailing_space_inside_a_fence_is_still_trimmed() -> None: + source = "```\ncode \n```\n" + + assert _convert(source) == "```\ncode\n```\n" + + +def test_empty_output_stays_empty() -> None: + assert _convert("") == "" + + +def test_hard_break_without_a_final_newline_is_kept() -> None: + source = "First line " + + assert _convert(source) == source + + +def test_markdown_files_keep_hard_line_breaks() -> None: + source = "First line \nSecond line\n" + + assert _convert_builtin(source, extension=".md", mimetype="text/markdown") == source + + +def test_markdown_crlf_file_keeps_the_hard_break() -> None: + source = "- item one \r\n continuation\r\n" + + assert ( + _convert_builtin(source, extension=".markdown", mimetype="text/markdown") + == "- item one \n continuation\n" + ) + + +def test_html_br_keeps_a_hard_line_break() -> None: + html = "

First
Second

Third
Fourth

" + + assert ( + _convert_builtin(html, extension=".html", mimetype="text/html") + == "First \nSecond\n\nThird \nFourth" + ) + + +def test_html_paragraphs_do_not_gain_trailing_spaces() -> None: + html = "

First

Second

" + + assert ( + _convert_builtin(html, extension=".html", mimetype="text/html") + == "First\n\nSecond" + ) From 75faa08aabbfc800142a4721d7ab928d29614ab0 Mon Sep 17 00:00:00 2001 From: MaciejZet Date: Mon, 12 Oct 2026 16:36:28 +0200 Subject: [PATCH 2/2] fix: limit hard break preservation to Markdown input --- .../markitdown/src/markitdown/_markitdown.py | 16 +- packages/markitdown/tests/test_cli_misc.py | 18 ++ packages/markitdown/tests/test_docx.py | 14 +- packages/markitdown/tests/test_html.py | 26 ++- .../tests/test_markdown_whitespace.py | 200 ------------------ packages/markitdown/tests/test_module_misc.py | 95 +++++++++ 6 files changed, 160 insertions(+), 209 deletions(-) delete mode 100644 packages/markitdown/tests/test_markdown_whitespace.py diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 1726e6882b..8e5f13ee79 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -99,12 +99,7 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes: def _rstrip_preserving_hard_break(line: str) -> str: - """Drop insignificant trailing whitespace, but keep a CommonMark hard break. - - Two or more trailing spaces before a newline are a hard line break. A line - that contains only whitespace is not one. Collapsing runs of blank lines is - unchanged. - """ + """Retain Markdown input's two-space break syntax on nonblank lines.""" if line.endswith(" ") and line.strip(): return line return line.rstrip() @@ -690,10 +685,19 @@ def _convert( file_stream.seek(cur_pos) if res is not None: + preserve_hard_breaks = isinstance( + converter, PlainTextConverter + ) and ( + (stream_info.extension or "").lower() in (".md", ".markdown") + or (stream_info.mimetype or "").split(";", 1)[0].strip().lower() + in ("text/markdown", "application/markdown") + ) # Normalize the content res.text_content = "\n".join( [ _rstrip_preserving_hard_break(line) + if preserve_hard_breaks + else line.rstrip() for line in re.split(r"\r?\n", res.text_content) ] ) diff --git a/packages/markitdown/tests/test_cli_misc.py b/packages/markitdown/tests/test_cli_misc.py index 5e42b16b7c..e4c2dc3fcf 100644 --- a/packages/markitdown/tests/test_cli_misc.py +++ b/packages/markitdown/tests/test_cli_misc.py @@ -20,6 +20,24 @@ # This includes things like help messages, version numbers, and invalid flags. +@pytest.mark.parametrize( + "extension,expected", [(".md", "First \nSecond\n"), (".txt", "First\nSecond\n")] +) +def test_cli_preserves_hard_breaks_only_for_markdown( + tmp_path, extension, expected +) -> None: + path = tmp_path / ("notes" + extension) + path.write_text("First \nSecond", encoding="utf-8") + result = subprocess.run( + [sys.executable, "-m", "markitdown", str(path)], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == expected + + def test_version() -> None: result = subprocess.run( [sys.executable, "-m", "markitdown", "--version"], diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py index bea4de59e5..57b40b54b9 100644 --- a/packages/markitdown/tests/test_docx.py +++ b/packages/markitdown/tests/test_docx.py @@ -534,6 +534,18 @@ def _docx(body: str = _BODY, *, embedded_style_map: Optional[str] = None) -> io. return stream +@pytest.mark.parametrize( + "breaks,expected", [(1, "First\nSecond"), (2, "First\n\nSecond")] +) +def test_docx_keeps_upstream_break_normalization(breaks, expected) -> None: + body = _paragraph(_text("First") + "" * breaks + _text("Second")) + result = MarkItDown(enable_plugins=False).convert_stream( + _docx(body), stream_info=StreamInfo(extension=".docx") + ) + + assert result.markdown == expected + + class _ImageConverter(DocxConverter): def __init__(self, render: Callable[..., Optional[str]]): super().__init__() @@ -1134,7 +1146,7 @@ def test_docx_underlined_text_is_preserved(tmp_path) -> None: (' ', "First Last"), ("", "First Last"), (" ", "First\u00a0Last"), - ("", "First \nLast"), + ("", "First\nLast"), ], ) def test_docx_underlined_whitespace_is_preserved( diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py index c5b3d90a96..123fca8ef1 100644 --- a/packages/markitdown/tests/test_html.py +++ b/packages/markitdown/tests/test_html.py @@ -22,6 +22,28 @@ def _convert_html(html: str, **kwargs) -> str: return result.markdown +@pytest.mark.parametrize( + "html,expected", + [ + ("

First
Second

", "First\nSecond"), + ("

First

Second

", "First\n\nSecond"), + ("

First
Second

", "First\nSecond"), + ("
First
Second
", "```\nFirst\nSecond\n```"), + ], +) +def test_html_keeps_upstream_break_normalization(html, expected) -> None: + assert _convert_html(html) == expected + + +def test_html_with_markdown_extension_keeps_upstream_normalization() -> None: + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(b"

First
Second

"), + stream_info=StreamInfo(extension=".md", mimetype="text/html", charset="utf-8"), + ) + + assert result.markdown == "First\nSecond" + + @pytest.mark.parametrize( "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "] ) @@ -38,7 +60,7 @@ def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None: (" ", "First Last"), ("\t", "First Last"), (" ", "First\u00a0Last"), - ("
", "First \nLast"), + ("
", "First\nLast"), ("word", "FirstwordLast"), (" word ", "First word Last"), ], @@ -289,7 +311,7 @@ def test_html_strikethrough_variants(tmp_path) -> None: # A line break inside the element is kept, and the markup # survives it because strikethrough may span a single newline "Newline I~~J\nK~~L.", - "Break M~~N \nO~~P.", + "Break M~~N\nO~~P.", ] ) diff --git a/packages/markitdown/tests/test_markdown_whitespace.py b/packages/markitdown/tests/test_markdown_whitespace.py deleted file mode 100644 index 45351e3a89..0000000000 --- a/packages/markitdown/tests/test_markdown_whitespace.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Trailing spaces that encode a hard break must survive normalization. - -``MarkItDown._convert`` strips trailing whitespace from every converter's -Markdown. Two or more trailing spaces are a CommonMark hard line break, so -those are kept. A single trailing space, a trailing tab, and a line that is -only whitespace are still removed. Runs of blank lines are still collapsed. -""" - -import io -from typing import Any, BinaryIO - -import pytest - -from markitdown import ( - DocumentConverter, - DocumentConverterResult, - MarkItDown, - StreamInfo, -) -from markitdown._markitdown import _rstrip_preserving_hard_break - - -class _PassthroughConverter(DocumentConverter): - def accepts( - self, - file_stream: BinaryIO, - stream_info: StreamInfo, - **kwargs: Any, - ) -> bool: - return (stream_info.extension or "").lower() == ".fixture" - - def convert( - self, - file_stream: BinaryIO, - stream_info: StreamInfo, - **kwargs: Any, - ) -> DocumentConverterResult: - return DocumentConverterResult(markdown=file_stream.read().decode("utf-8")) - - -def _convert(source: str) -> str: - markitdown = MarkItDown(enable_builtins=False, enable_plugins=False) - markitdown.register_converter(_PassthroughConverter(), priority=-1.0) - return markitdown.convert_stream( - io.BytesIO(source.encode("utf-8")), - stream_info=StreamInfo(extension=".fixture", charset="utf-8"), - ).markdown - - -def _convert_builtin(source: str, *, extension: str, mimetype: str) -> str: - return ( - MarkItDown(enable_plugins=False) - .convert_stream( - io.BytesIO(source.encode("utf-8")), - stream_info=StreamInfo( - extension=extension, mimetype=mimetype, charset="utf-8" - ), - ) - .markdown - ) - - -@pytest.mark.parametrize( - ("line", "expected"), - [ - ("First line ", "First line "), - ("First line ", "First line "), - ("First line ", "First line"), - ("First line\t", "First line"), - ("First line \t", "First line"), - ("First line \t", "First line"), - ("First line\t ", "First line\t "), - (" ", ""), - ("", ""), - ("First line\u00a0\u00a0", "First line"), - ], -) -def test_rstrip_keeps_only_a_hard_break(line: str, expected: str) -> None: - assert _rstrip_preserving_hard_break(line) == expected - - -def test_hard_line_breaks_survive_normalization() -> None: - source = "First line \nSecond line\n" - - assert _convert(source) == source - - -def test_single_trailing_space_is_still_trimmed() -> None: - source = "First line \nSecond line\n" - - assert _convert(source) == "First line\nSecond line\n" - - -def test_hard_line_breaks_with_more_than_two_spaces_are_preserved() -> None: - source = "First line \nSecond line\n" - - assert _convert(source) == source - - -def test_several_hard_breaks_in_one_document_are_preserved() -> None: - source = "One \nTwo \nThree\n" - - assert _convert(source) == source - - -def test_crlf_hard_break_becomes_lf_and_keeps_the_spaces() -> None: - source = "First line \r\nSecond line\r\n" - - assert _convert(source) == "First line \nSecond line\n" - - -def test_backslash_hard_break_is_unchanged() -> None: - source = "First line\\\nSecond line\n" - - assert _convert(source) == source - - -def test_trailing_tab_is_still_trimmed() -> None: - source = "First line\t\nSecond line\n" - - assert _convert(source) == "First line\nSecond line\n" - - -def test_spaces_followed_by_a_tab_are_not_a_hard_break() -> None: - source = "First line \t\nSecond line\n" - - assert _convert(source) == "First line\nSecond line\n" - - -def test_whitespace_only_line_is_still_trimmed() -> None: - source = "First line\n \nSecond line\n" - - assert _convert(source) == "First line\n\nSecond line\n" - - -def test_blank_line_runs_are_still_collapsed() -> None: - source = "First line\n\n\n\nSecond line\n" - - assert _convert(source) == "First line\n\nSecond line\n" - - -def test_hard_break_survives_beside_a_collapsed_blank_run() -> None: - source = "First line \n\n\n\nSecond line \nThird\n" - - assert _convert(source) == "First line \n\nSecond line \nThird\n" - - -def test_two_trailing_spaces_inside_a_fence_are_kept() -> None: - source = "```\ncode \n```\n" - - assert _convert(source) == source - - -def test_one_trailing_space_inside_a_fence_is_still_trimmed() -> None: - source = "```\ncode \n```\n" - - assert _convert(source) == "```\ncode\n```\n" - - -def test_empty_output_stays_empty() -> None: - assert _convert("") == "" - - -def test_hard_break_without_a_final_newline_is_kept() -> None: - source = "First line " - - assert _convert(source) == source - - -def test_markdown_files_keep_hard_line_breaks() -> None: - source = "First line \nSecond line\n" - - assert _convert_builtin(source, extension=".md", mimetype="text/markdown") == source - - -def test_markdown_crlf_file_keeps_the_hard_break() -> None: - source = "- item one \r\n continuation\r\n" - - assert ( - _convert_builtin(source, extension=".markdown", mimetype="text/markdown") - == "- item one \n continuation\n" - ) - - -def test_html_br_keeps_a_hard_line_break() -> None: - html = "

First
Second

Third
Fourth

" - - assert ( - _convert_builtin(html, extension=".html", mimetype="text/html") - == "First \nSecond\n\nThird \nFourth" - ) - - -def test_html_paragraphs_do_not_gain_trailing_spaces() -> None: - html = "

First

Second

" - - assert ( - _convert_builtin(html, extension=".html", mimetype="text/html") - == "First\n\nSecond" - ) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d6c9be59b..500dfabac9 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -14,6 +14,7 @@ import markitdown._uri_utils as uri_utils from markitdown import ( + DocumentConverterResult, FileConversionException, MarkItDown, StreamInfo, @@ -72,6 +73,100 @@ def validate_strings(result, expected_strings, exclude_strings=None): assert string not in text_content +@pytest.mark.parametrize( + "stream_info", + [ + StreamInfo(extension=".md"), + StreamInfo(extension=".markdown"), + StreamInfo(extension=".MD"), + StreamInfo(mimetype="text/markdown"), + StreamInfo(mimetype="application/markdown"), + StreamInfo(mimetype="Text/Markdown; charset=utf-8"), + StreamInfo(extension=".md", mimetype="text/plain"), + ], +) +def test_markdown_input_keeps_hard_breaks(stream_info: StreamInfo) -> None: + source = "First \nSecond \r\nThird\n" + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(source.encode()), + stream_info=stream_info.copy_and_update(charset="utf-8"), + ) + + assert result.markdown == "First \nSecond \nThird\n" + + +@pytest.mark.parametrize( + "extension,mimetype", + [(".txt", "text/plain"), (".text", "text/plain"), (None, "text/plain")], +) +def test_non_markdown_text_keeps_upstream_normalization(extension, mimetype) -> None: + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(b"First \nSecond \r\nThird \t\n\n\nLast\n"), + stream_info=StreamInfo(extension=extension, mimetype=mimetype, charset="utf-8"), + ) + + assert result.markdown == "First\nSecond\nThird\n\nLast\n" + + +@pytest.mark.parametrize( + "source,expected", + [ + ("First\\\nSecond\n", "First\\\nSecond\n"), + ("First \nSecond\n", "First\nSecond\n"), + ("First\t\nSecond\n", "First\nSecond\n"), + ("First \t\nSecond\n", "First\nSecond\n"), + ("First\n \n\n\nSecond\n", "First\n\nSecond\n"), + ("", ""), + ], +) +def test_markdown_other_normalization_is_unchanged(source, expected) -> None: + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(source.encode()), + stream_info=StreamInfo(extension=".md", charset="utf-8"), + ) + + assert result.markdown == expected + + +@pytest.mark.parametrize("entry_point", ["local", "file_uri", "data_uri", "response"]) +def test_markdown_hard_breaks_across_entry_points(tmp_path, entry_point) -> None: + source = b"First \nSecond\n" + markitdown = MarkItDown(enable_plugins=False) + path = tmp_path / "notes.md" + path.write_bytes(source) + + if entry_point == "local": + result = markitdown.convert(path) + elif entry_point == "file_uri": + result = markitdown.convert_uri(path.as_uri()) + elif entry_point == "data_uri": + result = markitdown.convert_uri("data:text/markdown,First%20%20%0ASecond%0A") + else: + response = requests.Response() + response.status_code = 200 + response.url = "https://example.com/notes" + response.headers["Content-Type"] = "text/markdown; charset=utf-8" + response.raw = io.BytesIO(source) + result = markitdown.convert_response(response) + + assert result.markdown == source.decode() + + +def test_custom_converter_keeps_upstream_normalization_for_markdown_input() -> None: + converter = MagicMock() + converter.accepts.return_value = True + converter.convert.return_value = DocumentConverterResult(markdown="First \nSecond") + markitdown = MarkItDown(enable_builtins=False, enable_plugins=False) + markitdown.register_converter(converter) + + result = markitdown.convert_stream( + io.BytesIO(b"First \nSecond"), + stream_info=StreamInfo(extension=".md", charset="utf-8"), + ) + + assert result.markdown == "First\nSecond" + + def test_stream_info_operations() -> None: """Test operations performed on StreamInfo objects."""