diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 8e2fe6a37c..8e5f13ee79 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -98,6 +98,13 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes: return sample +def _rstrip_preserving_hard_break(line: str) -> str: + """Retain Markdown input's two-space break syntax on nonblank lines.""" + if line.endswith(" ") and line.strip(): + return line + return line.rstrip() + + # Lower priority values are tried first. PRIORITY_SPECIFIC_FILE_FORMAT = ( 0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia @@ -678,9 +685,21 @@ def _convert( file_stream.seek(cur_pos) if res is not None: + preserve_hard_breaks = isinstance( + converter, PlainTextConverter + ) and ( + (stream_info.extension or "").lower() in (".md", ".markdown") + or (stream_info.mimetype or "").split(";", 1)[0].strip().lower() + in ("text/markdown", "application/markdown") + ) # Normalize the content res.text_content = "\n".join( - [line.rstrip() for line in re.split(r"\r?\n", res.text_content)] + [ + _rstrip_preserving_hard_break(line) + if preserve_hard_breaks + else line.rstrip() + for line in re.split(r"\r?\n", res.text_content) + ] ) res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content) return res diff --git a/packages/markitdown/tests/test_cli_misc.py b/packages/markitdown/tests/test_cli_misc.py index 5e42b16b7c..e4c2dc3fcf 100644 --- a/packages/markitdown/tests/test_cli_misc.py +++ b/packages/markitdown/tests/test_cli_misc.py @@ -20,6 +20,24 @@ # This includes things like help messages, version numbers, and invalid flags. +@pytest.mark.parametrize( + "extension,expected", [(".md", "First \nSecond\n"), (".txt", "First\nSecond\n")] +) +def test_cli_preserves_hard_breaks_only_for_markdown( + tmp_path, extension, expected +) -> None: + path = tmp_path / ("notes" + extension) + path.write_text("First \nSecond", encoding="utf-8") + result = subprocess.run( + [sys.executable, "-m", "markitdown", str(path)], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == expected + + def test_version() -> None: result = subprocess.run( [sys.executable, "-m", "markitdown", "--version"], diff --git a/packages/markitdown/tests/test_docx.py b/packages/markitdown/tests/test_docx.py index 4fa67684e7..57b40b54b9 100644 --- a/packages/markitdown/tests/test_docx.py +++ b/packages/markitdown/tests/test_docx.py @@ -534,6 +534,18 @@ def _docx(body: str = _BODY, *, embedded_style_map: Optional[str] = None) -> io. return stream +@pytest.mark.parametrize( + "breaks,expected", [(1, "First\nSecond"), (2, "First\n\nSecond")] +) +def test_docx_keeps_upstream_break_normalization(breaks, expected) -> None: + body = _paragraph(_text("First") + "" * breaks + _text("Second")) + result = MarkItDown(enable_plugins=False).convert_stream( + _docx(body), stream_info=StreamInfo(extension=".docx") + ) + + assert result.markdown == expected + + class _ImageConverter(DocxConverter): def __init__(self, render: Callable[..., Optional[str]]): super().__init__() diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py index 94cece47b9..123fca8ef1 100644 --- a/packages/markitdown/tests/test_html.py +++ b/packages/markitdown/tests/test_html.py @@ -22,6 +22,28 @@ def _convert_html(html: str, **kwargs) -> str: return result.markdown +@pytest.mark.parametrize( + "html,expected", + [ + ("

First
Second

", "First\nSecond"), + ("

First

Second

", "First\n\nSecond"), + ("

First
Second

", "First\nSecond"), + ("
First
Second
", "```\nFirst\nSecond\n```"), + ], +) +def test_html_keeps_upstream_break_normalization(html, expected) -> None: + assert _convert_html(html) == expected + + +def test_html_with_markdown_extension_keeps_upstream_normalization() -> None: + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(b"

First
Second

"), + stream_info=StreamInfo(extension=".md", mimetype="text/html", charset="utf-8"), + ) + + assert result.markdown == "First\nSecond" + + @pytest.mark.parametrize( "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "] ) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d6c9be59b..500dfabac9 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -14,6 +14,7 @@ import markitdown._uri_utils as uri_utils from markitdown import ( + DocumentConverterResult, FileConversionException, MarkItDown, StreamInfo, @@ -72,6 +73,100 @@ def validate_strings(result, expected_strings, exclude_strings=None): assert string not in text_content +@pytest.mark.parametrize( + "stream_info", + [ + StreamInfo(extension=".md"), + StreamInfo(extension=".markdown"), + StreamInfo(extension=".MD"), + StreamInfo(mimetype="text/markdown"), + StreamInfo(mimetype="application/markdown"), + StreamInfo(mimetype="Text/Markdown; charset=utf-8"), + StreamInfo(extension=".md", mimetype="text/plain"), + ], +) +def test_markdown_input_keeps_hard_breaks(stream_info: StreamInfo) -> None: + source = "First \nSecond \r\nThird\n" + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(source.encode()), + stream_info=stream_info.copy_and_update(charset="utf-8"), + ) + + assert result.markdown == "First \nSecond \nThird\n" + + +@pytest.mark.parametrize( + "extension,mimetype", + [(".txt", "text/plain"), (".text", "text/plain"), (None, "text/plain")], +) +def test_non_markdown_text_keeps_upstream_normalization(extension, mimetype) -> None: + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(b"First \nSecond \r\nThird \t\n\n\nLast\n"), + stream_info=StreamInfo(extension=extension, mimetype=mimetype, charset="utf-8"), + ) + + assert result.markdown == "First\nSecond\nThird\n\nLast\n" + + +@pytest.mark.parametrize( + "source,expected", + [ + ("First\\\nSecond\n", "First\\\nSecond\n"), + ("First \nSecond\n", "First\nSecond\n"), + ("First\t\nSecond\n", "First\nSecond\n"), + ("First \t\nSecond\n", "First\nSecond\n"), + ("First\n \n\n\nSecond\n", "First\n\nSecond\n"), + ("", ""), + ], +) +def test_markdown_other_normalization_is_unchanged(source, expected) -> None: + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(source.encode()), + stream_info=StreamInfo(extension=".md", charset="utf-8"), + ) + + assert result.markdown == expected + + +@pytest.mark.parametrize("entry_point", ["local", "file_uri", "data_uri", "response"]) +def test_markdown_hard_breaks_across_entry_points(tmp_path, entry_point) -> None: + source = b"First \nSecond\n" + markitdown = MarkItDown(enable_plugins=False) + path = tmp_path / "notes.md" + path.write_bytes(source) + + if entry_point == "local": + result = markitdown.convert(path) + elif entry_point == "file_uri": + result = markitdown.convert_uri(path.as_uri()) + elif entry_point == "data_uri": + result = markitdown.convert_uri("data:text/markdown,First%20%20%0ASecond%0A") + else: + response = requests.Response() + response.status_code = 200 + response.url = "https://example.com/notes" + response.headers["Content-Type"] = "text/markdown; charset=utf-8" + response.raw = io.BytesIO(source) + result = markitdown.convert_response(response) + + assert result.markdown == source.decode() + + +def test_custom_converter_keeps_upstream_normalization_for_markdown_input() -> None: + converter = MagicMock() + converter.accepts.return_value = True + converter.convert.return_value = DocumentConverterResult(markdown="First \nSecond") + markitdown = MarkItDown(enable_builtins=False, enable_plugins=False) + markitdown.register_converter(converter) + + result = markitdown.convert_stream( + io.BytesIO(b"First \nSecond"), + stream_info=StreamInfo(extension=".md", charset="utf-8"), + ) + + assert result.markdown == "First\nSecond" + + def test_stream_info_operations() -> None: """Test operations performed on StreamInfo objects."""