BeforetextAfter
"), + ("onetwo", "
Beforeone
twoAfter
BeforeA & B < CAfter
"), + ( + "one
two
", + "Before
one
two
After
", + ), + ( + "one
two
Before
one
two
After
", + ), + ( + "- one
- two
Before
- one
- two
After
", + ), + ( + "| Key |
|---|
| Value |
Before
| Key |
|---|
| Value |
After
", + ), + ( + "Title
Body
Before
Title
Body
" + "After
", + ), + ( + "leadblock
tail", + "Beforelead
block
tailAfter
", + ), + ( + "a < b\n *literal*Before
a < b\n *literal*After
", + ), + ], +) +def test_fragment_placement_precedes_shared_html_conversion( + monkeypatch: pytest.MonkeyPatch, fragment: str, expected: str +) -> None: + converter = _ImageConverter(Mock(return_value=fragment)) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + options = {"escape_asterisks": False, "custom_option": "forwarded"} + + result = converter.convert(_docx(), _INFO, **options) + + assert convert_html.call_args.args == (expected,) + assert convert_html.call_args.kwargs == options + assert ( + result.markdown + == HtmlConverter().convert_string(expected, escape_asterisks=False).markdown + ) + assert "data-markitdown-image" not in expected + + +@pytest.mark.parametrize( + ("style_map", "expected"), + [ + (None, "one
two
"), + ("p => ul > li:fresh", "one
two
"), + ("p => h2:fresh > strong", "one
two
one
two
"), + ], +) +def test_standalone_block_image_does_not_leave_empty_wrappers( + monkeypatch: pytest.MonkeyPatch, style_map: Optional[str], expected: str +) -> None: + converter = _ImageConverter(Mock(return_value="one
two
")) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + converter.convert(_docx(_paragraph(_image())), _INFO, style_map=style_map) + + assert convert_html.call_args.args == (expected,) + + +def test_block_image_keeps_its_table_cell( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = ( + "Serial: 12345
Status: active
") + ) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + result = converter.convert(_docx(body), _INFO) + + soup = BeautifulSoup(convert_html.call_args.args[0], "html.parser") + assert len(soup.find_all("tr")) == 2 + assert len(soup.find_all("td")) == 4 + assert ( + str(soup.find_all("td")[-1]) + == "Serial: 12345
Status: active
block
")) + convert_html = Mock(wraps=converter._html_converter.convert_string) + monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) + + converter.convert(_docx(), _INFO, style_map="p => p:fresh > em > strong") + + assert convert_html.call_args.args == ( + "Before
block
" + "After
", + ) + + +def test_hook_inherits_preprocessing_and_style_precedence() -> None: + body = ( + _BODY + + _paragraph( + 'Image {number}
" for number in range(12)]) + ) + body = _paragraph("".join(_image(number=number) for number in range(12))) + + result = converter.convert(_docx(body), _INFO) + + assert result.markdown == "\n\n".join(f"Image {number}" for number in range(12)) + + +def test_returned_image_and_literal_text_use_normal_html_rendering() -> None: + render = Mock( + return_value=( + "<literal> *not emphasis* & text
" + '
'
+ )
+ )
+ result = _ImageConverter(render).convert(_docx(_paragraph(_image())), _INFO)
+
+ assert (
+ result.markdown
+ == r"not a string
"]) +def test_invalid_return_is_not_silent_native_fallback(result: Any) -> None: + with pytest.raises(TypeError, match="HTML string or None"): + _ImageConverter(Mock(return_value=result)).convert(_docx(), _INFO) + + +@pytest.mark.parametrize( + "fragment", + [ + "document", + "content
", + ], +) +def test_full_document_is_not_a_valid_fragment(fragment: str) -> None: + with pytest.raises(ValueError, match="fragment, not a document"): + _ImageConverter(Mock(return_value=fragment)).convert(_docx(), _INFO) + + +@pytest.mark.parametrize("via_dispatcher", [False, True]) +def test_hook_errors_propagate_and_close_the_image_stream(via_dispatcher: bool) -> None: + streams = [] + error = RuntimeError("Image service failed") + + def render(stream, info, **kwargs): + streams.append(stream) + raise error + + converter = _ImageConverter(render) + if via_dispatcher: + markitdown = MarkItDown(enable_builtins=False) + markitdown.register_converter(converter, priority=-1) + with pytest.raises(FileConversionException) as caught: + markitdown.convert_stream(_docx(), stream_info=_INFO) + assert caught.value.attempts is not None + assert any( + attempt.exc_info and attempt.exc_info[1] is error + for attempt in caught.value.attempts + ) + else: + with pytest.raises(RuntimeError) as caught_direct: + converter.convert(_docx(), _INFO) + assert caught_direct.value is error + assert streams and all(stream.closed for stream in streams) + + +def test_dispatcher_can_fall_back_to_native_docx_after_hook_error() -> None: + render = Mock(side_effect=RuntimeError("Image service failed")) + markitdown = MarkItDown() + markitdown.register_converter(_ImageConverter(render), priority=-1) + + result = markitdown.convert_stream(_docx(), stream_info=_INFO) + + assert result.markdown == DocxConverter().convert(_docx(), _INFO).markdown + render.assert_called() + + +def test_mammoth_does_not_swallow_hook_file_reference_errors() -> None: + error = InvalidFileReferenceError("hook failed") + render = Mock(side_effect=error) + + with pytest.raises(RuntimeError, match="_image_to_html failed") as caught: + _ImageConverter(render).convert(_docx(), _INFO) + + assert caught.value.__cause__ is error + render.assert_called_once() + + +def test_per_call_options_and_fragments_do_not_leak_between_conversions() -> None: + converter = _ImageConverter(lambda stream, info, **kwargs: kwargs.get("image_text")) + first = converter.convert(_docx(), _INFO, image_text="first
") + second = converter.convert(_docx(), _INFO, image_text="second
") + third = converter.convert(_docx(), _INFO) + + assert first.markdown == "Before\n\nfirst\n\nAfter" + assert second.markdown == "Before\n\nsecond\n\nAfter" + assert third.markdown == DocxConverter().convert(_docx(), _INFO).markdown + + +def test_unreferenced_media_does_not_invoke_the_hook() -> None: + render = Mock(side_effect=AssertionError("unexpected image")) + + result = _ImageConverter(render).convert( + _docx(_paragraph(_text("Only text"))), _INFO + ) + + assert result.markdown == "Only text" + render.assert_not_called() + + +@pytest.mark.parametrize( + "fragment", + [ + 'new', + '', + ], +) +def test_image_html_does_not_create_nested_links( + monkeypatch: pytest.MonkeyPatch, fragment: str +) -> None: + body = _paragraph( + 'BeforetextAfter
"), - ("onetwo", "
Beforeone
twoAfter
BeforeA & B < CAfter
"), - ( - "one
two
", - "Before
one
two
After
", - ), - ( - "one
two
Before
one
two
After
", - ), - ( - "- one
- two
Before
- one
- two
After
", - ), - ( - "| Key |
|---|
| Value |
Before
| Key |
|---|
| Value |
After
", - ), - ( - "Title
Body
Before
Title
Body
" - "After
", - ), - ( - "leadblock
tail", - "Beforelead
block
tailAfter
", - ), - ( - "a < b\n *literal*Before
a < b\n *literal*After
", - ), - ], -) -def test_fragment_placement_precedes_shared_html_conversion( - monkeypatch: pytest.MonkeyPatch, fragment: str, expected: str -) -> None: - converter = _ImageConverter(Mock(return_value=fragment)) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - options = {"escape_asterisks": False, "custom_option": "forwarded"} - - result = converter.convert(_docx(), _INFO, **options) - - assert convert_html.call_args.args == (expected,) - assert convert_html.call_args.kwargs == options - assert ( - result.markdown - == HtmlConverter().convert_string(expected, escape_asterisks=False).markdown - ) - assert "data-markitdown-image" not in expected - - -@pytest.mark.parametrize( - ("style_map", "expected"), - [ - (None, "one
two
"), - ("p => ul > li:fresh", "one
two
"), - ("p => h2:fresh > strong", "one
two
one
two
"), - ], -) -def test_standalone_block_image_does_not_leave_empty_wrappers( - monkeypatch: pytest.MonkeyPatch, style_map: Optional[str], expected: str -) -> None: - converter = _ImageConverter(Mock(return_value="one
two
")) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - converter.convert(_docx(_paragraph(_image())), _INFO, style_map=style_map) - - assert convert_html.call_args.args == (expected,) - - -def test_block_image_keeps_its_table_cell( - monkeypatch: pytest.MonkeyPatch, -) -> None: - body = ( - "Serial: 12345
Status: active
") - ) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - result = converter.convert(_docx(body), _INFO) - - soup = BeautifulSoup(convert_html.call_args.args[0], "html.parser") - assert len(soup.find_all("tr")) == 2 - assert len(soup.find_all("td")) == 4 - assert ( - str(soup.find_all("td")[-1]) - == "Serial: 12345
Status: active
block
")) - convert_html = Mock(wraps=converter._html_converter.convert_string) - monkeypatch.setattr(converter._html_converter, "convert_string", convert_html) - - converter.convert(_docx(), _INFO, style_map="p => p:fresh > em > strong") - - assert convert_html.call_args.args == ( - "Before
block
" - "After
", - ) - - -def test_hook_inherits_preprocessing_and_style_precedence() -> None: - body = ( - _BODY - + _paragraph( - 'Image {number}
" for number in range(12)]) - ) - body = _paragraph("".join(_image(number=number) for number in range(12))) - - result = converter.convert(_docx(body), _INFO) - - assert result.markdown == "\n\n".join(f"Image {number}" for number in range(12)) - - -def test_returned_image_and_literal_text_use_normal_html_rendering() -> None: - render = Mock( - return_value=( - "<literal> *not emphasis* & text
" - ''
- )
- )
- result = _ImageConverter(render).convert(_docx(_paragraph(_image())), _INFO)
-
- assert (
- result.markdown
- == r"not a string
"]) -def test_invalid_return_is_not_silent_native_fallback(result: Any) -> None: - with pytest.raises(TypeError, match="HTML string or None"): - _ImageConverter(Mock(return_value=result)).convert(_docx(), _INFO) - - -@pytest.mark.parametrize( - "fragment", - [ - "document", - "content
", - ], -) -def test_full_document_is_not_a_valid_fragment(fragment: str) -> None: - with pytest.raises(ValueError, match="fragment, not a document"): - _ImageConverter(Mock(return_value=fragment)).convert(_docx(), _INFO) - - -@pytest.mark.parametrize("via_dispatcher", [False, True]) -def test_hook_errors_propagate_and_close_the_image_stream(via_dispatcher: bool) -> None: - streams = [] - error = RuntimeError("Image service failed") - - def render(stream, info, **kwargs): - streams.append(stream) - raise error - - converter = _ImageConverter(render) - if via_dispatcher: - markitdown = MarkItDown(enable_builtins=False) - markitdown.register_converter(converter, priority=-1) - with pytest.raises(FileConversionException) as caught: - markitdown.convert_stream(_docx(), stream_info=_INFO) - assert caught.value.attempts is not None - assert any( - attempt.exc_info and attempt.exc_info[1] is error - for attempt in caught.value.attempts - ) - else: - with pytest.raises(RuntimeError) as caught_direct: - converter.convert(_docx(), _INFO) - assert caught_direct.value is error - assert streams and all(stream.closed for stream in streams) - - -def test_dispatcher_can_fall_back_to_native_docx_after_hook_error() -> None: - render = Mock(side_effect=RuntimeError("Image service failed")) - markitdown = MarkItDown() - markitdown.register_converter(_ImageConverter(render), priority=-1) - - result = markitdown.convert_stream(_docx(), stream_info=_INFO) - - assert result.markdown == DocxConverter().convert(_docx(), _INFO).markdown - render.assert_called() - - -def test_mammoth_does_not_swallow_hook_file_reference_errors() -> None: - error = InvalidFileReferenceError("hook failed") - render = Mock(side_effect=error) - - with pytest.raises(RuntimeError, match="_image_to_html failed") as caught: - _ImageConverter(render).convert(_docx(), _INFO) - - assert caught.value.__cause__ is error - render.assert_called_once() - - -def test_per_call_options_and_fragments_do_not_leak_between_conversions() -> None: - converter = _ImageConverter(lambda stream, info, **kwargs: kwargs.get("image_text")) - first = converter.convert(_docx(), _INFO, image_text="first
") - second = converter.convert(_docx(), _INFO, image_text="second
") - third = converter.convert(_docx(), _INFO) - - assert first.markdown == "Before\n\nfirst\n\nAfter" - assert second.markdown == "Before\n\nsecond\n\nAfter" - assert third.markdown == DocxConverter().convert(_docx(), _INFO).markdown - - -def test_unreferenced_media_does_not_invoke_the_hook() -> None: - render = Mock(side_effect=AssertionError("unexpected image")) - - result = _ImageConverter(render).convert( - _docx(_paragraph(_text("Only text"))), _INFO - ) - - assert result.markdown == "Only text" - render.assert_not_called() - - -@pytest.mark.parametrize( - "fragment", - [ - 'new', - '', - ], -) -def test_image_html_does_not_create_nested_links( - monkeypatch: pytest.MonkeyPatch, fragment: str -) -> None: - body = _paragraph( - 'Chapter text.
" + '
'
+ ""
+)
+
+
+def _build_epub_with_data_uri() -> bytes:
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ zf.writestr("mimetype", "application/epub+zip")
+ zf.writestr("META-INF/container.xml", _EPUB_CONTAINER)
+ zf.writestr("OEBPS/content.opf", _EPUB_OPF)
+ zf.writestr("OEBPS/ch1.xhtml", _EPUB_CHAPTER)
+ return buf.getvalue()
+
+
+def test_epub_honors_keep_data_uris() -> None:
+ """EPUB chapters must be converted with the options the caller passed."""
+ result = MarkItDown().convert_stream(
+ io.BytesIO(_build_epub_with_data_uri()),
+ stream_info=StreamInfo(extension=".epub"),
+ keep_data_uris=True,
+ )
+
+ assert (
+ "" in result.markdown
+ )
+
+
+def test_epub_truncates_data_uris_by_default() -> None:
+ """Without the option, the default truncation must still apply."""
+ result = MarkItDown().convert_stream(
+ io.BytesIO(_build_epub_with_data_uri()),
+ stream_info=StreamInfo(extension=".epub"),
+ )
+
+ assert "" in result.markdown
+ assert "iVBORw0KGgo" not in result.markdown
+
+
+def test_epub_metadata_and_text_are_unchanged() -> None:
+ """The rest of the conversion must not move."""
+ result = MarkItDown().convert_stream(
+ io.BytesIO(_build_epub_with_data_uri()),
+ stream_info=StreamInfo(extension=".epub"),
+ )
+
+ assert result.title == "Example book"
+ assert "**Title:** Example book" in result.markdown
+ assert "Chapter text." in result.markdown
+
+
if __name__ == "__main__":
- test_percent_encoded_href_resolves_to_zip_entry()
- test_non_ascii_percent_encoded_href_resolves()
- test_literally_encoded_zip_entry_still_resolves()
- test_parent_relative_href_resolves()
- print("All tests passed")
+ raise SystemExit(pytest.main([__file__]))
diff --git a/packages/markitdown/tests/test_file_paths.py b/packages/markitdown/tests/test_file_paths.py
deleted file mode 100644
index 2930c2c522..0000000000
--- a/packages/markitdown/tests/test_file_paths.py
+++ /dev/null
@@ -1,138 +0,0 @@
-"""File URI validation must reject network paths before opening a file."""
-
-import os
-from pathlib import Path
-
-import pytest
-
-import markitdown._uri_utils as uri_utils
-from markitdown import MarkItDown
-from markitdown._uri_utils import _is_unc_or_device_path, file_uri_to_path
-
-
-UNC_URI_PATHS = [
- "//server.example/share/x.txt",
- "///server.example/share/x.txt",
- "////server.example/share/x.txt",
- r"/\server.example/share/x.txt",
- r"\/server.example/share/x.txt",
- r"\\server.example\share\x.txt",
- "/%2Fserver.example/share/x.txt",
- "/%2fserver.example/share/x.txt",
- "/%5Cserver.example/share/x.txt",
- "/%5cserver.example/share/x.txt",
- "%2F%2Fserver.example/share/x.txt",
- "%5C%5Cserver.example%5Cshare%5Cx.txt",
- "/%5C%3F%5CUNC%5Cserver.example%5Cshare%5Cx.txt",
- "/%5C.%5CUNC%5Cserver.example%5Cshare%5Cx.txt",
- "/%5C%3F%5CC:%5CTemp%5Cx.txt",
- "/%5C.%5CPhysicalDrive0",
-]
-
-BLOCKED_URIS = (
- [
- f"file://{authority}{path}"
- for authority in ("", "localhost")
- for path in UNC_URI_PATHS
- # An authority must be followed by a slash to introduce its path.
- if path.startswith("/")
- ]
- + [f"file:{path}" for path in UNC_URI_PATHS if not path.startswith("/")]
- + [
- "FILE:////server.example/share/x.txt",
- "file:////C:/Temp/x.txt",
- ]
-)
-
-
-@pytest.mark.parametrize("uri", BLOCKED_URIS)
-def test_file_uri_rejects_unc_or_device_path(uri: str) -> None:
- with pytest.raises(ValueError, match="UNC"):
- file_uri_to_path(uri)
-
-
-class _NoFileAccessMarkItDown(MarkItDown):
- def convert_local(self, *args, **kwargs):
- pytest.fail("Rejected file URIs must not reach convert_local()")
-
-
-@pytest.fixture(scope="module")
-def no_file_access_markitdown() -> MarkItDown:
- return _NoFileAccessMarkItDown(enable_builtins=False)
-
-
-@pytest.mark.parametrize("method", ["convert", "convert_uri"])
-@pytest.mark.parametrize("uri", BLOCKED_URIS + ["file://server.example/share/x.txt"])
-def test_convert_rejects_remote_file_uri_before_file_access(
- no_file_access_markitdown: MarkItDown, method: str, uri: str
-) -> None:
- with pytest.raises(ValueError, match="Unsupported file URI"):
- getattr(no_file_access_markitdown, method)(uri)
-
-
-@pytest.mark.parametrize(
- "path, blocked",
- [
- (r"\\server\share\x.txt", True),
- ("//server/share/x.txt", True),
- (r"/\server/share/x.txt", True),
- (r"\/server/share/x.txt", True),
- (r"\\?\UNC\server\share\x.txt", True),
- (r"\\?\C:\Temp\x.txt", True),
- (r"\\.\PhysicalDrive0", True),
- (r"C:\Temp\x.txt", False),
- ("C:/Temp/x.txt", False),
- ("/home/user/x.txt", False),
- ("relative.txt", False),
- ],
-)
-def test_resolved_path_validation(path: str, blocked: bool) -> None:
- assert _is_unc_or_device_path(path) is blocked
-
-
-def test_file_uri_rejects_unc_after_conversion(monkeypatch: pytest.MonkeyPatch) -> None:
- # Check the final guard independently of the URI prefix check. Replace only
- # the helper's converter binding, leaving the stdlib functions unchanged.
- monkeypatch.setattr(
- uri_utils, "url2pathname", lambda _: "//server.example/share/x.txt"
- )
-
- with pytest.raises(ValueError, match="UNC"):
- file_uri_to_path("file:///local.txt")
-
-
-@pytest.mark.parametrize("filename", ["notes.txt", "notes café.txt", "%5C%5Cx.txt"])
-@pytest.mark.parametrize("authority", ["", "localhost"])
-def test_local_file_uri_conversion(
- tmp_path: Path, filename: str, authority: str
-) -> None:
- local_file = tmp_path / filename
- local_file.write_text("Local file contents", encoding="utf-8")
- uri = local_file.as_uri().replace("file://", f"file://{authority}", 1)
-
- netloc, path = file_uri_to_path(uri)
-
- assert netloc == (authority or None)
- assert path == str(local_file)
- markitdown = MarkItDown()
- assert markitdown.convert(uri).markdown == "Local file contents"
- assert markitdown.convert(str(local_file)).markdown == "Local file contents"
-
-
-@pytest.mark.skipif(os.name != "nt", reason="Requires native Windows path conversion")
-@pytest.mark.parametrize("authority", ["", "localhost"])
-@pytest.mark.parametrize(
- "uri_path, expected",
- [
- ("/C:/Temp/notes.txt", r"C:\Temp\notes.txt"),
- ("/C%3A/Temp/notes.txt", r"C:\Temp\notes.txt"),
- ("/C%3a/Temp/notes.txt", r"C:\Temp\notes.txt"),
- ("/C:/Temp/notes%20caf%C3%A9.txt", "C:\\Temp\\notes café.txt"),
- ("/C:/Temp/%255C%255Cx.txt", r"C:\Temp\%5C%5Cx.txt"),
- ],
-)
-def test_windows_local_drive_uri(authority: str, uri_path: str, expected: str) -> None:
- netloc, path = file_uri_to_path(f"file://{authority}{uri_path}")
-
- assert netloc == (authority or None)
- assert path == expected
diff --git a/packages/markitdown/tests/test_html.py b/packages/markitdown/tests/test_html.py
new file mode 100644
index 0000000000..94cece47b9
--- /dev/null
+++ b/packages/markitdown/tests/test_html.py
@@ -0,0 +1,349 @@
+"""HTML conversion, link handling, and Wikipedia pages."""
+
+import io
+
+import pytest
+from bs4 import BeautifulSoup
+
+from markitdown import MarkItDown, StreamInfo
+from markitdown.converters import WikipediaConverter
+from markitdown.converters._markdownify import _CustomMarkdownify
+
+
+# HTML rendering
+
+
+def _convert_html(html: str, **kwargs) -> str:
+ result = MarkItDown().convert_stream(
+ io.BytesIO(html.encode("utf-8")),
+ file_extension=".html",
+ **kwargs,
+ )
+ return result.markdown
+
+
+@pytest.mark.parametrize(
+ "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "]
+)
+def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None:
+ element = BeautifulSoup("", "html.parser").u
+
+ assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace
+
+
+@pytest.mark.parametrize(
+ ("content", "expected"),
+ [
+ ("", "FirstLast"),
+ (" ", "First Last"),
+ ("\t", "First Last"),
+ (" ", "First\u00a0Last"),
+ ("", "First\nLast"), + ("word", "FirstwordLast"), + (" word ", "First word Last"), + ], +) +def test_html_underlined_content_is_preserved(content: str, expected: str) -> None: + assert _convert_html(f"
First{content}Last
") == expected + + +def test_preserves_non_utf8_percent_encoded_href_path() -> None: + href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" + html = f'example' + + markdown = _convert_html(html) + + assert f"[example]({href})" in markdown + assert "%EF%BF%BD" not in markdown + + +def test_html_href_still_quotes_raw_unicode_and_spaces() -> None: + href = "https://example.com/a path/日本語" + expected_href = "https://example.com/a%20path/" "%E6%97%A5%E6%9C%AC%E8%AA%9E" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_quotes_literal_percent_sign() -> None: + href = "https://example.com/100% complete" + expected_href = "https://example.com/100%25%20complete" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_quotes_malformed_percent_escape() -> None: + href = "https://example.com/items/%ZZ/%2F" + expected_href = "https://example.com/items/%25ZZ/%2F" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_preserves_encoded_slash() -> None: + href = "https://example.com/items/a%2Fb" + + markdown = _convert_html(f'example') + + assert f"[example]({href})" in markdown + + +def test_html_href_does_not_quote_query_or_fragment() -> None: + href = "https://example.com/a path?query=a b%20c#fragment with spaces" + expected_href = "https://example.com/a%20path?query=a b%20c#fragment with spaces" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_img_prefers_data_src_over_placeholder_data_uri() -> None: + placeholder = ( + "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" + ) + real_src = "https://example.com/photo.jpg" + html = ( + f'{heading}
+Body text.
Hello
Hello
'
+ stream_info = StreamInfo(mimetype="text/html", extension=".html")
+
+ result = markitdown.convert_stream(io.BytesIO(html), stream_info=stream_info)
+ assert result.markdown == ""
+ assert "AAAA" not in result.markdown
+
+ result = markitdown.convert_stream(
+ io.BytesIO(html), stream_info=stream_info, keep_data_uris=True
+ )
+ assert result.markdown == ""
+
+
+def test_html_strikethrough_variants(tmp_path) -> None:
+ html = """
+
+Plain s element after.
Plain del element after.
Plain strike element after.
Spaces A B C.
Runs D E F.
Empty GH.
Newline IJ
+KL.
Break MNP.
O
'
+ html += "
"
+ html += ""
+
+ try:
+ sys.setrecursionlimit(low_limit)
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ result = markitdown.convert_stream(
+ io.BytesIO(html.encode("utf-8")),
+ file_extension=".html",
+ )
+
+ # Should have emitted a warning about the fallback
+ recursion_warnings = [x for x in w if "deeply nested" in str(x.message)]
+ assert len(recursion_warnings) > 0
+
+ finally:
+ sys.setrecursionlimit(original_limit)
+
+ # The output should contain the text content, not raw HTML
+ assert "Deep content" in result.markdown
+ assert "bold text" in result.markdown
+ assert "Deep content with bold text
" + for _ in range(depth): + html += "", "First\nLast"), - ("word", "FirstwordLast"), - (" word ", "First word Last"), - ], -) -def test_html_underlined_content_is_preserved(content: str, expected: str) -> None: - assert _convert_html(f"
First{content}Last
") == expected - - -def test_preserves_non_utf8_percent_encoded_href_path() -> None: - href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" - html = f'example' - - markdown = _convert_html(html) - - assert f"[example]({href})" in markdown - assert "%EF%BF%BD" not in markdown - - -def test_html_href_still_quotes_raw_unicode_and_spaces() -> None: - href = "https://example.com/a path/日本語" - expected_href = "https://example.com/a%20path/" "%E6%97%A5%E6%9C%AC%E8%AA%9E" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_html_href_quotes_literal_percent_sign() -> None: - href = "https://example.com/100% complete" - expected_href = "https://example.com/100%25%20complete" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_html_href_quotes_malformed_percent_escape() -> None: - href = "https://example.com/items/%ZZ/%2F" - expected_href = "https://example.com/items/%25ZZ/%2F" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_html_href_preserves_encoded_slash() -> None: - href = "https://example.com/items/a%2Fb" - - markdown = _convert_html(f'example') - - assert f"[example]({href})" in markdown - - -def test_html_href_does_not_quote_query_or_fragment() -> None: - href = "https://example.com/a path?query=a b%20c#fragment with spaces" - expected_href = "https://example.com/a%20path?query=a b%20c#fragment with spaces" - - markdown = _convert_html(f'example') - - assert f"[example]({expected_href})" in markdown - - -def test_img_prefers_data_src_over_placeholder_data_uri() -> None: - placeholder = ( - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7" - ) - real_src = "https://example.com/photo.jpg" - html = ( - f'Hello
Hello
'
- stream_info = StreamInfo(mimetype="text/html", extension=".html")
-
- result = markitdown.convert_stream(io.BytesIO(html), stream_info=stream_info)
- assert result.markdown == ""
- assert "AAAA" not in result.markdown
-
- result = markitdown.convert_stream(
- io.BytesIO(html), stream_info=stream_info, keep_data_uris=True
- )
- assert result.markdown == ""
-
-
def test_file_uris() -> None:
expected_path = os.path.abspath("/path/to/file.txt")
@@ -378,460 +259,6 @@ def test_file_uri_with_percent_encoded_windows_drive(
assert path == r"C:\Temp\example.md"
-def test_docx_comments() -> None:
- # Test DOCX processing, with comments and setting style_map on init
- markitdown_with_style_map = MarkItDown(style_map="comment-reference => ")
- result = markitdown_with_style_map.convert(
- os.path.join(TEST_FILES_DIR, "test_with_comment.docx")
- )
- validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
-
-
-def _write_underlined_docx(
- path,
- embedded_style_map: Optional[str] = None,
- *,
- paragraph_xml: str = (
- "Plain s element after.
Plain del element after.
Plain strike element after.
Spaces A B C.
Runs D E F.
Empty GH.
Newline IJ
-KL.
Break MNP.
O
'
- html += "
"
- html += ""
-
- try:
- sys.setrecursionlimit(low_limit)
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- result = markitdown.convert_stream(
- io.BytesIO(html.encode("utf-8")),
- file_extension=".html",
- )
-
- # Should have emitted a warning about the fallback
- recursion_warnings = [x for x in w if "deeply nested" in str(x.message)]
- assert len(recursion_warnings) > 0
-
- finally:
- sys.setrecursionlimit(original_limit)
-
- # The output should contain the text content, not raw HTML
- assert "Deep content" in result.markdown
- assert "bold text" in result.markdown
- assert "Deep content with bold text
" - for _ in range(depth): - html += "'
- item_html += "
"
-
- rss = (
- ''
- 'Deep feed content with bold text
" - for _ in range(depth): - item_html += "