diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.py index bec42484fb..49aef4a3e5 100644 --- a/packages/markitdown/src/markitdown/converters/_rss_converter.py +++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py @@ -1,3 +1,5 @@ +import warnings + from defusedxml import minidom from xml.dom.minidom import Document, Element from typing import BinaryIO, Any, Union @@ -87,18 +89,23 @@ def convert( stream_info: StreamInfo, **kwargs: Any, # Options to pass to the converter ) -> DocumentConverterResult: + # Pop our own keyword before forwarding the rest to markdownify. + # strict=True raises RecursionError instead of falling back to plain text. + strict: bool = kwargs.pop("strict", False) self._kwargs = kwargs doc = minidom.parse(file_stream) feed_type = self._feed_type(doc) if feed_type == "rss": - return self._parse_rss_type(doc) + return self._parse_rss_type(doc, strict=strict) elif feed_type == "atom": - return self._parse_atom_type(doc) + return self._parse_atom_type(doc, strict=strict) else: raise ValueError("Unknown feed type") - def _parse_atom_type(self, doc: Document) -> DocumentConverterResult: + def _parse_atom_type( + self, doc: Document, *, strict: bool = False + ) -> DocumentConverterResult: """Parse the type of an Atom feed. Returns None if the feed type is not recognized or something goes wrong. @@ -121,16 +128,18 @@ def _parse_atom_type(self, doc: Document) -> DocumentConverterResult: if entry_updated: md_text += f"Updated on: {entry_updated}\n" if entry_summary: - md_text += self._parse_content(entry_summary) + md_text += self._parse_content(entry_summary, strict=strict) if entry_content: - md_text += self._parse_content(entry_content) + md_text += self._parse_content(entry_content, strict=strict) return DocumentConverterResult( markdown=md_text, title=title, ) - def _parse_rss_type(self, doc: Document) -> DocumentConverterResult: + def _parse_rss_type( + self, doc: Document, *, strict: bool = False + ) -> DocumentConverterResult: """Parse the type of an RSS feed. Returns None if the feed type is not recognized or something goes wrong. @@ -158,21 +167,34 @@ def _parse_rss_type(self, doc: Document) -> DocumentConverterResult: if pubDate: md_text += f"Published on: {pubDate}\n" if description: - md_text += self._parse_content(description) + md_text += self._parse_content(description, strict=strict) if content: - md_text += self._parse_content(content) + md_text += self._parse_content(content, strict=strict) return DocumentConverterResult( markdown=md_text, title=channel_title, ) - def _parse_content(self, content: str) -> str: + def _parse_content(self, content: str, *, strict: bool = False) -> str: """Parse the content of an RSS feed item""" try: # using bs4 because many RSS feeds have HTML-styled content soup = BeautifulSoup(content, "html.parser") return _CustomMarkdownify(**self._kwargs).convert_soup(soup) + except RecursionError: + if strict: + raise + # Deeply nested item content can exceed Python's recursion limit + # during markdownify's recursive DOM traversal. Fall back to + # BeautifulSoup's iterative get_text() so the caller still gets + # usable plain-text content instead of raw HTML. + warnings.warn( + "RSS item content is too deeply nested for markdown conversion " + "(RecursionError). Falling back to plain-text extraction.", + stacklevel=2, + ) + return BeautifulSoup(content, "html.parser").get_text("\n", strip=True) except BaseException as _: return content diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index dd04e142b4..a263801dcc 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -9,6 +9,7 @@ from markitdown._uri_utils import parse_data_uri, file_uri_to_path from markitdown._markitdown import _get_content_disposition_filename +from markitdown.converters import RssConverter from markitdown import ( MarkItDown, @@ -434,6 +435,7 @@ def test_deeply_nested_html_fallback() -> None: # Should have emitted a warning about the fallback recursion_warnings = [x for x in w if "deeply nested" in str(x.message)] assert len(recursion_warnings) > 0 + finally: sys.setrecursionlimit(original_limit) @@ -444,6 +446,96 @@ def test_deeply_nested_html_fallback() -> None: assert "

" not in result.markdown +def test_deeply_nested_rss_item_fallback() -> None: + """Deeply nested HTML inside an RSS item should fall back to plain-text + extraction instead of silently embedding raw unconverted HTML in the + markdown output (same failure class as the HTML converter fix in #1644). + + Note: This test uses sys.setrecursionlimit to guarantee a RecursionError + regardless of the host environment's default limit, making it deterministic + across different platforms and CI configurations. + """ + import sys + import warnings + + markitdown = MarkItDown() + + # Use a small recursion limit so the test is environment-independent. + # We restore the original limit in a finally block to avoid side-effects. + original_limit = sys.getrecursionlimit() + low_limit = 200 # well below markdownify's traversal depth for depth=500 + + # Build an RSS item whose content is deeply nested HTML + depth = 500 + item_html = "" + for _ in range(depth): + item_html += '

' + item_html += "

Deep feed content with bold text

" + for _ in range(depth): + item_html += "
" + + rss = ( + '' + '' + "" + "Test Feed" + "A test feed" + "" + "Deep Item" + f"" + "" + "" + "" + ) + atom = ( + '' + '' + "Test Feed" + "" + "Deep Entry" + f'' + "" + "" + ) + + try: + sys.setrecursionlimit(low_limit) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = markitdown.convert_stream( + io.BytesIO(rss.encode("utf-8")), + file_extension=".rss", + ) + + # Should have emitted a warning about the fallback + recursion_warnings = [x for x in w if "deeply nested" in str(x.message)] + assert len(recursion_warnings) > 0 + + # strict=True should expose the conversion failure rather than applying + # the plain-text fallback. + with pytest.raises(RecursionError): + RssConverter().convert( + io.BytesIO(rss.encode("utf-8")), + StreamInfo(extension=".rss"), + strict=True, + ) + with pytest.raises(RecursionError): + RssConverter().convert( + io.BytesIO(atom.encode("utf-8")), + StreamInfo(extension=".atom"), + strict=True, + ) + finally: + sys.setrecursionlimit(original_limit) + + # The output should contain the text content, not raw HTML + assert "Deep feed content" in result.markdown + assert "bold text" in result.markdown + assert "" not in result.markdown + + def test_doc_rlink() -> None: # Test for: CVE-2025-11849 markitdown = MarkItDown() @@ -644,7 +736,7 @@ def test_json_with_late_non_ascii_character(tmp_path) -> None: "title": "Example record", "abstract": "This is sample test. " * 500, }, - "notes": "non-ASCII character: è", + "notes": "non-ASCII character: è", } json_path = tmp_path / "input.json" json_path.write_text( @@ -653,7 +745,7 @@ def test_json_with_late_non_ascii_character(tmp_path) -> None: result = MarkItDown().convert(str(json_path)) - assert "non-ASCII character: è" in result.text_content + assert "non-ASCII character: è" in result.text_content if __name__ == "__main__":