Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 34 additions & 10 deletions packages/markitdown/src/markitdown/converters/_rss_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ class RssConverter(DocumentConverter):

def __init__(self):
super().__init__()
self._kwargs = {}

def accepts(
self,
Expand Down Expand Up @@ -147,20 +146,23 @@ def convert(
# Pop our own keyword before forwarding the rest to markdownify.
# strict=True raises RecursionError instead of falling back to plain text.
strict: bool = kwargs.pop("strict", False)
self._kwargs = kwargs
doc = minidom.parse(file_stream)
doc.documentURI = stream_info.url or kwargs.get("url")
feed_type = self._feed_type(doc)

if feed_type == "rss":
return self._parse_rss_type(doc, strict=strict)
return self._parse_rss_type(doc, strict=strict, markdown_options=kwargs)
elif feed_type == "atom":
return self._parse_atom_type(doc, strict=strict)
return self._parse_atom_type(doc, strict=strict, markdown_options=kwargs)
else:
raise ValueError("Unknown feed type")

def _parse_atom_type(
self, doc: Document, *, strict: bool = False
self,
doc: Document,
*,
strict: bool = False,
markdown_options: dict[str, Any] | None = None,
) -> DocumentConverterResult:
"""Parse the type of an Atom feed.

Expand Down Expand Up @@ -191,6 +193,7 @@ def _parse_atom_type(
is_markup=is_markup,
base_url=self._get_field_base_url(entry, tag_name),
strict=strict,
markdown_options=markdown_options,
)
for value, is_markup, tag_name in (
(entry_summary, summary_is_markup, "summary"),
Expand Down Expand Up @@ -242,14 +245,25 @@ def _get_atom_content(
return text.strip(), False

def _render_atom_content(
self, value: str, *, is_markup: bool, base_url: str = "", strict: bool = False
self,
value: str,
*,
is_markup: bool,
base_url: str = "",
strict: bool = False,
markdown_options: dict[str, Any] | None = None,
) -> str:
"""Render one Atom summary or content value as markdown."""
if not is_markup:
# Plain text is returned verbatim: routing it through the HTML
# parser drops tag-shaped text such as ``<job_id>`` entirely.
return value
return self._parse_content(value, base_url=base_url, strict=strict)
return self._parse_content(
value,
base_url=base_url,
strict=strict,
markdown_options=markdown_options,
)

def _get_flattened_text(
self, element: Element, tag_name: str, *, atom_text: bool = False
Expand Down Expand Up @@ -282,7 +296,11 @@ def _get_flattened_text(
return " ".join(part for part in parts if part) or None

def _parse_rss_type(
self, doc: Document, *, strict: bool = False
self,
doc: Document,
*,
strict: bool = False,
markdown_options: dict[str, Any] | None = None,
) -> DocumentConverterResult:
"""Parse the type of an RSS feed.

Expand Down Expand Up @@ -318,6 +336,7 @@ def _parse_rss_type(
value,
base_url=self._get_field_base_url(item, tag_name),
strict=strict,
markdown_options=markdown_options,
)
for value, tag_name in (
(description, "description"),
Expand All @@ -336,14 +355,19 @@ def _parse_rss_type(
)

def _parse_content(
self, content: str, *, base_url: str = "", strict: bool = False
self,
content: str,
*,
base_url: str = "",
strict: bool = False,
markdown_options: dict[str, Any] | None = None,
) -> str:
"""Parse the content of an RSS feed item"""
try:
# using bs4 because many RSS feeds have HTML-styled content
soup = BeautifulSoup(content, "html.parser")
self._resolve_content_links(soup, base_url)
return _CustomMarkdownify(**self._kwargs).convert_soup(soup)
return _CustomMarkdownify(**(markdown_options or {})).convert_soup(soup)
except RecursionError:
if strict:
raise
Expand Down
56 changes: 56 additions & 0 deletions packages/markitdown/tests/test_rss.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import io
import sys
import threading
from concurrent.futures import ThreadPoolExecutor

import pytest

Expand Down Expand Up @@ -81,6 +83,60 @@ def test_atom_without_namespace_is_still_supported() -> None:
assert "Entry content." in result.markdown


class _SynchronizedRssConverter(RssConverter):
"""Pause both conversions after their call-specific options are received."""

def __init__(self) -> None:
super().__init__()
self._barrier = threading.Barrier(2)

def _parse_content(self, content: str, **kwargs) -> str:
self._barrier.wait(timeout=5)
return super()._parse_content(content, **kwargs)


def test_concurrent_conversions_keep_call_specific_markdown_options() -> None:
embedded = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///w=="
feed = _feed_with_body(
"atom-content",
f'<h2>Section</h2><ul><li>item</li></ul><img src="{embedded}" alt="Pixel"/>',
)
converter = _SynchronizedRssConverter()
stream_info = StreamInfo(extension=".atom")

def convert(keep_data_uris: bool, heading_style: str, bullets: str) -> str:
return converter.convert(
io.BytesIO(feed),
stream_info,
keep_data_uris=keep_data_uris,
heading_style=heading_style,
bullets=bullets,
).markdown

with ThreadPoolExecutor(max_workers=2) as pool:
a = pool.submit(convert, True, "atx", "+")
b = pool.submit(convert, False, "underlined", "*")

a_md = a.result()
b_md = b.result()

# Dimensions are asserted together so a leak in any one of them fails the test,
# not just keep_data_uris (see the PR review note about single-dimension coverage).

# keep_data_uris
assert f"![Pixel]({embedded})" in a_md
assert "![Pixel](data:image/gif;base64...)" in b_md

# heading_style: atx ("##") on A, setext/underlined on B
assert "## Section" in a_md
assert "## Section" not in b_md
assert "Section\n------" in b_md

# bullets: "+" on A, "*" on B
assert "+ item" in a_md
assert "* item" in b_md and "+ item" not in b_md


def test_atom_ignores_elements_from_other_namespaces() -> None:
feed = b"""<a:feed xmlns:a="http://www.w3.org/2005/Atom" xmlns="urn:example:other">
<title>Unrelated title</title>
Expand Down