From df0dde6794b7123c71acbb9539020e44423512af Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Thu, 8 Oct 2026 16:03:20 +1100 Subject: [PATCH] Apply strip_document to completed Tag conversions Reuse the document newline policy after normal Tag processing when document conversion is available, and document the cache and validation effects. Cover exclusions, callbacks, resolver history and tree identity. --- README.rst | 12 ++ markdownify/__init__.py | 34 +++--- tests/test_soup.py | 242 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+), 13 deletions(-) create mode 100644 tests/test_soup.py diff --git a/README.rst b/README.rst index 059a68f..744abbd 100644 --- a/README.rst +++ b/README.rst @@ -181,6 +181,18 @@ nested ``Options`` class in ``MarkdownConverter`` subclasses. Converting BeautifulSoup objects ================================ +``convert_soup`` accepts a BeautifulSoup document or a Tag subtree. For Tag +roots, ``strip_document`` applies to the completed Markdown after the tag's +normal conversion. Use ``strip_document=None`` to retain boundary newlines. +Excluding ``[document]`` through ``strip`` or ``convert``, or resolving its +conversion function to ``None``, disables this final formatting. +Tag roots use the built-in ``strip_document`` policy. This final formatting +step does not invoke the resolved document conversion function; normal tag +dispatch and the original tree are retained. The document-function lookup is +cached, including ``None``, so Tag conversion can affect later document +resolution on the same converter. ``strip_document=None`` retains boundary +text but still performs this lookup. + .. code:: python from markdownify import MarkdownConverter diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 28cdaf6..242b344 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -79,6 +79,20 @@ def strip_pre(text): return text +def _strip_document(text, strip_mode): + """Remove only the selected document separation newlines.""" + if strip_mode == LSTRIP: + return text.lstrip('\n') + elif strip_mode == RSTRIP: + return text.rstrip('\n') + elif strip_mode == STRIP: + return text.strip('\n') + elif strip_mode is None: + return text + else: + raise ValueError('Invalid value for strip_document: %s' % strip_mode) + + def chomp(text): """ If the text in an inline tag like b, a, or em contains a leading or trailing @@ -223,7 +237,12 @@ def convert(self, html): return self.convert_soup(soup) def convert_soup(self, soup): - return self.process_tag(soup, parent_tags=set()) + text = self.process_tag(soup, parent_tags=set()) + if (isinstance(soup, Tag) + and soup.name.lower() != '[document]' + and self.get_conv_fn_cached('[document]') is not None): + text = _strip_document(text, self.options['strip_document']) + return text def process_element(self, node, parent_tags=None): if isinstance(node, NavigableString): @@ -327,18 +346,7 @@ def _can_ignore(el): def convert__document_(self, el, text, parent_tags): """Final document-level formatting for BeautifulSoup object (node.name == "[document]")""" - if self.options['strip_document'] == LSTRIP: - text = text.lstrip('\n') # remove leading separation newlines - elif self.options['strip_document'] == RSTRIP: - text = text.rstrip('\n') # remove trailing separation newlines - elif self.options['strip_document'] == STRIP: - text = text.strip('\n') # remove leading and trailing separation newlines - elif self.options['strip_document'] is None: - pass # leave leading and trailing separation newlines as-is - else: - raise ValueError('Invalid value for strip_document: %s' % self.options['strip_document']) - - return text + return _strip_document(text, self.options['strip_document']) def process_text(self, el, parent_tags=None): # For the top-level element, initialize the parent context with an empty set. diff --git a/tests/test_soup.py b/tests/test_soup.py new file mode 100644 index 0000000..98e3fbc --- /dev/null +++ b/tests/test_soup.py @@ -0,0 +1,242 @@ +import pytest # type: ignore[import-not-found] +from bs4 import BeautifulSoup +from markdownify import MarkdownConverter, LSTRIP, RSTRIP, STRIP, STRIP_ONE + + +@pytest.mark.parametrize('mode,expected', [ + (STRIP, 'Hello'), + (LSTRIP, 'Hello\n\n'), + (RSTRIP, '\n\n\nHello'), + (None, '\n\n\nHello\n\n'), +]) +def test_tag_strip_document(mode, expected): + soup = BeautifulSoup('\n

Hello

', 'html.parser') + assert MarkdownConverter(strip_document=mode).convert_soup(soup.html) == expected + + +@pytest.mark.parametrize('tag', ['html', 'body', 'div', 'p']) +def test_tag_default_strip_document(tag): + soup = BeautifulSoup('\n

Hello

', 'html.parser') + assert MarkdownConverter().convert_soup(soup.find(tag)) == 'Hello' + + +@pytest.mark.parametrize('options,expected', [ + ({'convert': ['p']}, '\n\nHello\n\n'), + ({'strip': ['[document]']}, '\n\nHello\n\n'), + ({'convert': []}, 'Hello'), +]) +@pytest.mark.parametrize('mode', [STRIP, LSTRIP, RSTRIP, None, 'invalid']) +def test_excluded_document_policy(options, expected, mode): + soup = BeautifulSoup('

Hello

', 'html.parser') + converter = MarkdownConverter(strip_document=mode, **options) + assert converter.convert_soup(soup.p) == expected + assert converter.convert_soup(soup) == expected + + +@pytest.mark.parametrize('options', [ + {'strip': []}, {'strip': ['div']}, {'convert': ['p', '[document]']}, +]) +def test_document_policy_enabled_independently_of_root_converter(options): + soup = BeautifulSoup('

Hello

', 'html.parser') + assert MarkdownConverter(**options).convert_soup(soup.div) == 'Hello' + + +@pytest.mark.parametrize('mode', ['invalid', STRIP_ONE]) +@pytest.mark.parametrize('html', ['

', '

Hello

']) +def test_invalid_tag_document_mode(html, mode): + soup = BeautifulSoup(html, 'html.parser') + with pytest.raises(ValueError, match='^Invalid value for strip_document: %s$' % mode): + MarkdownConverter(strip_document=mode).convert_soup(soup.p) + + +@pytest.mark.parametrize('mode,expected', [ + (STRIP, ' \tA\r\n\nB\t '), + (LSTRIP, ' \tA\r\n\nB\t \n\n'), + (RSTRIP, '\n\n \tA\r\n\nB\t '), + (None, '\n\n \tA\r\n\nB\t \n\n'), +]) +def test_custom_root_preserves_significant_whitespace(mode, expected): + soup = BeautifulSoup('child', 'html.parser') + calls = [] + + class Converter(MarkdownConverter): + def convert_root(self, el, text, parent_tags): + assert el is soup.root + assert text == '**child**' + assert parent_tags == set() + calls.append(el) + return '\n\n \tA\r\n\nB\t \n\n' + + assert Converter(strip_document=mode).convert_soup(soup.root) == expected + assert len(calls) == 1 + + +@pytest.mark.parametrize('mode', [STRIP, None, 'invalid']) +def test_replacing_document_callback_remains_document_only(mode): + soup = BeautifulSoup('

Hello

', 'html.parser') + calls = [] + + class Converter(MarkdownConverter): + def convert__document_(self, el, text, parent_tags): + assert el is soup + assert text == '\n\nHello\n\n' + assert parent_tags == set() + calls.append(el) + return '\n\nDOCUMENT\n\n' + + converter = Converter(strip_document=mode) + assert converter.convert_soup(soup) == '\n\nDOCUMENT\n\n' + if mode == 'invalid': + with pytest.raises(ValueError): + converter.convert_soup(soup.p) + else: + expected = 'Hello' if mode == STRIP else '\n\nHello\n\n' + assert converter.convert_soup(soup.p) == expected + assert len(calls) == 1 + + +def test_process_tag_override_preserves_document_output(): + soup = BeautifulSoup('

Hello

', 'html.parser') + calls = [] + + class Converter(MarkdownConverter): + def process_tag(self, node, parent_tags=None): + assert parent_tags == set() + calls.append(node) + return '\n\nCUSTOM\n\n' + + converter = Converter() + assert converter.convert_soup(soup) == '\n\nCUSTOM\n\n' + assert converter.convert_soup(soup.p) == 'CUSTOM' + assert len(calls) == 2 + assert calls[0] is soup and calls[1] is soup.p + soup.name = '[DOCUMENT]' + assert converter.convert_soup(soup) == '\n\nCUSTOM\n\n' + + +@pytest.mark.parametrize('available', [True, False]) +def test_document_lookup_cache_and_original_tag_dispatch(available): + soup = BeautifulSoup('

Hello

', 'html.parser') + resolved = [] + converted = [] + processed = [] + + class Converter(MarkdownConverter): + def get_conv_fn(self, name): + resolved.append(name) + if name == '[document]' and not available: + return None + return super().get_conv_fn(name) + + def convert_p(self, el, text, parent_tags): + converted.append((el, set(parent_tags))) + return super().convert_p(el, text, parent_tags) + + def process_element(self, node, parent_tags=None): + processed.append(node) + return super().process_element(node, parent_tags) + + converter = Converter() + expected = 'Hello' if available else '\n\nHello\n\n' + for root in (soup.p, soup.p, soup, soup.p): + assert converter.convert_soup(root) == expected + assert resolved.count('[document]') == 1 + assert resolved.count('p') == 1 + assert len(converted) == 4 + assert all(el is soup.p for el, _ in converted) + assert [parents for _, parents in converted] == [set(), set(), {'[document]'}, set()] + assert len(processed) == 5 + assert sum(node is soup.p for node in processed) == 1 + + +@pytest.mark.parametrize('mode', [STRIP, LSTRIP, RSTRIP, None, 'invalid']) +def test_attached_tree_relationships_survive_conversion(mode): + soup = BeautifulSoup('
before

Hello

after
', 'html.parser') + serialised = str(soup) + nodes = [soup] + list(soup.descendants) + links = [(node.parent, node.previous_sibling, node.next_sibling, + node.previous_element, node.next_element) for node in nodes] + converter = MarkdownConverter(strip_document=mode) + for _ in range(2): + if mode == 'invalid': + with pytest.raises(ValueError): + converter.convert_soup(soup.p) + else: + converter.convert_soup(soup.p) + current_nodes = [soup] + list(soup.descendants) + assert len(current_nodes) == len(nodes) + assert all(current is original for current, original in zip(current_nodes, nodes)) + for node, original in zip(nodes, links): + current = (node.parent, node.previous_sibling, node.next_sibling, + node.previous_element, node.next_element) + assert all(a is b for a, b in zip(current, original)) + assert str(soup) == serialised + + +@pytest.mark.parametrize('mode,prefix,suffix', [ + (STRIP, '', ''), (LSTRIP, '', '\n\n'), + (RSTRIP, '\n\n', ''), (None, '\n\n', '\n\n'), +]) +@pytest.mark.parametrize('pre_mode,code', [ + (STRIP, '```\n Hello\n```'), + (STRIP_ONE, '```\n \n Hello \n \n```'), + (None, '```\n \n \n Hello \n \n \n```'), +]) +@pytest.mark.parametrize('tag', ['section', 'pre']) +def test_pre_root_and_descendant_keep_code_whitespace(mode, prefix, suffix, pre_mode, code, tag): + soup = BeautifulSoup('
  \n  \n  Hello  \n  \n
', 'html.parser') + callbacks = [] + + def language(el): + assert el is soup.pre + callbacks.append(el) + + converter = MarkdownConverter(strip_document=mode, strip_pre=pre_mode, code_language_callback=language) + assert converter.convert_soup(soup.find(tag)) == prefix + code + suffix + assert len(callbacks) == 1 + + +@pytest.mark.parametrize('mode,expected', [(STRIP, 'ALIAS'), (None, '\n\nALIAS\n\n')]) +def test_document_method_alias_keeps_original_root_dispatch(mode, expected): + soup = BeautifulSoup('

Hello

', 'html.parser') + root = soup.p + root.name = '_document_' + calls = [] + + class Converter(MarkdownConverter): + def convert__document_(self, el, text, parent_tags): + assert el is root + assert text == 'Hello' + assert parent_tags == set() + calls.append(el) + return '\n\nALIAS\n\n' + + assert Converter(strip_document=mode).convert_soup(root) == expected + assert len(calls) == 1 + + +@pytest.mark.parametrize('mode', [STRIP, None]) +def test_tag_lookup_caches_document_unavailability_before_state_changes(mode): + soup = BeautifulSoup('

Hello

', 'html.parser') + resolved = [] + document_calls = [] + + class Converter(MarkdownConverter): + available = False + + def get_conv_fn(self, name): + resolved.append((name, self.available)) + if name == '[document]' and not self.available: + return None + return super().get_conv_fn(name) + + def convert__document_(self, el, text, parent_tags): + document_calls.append(el) + return super().convert__document_(el, text, parent_tags) + + converter = Converter(strip_document=mode) + assert converter.convert_soup(soup.p) == '\n\nHello\n\n' + converter.available = True + assert converter.convert_soup(soup) == '\n\nHello\n\n' + assert resolved == [('p', False), ('[document]', False)] + assert document_calls == []