diff --git a/README.rst b/README.rst index 059a68f..744abbd 100644 --- a/README.rst +++ b/README.rst @@ -181,6 +181,18 @@ nested ``Options`` class in ``MarkdownConverter`` subclasses. Converting BeautifulSoup objects ================================ +``convert_soup`` accepts a BeautifulSoup document or a Tag subtree. For Tag +roots, ``strip_document`` applies to the completed Markdown after the tag's +normal conversion. Use ``strip_document=None`` to retain boundary newlines. +Excluding ``[document]`` through ``strip`` or ``convert``, or resolving its +conversion function to ``None``, disables this final formatting. +Tag roots use the built-in ``strip_document`` policy. This final formatting +step does not invoke the resolved document conversion function; normal tag +dispatch and the original tree are retained. The document-function lookup is +cached, including ``None``, so Tag conversion can affect later document +resolution on the same converter. ``strip_document=None`` retains boundary +text but still performs this lookup. + .. code:: python from markdownify import MarkdownConverter diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 28cdaf6..242b344 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -79,6 +79,20 @@ def strip_pre(text): return text +def _strip_document(text, strip_mode): + """Remove only the selected document separation newlines.""" + if strip_mode == LSTRIP: + return text.lstrip('\n') + elif strip_mode == RSTRIP: + return text.rstrip('\n') + elif strip_mode == STRIP: + return text.strip('\n') + elif strip_mode is None: + return text + else: + raise ValueError('Invalid value for strip_document: %s' % strip_mode) + + def chomp(text): """ If the text in an inline tag like b, a, or em contains a leading or trailing @@ -223,7 +237,12 @@ def convert(self, html): return self.convert_soup(soup) def convert_soup(self, soup): - return self.process_tag(soup, parent_tags=set()) + text = self.process_tag(soup, parent_tags=set()) + if (isinstance(soup, Tag) + and soup.name.lower() != '[document]' + and self.get_conv_fn_cached('[document]') is not None): + text = _strip_document(text, self.options['strip_document']) + return text def process_element(self, node, parent_tags=None): if isinstance(node, NavigableString): @@ -327,18 +346,7 @@ def _can_ignore(el): def convert__document_(self, el, text, parent_tags): """Final document-level formatting for BeautifulSoup object (node.name == "[document]")""" - if self.options['strip_document'] == LSTRIP: - text = text.lstrip('\n') # remove leading separation newlines - elif self.options['strip_document'] == RSTRIP: - text = text.rstrip('\n') # remove trailing separation newlines - elif self.options['strip_document'] == STRIP: - text = text.strip('\n') # remove leading and trailing separation newlines - elif self.options['strip_document'] is None: - pass # leave leading and trailing separation newlines as-is - else: - raise ValueError('Invalid value for strip_document: %s' % self.options['strip_document']) - - return text + return _strip_document(text, self.options['strip_document']) def process_text(self, el, parent_tags=None): # For the top-level element, initialize the parent context with an empty set. diff --git a/tests/test_soup.py b/tests/test_soup.py new file mode 100644 index 0000000..98e3fbc --- /dev/null +++ b/tests/test_soup.py @@ -0,0 +1,242 @@ +import pytest # type: ignore[import-not-found] +from bs4 import BeautifulSoup +from markdownify import MarkdownConverter, LSTRIP, RSTRIP, STRIP, STRIP_ONE + + +@pytest.mark.parametrize('mode,expected', [ + (STRIP, 'Hello'), + (LSTRIP, 'Hello\n\n'), + (RSTRIP, '\n\n\nHello'), + (None, '\n\n\nHello\n\n'), +]) +def test_tag_strip_document(mode, expected): + soup = BeautifulSoup('\n
Hello
Hello
Hello
', 'html.parser') + converter = MarkdownConverter(strip_document=mode, **options) + assert converter.convert_soup(soup.p) == expected + assert converter.convert_soup(soup) == expected + + +@pytest.mark.parametrize('options', [ + {'strip': []}, {'strip': ['div']}, {'convert': ['p', '[document]']}, +]) +def test_document_policy_enabled_independently_of_root_converter(options): + soup = BeautifulSoup('Hello
Hello
']) +def test_invalid_tag_document_mode(html, mode): + soup = BeautifulSoup(html, 'html.parser') + with pytest.raises(ValueError, match='^Invalid value for strip_document: %s$' % mode): + MarkdownConverter(strip_document=mode).convert_soup(soup.p) + + +@pytest.mark.parametrize('mode,expected', [ + (STRIP, ' \tA\r\n\nB\t '), + (LSTRIP, ' \tA\r\n\nB\t \n\n'), + (RSTRIP, '\n\n \tA\r\n\nB\t '), + (None, '\n\n \tA\r\n\nB\t \n\n'), +]) +def test_custom_root_preserves_significant_whitespace(mode, expected): + soup = BeautifulSoup('Hello
', 'html.parser') + calls = [] + + class Converter(MarkdownConverter): + def convert__document_(self, el, text, parent_tags): + assert el is soup + assert text == '\n\nHello\n\n' + assert parent_tags == set() + calls.append(el) + return '\n\nDOCUMENT\n\n' + + converter = Converter(strip_document=mode) + assert converter.convert_soup(soup) == '\n\nDOCUMENT\n\n' + if mode == 'invalid': + with pytest.raises(ValueError): + converter.convert_soup(soup.p) + else: + expected = 'Hello' if mode == STRIP else '\n\nHello\n\n' + assert converter.convert_soup(soup.p) == expected + assert len(calls) == 1 + + +def test_process_tag_override_preserves_document_output(): + soup = BeautifulSoup('Hello
', 'html.parser') + calls = [] + + class Converter(MarkdownConverter): + def process_tag(self, node, parent_tags=None): + assert parent_tags == set() + calls.append(node) + return '\n\nCUSTOM\n\n' + + converter = Converter() + assert converter.convert_soup(soup) == '\n\nCUSTOM\n\n' + assert converter.convert_soup(soup.p) == 'CUSTOM' + assert len(calls) == 2 + assert calls[0] is soup and calls[1] is soup.p + soup.name = '[DOCUMENT]' + assert converter.convert_soup(soup) == '\n\nCUSTOM\n\n' + + +@pytest.mark.parametrize('available', [True, False]) +def test_document_lookup_cache_and_original_tag_dispatch(available): + soup = BeautifulSoup('Hello
', 'html.parser') + resolved = [] + converted = [] + processed = [] + + class Converter(MarkdownConverter): + def get_conv_fn(self, name): + resolved.append(name) + if name == '[document]' and not available: + return None + return super().get_conv_fn(name) + + def convert_p(self, el, text, parent_tags): + converted.append((el, set(parent_tags))) + return super().convert_p(el, text, parent_tags) + + def process_element(self, node, parent_tags=None): + processed.append(node) + return super().process_element(node, parent_tags) + + converter = Converter() + expected = 'Hello' if available else '\n\nHello\n\n' + for root in (soup.p, soup.p, soup, soup.p): + assert converter.convert_soup(root) == expected + assert resolved.count('[document]') == 1 + assert resolved.count('p') == 1 + assert len(converted) == 4 + assert all(el is soup.p for el, _ in converted) + assert [parents for _, parents in converted] == [set(), set(), {'[document]'}, set()] + assert len(processed) == 5 + assert sum(node is soup.p for node in processed) == 1 + + +@pytest.mark.parametrize('mode', [STRIP, LSTRIP, RSTRIP, None, 'invalid']) +def test_attached_tree_relationships_survive_conversion(mode): + soup = BeautifulSoup('Hello
after\n \n Hello \n \n
Hello
', 'html.parser') + root = soup.p + root.name = '_document_' + calls = [] + + class Converter(MarkdownConverter): + def convert__document_(self, el, text, parent_tags): + assert el is root + assert text == 'Hello' + assert parent_tags == set() + calls.append(el) + return '\n\nALIAS\n\n' + + assert Converter(strip_document=mode).convert_soup(root) == expected + assert len(calls) == 1 + + +@pytest.mark.parametrize('mode', [STRIP, None]) +def test_tag_lookup_caches_document_unavailability_before_state_changes(mode): + soup = BeautifulSoup('Hello
', 'html.parser') + resolved = [] + document_calls = [] + + class Converter(MarkdownConverter): + available = False + + def get_conv_fn(self, name): + resolved.append((name, self.available)) + if name == '[document]' and not self.available: + return None + return super().get_conv_fn(name) + + def convert__document_(self, el, text, parent_tags): + document_calls.append(el) + return super().convert__document_(el, text, parent_tags) + + converter = Converter(strip_document=mode) + assert converter.convert_soup(soup.p) == '\n\nHello\n\n' + converter.available = True + assert converter.convert_soup(soup) == '\n\nHello\n\n' + assert resolved == [('p', False), ('[document]', False)] + assert document_calls == []