|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +import base64 |
| 4 | +import binascii |
| 5 | +import re |
| 6 | +import warnings |
| 7 | +from typing import TYPE_CHECKING, BinaryIO, Final |
| 8 | +from urllib.parse import parse_qs, urljoin, urlparse |
| 9 | + |
| 10 | +from bs4 import BeautifulSoup, Tag |
| 11 | + |
| 12 | +from .._base_converter import DocumentConverterResult |
| 13 | +from .._stream_info import StreamInfo |
| 14 | +from ._markdownify import _CustomMarkdownify |
| 15 | + |
| 16 | +if TYPE_CHECKING: |
| 17 | + from ._markdown_options import _MarkdownOptions |
| 18 | + |
| 19 | + |
| 20 | +def convert_html( |
| 21 | + stream: BinaryIO, info: StreamInfo, options: _MarkdownOptions |
| 22 | +) -> DocumentConverterResult: |
| 23 | + soup: Final = _parse_html(stream, info) |
| 24 | + return DocumentConverterResult( |
| 25 | + markdown=_convert(soup.select_one("body") or soup, options).strip(), |
| 26 | + title=soup.title.string if soup.title else None, |
| 27 | + ) |
| 28 | + |
| 29 | + |
| 30 | +def convert_bing( |
| 31 | + stream: BinaryIO, info: StreamInfo, options: _MarkdownOptions |
| 32 | +) -> DocumentConverterResult: |
| 33 | + soup: Final = BeautifulSoup( |
| 34 | + stream, "html.parser", from_encoding=info.charset or "utf-8" |
| 35 | + ) |
| 36 | + for title in soup.select(".tptt"): |
| 37 | + if title.string: |
| 38 | + title.string += " " |
| 39 | + for slug in soup.select(".algoSlug_icon"): |
| 40 | + slug.extract() |
| 41 | + results: Final[list[str]] = [] |
| 42 | + for result in soup.select(".b_algo"): |
| 43 | + for anchor in result.select("a[href]"): |
| 44 | + parameters = parse_qs(urlparse(str(anchor["href"])).query) |
| 45 | + # Bing prefixes the Base64URL destination with two characters. |
| 46 | + if destination := parameters.get("u"): |
| 47 | + try: |
| 48 | + anchor["href"] = base64.b64decode( |
| 49 | + destination[0][2:].strip() + "==", altchars="-_" |
| 50 | + ).decode("utf-8") |
| 51 | + except (UnicodeDecodeError, binascii.Error): |
| 52 | + pass |
| 53 | + results.append( |
| 54 | + "\n".join( |
| 55 | + line.strip() |
| 56 | + for line in re.split( |
| 57 | + r"\n+", _CustomMarkdownify(**options).convert_soup(result).strip() |
| 58 | + ) |
| 59 | + if line.strip() |
| 60 | + ) |
| 61 | + ) |
| 62 | + query: Final = parse_qs(urlparse(info.url or "").query).get("q", [""])[0] |
| 63 | + return DocumentConverterResult( |
| 64 | + markdown=f"## A Bing search for '{query}' found the following results:\n\n" |
| 65 | + + "\n\n".join(results), |
| 66 | + title=soup.title.string if soup.title else None, |
| 67 | + ) |
| 68 | + |
| 69 | + |
| 70 | +def convert_wikipedia( |
| 71 | + stream: BinaryIO, info: StreamInfo, options: _MarkdownOptions |
| 72 | +) -> DocumentConverterResult: |
| 73 | + soup: Final = _parse_html(stream, info) |
| 74 | + body: Final = soup.select_one("div#mw-content-text") |
| 75 | + title = soup.title.string if soup.title else None |
| 76 | + if body: |
| 77 | + if heading := soup.select_one("span.mw-page-title-main") or soup.select_one( |
| 78 | + "h1#firstHeading" |
| 79 | + ): |
| 80 | + title = heading.get_text() |
| 81 | + title = title.strip() or None if title else None |
| 82 | + markdown = (f"# {title}\n\n" if title else "") + _CustomMarkdownify( |
| 83 | + **options |
| 84 | + ).convert_soup(body) |
| 85 | + else: |
| 86 | + markdown = _CustomMarkdownify(**options).convert_soup(soup) |
| 87 | + return DocumentConverterResult(markdown=markdown, title=title) |
| 88 | + |
| 89 | + |
| 90 | +def convert_feed(content: str, base_url: str, options: _MarkdownOptions) -> str: |
| 91 | + soup: Final = BeautifulSoup(content, "html.parser") |
| 92 | + pending: Final[list[tuple[Tag, str]]] = [(soup, base_url)] |
| 93 | + while pending: |
| 94 | + node, inherited = pending.pop() |
| 95 | + override = node.get("xml:base") |
| 96 | + current = ( |
| 97 | + _resolve_url(inherited, override) |
| 98 | + if isinstance(override, str) |
| 99 | + else inherited |
| 100 | + ) |
| 101 | + attributes = ( |
| 102 | + ("href",) |
| 103 | + if node.name == "a" |
| 104 | + else ("src", "data-src") |
| 105 | + if node.name == "img" |
| 106 | + else () |
| 107 | + ) |
| 108 | + for attribute in attributes: |
| 109 | + reference = node.get(attribute) |
| 110 | + if isinstance(reference, str) and (reference or attribute == "href"): |
| 111 | + node[attribute] = _resolve_url(current, reference) |
| 112 | + pending.extend( |
| 113 | + (child, current) for child in node.children if isinstance(child, Tag) |
| 114 | + ) |
| 115 | + return _convert(soup, options) |
| 116 | + |
| 117 | + |
| 118 | +def _parse_html(stream: BinaryIO, info: StreamInfo) -> BeautifulSoup: |
| 119 | + soup: Final = BeautifulSoup( |
| 120 | + stream, "html.parser", from_encoding=info.charset or "utf-8" |
| 121 | + ) |
| 122 | + for element in soup.select("script, style"): |
| 123 | + element.extract() |
| 124 | + return soup |
| 125 | + |
| 126 | + |
| 127 | +def _convert(soup: Tag, options: _MarkdownOptions) -> str: |
| 128 | + try: |
| 129 | + return _CustomMarkdownify(**options).convert_soup(soup) |
| 130 | + except RecursionError: |
| 131 | + if options.get("strict", False): |
| 132 | + raise |
| 133 | + warnings.warn( |
| 134 | + "HTML content is too deeply nested for markdown conversion (RecursionError). " |
| 135 | + "Falling back to plain-text extraction.", |
| 136 | + stacklevel=2, |
| 137 | + ) |
| 138 | + return soup.get_text("\n", strip=True) |
| 139 | + |
| 140 | + |
| 141 | +def _resolve_url(base: str, reference: str) -> str: |
| 142 | + try: |
| 143 | + return urljoin(base, reference) |
| 144 | + except ValueError: |
| 145 | + return reference |
| 146 | + |
| 147 | + |
| 148 | +__all__ = ["convert_bing", "convert_feed", "convert_html", "convert_wikipedia"] |
0 commit comments