fix(pdf): preserve positioned word boundaries without discarding tables

Author: cagdasyurekli

packages/markitdown/src/markitdown/converters/_pdf_converter.py

@@ -1,6 +1,7 @@
 import sys
 import io
 import re
+import logging
 from typing import BinaryIO, Any
 
 from .._base_converter import DocumentConverter, DocumentConverterResult
@@ -9,6 +10,12 @@
 
 # Pattern for MasterFormat-style partial numbering (e.g., ".1", ".2", ".10")
 PARTIAL_NUMBERING_PATTERN = re.compile(r"^\.\d+$")
+LONG_ASCII_WORD_PATTERN = re.compile(r"[A-Za-z]{30,}")
+
+# pdfplumber's default x_tolerance can merge adjacent positioned words in PDFs
+# that omit literal space glyphs. A smaller tolerance preserves those spaces
+# while still allowing table/form detection to operate on word-level positions.
+PDFPLUMBER_TEXT_X_TOLERANCE = 1
 
 
 def _merge_partial_numbering_lines(text: str) -> str:
@@ -57,6 +64,69 @@ def _merge_partial_numbering_lines(text: str) -> str:
     return "\n".join(result_lines)
 
 
+def _has_space_starved_text(text: str) -> bool:
+    """
+    Return True when extracted text appears to have lost most word boundaries.
+
+    Some PDFs position words separately instead of storing literal space glyphs.
+    pdfminer can then produce long alphabetic runs such as
+    "NaturalLanguageProcessing" throughout the document. This intentionally
+    uses a conservative threshold so ordinary identifiers/URLs do not trigger
+    a fallback choice by themselves.
+    """
+    ascii_letters = sum(1 for char in text if char.isascii() and char.isalpha())
+    if ascii_letters < 500:
+        return False
+
+    whitespace_ratio = sum(1 for char in text if char.isspace()) / ascii_letters
+    long_word_count = len(LONG_ASCII_WORD_PATTERN.findall(text))
+
+    return whitespace_ratio < 0.03 and long_word_count >= 5
+
+
+def _has_space_starved_words(words: list[dict]) -> bool:
+    alpha_word_count = sum(
+        1
+        for word in words
+        if any(char.isascii() and char.isalpha() for char in word.get("text", ""))
+    )
+    long_word_count = sum(
+        1 for word in words if LONG_ASCII_WORD_PATTERN.search(word.get("text", ""))
+    )
+
+    if long_word_count < 5 or long_word_count / alpha_word_count < 0.025:
+        return False
+
+    # Long cells and separate text columns are not evidence of missing spaces.
+    # Preserve layouts with repeated wide gutters between aligned runs.
+    rows: dict[float, list[dict]] = {}
+    for word in words:
+        if not all(key in word for key in ("top", "x0", "x1")):
+            continue
+        rows.setdefault(round(word["top"] / 5) * 5, []).append(word)
+    gutters: dict[float, int] = {}
+    for row in rows.values():
+        row.sort(key=lambda word: word["x0"])
+        row_gutters = {
+            round((left["x1"] + right["x0"]) / 10) * 10
+            for left, right in zip(row, row[1:])
+            if right["x0"] - left["x1"] >= 20
+        }
+        for gutter in row_gutters:
+            gutters[gutter] = gutters.get(gutter, 0) + 1
+    return not any(
+        count >= 3 and count >= len(rows) * 0.5 for count in gutters.values()
+    )
+
+
+def _select_plain_text_extraction(pdfminer_text: str, pdfplumber_text: str) -> str:
+    """Choose the better plain-text PDF extraction result."""
+    if pdfplumber_text.strip() and _has_space_starved_text(pdfminer_text):
+        return pdfplumber_text
+
+    return pdfminer_text
+
+
 # Load dependencies
 _dependency_exc_info = None
 try:
@@ -117,7 +187,9 @@ def fmt_row(row: list[str]) -> str:
     return "\n".join(md)
 
 
-def _extract_form_content_from_words(page: Any) -> str | None:
+def _extract_form_content_from_words(
+    page: Any, words: list[dict] | None = None, *, space_starved: bool = False
+) -> str | None:
     """
     Extract form-style content from a PDF page by analyzing word positions.
     This handles borderless forms/tables where words are aligned in columns.
@@ -129,10 +201,14 @@ def _extract_form_content_from_words(page: Any) -> str | None:
     Returns None if the page doesn't appear to be a form-style document,
     indicating that pdfminer should be used instead for better text spacing.
     """
-    words = page.extract_words(keep_blank_chars=True, x_tolerance=3, y_tolerance=3)
+    if words is None:
+        words = page.extract_words(keep_blank_chars=True, x_tolerance=3, y_tolerance=3)
     if not words:
         return None
 
+    if space_starved or _has_space_starved_words(words):
+        return None
+
     # Group words by their Y position (rows)
     y_tolerance = 5
     rows_by_y: dict[float, list[dict]] = {}
@@ -551,29 +627,57 @@ def convert(
 
             with pdfplumber.open(pdf_bytes) as pdf:
                 for page_idx, page in enumerate(pdf.pages):
-                    page_content = _extract_form_content_from_words(page)
+                    words = page.extract_words(
+                        keep_blank_chars=True, x_tolerance=3, y_tolerance=3
+                    )
+                    page_is_space_starved = _has_space_starved_words(words)
+                    if page_is_space_starved:
+                        words = page.extract_words(
+                            keep_blank_chars=True,
+                            x_tolerance=PDFPLUMBER_TEXT_X_TOLERANCE,
+                            y_tolerance=3,
+                        )
+                    # Tight words no longer contain the collapsed runs, so retain
+                    # the original page classification when checking for forms.
+                    page_content = _extract_form_content_from_words(
+                        page, words=words, space_starved=page_is_space_starved
+                    )
 
                     if page_content is not None:
                         form_page_count += 1
                         if page_content.strip():
                             markdown_chunks.append(page_content)
                     else:
                         plain_page_indices.append(page_idx)
-                        text = page.extract_text()
+                        if page_is_space_starved:
+                            text = page.extract_text(
+                                x_tolerance=PDFPLUMBER_TEXT_X_TOLERANCE
+                            )
+                        else:
+                            text = page.extract_text()
                         if text and text.strip():
                             markdown_chunks.append(text.strip())
 
                     page.close()  # Free cached page data immediately
 
             # If no pages had form-style content, use pdfminer for
-            # the whole document (better text spacing for prose).
+            # the whole document when it has good spacing. Some PDFs
+            # encode words by position without literal spaces; in that
+            # case pdfplumber with a tight x_tolerance preserves word
+            # boundaries better.
+            pdfplumber_markdown = "\n\n".join(markdown_chunks).strip()
             if form_page_count == 0:
                 pdf_bytes.seek(0)
-                markdown = pdfminer.high_level.extract_text(pdf_bytes)
+                pdfminer_markdown = pdfminer.high_level.extract_text(pdf_bytes)
+                markdown = _select_plain_text_extraction(
+                    pdfminer_markdown, pdfplumber_markdown
+                )
             else:
-                markdown = "\n\n".join(markdown_chunks).strip()
+                markdown = pdfplumber_markdown
 
-        except Exception:
+        except Exception as exc:
+            # Preserve fallback behavior, but expose unexpected parser failures.
+            logging.getLogger(__name__).warning("PDF table extraction failed: %s", exc)
             # Fallback if pdfplumber fails
             pdf_bytes.seek(0)
             markdown = pdfminer.high_level.extract_text(pdf_bytes)