Trim comma-padded blank rows in CSV table conversion

packages/markitdown/src/markitdown/converters/_csv_converter.py

@@ -29,8 +29,24 @@ def _escape_table_cell(value: str) -> str:
     return value.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
 
 
+def _is_padding_row(row: list[str]) -> bool:
+    """Whether a parsed row carries only padded, empty cells.
+
+    Writers that pad every record to the column width emit a blank row as
+    "," or ",," rather than as an empty line -- ``pandas.DataFrame.to_csv``
+    does this for all-NaN rows -- so a run of empty (or whitespace-only)
+    cells is the same blank row. This is the same reading the PDF table
+    builder already uses in ``_pdf_converter._to_markdown_table``.
+
+    Only rows that follow the header can be judged this way: in the first
+    row, empty fields are indistinguishable from a header whose column
+    names are empty, so the leading run keeps the "no fields at all" test.
+    """
+    return all(not cell.strip() for cell in row)
+
+
 def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
-    """Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""
+    """Remove empty rows from the beginning and end, and immediately after the header. Once the header is known, a row of only empty cells counts as empty just as an absent one does. This operation is performed in-place."""
     start = 0
     while start < len(rows) and not rows[start]:
         start += 1
@@ -41,11 +57,11 @@ def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
 
     header_index = start
     start += 1
-    while start < len(rows) and not rows[start]:
+    while start < len(rows) and _is_padding_row(rows[start]):
         start += 1
 
     end = len(rows)
-    while end > start and not rows[end - 1]:
+    while end > start and _is_padding_row(rows[end - 1]):
         end -= 1
 
     # Remove each blank run at once, rather than shifting the list per row.

packages/markitdown/tests/test_csv.py

@@ -144,6 +144,66 @@ def test_csv_all_blank_input_returns_empty_markdown() -> None:
     assert result == ""
 
 
+def test_csv_comma_padded_trailing_rows_are_trimmed() -> None:
+    # A writer that pads every record to the column width writes a blank row
+    # as "," rather than as an empty line; pandas' to_csv does this for
+    # all-NaN rows. Past the header such a row is blank just the same, and
+    # must not survive as an empty data row that widens the table.
+    result = _convert_csv(b"name,age\nAlice,30\n,\n,\n")
+
+    assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"
+
+
+def test_csv_padded_row_after_header_is_trimmed() -> None:
+    result = _convert_csv(b"name,age\n,\nAlice,30\n")
+
+    assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"
+
+
+def test_csv_whitespace_padded_trailing_row_is_trimmed() -> None:
+    result = _convert_csv(b"name,age\nAlice,30\n , \n")
+
+    assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"
+
+
+def test_csv_unnamed_header_is_kept() -> None:
+    # In the first row, empty fields cannot be told apart from a header whose
+    # column names are empty -- pandas.read_csv reads this same record as
+    # "Unnamed: 0"/"Unnamed: 1" -- so the leading run keeps the old meaning of
+    # "no fields at all".
+    result = _convert_csv(b",\nAlice,30\nBob,40\n")
+
+    assert result == "|  |  |\n| --- | --- |\n| Alice | 30 |\n| Bob | 40 |"
+
+
+def test_csv_quoted_unnamed_header_is_kept() -> None:
+    result = _convert_csv(b'"",""\nAlice,30\nBob,40\n')
+
+    assert result == "|  |  |\n| --- | --- |\n| Alice | 30 |\n| Bob | 40 |"
+
+
+def test_csv_unnamed_header_keeps_its_column_count() -> None:
+    result = _convert_csv(b",,\nAlice,30\nBob,40\n")
+
+    assert (
+        result == "|  |  |  |\n"
+        "| --- | --- | --- |\n"
+        "| Alice | 30 |  |\n"
+        "| Bob | 40 |  |"
+    )
+
+
+def test_csv_comma_padded_row_between_data_rows_is_kept() -> None:
+    # Only the trailing run and the run right after the header are trimmed; an
+    # internal padded row keeps its place, exactly as the existing blank-line
+    # test pins for unpadded blank rows.
+    result = _convert_csv(b"name,age\nAlice,30\n,\nBob,40\n")
+
+    assert (
+        result == "| name | age |\n| --- | --- |\n| Alice | 30 |\n|  |  |\n| Bob | 40 |"
+    )
+
+
 @pytest.mark.parametrize(
     "data,expected",
     [