Mitigate UnicodeDecodeError due to wrong ASCII charset guess for long files (#2360)

* Add json-bug test

* Harden ipyn_ converter accepts

* Extend the window used to guess the charset

* Add help message in case of UnicodeDecodeError

* Fix style

* Remove leftover print

Author: fcatalan92

packages/markitdown/src/markitdown/_exceptions.py

@@ -73,4 +73,10 @@ def __init__(
                     else:
                         message += f" - {type(attempt.converter).__name__} threw {attempt.exc_info[0].__name__} with message: {attempt.exc_info[1]}\n"
 
+                        if isinstance(attempt.exc_info[1], UnicodeDecodeError):
+                            message += (
+                                "The charset may have been automatically guessed incorrectly. "
+                                "If you didn't specify the charset, please retry with an explicitly defined one.\n"
+                            )
+
         super().__init__(message)

packages/markitdown/src/markitdown/_markitdown.py

@@ -726,9 +726,9 @@ def _get_stream_info_guesses(
                 # If it's text, also guess the charset
                 charset = None
                 if result.prediction.output.is_text:
-                    # Read the first 4k to guess the charset
+                    # Read the first 64k to guess the charset
                     file_stream.seek(cur_pos)
-                    stream_page = file_stream.read(4096)
+                    stream_page = file_stream.read(65536)
                     charset_result = charset_normalizer.from_bytes(stream_page).best()
 
                     if charset_result is not None:

packages/markitdown/src/markitdown/converters/_ipynb_converter.py

@@ -38,6 +38,8 @@ def accepts(
                         "nbformat" in notebook_content
                         and "nbformat_minor" in notebook_content
                     )
+                except Exception as e:
+                    pass
                 finally:
                     file_stream.seek(cur_pos)
 

packages/markitdown/tests/test_module_misc.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3 -m pytest
 import io
+import json
 import os
 import re
 import shutil
@@ -587,6 +588,24 @@ def test_markitdown_llm() -> None:
     validate_strings(result, PPTX_TEST_STRINGS)
 
 
+def test_json_with_late_non_ascii_character(tmp_path) -> None:
+    payload = {
+        "record": {
+            "title": "Example record",
+            "abstract": "This is sample test. " * 500,
+        },
+        "notes": "non-ASCII character: è",
+    }
+    json_path = tmp_path / "input.json"
+    json_path.write_text(
+        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+
+    result = MarkItDown().convert(str(json_path))
+
+    assert "non-ASCII character: è" in result.text_content
+
+
 if __name__ == "__main__":
     """Runs this file's tests from the command line."""
     for test in [