diff --git a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py index 6bc1c1673f..d9ab4db82a 100644 --- a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py +++ b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py @@ -12,6 +12,23 @@ ACCEPTED_FILE_EXTENSIONS = [".ipynb"] +def _iter_cells(notebook_content: dict): + """Yield a notebook's cells, whichever nbformat version it uses. + + nbformat 4 keeps a flat ``cells`` list at the top level. nbformat 3 (IPython 3 + and Jupyter 4.0, still common in archives) nests them under + ``worksheets[*].cells`` instead, so reading only the top level silently yielded + nothing and the whole notebook converted to an empty document. + """ + cells = notebook_content.get("cells") + if cells is not None: + yield from cells + return + + for worksheet in notebook_content.get("worksheets", []): + yield from worksheet.get("cells", []) + + class IpynbConverter(DocumentConverter): """Converts Jupyter Notebook (.ipynb) files to Markdown.""" @@ -67,25 +84,51 @@ def _convert(self, notebook_content: dict) -> DocumentConverterResult: md_output = [] title = None - for cell in notebook_content.get("cells", []): + for cell in _iter_cells(notebook_content): cell_type = cell.get("cell_type", "") - source_lines = cell.get("source", []) + # nbformat 3 stores code cell text under "input"; nbformat 4 uses + # "source" for every cell type. Reading only "source" dropped the + # content of every code cell in an nbformat 3 notebook. + source = cell.get("source") + if source is None: + source = cell.get("input", []) + # Both shapes are valid on disk, and the title match below + # iterates per-line, so a plain string would iterate characters. + if isinstance(source, str): + line_source = source.splitlines(keepends=True) + else: + line_source = source + cell_source = "".join(line_source) if cell_type == "markdown": - md_output.append("".join(source_lines)) + md_output.append(cell_source) # Extract the first # heading as title if not already found if title is None: - for line in source_lines: + for line in line_source: if line.startswith("# "): title = line.removeprefix("# ").strip() break + elif cell_type == "heading": + # nbformat 3 removed heading cells in 4.0 in favour of a + # markdown '#' line; map them back so they are not dropped. + level = cell.get("level", 1) + try: + level = int(level) + except (TypeError, ValueError): + level = 1 + level = max(level, 1) + head_text = cell_source.strip() + md_output.append(f"{'#' * level} {head_text}") + if title is None and head_text: + title = head_text + elif cell_type == "code": # Code cells are wrapped in Markdown code blocks - md_output.append(f"```python\n{''.join(source_lines)}\n```") + md_output.append(f"```python\n{cell_source}\n```") elif cell_type == "raw": - md_output.append(f"```\n{''.join(source_lines)}\n```") + md_output.append(f"```\n{cell_source}\n```") md_text = "\n\n".join(md_output) diff --git a/packages/markitdown/tests/test_ipynb_nbformat3.py b/packages/markitdown/tests/test_ipynb_nbformat3.py new file mode 100644 index 0000000000..29d7cc9963 --- /dev/null +++ b/packages/markitdown/tests/test_ipynb_nbformat3.py @@ -0,0 +1,189 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Regression tests for reading notebooks saved in the nbformat 3 layout. + +nbformat 3 (IPython 3, Jupyter 4.0) nests cells under ``worksheets[*].cells`` and +stores code cell text under ``input``. Reading only the nbformat 4 layout produced +an empty document with a success exit code, so every cell was silently dropped. +""" + +import io +import json + +from markitdown import MarkItDown, StreamInfo +from markitdown.converters._ipynb_converter import IpynbConverter + +_NBFORMAT_3 = { + "metadata": {}, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "language": "python", + "input": ["print('hello')\n"], + "outputs": [], + }, + { + "cell_type": "markdown", + "source": ["# Quarterly notes\n", "\n", "Revenue grew.\n"], + }, + ] + } + ], +} + +# The same notebook in the nbformat 4 layout: flat `cells`, and `source` everywhere. +_NBFORMAT_4 = { + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + "cells": [ + {"cell_type": "code", "source": ["print('hello')\n"], "outputs": []}, + { + "cell_type": "markdown", + "source": ["# Quarterly notes\n", "\n", "Revenue grew.\n"], + }, + ], +} + + +def _convert(notebook: dict): + data = json.dumps(notebook).encode("utf-8") + return IpynbConverter().convert( + io.BytesIO(data), StreamInfo(extension=".ipynb", charset="utf-8") + ) + + +def test_nbformat_3_notebook_is_not_empty() -> None: + """An accepted notebook must never convert to an empty document.""" + result = _convert(_NBFORMAT_3) + + assert result.markdown.strip(), "nbformat 3 notebook converted to nothing" + + +def test_nbformat_3_code_cells_use_their_input_key() -> None: + result = _convert(_NBFORMAT_3) + + assert "```python\nprint('hello')" in result.markdown + + +def test_nbformat_3_markdown_cells_are_kept_and_title_extracted() -> None: + result = _convert(_NBFORMAT_3) + + assert "# Quarterly notes" in result.markdown + assert "Revenue grew." in result.markdown + assert result.title == "Quarterly notes" + + +def test_nbformat_3_matches_nbformat_4_for_the_same_notebook() -> None: + """The two layouts describe the same document and must produce the same Markdown.""" + assert _convert(_NBFORMAT_3).markdown == _convert(_NBFORMAT_4).markdown + + +def test_nbformat_3_converts_through_the_public_entry_point() -> None: + """The whole stack must see the content, not fall through to plain text.""" + data = json.dumps(_NBFORMAT_3).encode("utf-8") + + result = MarkItDown().convert_stream( + io.BytesIO(data), stream_info=StreamInfo(extension=".ipynb") + ) + + assert "print('hello')" in result.markdown + assert "Revenue grew." in result.markdown + + +def test_nbformat_3_with_several_worksheets_keeps_every_cell() -> None: + notebook = { + "metadata": {}, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + {"cells": [{"cell_type": "code", "input": ["first = 1\n"], "outputs": []}]}, + { + "cells": [ + {"cell_type": "code", "input": ["second = 2\n"], "outputs": []} + ] + }, + ], + } + + markdown = _convert(notebook).markdown + + assert "first = 1" in markdown + assert "second = 2" in markdown + + +def test_nbformat_3_title_from_metadata_still_wins() -> None: + notebook = dict(_NBFORMAT_3, metadata={"title": "From metadata"}) + + assert _convert(notebook).title == "From metadata" + + +def test_nbformat_4_is_unchanged() -> None: + """The nbformat 4 path must be untouched by this change.""" + result = _convert(_NBFORMAT_4) + + assert "```python\nprint('hello')" in result.markdown + assert result.title == "Quarterly notes" + + +def test_nbformat_3_heading_cells_are_kept_as_markdown_headings() -> None: + """v3 heading cells must not vanish; they map to the '#' form that v4 uses.""" + notebook = { + "metadata": {}, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "level": 1, + "source": ["Quarterly notes\n"], + }, + {"cell_type": "code", "input": ["print('hello')\n"], "outputs": []}, + ] + } + ], + } + + result = _convert(notebook) + + assert "# Quarterly notes" in result.markdown + assert result.title == "Quarterly notes" + + +def test_nbformat_3_single_string_source_still_yields_title() -> None: + notebook = { + "metadata": {}, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "source": "# Quarterly notes\n\nRevenue grew.\n", + } + ] + } + ], + } + + result = _convert(notebook) + + assert result.title == "Quarterly notes" + assert "Revenue grew." in result.markdown + + +def test_null_cells_fall_back_to_worksheets() -> None: + notebook = dict(_NBFORMAT_3, cells=None) + + result = _convert(notebook) + + assert "print('hello')" in result.markdown + assert "Revenue grew." in result.markdown