From 6fc5d8716ca11b77cdc010bc73251a25bbc72157 Mon Sep 17 00:00:00 2001 From: lyle <1179454895@qq.com> Date: Thu, 6 Aug 2026 10:37:24 +0800 Subject: [PATCH] fix: support page:last doclib locators --- mineru/doclib/server.py | 22 +++++- tests/unittest/test_doclib_cache_semantics.py | 71 +++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/mineru/doclib/server.py b/mineru/doclib/server.py index 2db1a68a6d..006d9d619a 100644 --- a/mineru/doclib/server.py +++ b/mineru/doclib/server.py @@ -10,7 +10,7 @@ import sys import time from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from io import BytesIO from pathlib import Path from typing import Any, Iterator, cast @@ -219,6 +219,7 @@ class _LocatorParts: page_no: int | None = None block_no: int | None = None char_offset: int | None = None + is_last_page: bool = False class DoclibServer(AsyncDoclibInterface): @@ -701,6 +702,7 @@ async def _build_read_plan_from_locator( "Text files do not require MinerU parsing. Read the file directly.", "locator", ) + cursor = _resolve_last_page(cursor, doc) tier = cursor.tier or await self._default_read_tier(doc["sha256"]) if tier is None: @@ -1480,7 +1482,7 @@ def _render_sidecar_image_asset(self, plan: _ReadPlan, sidecar_path: Path) -> Co _READ_LOCATOR_RE = re.compile( r"^doc:(?P[0-9a-fA-F]+)" r"(?:/tier:(?Pflash|basic|standard|advanced)" - r"(?:/page:(?P[1-9][0-9]*)" + r"(?:/page:(?Plast|[1-9][0-9]*)" r"(?:/block:(?P[1-9][0-9]*)(?:/char:(?P0|[1-9][0-9]*))?)?)?)?$" ) @@ -1495,12 +1497,26 @@ def _parse_doc_locator(locator: str) -> _LocatorParts: return _LocatorParts( short_id=match.group("short_id"), tier=cast(Tier | None, match.group("tier")), - page_no=int(page_no) if page_no is not None else None, + page_no=int(page_no) if page_no not in (None, "last") else None, block_no=int(block_no) if block_no is not None else None, char_offset=int(char_offset) if char_offset is not None else None, + is_last_page=page_no == "last", ) +def _resolve_last_page(locator: _LocatorParts, doc: DocRow) -> _LocatorParts: + if not locator.is_last_page: + return locator + page_count = doc.get("page_count") + if page_count is None or page_count < 1: + raise InvalidRequestError( + "invalid_locator", + "Cannot resolve page:last because the document page count is unavailable.", + "locator", + ) + return replace(locator, page_no=page_count, is_last_page=False) + + def _canonical_locator(short_id: str, tier: Tier, locator: _LocatorParts) -> str: if locator.page_no is None: return f"doc:{short_id}/tier:{tier}" diff --git a/tests/unittest/test_doclib_cache_semantics.py b/tests/unittest/test_doclib_cache_semantics.py index 64be3241e4..18ff8699f2 100644 --- a/tests/unittest/test_doclib_cache_semantics.py +++ b/tests/unittest/test_doclib_cache_semantics.py @@ -3309,6 +3309,77 @@ async def _run() -> None: asyncio.run(_run()) +def test_read_last_page_locator_resolves_to_canonical_numeric_locator(tmp_path: Path) -> None: + async def _run() -> None: + db = DatabaseManager(str(tmp_path / "doclib.sqlite")) + await db.initialize() + server = DoclibServer(SimpleNamespace(db=db, data_dir=str(tmp_path))) + now = 1000 + sha256 = "a" * 64 + short_id = "aaaaaaa" + await db.execute( + "INSERT INTO docs (sha256, short_id, size_bytes, file_type, page_count, first_seen_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (sha256, short_id, 12, "pdf", 4, now, now), + ) + await db.execute( + "INSERT INTO parses (sha256, tier, page_range, status, privacy, done_at, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (sha256, "flash", "4", "done", "local", now, now, now), + ) + last_page = _text_page("last page content") + last_page.page_idx = 3 + _write_batch( + tmp_path, + sha256, + "flash", + "4", + now, + ParseResult([last_page]).to_dict(skip_defaults=True)["pages"], + ) + + try: + response = await server.read_content(f"doc:{short_id}/tier:flash/page:last") + + assert "last page content" in response.content + assert response.request_scope.page_range == "4" + assert response.request_scope.locator == f"doc:{short_id}/tier:flash/page:4" + assert response.content_ranges[0].page_range == "4" + assert response.content_ranges[0].start == f"doc:{short_id}/tier:flash/page:4" + assert response.content_ranges[0].end == f"doc:{short_id}/tier:flash/page:4/block:1" + finally: + await db.close() + + asyncio.run(_run()) + + +def test_read_last_page_locator_rejects_unknown_page_count(tmp_path: Path) -> None: + async def _run() -> None: + db = DatabaseManager(str(tmp_path / "doclib.sqlite")) + await db.initialize() + server = DoclibServer(SimpleNamespace(db=db, data_dir=str(tmp_path))) + now = 1000 + sha256 = "a" * 64 + short_id = "aaaaaaa" + await db.execute( + "INSERT INTO docs (sha256, short_id, size_bytes, file_type, page_count, first_seen_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (sha256, short_id, 12, "pdf", None, now, now), + ) + + try: + with pytest.raises(InvalidRequestError) as exc_info: + await server.read_content(f"doc:{short_id}/tier:flash/page:last") + + assert exc_info.value.code == "invalid_locator" + assert exc_info.value.param == "locator" + assert "page count is unavailable" in exc_info.value.message + finally: + await db.close() + + asyncio.run(_run()) + + def test_get_file_by_path_missing_doclib_record_message_is_not_disk_file_not_found(tmp_path: Path) -> None: class _ParseSvc: async def ensure_ingested(self, path: str) -> None: