From 84dc66ce2abcc2a12469859aacbe78199b5af9db Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 17 Oct 2026 01:29:46 -0700 Subject: [PATCH 1/6] perf(local): read scans and photos with OCR alone before Docling RapidOCR, the OCR inside Docling, reads a scanned page without Docling's layout model and TableFormer, and Apple's Vision does on a Mac. On 499 real pages (forms, receipts, DocVQA scans, books, court and archive scans) both found more of the page than Docling did, and RapidOCR took about 2 s and 8 CPU-s a page where Docling took 4.9 s and 39. Docling still reads a page whose OCR came out garbled, through the splice a mixed PDF's flagged pages take. SURFSENSE_LOCAL_SCAN_OCR=off reads every scan with Docling as before. --- .github/workflows/release-local.yml | 1 + surfsense_local/backend/bundling/worker.spec | 8 + surfsense_local/backend/pyproject.toml | 4 + surfsense_local/backend/tests/conftest.py | 7 + .../integration/worker/test_ocr_first.py | 149 ++++++++ .../tests/integration/worker/test_parsing.py | 2 + .../worker/test_rapidocr_engine.py | 83 +++++ .../tests/unit/worker/test_scan_ocr.py | 332 ++++++++++++++++++ surfsense_local/backend/uv.lock | 96 +++++ surfsense_local/backend/worker.py | 4 + .../backend/worker/ingestion/parsing.py | 5 + .../worker/ingestion/pdf_reading/ocr_first.py | 134 +++++++ .../worker/ingestion/pdf_reading/route.py | 41 ++- .../worker/ingestion/scan_ocr/__init__.py | 4 + .../worker/ingestion/scan_ocr/check.py | 51 +++ .../worker/ingestion/scan_ocr/frames.py | 75 ++++ .../worker/ingestion/scan_ocr/images.py | 51 +++ .../backend/worker/ingestion/scan_ocr/page.py | 173 +++++++++ .../worker/ingestion/scan_ocr/rapid.py | 156 ++++++++ .../worker/ingestion/scan_ocr/reader.py | 58 +++ .../ingestion/scan_ocr/reading_order.py | 164 +++++++++ .../worker/ingestion/scan_ocr/switch.py | 44 +++ .../worker/ingestion/scan_ocr/upright.py | 34 ++ .../worker/ingestion/scan_ocr/vision.py | 122 +++++++ .../scripts/notices/allowed-without-text.json | 18 + 25 files changed, 1814 insertions(+), 2 deletions(-) create mode 100644 surfsense_local/backend/tests/integration/worker/test_ocr_first.py create mode 100644 surfsense_local/backend/tests/integration/worker/test_rapidocr_engine.py create mode 100644 surfsense_local/backend/tests/unit/worker/test_scan_ocr.py create mode 100644 surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/__init__.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/check.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/frames.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/images.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/page.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/reader.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/switch.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/upright.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/vision.py diff --git a/.github/workflows/release-local.yml b/.github/workflows/release-local.yml index f2dfe68690..4576a2a42d 100644 --- a/.github/workflows/release-local.yml +++ b/.github/workflows/release-local.yml @@ -171,6 +171,7 @@ jobs: WORKER="./surfsense_local/backend/dist/worker/worker" if [ "$RUNNER_OS" = "Windows" ]; then WORKER="${WORKER}.exe"; fi "$WORKER" --check-vision-runtime + "$WORKER" --check-scan-ocr # A script is not analysed when the worker is frozen, so a library the # spec leaves out shows only when a script runs in the shipped worker; diff --git a/surfsense_local/backend/bundling/worker.spec b/surfsense_local/backend/bundling/worker.spec index 2e232fa478..2c2678ce89 100644 --- a/surfsense_local/backend/bundling/worker.spec +++ b/surfsense_local/backend/bundling/worker.spec @@ -121,6 +121,14 @@ hiddenimports += [ "pandas", ] +# Apple's Vision (worker/ingestion/scan_ocr/vision.py) is imported inside +# functions, which the analyser follows; named so a refactor cannot drop it +# unseen. `worker.py --check-scan-ocr` checks the frozen worker on every release. +if sys.platform == "darwin": + # Not collect_all("objc"): it would ship pyobjc-core's 23 MB of .dSYM bundles + # and test extensions into a notarised app. + hiddenimports += ["objc", "Foundation", "Quartz", "Vision"] + a = Analysis( [str(BACKEND / "worker.py")], pathex=[str(BACKEND)], diff --git a/surfsense_local/backend/pyproject.toml b/surfsense_local/backend/pyproject.toml index bd899ffd11..9c47fae026 100644 --- a/surfsense_local/backend/pyproject.toml +++ b/surfsense_local/backend/pyproject.toml @@ -65,6 +65,10 @@ dependencies = [ "xlsxwriter>=3.2.9", "lameenc>=1.8.4", "mcp>=2.3,<2.4", + # Apple's Vision reads scans and photos on a Mac before Docling + # (worker/ingestion/scan_ocr). Not ocrmac: Docling would then switch its own OCR. + "pyobjc-framework-Quartz>=12.2.2; sys_platform == 'darwin'", + "pyobjc-framework-Vision>=12.2.2; sys_platform == 'darwin'", ] [dependency-groups] diff --git a/surfsense_local/backend/tests/conftest.py b/surfsense_local/backend/tests/conftest.py index 5a70d8f733..586e349506 100644 --- a/surfsense_local/backend/tests/conftest.py +++ b/surfsense_local/backend/tests/conftest.py @@ -111,6 +111,13 @@ def no_office_support(monkeypatch: pytest.MonkeyPatch) -> None: office_off(monkeypatch) +@pytest.fixture(autouse=True) +def no_system_ocr(monkeypatch: pytest.MonkeyPatch) -> None: + """Scans and photos go to Docling, as the tests of Docling's readings expect, + whatever OCR this machine has; a test of the system's OCR turns it on.""" + monkeypatch.setenv("SURFSENSE_LOCAL_SCAN_OCR", "off") + + @pytest.fixture def real_model(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: """Point the encoder at the model fetch_embedding_model.py downloaded, or skip.""" diff --git a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py new file mode 100644 index 0000000000..1c79db8b5d --- /dev/null +++ b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py @@ -0,0 +1,149 @@ +"""A scan or a photo read by OCR before Docling: the source is ready with OCR's +text, Docling reads only the pages OCR could not read, and with no OCR here Docling +reads as before. The OCR is a stand-in that returns pages of boxes.""" + +import io +from collections.abc import Iterable, Iterator + +import pytest +from PIL import Image +from sqlalchemy.orm import Session + +from modules.documents.models import DocumentStatus +from modules.documents.tasks import PRIORITY_IDLE +from shared.queue import ingest_queue +from tests.integration.worker.paged_docling_stand_in import PagedDocling +from tests.integration.worker.test_mixed_pdfs import folder_of, scans, session, upload +from worker.ingestion import run +from worker.ingestion.pdf_reading import ocr_first +from worker.ingestion.scan_ocr import images +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import Line, OcrPage, Word + +pytestmark = pytest.mark.integration + +__all__ = ["session"] + + +def prose(page: int) -> OcrPage: + lines = tuple( + Line( + f"Scanned page {page} line {n} says the harbour board met.", + tuple( + Word(word, 100 + 120 * i, 200 + 45 * n, 100, 30) + for i, word in enumerate( + f"Scanned page {page} line {n} says the harbour board met.".split() + ) + ), + ) + for n in range(8) + ) + return OcrPage(1786, 2526, lines, 0.08, "en-US") + + +def garbled(page: int) -> OcrPage: + """A page the engine was unsure of: it goes to Docling.""" + return OcrPage(1786, 2526, prose(page).lines, 0.08, "en-US", confidence=0.5) + + +class StandInOcr: + name = "Stand-in OCR" + language = "en-US" + max_side = 2000 + + def __init__(self) -> None: + self.garbled: set[int] = set() + self.read_pages: list[int] = [] + + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + for frame in frames: + self.read_pages.append(frame.number) + n = frame.number + yield frame, garbled(n) if n in self.garbled else prose(n) + + +@pytest.fixture +def ocr(monkeypatch: pytest.MonkeyPatch) -> StandInOcr: + reader = StandInOcr() + monkeypatch.setattr(ocr_first, "page_reader", lambda: reader) + monkeypatch.setattr(images, "page_reader", lambda: reader) + return reader + + +def test_a_scan_is_ready_with_ocrs_text_and_docling_is_never_asked( + session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr +) -> None: + source = upload(session, scans(3)) + + run(source.id) + + session.expire_all() + assert source.status is DocumentStatus.READY + assert paged_docling.asked == [] + assert ocr.read_pages == [1, 2, 3] + assert "Scanned page 2 line 0 says the harbour board met." in (source.content or "") + + +def test_a_garbled_page_of_a_scan_is_read_by_docling_in_its_place( + session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr +) -> None: + ocr.garbled = {2} + source = upload(session, scans(3)) + + run(source.id) + + session.expire_all() + assert paged_docling.asked == [(2, 2)] + parts = (source.content or "").split("\n\n") + assert paged_docling.markdown(2, 2) in source.content + assert parts.index(paged_docling.markdown(2, 2)) > 0 + assert "Scanned page 3 line 7" in parts[-1] + + +def test_a_short_scan_still_gets_its_figures_after_it_is_ready( + session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr +) -> None: + """From the figures-only pass, queued at idle, as for a text-read PDF.""" + source = upload(session, scans(2)) + + run(source.id) + + assert [(t.name, t.args, t.priority) for t in ingest_queue.pending()] == [ + ("extract_figures", (source.id,), PRIORITY_IDLE) + ] + + +def test_with_no_ocr_here_docling_reads_the_scan_whole( + session: Session, + stub_model: None, + paged_docling: PagedDocling, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ocr_first, "page_reader", lambda: None) + source = upload(session, scans(3)) + + run(source.id) + + session.expire_all() + assert source.status is DocumentStatus.READY + assert paged_docling.asked == [None] + + +def photo() -> bytes: + pixels = io.BytesIO() + Image.new("RGB", (900, 1200), "white").save(pixels, "JPEG") + return pixels.getvalue() + + +def test_a_photo_is_read_by_ocr_and_is_its_own_figure( + session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr +) -> None: + source = upload(session, photo(), name="receipt.jpg") + + run(source.id) + + session.expire_all() + assert source.status is DocumentStatus.READY + assert paged_docling.asked == [] + assert "Scanned page 1 line 0" in (source.content or "") + assert (folder_of(source) / "figures" / "figures.json").is_file() diff --git a/surfsense_local/backend/tests/integration/worker/test_parsing.py b/surfsense_local/backend/tests/integration/worker/test_parsing.py index df58dc1379..d918944822 100644 --- a/surfsense_local/backend/tests/integration/worker/test_parsing.py +++ b/surfsense_local/backend/tests/integration/worker/test_parsing.py @@ -162,6 +162,8 @@ def parsed(tmp_path_factory: pytest.TempPathFactory) -> dict[str, str]: # A fresh install has no Hugging Face cache and every sidecar starts offline. "HF_HOME": str(empty_cache), "HF_HUB_OFFLINE": "1", + # This reads through Docling; the system's OCR has tests of its own. + "SURFSENSE_LOCAL_SCAN_OCR": "off", } child = subprocess.run( [sys.executable, "-c", CHILD, *files], diff --git a/surfsense_local/backend/tests/integration/worker/test_rapidocr_engine.py b/surfsense_local/backend/tests/integration/worker/test_rapidocr_engine.py new file mode 100644 index 0000000000..271166172f --- /dev/null +++ b/surfsense_local/backend/tests/integration/worker/test_rapidocr_engine.py @@ -0,0 +1,83 @@ +"""RapidOCR itself, on every system CI runs: it reads a rendered page, keeps a +table's rows on their lines, and turns a page it was handed sideways upright. + +Its models ship in the rapidocr wheel, so no pack is needed here. +""" + +from collections.abc import Iterator + +import pytest +from PIL import Image, ImageDraw, ImageFont + +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import PageVerdict +from worker.ingestion.scan_ocr.rapid import RapidOcr +from worker.ingestion.scan_ocr.upright import read_upright + +pytestmark = pytest.mark.integration + +_FONTS = ( + "arial.ttf", + "/System/Library/Fonts/Helvetica.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", +) +PROSE = [ + "The harbour board met on Tuesday to agree the budget", + "for the coming year and the plan for the new quay.", + "Members asked for the figures to be published in full", + "before the next meeting of the board in the spring.", +] +ROWS = [("Fuel", "1,250"), ("Wages", "8,400"), ("Repairs", "2,075")] + + +def font(size: int = 40) -> ImageFont.FreeTypeFont | ImageFont.ImageFont: + """A scalable font, so the text is large enough to read; Pillow's own as a last resort.""" + for name in _FONTS: + try: + return ImageFont.truetype(name, size) + except OSError: + continue + return ImageFont.load_default(size=size) + + +def a_page() -> Image.Image: + """Prose, then a small table whose labels and amounts stand far apart.""" + image = Image.new("L", (1700, 1100), 255) + draw = ImageDraw.Draw(image) + for n, text in enumerate(PROSE): + draw.text((120, 120 + 70 * n), text, fill=0, font=font()) + for n, (label, amount) in enumerate(ROWS): + draw.text((120, 520 + 70 * n), label, fill=0, font=font()) + draw.text((1200, 520 + 70 * n), amount, fill=0, font=font()) + return image + + +@pytest.fixture(scope="module") +def rapid() -> Iterator[RapidOcr]: + """RapidOCR built once for the module, with the wheel's models when no pack is here.""" + reader = RapidOcr.create() + if reader is None: + pytest.skip("rapidocr is not installed") + yield reader + + +def test_rapidocr_reads_a_page_and_keeps_a_rows_cells_together(rapid: RapidOcr) -> None: + """A row's label and amount, far apart on the page, read as one line.""" + [(_, page)] = list(rapid.read([Frame(1, a_page(), 0.05)])) + text = page.text() + + assert page.verdict() is PageVerdict.TEXT + assert page.confidence is not None and page.confidence > 0.8 + assert "harbour board met on Tuesday" in text + assert any("Wages" in line and "8,400" in line for line in text.splitlines()) + + +def test_a_page_handed_over_sideways_is_read_upright(rapid: RapidOcr) -> None: + """The engine is unsure of a sideways page, so it is read again turned.""" + sideways = a_page().rotate(90, expand=True) + + [(frame, page)] = list(read_upright(rapid, [Frame(1, sideways, 0.05)])) + + assert page.verdict() is not PageVerdict.GARBLED + assert "harbour board" in page.text() + assert frame.image.size == (1700, 1100) diff --git a/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py b/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py new file mode 100644 index 0000000000..330ee6c074 --- /dev/null +++ b/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py @@ -0,0 +1,332 @@ +"""OCR-first on scans: which pages its text stands for, how that text reads, and +which pages still go to Docling. No OCR runs: pages are made of boxes.""" + +from collections.abc import Iterable, Iterator +from pathlib import Path + +import pytest +from PIL import Image + +from worker.ingestion.pdf_reading import ocr_first +from worker.ingestion.pdf_text_layer import LayerPage, TextLayer +from worker.ingestion.pdf_text_layer.page_triage import PageKind +from worker.ingestion.scan_ocr import switch +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import Line, OcrPage, PageVerdict, Word +from worker.ingestion.scan_ocr.upright import read_upright + +pytestmark = pytest.mark.unit + +WIDTH, HEIGHT = 1786, 2526 +WORD_H = 30.0 + + +def line(text: str, y: float, x: float = 100.0, gap: float = 12.0) -> Line: + """Words left to right on one line, `gap` pixels apart.""" + words = [] + at = x + for piece in text.split(): + w = 18.0 * len(piece) + words.append(Word(piece, at, y, w, WORD_H)) + at += w + gap + return Line(text, tuple(words)) + + +def page(*lines: Line, ink: float = 0.08, confidence: float | None = None) -> OcrPage: + """A page of these lines, as an engine reads it.""" + return OcrPage(WIDTH, HEIGHT, tuple(lines), ink, "en-US", confidence) + + +def prose(lines: int = 12, start: float = 200.0) -> list[Line]: + """Lines of running text, one under another.""" + return [ + line( + f"The council heard the report on line {n} and voted to adopt it.", + start + 45 * n, + ) + for n in range(lines) + ] + + +def test_running_text_is_text() -> None: + """Running text is what OCR's text can stand for.""" + assert page(*prose()).verdict() is PageVerdict.TEXT + + +def test_prose_with_amounts_and_dates_is_still_text() -> None: + """Justified prose with $5,000 and 12/10/98 in it is not a table.""" + lines = [ + *prose(8), + line("The grant of $5,000 was paid on 12/10/98 to the district.", 600), + line("A further $1,250 followed in 1999, as the board agreed.", 645), + ] + assert page(*lines).verdict() is PageVerdict.TEXT + + +def rows(n: int = 7) -> list[Line]: + """A table's rows, each kept as one line with its cells far apart.""" + return [ + line(f"Item {k} 1,{k}00 2,{k}50 3,{k}75", 300 + 45 * k, gap=200.0) + for k in range(1, n + 1) + ] + + +def test_a_table_stays_with_ocr() -> None: + """Its rows answered better than Docling's Markdown, which drops text.""" + assert page(*prose(3), *rows(), confidence=0.97).verdict() is PageVerdict.TEXT + + +def test_too_little_text_on_an_inked_page_is_too_little() -> None: + """Handwriting or a photo: Docling runs the same OCR, so it stays.""" + assert page(line("Fig. 3", 300), ink=0.2).verdict() is PageVerdict.TOO_LITTLE + + +def test_a_blank_page_is_blank() -> None: + """No text and almost no ink: blank paper, not a misread.""" + assert page(ink=0.001).verdict() is PageVerdict.BLANK + + +def test_garbled_output_goes_to_docling() -> None: + """Mostly symbols: the engine misread the page.""" + noise = [ + line("~~ ^^ ;; || ~~ ^^ ;; || ~~ ^^ ;; ||", 300 + 45 * n) for n in range(5) + ] + assert page(*noise).verdict() is PageVerdict.GARBLED + + +def sideways() -> OcrPage: + """What an engine with no confidence made of a sideways photo of prose.""" + strokes = [line("O 00 o c 0 O o 00 c O 0 o c 00 O", 300 + 45 * n) for n in range(6)] + return page(*strokes) + + +def test_a_page_read_sideways_is_garbled_and_looks_sideways() -> None: + """Letters and digits all, so the alnum guard passes it; few are words.""" + assert sideways().verdict() is PageVerdict.GARBLED + assert sideways().looks_sideways() + assert not page(*prose()).looks_sideways() + + +def test_a_page_of_figures_stands_on_the_engines_confidence() -> None: + """Few words but long tokens: not sideways; with no confidence the word share + sends it to Docling, as today.""" + figures = [ + line(f"{n},100 {n},250 {n}.5% 1,{n}00", 300 + 45 * n, gap=200.0) + for n in range(8) + ] + assert page(*figures, confidence=0.97).verdict() is PageVerdict.TEXT + assert page(*figures).verdict() is PageVerdict.GARBLED + assert not page(*figures).looks_sideways() + + +def test_an_unsure_engine_means_a_garbled_page_read_sideways() -> None: + """RapidOCR's confidence fell to 0.56-0.79 on sideways photos.""" + assert page(*prose(), confidence=0.6).verdict() is PageVerdict.GARBLED + assert page(*prose(), confidence=0.6).looks_sideways() + assert not page(*prose(), confidence=0.97).looks_sideways() + + +def test_a_sideways_page_is_read_again_turned(monkeypatch: pytest.MonkeyPatch) -> None: + """The first turn that reads as text wins; an upright page is read once.""" + readings = iter([sideways(), page(*prose()), sideways(), sideways()]) + reader = StubReader({}) + reader.read = lambda frames: ((f, next(readings)) for f in frames) # type: ignore[method-assign] + frame = Frame(1, Image.new("L", (40, 20), 0), 0.5) + + [(turned, read)] = list(read_upright(reader, [frame])) + + assert read.verdict() is PageVerdict.TEXT + assert turned.image.size == (20, 40) + + +def test_a_hyphen_at_a_line_end_joins_the_next_word() -> None: + """A word broken over a line reads whole, as the chunker and search want it.""" + lines = [line("The board approved the ap-", 200), line("plication in full.", 245)] + assert page(*lines).text() == "The board approved the application in full." + + +def test_a_wide_gap_starts_a_paragraph() -> None: + """A blank line where the page has one, so passages break there.""" + first = [line(f"First paragraph line {n}.", 200 + 45 * n) for n in range(4)] + second = [line(f"Second paragraph line {n}.", 500 + 45 * n) for n in range(4)] + text = page(*first, *second).text() + + assert "line 3.\n\nSecond paragraph line 0." in text + assert "line 0.\nFirst paragraph line 1." in text + + +def test_the_switch_picks_the_reader_and_is_off_when_unclear( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Any case and spaces; a value nobody meant turns OCR-first off.""" + monkeypatch.delenv("SURFSENSE_LOCAL_SCAN_OCR", raising=False) + assert switch.scan_ocr_choice() is switch.ScanOcr.AUTO + monkeypatch.setenv("SURFSENSE_LOCAL_SCAN_OCR", " RapidOCR ") + assert switch.scan_ocr_choice() is switch.ScanOcr.RAPIDOCR + monkeypatch.setenv("SURFSENSE_LOCAL_SCAN_OCR", "OFF") + assert switch.scan_ocr_choice() is switch.ScanOcr.OFF + monkeypatch.setenv("SURFSENSE_LOCAL_SCAN_OCR", "system") + assert switch.scan_ocr_choice() is switch.ScanOcr.OFF + + +class StubReader: + """OCR's pages by page number, as a system engine would give them.""" + + name = "Stub OCR" + language = "en-US" + max_side = 10_000 + + def __init__(self, pages: dict[int, OcrPage]) -> None: + self.pages = pages + self.asked: list[int] = [] + + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + for frame in frames: + self.asked.append(frame.number) + yield frame, self.pages[frame.number] + + +@pytest.fixture +def ocr(monkeypatch: pytest.MonkeyPatch) -> dict[int, OcrPage]: + """Pages the stub reads; frames are blank pictures, never a real render.""" + pages: dict[int, OcrPage] = {} + reader = StubReader(pages) + + def frames(_original: Path, numbers: list[int], _max_side: int) -> Iterator[Frame]: + for number in numbers: + yield Frame(number, Image.new("L", (10, 10), 255), 0.0) + + monkeypatch.setattr(ocr_first, "page_reader", lambda: reader) + monkeypatch.setattr(ocr_first, "pdf_frames", frames) + monkeypatch.setattr( + ocr_first, "scanned_pages", lambda _pdf: [True, True, True, True] + ) + return pages + + +def scanned(n: int, kind: PageKind = PageKind.NO_TEXT) -> LayerPage: + """A page with no text layer, as a scanner writes it.""" + return LayerPage(None, f"page {n} is {kind}", kind, 1) + + +def text_page(n: int) -> LayerPage: + """A page of running text in the PDF's own text layer.""" + return LayerPage(f"Page {n} text.", None, PageKind.TEXT) + + +def layer_of(*pages: LayerPage) -> TextLayer: + """The text layer of a file with these pages, as reading it leaves it.""" + return TextLayer(None, "page 1 is ocr:no-text", sum(p.images for p in pages), pages) + + +def test_a_scan_of_running_text_is_read_by_ocr_alone(ocr: dict[int, OcrPage]) -> None: + """Every page stands, so Docling has nothing to read.""" + ocr.update({1: page(*prose()), 2: page(*prose())}) + + reading = ocr_first.ocr_reading(Path("scan.pdf"), layer_of(scanned(1), scanned(2))) + + assert reading is not None + assert reading.read == 2 and reading.docling == {} + assert all("voted to adopt" in (p.text or "") for p in reading.layer.pages) + + +def test_a_table_page_of_a_scan_stays_with_ocr(ocr: dict[int, OcrPage]) -> None: + """Its rows keep their cells together, and Docling is not asked.""" + ocr.update({1: page(*prose()), 2: page(*rows(), confidence=0.97)}) + + reading = ocr_first.ocr_reading(Path("scan.pdf"), layer_of(scanned(1), scanned(2))) + + assert reading is not None + assert reading.docling == {} + assert "Item 3 1,300 2,350 3,375" in (reading.layer.pages[1].text or "") + + +def test_a_garbled_page_of_a_scan_goes_to_docling(ocr: dict[int, OcrPage]) -> None: + """Only that page is left for Docling, in its place.""" + ocr.update( + {1: page(*prose()), 2: page(*prose(), confidence=0.5), 3: page(*prose())} + ) + + reading = ocr_first.ocr_reading( + Path("scan.pdf"), layer_of(scanned(1), scanned(2), scanned(3)) + ) + + assert reading is not None + assert reading.docling == {2: "page 2 is garbled"} + assert reading.layer.pages[1].text is None + + +def test_a_mixed_pdf_keeps_its_text_pages_and_ocrs_the_rest( + ocr: dict[int, OcrPage], +) -> None: + """The text layer's pages stay as they were read.""" + ocr.update({2: page(*prose())}) + + reading = ocr_first.ocr_reading( + Path("mixed.pdf"), layer_of(text_page(1), scanned(2), text_page(3)) + ) + + assert reading is not None + assert [p.text is not None for p in reading.layer.pages] == [True, True, True] + assert reading.layer.pages[0].text == "Page 1 text." + + +def test_a_scanner_layer_is_read_again(ocr: dict[int, OcrPage]) -> None: + """A scanner's own invisible OCR is distrusted, as the text layer distrusts it.""" + ocr.update({1: page(*prose())}) + + reading = ocr_first.ocr_reading( + Path("scan.pdf"), layer_of(scanned(1, PageKind.SCANNER_LAYER)) + ) + + assert reading is not None and reading.read == 1 + + +def test_a_turned_scan_is_read_and_a_turned_text_page_is_not( + ocr: dict[int, OcrPage], monkeypatch: pytest.MonkeyPatch +) -> None: + """pdfium renders a turned page upright; a turned born-digital page stays Docling's.""" + monkeypatch.setattr(ocr_first, "scanned_pages", lambda _pdf: [True, False]) + turned = [ + LayerPage(None, f"page {n} is turned 90 degrees", None, 1) for n in (1, 2) + ] + ocr.update({1: page(*prose())}) + + reading = ocr_first.ocr_reading(Path("turned.pdf"), layer_of(*turned)) + + assert reading is not None + assert reading.docling == {2: "page 2 is turned 90 degrees"} + + +def test_a_scan_ocr_cannot_stand_for_at_all_is_left_to_docling( + ocr: dict[int, OcrPage], +) -> None: + """Docling reads it whole, as before, and keeps its figures as it reads.""" + ocr.update({1: page(*prose(), confidence=0.5)}) + + assert ocr_first.ocr_reading(Path("scan.pdf"), layer_of(scanned(1))) is None + + +def test_no_reader_or_a_failing_one_leaves_it_to_docling( + ocr: dict[int, OcrPage], monkeypatch: pytest.MonkeyPatch +) -> None: + """OCR-first never fails an ingest: Docling reads instead.""" + pages = layer_of(scanned(1)) + monkeypatch.setattr(ocr_first, "page_reader", lambda: None) + assert ocr_first.ocr_reading(Path("scan.pdf"), pages) is None + + class Failing(StubReader): + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + raise RuntimeError("onnxruntime could not run the detector") + + monkeypatch.setattr(ocr_first, "page_reader", lambda: Failing({})) + assert ocr_first.ocr_reading(Path("scan.pdf"), pages) is None + + +def test_a_form_or_an_unreadable_pdf_is_left_to_docling( + ocr: dict[int, OcrPage], +) -> None: + """No pages in the layer: Docling reads the whole file, as before.""" + assert ( + ocr_first.ocr_reading(Path("form.pdf"), TextLayer(None, "a form field")) is None + ) diff --git a/surfsense_local/backend/uv.lock b/surfsense_local/backend/uv.lock index 25008cf00a..f1bdd01c97 100644 --- a/surfsense_local/backend/uv.lock +++ b/surfsense_local/backend/uv.lock @@ -2345,6 +2345,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/06/3d67bd912ef337aa4856b466121d03f304aa1bb4d804f9298b6227cd227e/pylatexenc-2.11-py2.py3-none-any.whl", hash = "sha256:e78e7391d6c104f1ed150e21cfaa58016cdb50aa54406a2eecb793649ffdfdd0", size = 137533, upload-time = "2026-07-25T17:26:30.141Z" }, ] +[[package]] +name = "pyobjc-core" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/78/abc4ce5920305780aeb36b4067a86253378b36e29ba96673a3deb02eb03a/pyobjc_core-12.2.2.tar.gz", hash = "sha256:3906452339cd06a3bb07df103c2511d4cb0f7a22d8771c0b802eba15d9a642b6", size = 1067701, upload-time = "2026-08-11T19:43:39.059Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/b2/bbf7f049880ab40d110e66f25122342a1f6c98d6fe3c59bb98985503c660/pyobjc_core-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:122e6ad302a2abf5d4d4adb0156db751600ddf2768441696cba17b31323085e7", size = 6427637, upload-time = "2026-08-11T14:51:36.038Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ed/a8bf040caf3704023d74086b7fb96cf4ed2e844e24bd94e5248ba214b700/pyobjc_core-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:950bd2d9c74634398c4e3d24ef2f213d4e23d705083697464fa67afedc53c1ad", size = 6427113, upload-time = "2026-08-11T15:04:39.424Z" }, + { url = "https://files.pythonhosted.org/packages/e7/5a/760f8b9e116edd43c57e33844dc17619158fbdd311250d4209910192d72d/pyobjc_core-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:3772b406edb3ff78171530a17cda1c4a7817f87b87ded0d8715b3fa664df16db", size = 6666817, upload-time = "2026-08-11T19:30:17.01Z" }, + { url = "https://files.pythonhosted.org/packages/13/37/486d38a173b0b8dce973a3e13c74cf402ed1b8621586b5963bc9efd49a48/pyobjc_core-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2062e8ad30a310441cd022544a897553408bebeaa7820d5edba3c96fd7fd693b", size = 6421184, upload-time = "2026-08-11T19:30:21.081Z" }, + { url = "https://files.pythonhosted.org/packages/04/f1/d138fd9b9a66ea8db56a8138b77d3413b85da3defe13363a19f364f85529/pyobjc_core-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:2c7ef3d2f865b4b3ebb14ec3556f7a3e8abb6d130c67275cd9daa08dbd6e4e4e", size = 6671824, upload-time = "2026-08-11T19:30:25.005Z" }, + { url = "https://files.pythonhosted.org/packages/d5/85/577e2265cccf59daf48c460f0a8deeaf7dbe2991227a8859ab1eeab4945e/pyobjc_core-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:89acc6bc13aaa6e3f52b0ce652ede7e201edb6bf062741b246b0c5a44582f25f", size = 6477694, upload-time = "2026-08-11T19:30:28.821Z" }, + { url = "https://files.pythonhosted.org/packages/77/0a/bd9f830c64c6f334530831e75c01bfe0a770a3fbb00fddc70329223118b3/pyobjc_core-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59a77038ebe0ab1240f61c341e7fb67b8674f2b4cd41bc71a6472511a12b50f7", size = 6712233, upload-time = "2026-08-11T19:30:33.032Z" }, +] + +[[package]] +name = "pyobjc-framework-cocoa" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/76/49c6da2c6a831020b4854ba20079d5a1030474bffc776b7b73c2eeff8c15/pyobjc_framework_cocoa-12.2.2.tar.gz", hash = "sha256:c96c0ef69a71afbbb0e6a7d594b455c5fe47d62e0db376ee7a2b4b828c16ace9", size = 3132831, upload-time = "2026-08-11T19:44:02.288Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/2f/b67e73d8bc367e03fe7861cd9c49fff9dcfa6db83bc0630c0adcfb25b7fa/pyobjc_framework_cocoa-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e106f395531e67694376b0f1184612cbeea3ec8b9bf56b55ef41d026171d2a2d", size = 388117, upload-time = "2026-08-11T19:32:43.161Z" }, + { url = "https://files.pythonhosted.org/packages/db/e1/5d9b04ebb60042b9cb49adc2d33115e2f2c2e4ff7d548017bfaff8b7f536/pyobjc_framework_cocoa-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:600b1723184ca094931330e79355274949965460e23de38628d601b5a967baf9", size = 388181, upload-time = "2026-08-11T19:32:44.537Z" }, + { url = "https://files.pythonhosted.org/packages/b4/25/2a343357d5fe09bbe9c0e294dc03450866a0d6c1792fad36b6bcc00174c0/pyobjc_framework_cocoa-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:875f2aad73963faa81a6b36ae674fd494a4658d6d999e1075e0e2aca3d2391df", size = 392275, upload-time = "2026-08-11T19:32:45.631Z" }, + { url = "https://files.pythonhosted.org/packages/1f/1a/b99521999b9f54b89aad928ddff0faad507abfe33bc46599454bfa48a4b2/pyobjc_framework_cocoa-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:889d7bbd4ba2d4941078bfbbfb882138e51dbead27df006abfe0f2e0d49b5b2e", size = 388368, upload-time = "2026-08-11T19:32:46.781Z" }, + { url = "https://files.pythonhosted.org/packages/6d/26/0c697dbc73dcc76bc0f68ea5aeed25bf7b05217df5102659e878501b2d5f/pyobjc_framework_cocoa-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:de69c5933750f3a4599ed962eccd92b6a71914c7e4318dacc7895738a8ae60d7", size = 392404, upload-time = "2026-08-11T19:32:47.918Z" }, + { url = "https://files.pythonhosted.org/packages/df/82/502f740fd8f4e9ef741c9d40ba67467ab2c8196f2c09dcba12936d28a4fd/pyobjc_framework_cocoa-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:0e8ace0d44a00d281281a723d17fcd05eea7544a38a6a512e1fd018ddb7aece2", size = 388585, upload-time = "2026-08-11T19:32:49.171Z" }, + { url = "https://files.pythonhosted.org/packages/7d/3b/07ce3c0ab8d1e9e1bed74fea1bf1cce73527a365a7a23c755051d3be9865/pyobjc_framework_cocoa-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8fe5b2e79c9530f667b4e58a87a3a15ea62f86a5d19eec405517ecbd4f454868", size = 392693, upload-time = "2026-08-11T19:32:50.283Z" }, +] + +[[package]] +name = "pyobjc-framework-coreml" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/3b/c835535e7aef41afc953e5597ea41e693b7bae80d753ab74b9721e07cba5/pyobjc_framework_coreml-12.2.2.tar.gz", hash = "sha256:3e6abe134634adbcc0c1e843826e42df86e63beb3bc7e8e4a914e93179ae1e75", size = 49750, upload-time = "2026-08-11T19:44:14.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/ab/e02409bf4e13b88812a198118e4a0e5cce35e3bd0957a2a3f879ab06ca90/pyobjc_framework_coreml-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94c129d9c1955eafc6783683ccaa08590f4cd7ef229515c8a28324032351871b", size = 11971, upload-time = "2026-08-11T19:34:12.422Z" }, + { url = "https://files.pythonhosted.org/packages/21/7a/46cd02ceb310f2f20e4a5fb78b4f2e456ea0c54799c773343ca575ccfb04/pyobjc_framework_coreml-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:70ae22934f2432677bbbcee04a21db2b0301056cd8390e61101fb59e1d374174", size = 11993, upload-time = "2026-08-11T19:34:13.151Z" }, + { url = "https://files.pythonhosted.org/packages/65/80/7212482d1ce6c163c1335f1ec11ebc819d35a90b3ac898e56d2fea9b575c/pyobjc_framework_coreml-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:47b635a2ccc1c2b365b3e0d5a7ff1c5b03c351a9f350418a2d458dad00114429", size = 12211, upload-time = "2026-08-11T19:34:13.876Z" }, + { url = "https://files.pythonhosted.org/packages/c5/2a/e24b588906a6cec690cf8147358290e793d8b616aa508f743f52ddfa86e1/pyobjc_framework_coreml-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:7b263a80158d4380b208ffb8cbf3b276ed4daf3753723dbbd7e561a25ee2048b", size = 12031, upload-time = "2026-08-11T19:34:14.593Z" }, + { url = "https://files.pythonhosted.org/packages/5a/15/bb1c7476a050f9759827dafc8c7294a37a8ecf74728f003ea2ccbe9502c1/pyobjc_framework_coreml-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5971bdf03ed6002ab6e913ac8003077172d8423cc13206246f343518d1515e10", size = 12209, upload-time = "2026-08-11T19:34:15.306Z" }, + { url = "https://files.pythonhosted.org/packages/6c/90/49fa6d98787da09f9ebd498f7e7252c3243c7e294a975bf77b860304ec11/pyobjc_framework_coreml-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:20dba373eac775cb487d28a92d02e78903d464367d67dd69aa17543515234183", size = 12025, upload-time = "2026-08-11T19:34:16.043Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ee/2253e698cd41ee7ea9b3f1fd796cad822722d6e351f55731809c52005421/pyobjc_framework_coreml-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:b3711abe3380c3343d57dfa490578ef3c87ca0cdc8a403d81bc35e750ef48d49", size = 12201, upload-time = "2026-08-11T19:34:17.221Z" }, +] + +[[package]] +name = "pyobjc-framework-quartz" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/b1/426a37c7ae37280b3ffca2571fb48f211946aee2f4ca31a603ed1943c4a7/pyobjc_framework_quartz-12.2.2.tar.gz", hash = "sha256:810f97b210cfd93704d240860286dfd6df09f9f1c52525fc5c2166723aea3f9e", size = 3218295, upload-time = "2026-08-11T19:45:15.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/e4/8be95d2ff850f82fb55b44c63333a00a920bf8a73642e7d9c2f3638a26d2/pyobjc_framework_quartz-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7f668979d0c7320bf8f7ed6e030da578f93ab0f5dd619b295ec735cd8d5faa34", size = 219003, upload-time = "2026-08-11T19:40:24.425Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ae/b515852dbe491171f2f2e2eb7739588a5eb7f36720a739545337b8c0d706/pyobjc_framework_quartz-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0ec9751904ef975bf0789d760dc4fadcb400edc4ffe4a736eb54971968babe5c", size = 219404, upload-time = "2026-08-11T19:40:25.461Z" }, + { url = "https://files.pythonhosted.org/packages/ff/5f/c7ee66f4483385396d91f65036c62f3dd20bccaa07354643ef17b259aa75/pyobjc_framework_quartz-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:63f6f0f3233dcf650aac1374781e78961b0b17b33e3351953bacf8bd0c430593", size = 224462, upload-time = "2026-08-11T19:40:26.647Z" }, + { url = "https://files.pythonhosted.org/packages/b1/33/230ae7777b0909fe2c24f28413c51c860faebf762d824b097a0e2fb48304/pyobjc_framework_quartz-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:1f7f3d9010e38f03ea1fa266664c10ea349cd7492bd603b403584f49d713dbed", size = 219773, upload-time = "2026-08-11T19:40:27.64Z" }, + { url = "https://files.pythonhosted.org/packages/25/eb/7482fdd384521916e98a6164be220494b1f7794792b1d60fa3f5207d84a4/pyobjc_framework_quartz-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ebf8167ca2096cf3a05199decfa517be0df4c56048f49cf132bd6b1a6ab9c086", size = 224726, upload-time = "2026-08-11T19:40:28.718Z" }, + { url = "https://files.pythonhosted.org/packages/33/67/b4b0ffc486b08492bfef4b33731d044b295f1696c8fb7d119c8367079139/pyobjc_framework_quartz-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:cee63b891c2b6b7ccf98f233175411529f3e80286f58438793b3634af79858f1", size = 219827, upload-time = "2026-08-11T19:40:29.764Z" }, + { url = "https://files.pythonhosted.org/packages/aa/eb/5fb627c2457046883c6fd12d25c44db40c12bcde4622dc0f30851108e106/pyobjc_framework_quartz-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8f58c589b5a76ba98f186b1f3b19fb1c8b730e82351f81fb62e6194f64a71622", size = 224767, upload-time = "2026-08-11T19:40:30.991Z" }, +] + +[[package]] +name = "pyobjc-framework-vision" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-framework-coreml" }, + { name = "pyobjc-framework-quartz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/bf/31e8bcae94047b365592ab7533096a4025c0306a8a312b65bcbf57e073ec/pyobjc_framework_vision-12.2.2.tar.gz", hash = "sha256:a6bf78e8dca145c6a78fd8d5f925b6da54649a60a1464b81ff405bca4a08406b", size = 73289, upload-time = "2026-08-11T19:45:41.877Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5f/7dc08bcbc802a576dafd70a838b60c369d7f1b564eb4b2fd96f328438498/pyobjc_framework_vision-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1fcb99d44ef4df33a7582467e0cd059515f1137ccecb3648ac8a59272851506a", size = 16948, upload-time = "2026-08-11T19:43:23.808Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f6/e3cb58a12ab51ff9ac159988fc3c6410a60234dc7b448223fc090cd3d1f7/pyobjc_framework_vision-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:306e536e12d3565e53649818e3324e801fbb1696ea75ba751852dffbacba8b14", size = 16961, upload-time = "2026-08-11T19:43:24.668Z" }, + { url = "https://files.pythonhosted.org/packages/03/16/2791ee2985b3ce4dde1e05e355d890583a4db19729f1b8dda3549510ef91/pyobjc_framework_vision-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:bd02315e62b16397f5fbcd0dcc23744bdf86c9ce5ff94970a2befbf14a8c2ea7", size = 17105, upload-time = "2026-08-11T19:43:25.722Z" }, + { url = "https://files.pythonhosted.org/packages/28/c1/2951e9182e2c077d2c23c3f0772a1cfbf6bd627247388ac795a53888b08a/pyobjc_framework_vision-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:66eab9ecdc741f4ee72cc287c5ffccfe6f81fcdd8544bec98e5cc6c50b901865", size = 16940, upload-time = "2026-08-11T19:43:26.513Z" }, + { url = "https://files.pythonhosted.org/packages/57/fc/7431c063eeb9f44b75fcdf213dbfd50c7822af1eba8e40df31799a095ac3/pyobjc_framework_vision-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6dd4c8a804a89585ae170f0b5eb5265538d75fc76092e9f33a67386304d19aed", size = 17096, upload-time = "2026-08-11T19:43:27.272Z" }, + { url = "https://files.pythonhosted.org/packages/6b/90/85f349d583f691bf64473b52ae96ba52320226816ae3e54f4d4eeeb7ba27/pyobjc_framework_vision-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:60b61390c250640b0dbc1b88861bb80edf6dfa699e90b91a5b4bfd37d4a71b99", size = 16930, upload-time = "2026-08-11T19:43:28.029Z" }, + { url = "https://files.pythonhosted.org/packages/6f/9d/fa24a7234a7d1100225582968ae29ad3aeb2616f1528fc877384895a3e9d/pyobjc_framework_vision-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:c323455ddabc5fab868c00797fedd5e338688054c776cd6a7801d36f045e9d26", size = 17096, upload-time = "2026-08-11T19:43:28.828Z" }, +] + [[package]] name = "pyparsing" version = "3.3.3" @@ -3184,6 +3276,8 @@ dependencies = [ { name = "pillow" }, { name = "psutil" }, { name = "pydantic-settings" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, { name = "pypdf" }, { name = "pypdfium2" }, { name = "python-docx" }, @@ -3231,6 +3325,8 @@ requires-dist = [ { name = "pillow", specifier = ">=12" }, { name = "psutil", specifier = ">=7" }, { name = "pydantic-settings", specifier = ">=2.7.0" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'", specifier = ">=12.2.2" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'", specifier = ">=12.2.2" }, { name = "pypdf", specifier = ">=6" }, { name = "pypdfium2", specifier = ">=5.13.0" }, { name = "python-docx", specifier = ">=1.2.0" }, diff --git a/surfsense_local/backend/worker.py b/surfsense_local/backend/worker.py index ff3dbf39cb..1e2567e0e9 100644 --- a/surfsense_local/backend/worker.py +++ b/surfsense_local/backend/worker.py @@ -18,6 +18,10 @@ def check_vision_runtime() -> None: multiprocessing.freeze_support() if sys.argv[1:] == ["--check-vision-runtime"]: check_vision_runtime() + elif sys.argv[1:] == ["--check-scan-ocr"]: + from worker.ingestion.scan_ocr.check import check_scan_ocr + + sys.stdout.write(check_scan_ocr() + "\n") elif len(sys.argv) == 3 and sys.argv[1] == "--run-document-script": # Ahead of the consumer's imports: a script process loads no queue, # database or settings. diff --git a/surfsense_local/backend/worker/ingestion/parsing.py b/surfsense_local/backend/worker/ingestion/parsing.py index 96ded7a5cc..91bf88b369 100644 --- a/surfsense_local/backend/worker/ingestion/parsing.py +++ b/surfsense_local/backend/worker/ingestion/parsing.py @@ -17,6 +17,7 @@ from worker.ingestion.figures.word_figures_pass import leave_figures_to_the_pass from worker.ingestion.image_page import IMAGE_SUFFIXES, as_page from worker.ingestion.parser_pack import missing_parser_folders, parser_dir +from worker.ingestion.scan_ocr.images import image_markdown from worker.ingestion.slideless_deck import describe_slideless_deck from worker.ingestion.word_markdown import markdown_without_docling @@ -53,6 +54,10 @@ def _markdown_from(path: Path) -> str: # Its figures come from the figures-only pass, which still asks Docling. leave_figures_to_the_pass(path) return markdown + if suffix in IMAGE_SUFFIXES and (markdown := image_markdown(path)) is not None: + # An image is its own one figure; Docling's reading adds nothing to it. + keep_figures(path, None) + return markdown converted = convert(path) keep_figures(path, converted) diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py new file mode 100644 index 0000000000..912e544f95 --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py @@ -0,0 +1,134 @@ +"""A scanned page read by OCR alone before Docling is asked. + +Docling took 4.9 s and 39 CPU-s a page on the corpus's real scans; RapidOCR alone +about 2 s and 8 CPU-s, and found more of each page. A page whose OCR came out +garbled still goes to Docling, through the same splice a mixed PDF's flagged +pages take. +""" + +import logging +import time +from dataclasses import dataclass, replace +from pathlib import Path + +from sqlalchemy.orm import object_session + +from modules.documents.models import Document +from worker.ingestion.docling_batches.queue_turn import reading_turn +from worker.ingestion.figures.scanned_pages import scanned_pages +from worker.ingestion.pdf_text_layer import LayerPage, TextLayer +from worker.ingestion.pdf_text_layer.page_triage import PageKind +from worker.ingestion.scan_ocr.frames import pdf_frames +from worker.ingestion.scan_ocr.page import KEPT, PageVerdict +from worker.ingestion.scan_ocr.reader import PageReader, page_reader +from worker.ingestion.scan_ocr.upright import read_upright +from worker.jobs import JobCancelledError + +logger = logging.getLogger(__name__) + +# Pages whose words only pixels hold: no text layer, a scanner's own OCR layer +# (re-read, as the text layer distrusts it), or text pdfium cannot map. +OCR_KINDS = frozenset({PageKind.NO_TEXT, PageKind.SCANNER_LAYER, PageKind.GARBLED}) +# With no page of running text, a cover's title is all the layer has: the +# pixels of these pages hold the rest. +_PICTURE_KINDS = frozenset({PageKind.BIG_IMAGE, PageKind.SPARSE}) +# Pages between checks for a cancel, as a Docling batch is 8 pages. +CANCEL_EVERY = 8 + + +@dataclass(frozen=True) +class OcrReading: + """The layer with OCR's text on the pages it stands for, and why each page + left for Docling went there.""" + + layer: TextLayer + docling: dict[int, str] + read: int + reader: str + seconds: float + + +def ocr_reading( + original: Path, layer: TextLayer, document: Document | None = None +) -> OcrReading | None: + """None when there is nothing for OCR here, no OCR on this computer, or it + failed: Docling then reads as before. Raises only JobCancelledError.""" + if not layer.pages: + return None + reader = page_reader() + if reader is None: + return None + try: + candidates = _candidates(original, layer) + if not candidates: + return None + return _read(reader, original, layer, candidates, document) + except JobCancelledError: + raise + except Exception: + logger.warning( + "%s could not read %s; Docling reads it", + reader.name, + original.name, + exc_info=True, + ) + return None + + +def _candidates(original: Path, layer: TextLayer) -> list[int]: + pages = list(enumerate(layer.pages, start=1)) + picked = {n for n, page in pages if page.text is None and page.kind in OCR_KINDS} + turned = [n for n, page in pages if page.text is None and page.kind is None] + if turned: + # A turned page is OCR's only when it is a scan; pdfium renders it upright. + scans = scanned_pages(original) or [] + picked |= {n for n in turned if n <= len(scans) and scans[n - 1]} + if not any( + page.kind is PageKind.TEXT for _, page in pages if page.text is not None + ): + picked |= {n for n, page in pages if page.kind in _PICTURE_KINDS} + return sorted(picked) + + +def _read( + reader: PageReader, + original: Path, + layer: TextLayer, + candidates: list[int], + document: Document | None, +) -> OcrReading | None: + started = time.perf_counter() + # The ingest job's own session; a reading outside one has nothing to check. + session = object_session(document) if document is not None else None + turn = reading_turn(session, document) if session and document else None + pages: list[LayerPage] = list(layer.pages) + docling: dict[int, str] = {} + read = 0 + frames = pdf_frames(original, candidates, reader.max_side) + for count, (frame, page) in enumerate(read_upright(reader, frames), start=1): + if turn is not None and count % CANCEL_EVERY == 0: + turn.check_cancelled() + verdict = page.verdict() + if verdict in KEPT: + # A blank render keeps what little the layer had, a cover's title. + own = pages[frame.number - 1].text or "" + text = own if verdict is PageVerdict.BLANK else page.text() + pages[frame.number - 1] = replace( + pages[frame.number - 1], text=text, refused_because=None + ) + read += 1 + else: + docling[frame.number] = f"page {frame.number} is {verdict}" + if not read: + # Docling would read every page of it anyway; let it read the file whole. + return None + for n, page in enumerate(pages, start=1): + if page.text is None and n not in docling: + docling[n] = page.refused_because or f"page {n} is for Docling" + return OcrReading( + replace(layer, pages=tuple(pages)), + docling, + read, + reader.name, + time.perf_counter() - started, + ) diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/route.py b/surfsense_local/backend/worker/ingestion/pdf_reading/route.py index 889024a7b3..233999e6da 100644 --- a/surfsense_local/backend/worker/ingestion/pdf_reading/route.py +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/route.py @@ -6,13 +6,20 @@ from modules.documents.models import Document from worker.ingestion.docling_batches.joined import joined -from worker.ingestion.docling_batches.page_runs import PageRun, batches_of, described +from worker.ingestion.docling_batches.page_runs import ( + PageRun, + batches_of, + described, + runs_of, +) from worker.ingestion.docling_batches.reading import read_batches from worker.ingestion.figures.idle_pass import queue_idle_figures_pass from worker.ingestion.pdf_reading.docling_pages import pages_for_docling +from worker.ingestion.pdf_reading.ocr_first import OcrReading, ocr_reading from worker.ingestion.pdf_reading.spliced_text import spliced from worker.ingestion.pdf_reading.whole_reading import whole_markdown from worker.ingestion.pdf_text_layer import TextLayer, read_text_layer +from worker.ingestion.pdf_text_layer.reading import PAGE_BREAK logger = logging.getLogger(__name__) @@ -21,11 +28,14 @@ def pdf_markdown(document: Document, original: Path) -> str: - """Raises JobYieldedError when a long Docling reading gives the queue up.""" + """Raises JobYieldedError when a long Docling reading gives the queue up, and + JobCancelledError when a reading is cancelled.""" layer = read_text_layer(original) if layer.text is not None: queue_idle_figures_pass(document, original, layer) return layer.text + if (reading := ocr_reading(original, layer, document)) is not None: + return _ocr_first_markdown(document, original, reading) runs = pages_for_docling(layer) if runs is None: logger.info("Docling reads %s: %s", original.name, layer.refused_because) @@ -43,6 +53,29 @@ def pdf_markdown(document: Document, original: Path) -> str: return markdown +def _ocr_first_markdown(document: Document, original: Path, reading: OcrReading) -> str: + """OCR's pages with Docling's markdown in place of the pages it reads.""" + runs = runs_of(sorted(reading.docling)) + logger.info( + "%s reads %d of %d pages of %s in %.1f s%s", + reading.reader, + reading.read, + len(reading.layer.pages), + original.name, + reading.seconds, + f"; Docling reads {described(runs)}: {_shown(list(reading.docling.values()))}" + if runs + else "", + ) + if runs: + markdown = spliced(reading.layer, _docling_markdown(document, original, runs)) + else: + markdown = PAGE_BREAK.join(page.text or "" for page in reading.layer.pages) + # A scan's figures come from the figures-only pass too, after it is ready. + queue_idle_figures_pass(document, original, reading.layer) + return markdown + + def _docling_markdown( document: Document, original: Path, runs: Sequence[PageRun] ) -> list[tuple[PageRun, str]]: @@ -60,6 +93,10 @@ def _docling_markdown( def _reasons(layer: TextLayer, runs: Sequence[PageRun]) -> str: reasons = [layer.pages[n - 1].refused_because for run in runs for n in run.pages] + return _shown(reasons) + + +def _shown(reasons: Sequence[str | None]) -> str: shown = "; ".join(reason or "" for reason in reasons[:_REASONS_SHOWN]) more = len(reasons) - _REASONS_SHOWN return f"{shown}; and {more} more" if more > 0 else shown diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/__init__.py b/surfsense_local/backend/worker/ingestion/scan_ocr/__init__.py new file mode 100644 index 0000000000..617fc59861 --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/__init__.py @@ -0,0 +1,4 @@ +"""Scans and photos read by OCR alone before Docling: RapidOCR, Docling's own OCR +without its layout and table models, or Apple's Vision on a Mac. On the corpus's +real pages both found more of a page than Docling, in a fraction of its time; +Docling still reads a page whose OCR came out garbled.""" diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/check.py b/surfsense_local/backend/worker/ingestion/scan_ocr/check.py new file mode 100644 index 0000000000..0fed6c7e8c --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/check.py @@ -0,0 +1,51 @@ +"""For the release build: the frozen worker carries its OCR readers and reads a +line with each. Their imports are lazy, so a spec that drops them fails only here.""" + +import sys + +from PIL import Image, ImageDraw, ImageFont + +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.reader import PageReader, rapid_reader, vision_reader + +SAMPLE = "SurfSense reads scanned pages" +_FONTS = ( + "arial.ttf", + "/System/Library/Fonts/Helvetica.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", +) + + +def check_scan_ocr() -> str: + """What each reader read; raises when one cannot be imported or started.""" + import rapidocr # noqa: F401 + + readers: list[PageReader | None] = [rapid_reader()] + if sys.platform == "darwin": + import Quartz # noqa: F401 + import Vision # noqa: F401 + + readers.append(vision_reader()) + if any(reader is None for reader in readers): + raise RuntimeError("an OCR reader could not start") + return "; ".join(_read(reader) for reader in readers if reader is not None) + + +def _read(reader: PageReader) -> str: + [(_, page)] = list(reader.read([Frame(1, _sample(), 0.05)])) + return f"{reader.name} read {page.text()!r}" + + +def _sample() -> Image.Image: + image = Image.new("L", (1400, 200), 255) + ImageDraw.Draw(image).text((40, 60), SAMPLE, fill=0, font=_font()) + return image + + +def _font() -> ImageFont.FreeTypeFont | ImageFont.ImageFont: + for name in _FONTS: + try: + return ImageFont.truetype(name, 64) + except OSError: + continue + return ImageFont.load_default(size=64) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py b/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py new file mode 100644 index 0000000000..334f82dc14 --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py @@ -0,0 +1,75 @@ +"""Pages as greyscale pictures for OCR: a PDF's pages rendered, an image's frames. + +216 dpi, the resolution Docling OCRs at: 144 dpi lost table numbers, and 300 dpi +read nothing more on real forms, 1.5x slower. +""" + +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path + +import pypdfium2 +from PIL import Image, ImageOps + +from shared import pdfium +from worker.ingestion.image_page import A4_HEIGHT_INCHES, OCR_DPI + +# Darker than this counts as ink, for telling blank paper from an unread page. +_INK_LEVEL = 128 + + +@dataclass(frozen=True) +class Frame: + """Page `number`, counted from 1, as an 8-bit greyscale picture.""" + + number: int + image: Image.Image + ink: float + + +def pdf_frames(original: Path, pages: Sequence[int], max_side: int) -> Iterator[Frame]: + """The pages rendered upright (pdfium applies /Rotate), holding pdfium's lock + only while it renders, so the system's OCR reads one page as the next renders.""" + with pdfium.lock: + pdf = pypdfium2.PdfDocument(original) + try: + for number in pages: + with pdfium.lock: + page = pdf[number - 1] + try: + width, height = page.get_size() + scale = min(OCR_DPI / 72, max_side / max(width, height, 1) * 0.999) + bitmap = page.render(scale=scale, grayscale=True) + try: + image = bitmap.to_pil().convert("L") + finally: + bitmap.close() + finally: + page.close() + yield Frame(number, image, _ink(image)) + finally: + with pdfium.lock: + pdf.close() + + +def image_frames(path: Path, max_side: int) -> Iterator[Frame]: + """Each frame upright by its EXIF and sized as an A4 page at 216 dpi, never + below its own pixels: real 754x1000 forms read 0.52 of their words at their + own size and 0.67 at A4 size.""" + with Image.open(path) as image: + for index in range(getattr(image, "n_frames", 1)): + image.seek(index) + frame = ImageOps.exif_transpose(image).convert("L") + longest = max(frame.size) + scale = min( + max(1.0, OCR_DPI * A4_HEIGHT_INCHES / longest), max_side / longest + ) + if abs(scale - 1.0) > 0.01: + size = (round(frame.width * scale), round(frame.height * scale)) + frame = frame.resize(size, Image.Resampling.LANCZOS) + yield Frame(index + 1, frame, _ink(frame)) + + +def _ink(image: Image.Image) -> float: + histogram = image.histogram() + return sum(histogram[:_INK_LEVEL]) / max(1, image.width * image.height) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/images.py b/surfsense_local/backend/worker/ingestion/scan_ocr/images.py new file mode 100644 index 0000000000..b804408d6e --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/images.py @@ -0,0 +1,51 @@ +"""An uploaded image read by OCR alone, before Docling is asked. + +All of it or none: Docling reads a multi-frame TIFF as one stream, so a frame +OCR's text cannot stand for sends the whole file to Docling, as before. +""" + +import logging +import time +from pathlib import Path + +from worker.ingestion.pdf_text_layer.reading import PAGE_BREAK +from worker.ingestion.scan_ocr.frames import image_frames +from worker.ingestion.scan_ocr.page import KEPT, PageVerdict +from worker.ingestion.scan_ocr.reader import page_reader +from worker.ingestion.scan_ocr.upright import read_upright + +logger = logging.getLogger(__name__) + + +def image_markdown(path: Path) -> str | None: + """None when Docling should read it: no OCR here, a garbled frame, nothing + but blank frames, or a failure. Never raises.""" + reader = page_reader() + if reader is None: + return None + started = time.perf_counter() + texts: list[str] = [] + try: + for frame, page in read_upright(reader, image_frames(path, reader.max_side)): + verdict = page.verdict() + if verdict not in KEPT: + logger.info( + "Docling reads %s: frame %d is %s", path.name, frame.number, verdict + ) + return None + if verdict is not PageVerdict.BLANK: + texts.append(page.text()) + except Exception: + logger.warning( + "%s could not read %s; Docling reads it", + reader.name, + path.name, + exc_info=True, + ) + return None + if not texts: + return None + logger.info( + "%s reads %s in %.1f s", reader.name, path.name, time.perf_counter() - started + ) + return PAGE_BREAK.join(texts) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/page.py b/surfsense_local/backend/worker/ingestion/scan_ocr/page.py new file mode 100644 index 0000000000..8c7b28b7bb --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/page.py @@ -0,0 +1,173 @@ +"""One page as OCR read it, and whether its text can stand. + +Every reader gives the same shape: lines of words with pixel boxes, top-left +origin. Tables stay with OCR: on the corpus's real scans RapidOCR's rows answered +better than Docling's Markdown (Haiku, 0.79 of 374 DocVQA questions against 0.60), +whose layout step drops text. Docling runs the same OCR inside, so only a page +OCR could not read at all is worth its try. +""" + +import re +import statistics +from dataclasses import dataclass +from enum import StrEnum +from itertools import pairwise + +from worker.ingestion.pdf_text_layer.page_triage import GARBLED_ALNUM + +# Fewer characters than this say nothing on their own: a stamp, a page number, +# or handwriting the engine could not read. +MIN_CHARS = 20 +# Share of dark pixels below which a page with no text is blank paper. +BLANK_INK = 0.01 +# A gap above a line this many median line pitches starts a paragraph, the +# text layer's own factor. +PARAGRAPH_PITCH = 1.3 +# An engine's own confidence, a character's share of the page apiece: RapidOCR's +# was 0.85 or more on every upright real page of the corpus's forms, receipts, +# books and DocVQA (median 0.98-0.99), and 0.56 to 0.79 on sideways photos. +LOW_CONFIDENCE = 0.8 +# With no confidence, the share of characters in words of 3 letters or more: no +# upright real page of the corpus fell below 0.32 as Vision read it, nor 0.36 as +# Windows OCR did; read sideways, pages gave 0 to 0.22, "O 00 o c". +WORD_SHARE = 0.3 +# Characters a token, on average, with no confidence: 2.8 to 5.5 on real pages, +# 1.0 to 1.8 on a page read sideways, its letters' strokes one by one. +SIDEWAYS_TOKEN = 2.2 +_WORD = re.compile(r"[^\W\d_]{3,}") + + +@dataclass(frozen=True) +class Word: + text: str + x: float + y: float + w: float + h: float + + +@dataclass(frozen=True) +class Line: + text: str + words: tuple[Word, ...] = () + + @property + def box(self) -> tuple[float, float, float, float]: + """x, y, right, bottom over its words.""" + if not self.words: + return (0.0, 0.0, 0.0, 0.0) + return ( + min(word.x for word in self.words), + min(word.y for word in self.words), + max(word.x + word.w for word in self.words), + max(word.y + word.h for word in self.words), + ) + + +class PageVerdict(StrEnum): + TEXT = "text" + BLANK = "blank" + TOO_LITTLE = "too little text" + GARBLED = "garbled" + + +# What OCR's text stands for; a garbled page goes to Docling. +KEPT = frozenset({PageVerdict.TEXT, PageVerdict.BLANK, PageVerdict.TOO_LITTLE}) + + +@dataclass(frozen=True) +class OcrPage: + width: int + height: int + lines: tuple[Line, ...] = () + # Share of dark pixels, to tell blank paper from an unreadable page. + ink: float = 0.0 + language: str | None = None + # The engine's own, when it gives one: RapidOCR's; Vision's is not used. + confidence: float | None = None + + @property + def chars(self) -> int: + return sum(len(line.text.strip()) for line in self.lines) + + def text(self) -> str: + """Lines in reading order: a hyphen at a line end joins the next line's + lowercase word, and a wide gap above a line starts a paragraph.""" + lines = [line for line in self.lines if line.text.strip()] + if not lines: + return "" + breaks = _paragraph_starts(lines) + out: list[str] = [lines[0].text.strip()] + for index, line in enumerate(lines[1:], start=1): + text = line.text.strip() + if index in breaks: + out.append("\n\n" + text) + elif _hyphenated(out[-1]) and text[:1].islower(): + out[-1] = out[-1][:-1] + text + else: + out.append("\n" + text) + return "".join(out) + + def verdict(self) -> PageVerdict: + if self.chars < MIN_CHARS: + return PageVerdict.BLANK if self.ink < BLANK_INK else PageVerdict.TOO_LITTLE + letters = "".join(line.text for line in self.lines) + visible = [c for c in letters if not c.isspace()] + if sum(c.isalnum() for c in visible) < GARBLED_ALNUM * len(visible): + return PageVerdict.GARBLED + if self.confidence is not None: + # A page of figures has few words; the engine knows what it read. + if self.confidence < LOW_CONFIDENCE: + return PageVerdict.GARBLED + elif self.word_share() < WORD_SHARE: + return PageVerdict.GARBLED + return PageVerdict.TEXT + + def word_share(self) -> float: + text = " ".join(line.text for line in self.lines) + visible = sum(not c.isspace() for c in text) + return sum(len(m.group()) for m in _WORD.finditer(text)) / max(1, visible) + + def looks_sideways(self) -> bool: + """Read turned a quarter or half round: the engine unsure of its text, + or, with no confidence given, one or two characters a token.""" + tokens = " ".join(line.text for line in self.lines).split() + if self.chars < MIN_CHARS or not tokens: + # Little text says nothing of which way up: a photo, a signature. + return False + if self.confidence is not None: + return self.confidence < LOW_CONFIDENCE + return sum(map(len, tokens)) / len(tokens) < SIDEWAYS_TOKEN + + +def _hyphenated(text: str) -> bool: + return len(text) > 1 and text.endswith("-") and text[-2].isalpha() + + +def _paragraph_starts(lines: list[Line]) -> set[int]: + """Indexes of lines with a wide gap above, or that jump back up: a new column.""" + tops = [line.box[1] for line in lines] + pitches = [b - a for a, b in pairwise(tops) if b > a] + if len(pitches) < 3 or not any(line.words for line in lines): + return set() + pitch = statistics.median(pitches) + starts: set[int] = set() + for index in range(1, len(lines)): + step = tops[index] - tops[index - 1] + if step < 0 or step > PARAGRAPH_PITCH * pitch: + starts.add(index) + return starts + + +def spread_words(text: str, x: float, y: float, w: float, h: float) -> tuple[Word, ...]: + """A line an engine boxed whole, its words placed by character count.""" + if w <= 0 or not text: + return () + per_char = w / len(text) + words: list[Word] = [] + at = x + for piece in text.split(" "): + if piece: + words.append(Word(piece, at, y, per_char * len(piece), h)) + at += per_char * (len(piece) + 1) + return tuple(words) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py b/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py new file mode 100644 index 0000000000..0fa54f089e --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py @@ -0,0 +1,156 @@ +"""RapidOCR on its own: the OCR inside Docling, without Docling's layout model, +TableFormer or torch. + +The same PP-OCRv6 files Docling reads, and more of the page: on 499 real pages it +found 0.80 of a form's words where Docling found 0.72, and the answer to 0.88 of +374 DocVQA questions where Docling's text held 0.70. A page took about 2 s and +8 CPU-s on a Ryzen 5800X, where Docling took 4.9 s and 39 CPU-s. +""" + +import logging +from collections import deque +from collections.abc import Iterable, Iterator +from concurrent.futures import Future, ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import psutil + +from worker.ingestion.parser_pack import parser_dir +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import OcrPage +from worker.ingestion.scan_ocr.reading_order import Box, lines_in_order + +log = logging.getLogger(__name__) + +DET = "PP-OCRv6_det_small.onnx" +REC = "PP-OCRv6_rec_small.onnx" +CLS = "ch_ppocr_mobile_v2.0_cls_mobile.onnx" +# RapidOCR shrinks anything longer to this before reading; rendering larger only +# costs time. +MAX_SIDE = 2000 +# Pages read at once, one onnxruntime thread each: its CPU-s a page grow with its +# threads (14.9 at 1, 36.6 at 8), so several pages at one thread read more pages +# a CPU-second. At most 4, and half the physical cores, as chat may need the rest. +MAX_IN_FLIGHT = 4 + + +class RapidOcr: + name = "RapidOCR" + language = "multi" + max_side = MAX_SIDE + + def __init__(self, engine: Any) -> None: + self._engine = engine + + @classmethod + def create(cls) -> "RapidOcr | None": + models = _models() + if models is None: + log.info("RapidOCR's models are not here") + return None + try: + from rapidocr import RapidOCR + + _no_spinning() + engine = RapidOCR( + params={ + # What Docling passes for the app: RapidOcrOptions(use_cls=False). + "Global.text_score": 0.5, + "Global.use_cls": False, + "Global.log_level": "error", + "EngineConfig.onnxruntime.intra_op_num_threads": 1, + "Det.model_path": str(models / DET), + "Rec.model_path": str(models / REC), + # Built even when off; pointed at the pack so nothing downloads. + "Cls.model_path": str(models / CLS), + } + ) + except Exception: + log.info("RapidOCR could not start", exc_info=True) + return None + return cls(engine) + + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + """Each frame's page, in order, several read at once on one engine: its + only per-call state is harmless (rapidocr 3.9.2).""" + in_flight = max( + 1, min(MAX_IN_FLIGHT, (psutil.cpu_count(logical=False) or 2) // 2) + ) + with ThreadPoolExecutor(in_flight, thread_name_prefix="rapidocr") as pool: + pending: deque[tuple[Frame, Future[OcrPage]]] = deque() + for frame in frames: + if len(pending) >= in_flight: + done, future = pending.popleft() + yield done, future.result() + pending.append((frame, pool.submit(self._recognise, frame))) + while pending: + done, future = pending.popleft() + yield done, future.result() + + def _recognise(self, frame: Frame) -> OcrPage: + import numpy + + image = frame.image + result = self._engine(numpy.asarray(image)) + boxes: list[Box] = [] + weighed = chars = 0.0 + if getattr(result, "boxes", None) is not None and result.txts: + scores = list(result.scores or []) + for index, (corners, text) in enumerate( + zip(result.boxes.tolist(), result.txts, strict=False) + ): + text = (text or "").strip() + if text: + xs = [point[0] for point in corners] + ys = [point[1] for point in corners] + boxes.append(Box(min(xs), min(ys), max(xs), max(ys), text)) + if index < len(scores): + weighed += float(scores[index]) * len(text) + chars += len(text) + return OcrPage( + image.width, + image.height, + lines_in_order(boxes), + frame.ink, + self.language, + confidence=weighed / chars if chars else None, + ) + + +def _models() -> Path | None: + """The parser pack's RapidOcr folder, else the identical files the rapidocr + wheel carries.""" + candidates = [parser_dir() / "RapidOcr"] + try: + import rapidocr + + candidates.append(Path(rapidocr.__file__).parent / "models") + except ImportError: + return None + for folder in candidates: + if all((folder / name).is_file() for name in (DET, REC, CLS)): + return folder + return None + + +def _no_spinning() -> None: + """onnxruntime's idle threads spin, burning CPU-seconds a page never used: + 13 CPU-s a page at 8 threads in the corpus run. RapidOCR has no switch for it.""" + try: + from rapidocr.inference_engine.onnxruntime import main as ort_main + + build = ort_main.OrtInferSession._init_sess_opts + if getattr(build, "_no_spin", False): + return + + def quiet(config: Any) -> Any: + options = build(config) + options.add_session_config_entry("session.intra_op.allow_spinning", "0") + options.add_session_config_entry("session.inter_op.allow_spinning", "0") + return options + + quiet._no_spin = True # type: ignore[attr-defined] + ort_main.OrtInferSession._init_sess_opts = staticmethod(quiet) + except Exception: + log.debug("onnxruntime's threads keep spinning", exc_info=True) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/reader.py b/surfsense_local/backend/worker/ingestion/scan_ocr/reader.py new file mode 100644 index 0000000000..1af6636c7d --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/reader.py @@ -0,0 +1,58 @@ +"""The OCR that reads scans and photos before Docling, or None: then Docling reads +as before.""" + +import logging +import sys +from collections.abc import Iterable, Iterator +from functools import lru_cache +from typing import Protocol + +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import OcrPage +from worker.ingestion.scan_ocr.switch import ScanOcr, scan_ocr_choice + +log = logging.getLogger(__name__) + + +class PageReader(Protocol): + name: str + language: str | None + # The longest side a frame may have. + max_side: int + + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: ... + + +def page_reader() -> PageReader | None: + """None when switched off, or when nothing here can read a page.""" + choice = scan_ocr_choice() + if choice is ScanOcr.OFF: + return None + if choice is ScanOcr.AUTO and sys.platform == "darwin": + return vision_reader() or rapid_reader() + return rapid_reader() + + +@lru_cache(maxsize=1) +def rapid_reader() -> PageReader | None: + """Once per process, so a missing model says so only the first time.""" + from worker.ingestion.scan_ocr.rapid import RapidOcr + + return _announced(RapidOcr.create(), RapidOcr.name) + + +@lru_cache(maxsize=1) +def vision_reader() -> PageReader | None: + """Apple's Vision: on the corpus's real pages as good as RapidOCR but on + forms, and the only reader of camera photos taken at an angle.""" + from worker.ingestion.scan_ocr.vision import VisionOcr + + return _announced(VisionOcr.create(), VisionOcr.name) + + +def _announced(reader: PageReader | None, name: str) -> PageReader | None: + if reader is None: + log.info("%s cannot read scans here", name) + else: + log.info("scans and photos are read by %s first", name) + return reader diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py b/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py new file mode 100644 index 0000000000..b53ceaa53a --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py @@ -0,0 +1,164 @@ +"""Boxes an engine read one by one, as lines in reading order. + +RapidOCR boxes each run of text alone. A visual row of boxes reads as one line, +left to right, so a receipt's item and price stay together; two columns of prose +side by side read one after the other, not across. +""" + +import statistics +from dataclasses import dataclass, field + +from worker.ingestion.scan_ocr.page import Line, spread_words + +# A gutter at least this many text heights wide can split two columns. +_GUTTER = 0.75 + + +@dataclass(frozen=True) +class Box: + x0: float + y0: float + x1: float + y1: float + text: str + + @property + def h(self) -> float: + return max(1.0, self.y1 - self.y0) + + @property + def cy(self) -> float: + return (self.y0 + self.y1) / 2 + + +@dataclass +class _Row: + boxes: list[Box] = field(default_factory=list) + + @property + def cy(self) -> float: + return sum(box.cy for box in self.boxes) / len(self.boxes) + + @property + def h(self) -> float: + return statistics.median(box.h for box in self.boxes) + + +def lines_in_order(boxes: list[Box]) -> tuple[Line, ...]: + if not boxes: + return () + unit = statistics.median(box.h for box in boxes) + rows = [row for block in _blocks(boxes, unit) for row in _rows(block)] + return tuple( + Line( + " ".join(box.text for box in row.boxes), + tuple( + word + for box in row.boxes + for word in spread_words( + box.text, box.x0, box.y0, box.x1 - box.x0, box.h + ) + ), + ) + for row in rows + ) + + +def _rows(boxes: list[Box]) -> list[_Row]: + """A box joins the row above when their centres are within half the smaller + height.""" + rows: list[_Row] = [] + for box in sorted(boxes, key=lambda b: (b.cy, b.x0)): + if rows and abs(box.cy - rows[-1].cy) <= 0.5 * min(box.h, rows[-1].h): + rows[-1].boxes.append(box) + else: + rows.append(_Row([box])) + for row in rows: + row.boxes.sort(key=lambda b: b.x0) + return rows + + +def _gaps(spans: list[tuple[float, float]]) -> list[tuple[float, float]]: + ordered = sorted(spans) + gaps: list[tuple[float, float]] = [] + end = ordered[0][1] + for start, stop in ordered[1:]: + if start > end: + gaps.append((end, start)) + end = max(end, stop) + return gaps + + +def _column_like(side: list[Box], width: float) -> bool: + """Several rows of long text: prose, not a table's or a receipt's column.""" + if len(side) < 3: + return False + if max(b.x1 for b in side) - min(b.x0 for b in side) < 0.15 * width: + return False + if statistics.median(len(b.text) for b in side) < 10: + return False + return len(_rows(side)) >= 3 + + +def _vertical_cut(boxes: list[Box], unit: float) -> tuple[list[Box], list[Box]] | None: + width = max(1.0, max(b.x1 for b in boxes) - min(b.x0 for b in boxes)) + best: tuple[float, list[Box], list[Box]] | None = None + for g0, g1 in _gaps([(b.x0, b.x1) for b in boxes]): + if g1 - g0 < _GUTTER * unit: + continue + left = [b for b in boxes if b.x1 <= g0] + right = [b for b in boxes if b.x0 >= g1] + if ( + _column_like(left, width) + and _column_like(right, width) + and (best is None or g1 - g0 > best[0]) + ): + best = (g1 - g0, left, right) + return None if best is None else (best[1], best[2]) + + +def _bands(boxes: list[Box], unit: float) -> list[list[Box]]: + """Strips across the page with blank space between them; bands that share a + gutter stay together, as two columns whose lines happen to align would.""" + gaps = _gaps([(b.y0, b.y1) for b in boxes]) + if not gaps: + return [boxes] + cuts = [(a + b) / 2 for a, b in gaps] + bands: list[list[Box]] = [[] for _ in range(len(cuts) + 1)] + for box in boxes: + bands[sum(box.cy > cut for cut in cuts)].append(box) + bands = [band for band in bands if band] + groups: list[list[Box]] = [] + shared: tuple[float, float] | None = None + for band in bands: + gutter = _band_gutter(band, unit) + if groups and shared is not None and gutter is not None: + lo, hi = max(shared[0], gutter[0]), min(shared[1], gutter[1]) + if hi - lo >= _GUTTER * unit: + groups[-1].extend(band) + shared = (lo, hi) + continue + groups.append(list(band)) + shared = gutter + return bands if len(groups) == 1 else groups + + +def _band_gutter(band: list[Box], unit: float) -> tuple[float, float] | None: + gaps = [ + g for g in _gaps([(b.x0, b.x1) for b in band]) if g[1] - g[0] >= _GUTTER * unit + ] + return max(gaps, key=lambda g: g[1] - g[0]) if gaps else None + + +def _blocks(boxes: list[Box], unit: float, depth: int = 0) -> list[list[Box]]: + """Columns left to right where prose stands side by side, else bands top to + bottom (an XY cut).""" + if len(boxes) < 6 or depth > 24: + return [boxes] + cut = _vertical_cut(boxes, unit) + if cut is not None: + return _blocks(cut[0], unit, depth + 1) + _blocks(cut[1], unit, depth + 1) + bands = _bands(boxes, unit) + if len(bands) == 1: + return [boxes] + return [block for band in bands for block in _blocks(band, unit, depth + 1)] diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/switch.py b/surfsense_local/backend/worker/ingestion/scan_ocr/switch.py new file mode 100644 index 0000000000..9f162743f1 --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/switch.py @@ -0,0 +1,44 @@ +"""SURFSENSE_LOCAL_SCAN_OCR: which OCR reads a scan or a photo before Docling. + +auto the reader measured best here: Apple's Vision on a Mac, RapidOCR alone + everywhere else +rapidocr RapidOCR alone, Docling's own OCR without its layout and table models +off Docling reads every scan and photo whole, as before +""" + +import logging +from enum import StrEnum + +from pydantic import ValidationError, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +log = logging.getLogger(__name__) + + +class ScanOcr(StrEnum): + AUTO = "auto" + RAPIDOCR = "rapidocr" + OFF = "off" + + +class ScanOcrSettings(BaseSettings): + """An escape hatch back to Docling, should OCR-first misread scans.""" + + model_config = SettingsConfigDict(env_prefix="SURFSENSE_LOCAL_") + + scan_ocr: ScanOcr = ScanOcr.AUTO + + @field_validator("scan_ocr", mode="before") + @classmethod + def _any_case(cls, value: object) -> object: + return value.strip().lower() if isinstance(value, str) else value + + +def scan_ocr_choice() -> ScanOcr: + # Not cached: tests flip it per case, and it costs microseconds per file. + try: + return ScanOcrSettings().scan_ocr + except ValidationError: + # Whoever set the switch most likely meant to turn this reader off. + log.warning("SURFSENSE_LOCAL_SCAN_OCR is not auto, rapidocr or off: off") + return ScanOcr.OFF diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py b/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py new file mode 100644 index 0000000000..f531e4b23f --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py @@ -0,0 +1,34 @@ +"""A page the system's OCR read sideways, read again turned upright. + +The engines read upright text only. A phone photo with no EXIF turn, or a scan +fed in sideways, came back as "O 00 o c": all 20 real camera captures in the +corpus did. Turning costs a reading or three, and only on such pages. +""" + +from collections.abc import Iterable, Iterator + +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import OcrPage, PageVerdict +from worker.ingestion.scan_ocr.reader import PageReader + +# A quarter turn first: sideways is far commoner than upside down. +TURNS = (90, 270, 180) + + +def read_upright( + reader: PageReader, frames: Iterable[Frame] +) -> Iterator[tuple[Frame, OcrPage]]: + """Each frame's page, turned to whichever way reads best when it looks sideways.""" + for frame, page in reader.read(frames): + if not page.looks_sideways(): + yield frame, page + continue + turned = [ + Frame(frame.number, frame.image.rotate(angle, expand=True), frame.ink) + for angle in TURNS + ] + best = max(reader.read(turned), key=lambda read: read[1].word_share()) + if best[1].verdict() in (PageVerdict.GARBLED, PageVerdict.TOO_LITTLE): + yield frame, page + else: + yield best diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py b/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py new file mode 100644 index 0000000000..f25d9d5fe6 --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py @@ -0,0 +1,122 @@ +"""macOS's own OCR (Vision's VNRecognizeTextRequest, accurate), through PyObjC. + +On a GitHub macOS runner (a 3-core M1 virtual machine, no Neural Engine) it read +the corpus's 499 real pages at 2.4 s a page, as well as RapidOCR on books, +receipts and DocVQA but for forms (0.71 of their words against 0.80), and read +the camera photos neither RapidOCR nor Docling could (0.94 against 0.05). Its +fast level found 0.27 of all words: not used. + +Called directly rather than through ocrmac: installing ocrmac would switch +Docling's own OCR on a Mac with no parser pack. +""" + +import logging +from collections import deque +from collections.abc import Iterable, Iterator +from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any + +import psutil + +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import OcrPage +from worker.ingestion.scan_ocr.reading_order import Box, lines_in_order + +log = logging.getLogger(__name__) + +# Throughput stops growing past the performance cores; leave room for chat. +MAX_IN_FLIGHT = 4 +# Vision takes far larger images than a 216 dpi page; this only bounds memory. +MAX_SIDE = 10_000 + + +class VisionOcr: + name = "Vision" + language = "auto" + max_side = MAX_SIDE + + @classmethod + def create(cls) -> "VisionOcr | None": + try: + import Vision + + Vision.VNRecognizeTextRequest.alloc().init() + except Exception: + log.info("Vision is not available", exc_info=True) + return None + return cls() + + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + """Each frame's page, in order. PyObjC lets go of the GIL inside every + Objective-C call, so pages in flight do overlap.""" + in_flight = max( + 1, min(MAX_IN_FLIGHT, (psutil.cpu_count(logical=False) or 2) // 2) + ) + with ThreadPoolExecutor(in_flight, thread_name_prefix="vision-ocr") as pool: + pending: deque[tuple[Frame, Future[OcrPage]]] = deque() + for frame in frames: + if len(pending) >= in_flight: + done, future = pending.popleft() + yield done, future.result() + pending.append((frame, pool.submit(_recognise, frame))) + while pending: + done, future = pending.popleft() + yield done, future.result() + + +def _recognise(frame: Frame) -> OcrPage: + import Foundation + import objc + import Quartz + import Vision + + image = frame.image + width, height = image.size + with objc.autorelease_pool(): + pixels = image.tobytes() + data = Foundation.NSData.dataWithBytes_length_(pixels, len(pixels)) + provider = Quartz.CGDataProviderCreateWithCFData(data) + picture = Quartz.CGImageCreate( + width, + height, + 8, + 8, + width, + Quartz.CGColorSpaceCreateDeviceGray(), + Quartz.kCGImageAlphaNone, + provider, + None, + False, + Quartz.kCGRenderingIntentDefault, + ) + handler = Vision.VNImageRequestHandler.alloc().initWithCGImage_options_( + picture, {} + ) + request = Vision.VNRecognizeTextRequest.alloc().init() + request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate) + request.setUsesLanguageCorrection_(True) + # macOS 13, the app's own minimum: Vision picks the language. + if hasattr(request, "setAutomaticallyDetectsLanguage_"): + request.setAutomaticallyDetectsLanguage_(True) + outcome = handler.performRequests_error_([request], None) + ok, error = outcome if isinstance(outcome, tuple) else (outcome, None) + if not ok: + raise RuntimeError(f"Vision could not read the page: {error}") + boxes = [ + box + for observation in request.results() or () + if (box := _box(observation, width, height)) is not None + ] + # No confidence: Vision's was never measured against the corpus, and the + # page verdict's threshold is RapidOCR's. + return OcrPage(width, height, lines_in_order(boxes), frame.ink, "auto") + + +def _box(observation: Any, width: int, height: int) -> Box | None: + """Vision's box is normalised with its origin bottom left.""" + text = str(observation.topCandidates_(1)[0].string()).strip() + if not text: + return None + box = observation.boundingBox() + x, y = box.origin.x * width, (1 - box.origin.y - box.size.height) * height + return Box(x, y, x + box.size.width * width, y + box.size.height * height, text) diff --git a/surfsense_local/electron/scripts/notices/allowed-without-text.json b/surfsense_local/electron/scripts/notices/allowed-without-text.json index fe2759afe4..5441de28f9 100644 --- a/surfsense_local/electron/scripts/notices/allowed-without-text.json +++ b/surfsense_local/electron/scripts/notices/allowed-without-text.json @@ -131,6 +131,24 @@ "version": "3.81.0", "reason": "The wheel ships no licence file. Its metadata declares MIT, by Ronie Martinez." }, + { + "tree": "python", + "name": "pyobjc-core", + "version": "12.2.2", + "reason": "macOS only. The wheel ships no licence file. Its metadata declares MIT, by Ronald Oussoren and the PyObjC contributors, whose text pyobjc-framework-Cocoa and -Quartz ship." + }, + { + "tree": "python", + "name": "pyobjc-framework-CoreML", + "version": "12.2.2", + "reason": "macOS only. The wheel ships no licence file. Its metadata declares MIT, by Ronald Oussoren and the PyObjC contributors, whose text pyobjc-framework-Cocoa and -Quartz ship." + }, + { + "tree": "python", + "name": "pyobjc-framework-Vision", + "version": "12.2.2", + "reason": "macOS only. The wheel ships no licence file. Its metadata declares MIT, by Ronald Oussoren and the PyObjC contributors, whose text pyobjc-framework-Cocoa and -Quartz ship." + }, { "tree": "python", "name": "rapidocr", From 6e07853d38178da9d0408f9604deba0afcd8d627 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 17 Oct 2026 02:03:39 -0700 Subject: [PATCH 2/6] fix(local): scan OCR review fixes: close renders, one cap, upright turns, resume, figures on ask Renders close on the ingest thread whatever stops a reading; pages and a sideways page's turns share one cap; a turn replaces a page only when it reads as text, and lines standing on end count as sideways; a cover keeps its own title over OCR's scraps; blank pages alone send the file to Docling; OCR's pages are kept while Docling's part of the file gives the queue up; a scan's figures wait to be asked; no process-wide onnxruntime patch. --- ...can-is-read-by-ocr-alone-before-docling.md | 51 ++++++++++ docs/adr/README.md | 1 + docs/architecture/documents.md | 23 ++++- .../integration/worker/test_ocr_first.py | 50 +++++++++- .../integration/worker/test_vision_engine.py | 42 ++++++++ .../tests/unit/worker/test_scan_ocr.py | 97 +++++++++++++++++++ .../worker/ingestion/pdf_reading/ocr_first.py | 76 ++++++++++----- .../worker/ingestion/pdf_reading/ocr_kept.py | 61 ++++++++++++ .../worker/ingestion/pdf_reading/route.py | 13 ++- .../worker/ingestion/scan_ocr/frames.py | 4 +- .../worker/ingestion/scan_ocr/images.py | 23 +++-- .../backend/worker/ingestion/scan_ocr/page.py | 12 ++- .../ingestion/scan_ocr/pages_at_once.py | 53 ++++++++++ .../worker/ingestion/scan_ocr/rapid.py | 60 +++--------- .../ingestion/scan_ocr/reading_order.py | 16 +++ .../worker/ingestion/scan_ocr/upright.py | 35 +++---- .../worker/ingestion/scan_ocr/vision.py | 39 +++----- 17 files changed, 523 insertions(+), 133 deletions(-) create mode 100644 docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md create mode 100644 surfsense_local/backend/tests/integration/worker/test_vision_engine.py create mode 100644 surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py create mode 100644 surfsense_local/backend/worker/ingestion/scan_ocr/pages_at_once.py diff --git a/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md b/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md new file mode 100644 index 0000000000..b2586d6146 --- /dev/null +++ b/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md @@ -0,0 +1,51 @@ +# ADR 0060: A scan or a photo is read by OCR alone, RapidOCR or Apple's Vision, and Docling reads only a page OCR could not + +- **Status:** Accepted +- **Date:** 2026-10-17 +- **Source:** measured on a Ryzen 7 5800X under Windows 11, on a GitHub macOS runner (a 3-core M1 virtual machine), and with Haiku 5.5 for answers; recorded in [documents](../architecture/documents.md#scans-and-photos) + +## Context + +A scanned PDF, a PDF page with no text layer and an uploaded image went to Docling with OCR and table structure on. On 499 real pages it took 4.9 s and 39 CPU-s a page, plus 8 to 17 s to import Docling and load its models in a worker's first job: a 20-page scan was ready in 98.5 s, and [ADR 0059](0059-a-born-digital-pdf-is-read-from-its-text-layer.md)'s text layer, which cut born-digital PDFs to seconds, could do nothing for it. + +The pages were real, not rendered from born-digital files: 80 scanned business forms (FUNSD) and 80 DocVQA pages (letters, tables and reports from the UCSF Industry Documents Library, with 374 human questions and answers), 60 scanned receipts (SROIE) and 60 phone photos of receipts (CORD), 36 pages of three Internet Archive book scans with their Project Gutenberg text, one in French, 20 camera captures of printed pages (SmartDoc 2015), and six public scanned PDFs of 13 to 40 pages from the National Archives, govinfo and RECAP. Every reader was scored on the same pages, against human ground truth. + +Docling's time on a scan went mostly to its layout model and TableFormer, not to OCR: RapidOCR, which Docling runs inside on each text region with the PP-OCRv6 small models, reads a whole page alone in about 2 s and 8 CPU-s. Read alone, it also found more of each page. Docling's layout step keeps only the text of the regions it classes, and drops running headers and page numbers outright: it kept 0.04 of the book scans' numbers, where RapidOCR alone kept 0.93. + +| Real pages | RapidOCR alone | Docling | Windows OCR | Vision (accurate) | +|---|---|---|---|---| +| Words found, forms | 0.80 | 0.72 | 0.61 | 0.71 | +| Words found, receipt scans | 0.90 | 0.87 | 0.69 | 0.89 | +| Words found, receipt photos | 0.83 | 0.79 | 0.43 | 0.80 | +| Words found, books (English / French) | 0.98 / 0.92 | 0.97 / 0.90 | 0.98 / 0.83 | 0.99 / 0.92 | +| Words found, camera captures | 0.05 | 0.05 | 0.00 | 0.94 | +| Numbers found, all pages | 0.65 | 0.59 | 0.40 | 0.67 | +| DocVQA answers in the text | 0.88 | 0.70 | 0.70 | 0.86 | +| Seconds a page | 2.0 | 4.9 | 0.24 | 2.4 on the VM | + +Answers followed: Haiku 5.5, given each page's text whole, answered 0.79 of the 374 DocVQA questions exactly from RapidOCR's text and 0.60 from Docling's Markdown, and 0.97 of 120 receipt totals and dates against 0.93. It replied that the text did not say for 34 questions against 104. Tables included: the DocVQA pages hold most of the corpus's tables, and RapidOCR's lines, ordered into rows, read them better than Docling's pipe tables. + +## Decision + +- **What OCR reads first.** A PDF's pages with no text layer, with a scanner's invisible OCR layer, with text pdfium cannot map, a turned page that is a scan, and, when no page is running text, its cover and picture pages; and every uploaded image. They are rendered greyscale at 216 dpi, an image as an A4 page as before, and read by the reader below. The code is [`scan_ocr/`](../../surfsense_local/backend/worker/ingestion/scan_ocr/) and [`pdf_reading/ocr_first.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py). +- **The reader.** RapidOCR alone on Windows and Linux: the parser pack's own PP-OCRv6 small files, or the identical ones in the rapidocr wheel, with the line classifier off as Docling has it, 4 pages at a time at one onnxruntime thread each, which spends the least CPU a page. On a Mac, Apple's Vision at its accurate level, through PyObjC; RapidOCR if Vision cannot start. Vision's fast level found 0.27 of all words and is not used. Each reader's boxes are put in reading order the same way: a visual row reads left to right as one line, so a table's label and amount stay together, and two columns of prose read one after the other. +- **Which pages stand.** A page stands unless its OCR is garbled: mostly symbols, RapidOCR's own confidence under 0.8 (0.85 or more on every upright real page, 0.56 to 0.79 on sideways photos), or, for Vision, which gives none used here, under 30% of its characters in words of three letters. A page that looks read sideways, its text unsure or its lines standing on end, is read again turned a quarter, three quarters and half round, one turn at a time, and the first turn that reads as upright text replaces it; all 20 camera captures were sideways. Docling reads a garbled page, in its place, through [ADR 0059](0059-a-born-digital-pdf-is-read-from-its-text-layer.md)'s splice; a scan with no page that stands goes to Docling whole, as before, and an image with a garbled frame too. Tables stay with OCR. +- **Figures.** Ingest keeps none for a scan read this way, and a scan's figures wait until the agent or Studio asks, as a PDF over 40 pages does: the figures-only pass reads scanned pages with Docling in full, about 39 CPU-s a page, which would give back the CPU this saves. A PDF with born-digital pages beside its scans still gets the pass queued at idle. An image is still its own figure. +- `SURFSENSE_LOCAL_SCAN_OCR=off` reads every scan and photo with Docling, as before; `rapidocr` uses RapidOCR on a Mac too. A value it cannot read also means off. `worker --check-scan-ocr` reads a rendered line with each reader in the frozen worker on every release build. +- **Rejected: Windows' own OCR** (Windows.Media.Ocr, through pywinrt). It read a page in 0.13 to 0.24 s, and worked frozen, but found 0.61 of a form's words and 0.43 of a receipt photo's, writes a table column by column, and nothing in its output told its good pages from its bad ones: keeping only its cleanest-looking pages still lost 7 to 10 points of words against RapidOCR. +- **Rejected: Docling for table pages.** Its Markdown answered table questions worse than RapidOCR's rows, and paid its cold start and 39 CPU-s a page. +- **Rejected: page-parallel tuning beyond 4 pages.** RapidOCR costs about 8 CPU-s a page however it is split; 8 pages at once took 184 s for 100 pages against 209 s for 4 and 192 s for one page on 8 threads, at more CPU. + +## Consequences + +- A scan's pages no longer import Docling unless one is garbled, so a worker that sees only scans skips Docling's cold start and its 1 GB, until a figures-only pass runs. +- PENDING: time to ready through the upload route. +- Docling's headings and pipe tables are gone from a scan's text; ADR 0059 found the same trade for text-layer PDFs to the app's benefit. +- An image's text is RapidOCR's or Vision's whole, or Docling's whole: Docling cannot be given one frame of a multi-frame TIFF. + +## Where the code stands + +- Speed is measured on one Windows machine; RapidOCR on Linux runs the same onnxruntime CPU code, and in CI's tests on Ubuntu, macOS and Windows. Vision's speed is from a virtual Mac with no Neural Engine, so a real Mac should read faster; its accuracy is from that runner. +- A long OCR-first reading checks for a cancel every 8 pages but does not give way to other uploads, as a long Docling reading does: a 100-page scan holds the queue for about 3 minutes. When its garbled pages go to a Docling reading that gives way, OCR's pages are kept, so the resumed run does not read them again. +- Camera captures shot at an angle are read upright, but RapidOCR's rows can still mix lines a perspective skews; Vision read them well. +- PP-OCRv6 covers Latin, Chinese and Japanese, as Docling's did, so scanned Korean, Cyrillic, Devanagari or Arabic text is misread, and Vision's languages are the system's. diff --git a/docs/adr/README.md b/docs/adr/README.md index 6590436506..458b878cd8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -87,3 +87,4 @@ The Date is the day the decision was first written down. Source links are permal | 0057 | [A Claude plan is reached through the user's own Claude Code, and SurfSense holds no Claude credential](0057-a-claude-plan-runs-through-the-users-own-claude-code.md) | Accepted | | 0058 | [The bundled embedder runs at FP32 on x86-64, upcast in memory, and its vector space is still the pinned FP16 file](0058-the-bundled-embedder-runs-at-fp32-on-x86-64.md) | Accepted | | 0059 | [A born-digital PDF is read from its own text layer, and Docling reads only a PDF with a page that needs it](0059-a-born-digital-pdf-is-read-from-its-text-layer.md) | Accepted | +| 0060 | [A scan or a photo is read by OCR alone, RapidOCR or Apple's Vision, and Docling reads only a page OCR could not](0060-a-scan-is-read-by-ocr-alone-before-docling.md) | Accepted | diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index 9ac677c79e..a8c7fb6c16 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -124,7 +124,7 @@ The desktop app previews an original in place of the left sidebar when its MIME `ingest_document(document_id)` is declared in [`modules/documents/tasks.py`](../../surfsense_local/backend/modules/documents/tasks.py), because Huey binds a task to its queue when it is decorated and the API is what enqueues it. The task body imports the pipeline lazily and Docling is imported only when the converter is first built, so the API never loads Docling or torch. Each job opens its own database engine. 1. **Start.** `begin_job` marks the row `processing` unless it was cancelled, and the worker notifies the API. The job then reads the active embedding index; with none, because onboarding has not chosen an embedder, it fails with `no embedding model has been chosen yet`. -2. **Parse** ([`parsing.py`](../../surfsense_local/backend/worker/ingestion/parsing.py)). A `NOTE` or `ARTIFACT` already has its markdown in `content`. A recording, a `FILE` whose `mime_type` is `audio/*`, is transcribed before Docling is reached, which must never read one ([Recordings](#recordings)). A `FILE` with a text extension (`.md`, `.markdown`, `.txt`, `.text`) is read as UTF-8. A `.pdf` is read from its own pdfium text layer when every page passes the triage, and Docling reads only the pages that fail it when they are at most half the file ([PDFs with a text layer](#pdfs-with-a-text-layer)); a `.docx` is read without Docling (below); any other PDF, and anything else, goes to Docling, a PDF longer than 8 pages a batch of pages at a time ([Long Docling readings](#long-docling-readings)). A `.pptx` with no slides, such as a template, skips Docling, which refuses a deck with no pages: its text is `PowerPoint template with no slides.`, its slide size and its slide layout names in order, one line each, read with python-pptx ([`slideless_deck.py`](../../surfsense_local/backend/worker/ingestion/slideless_deck.py)), so search finds it and the agent can start a deck from it as a template ([studio](studio.md)). A `.docx` is read as markdown with python-docx and lxml ([`word_markdown/`](../../surfsense_local/backend/worker/ingestion/word_markdown/)): 0.13 s for the longest reference contract, which Docling reads in 5.1 s warm, and 0.05 s for a fresh worker's first Word file, which costs 4.4 s through Docling, its import mostly. The body comes in order: headings as Docling decides them (a style id or name with "heading", or the style's own `w:outlineLvl`; Title is `#`, Heading N is N+1 `#`s), list items and numbered headings with the numbers Word renders from `numbering.xml`, tables one compact line per row with a merged cell's text once and a nested table flattened into its cell, links as markdown links to their address, a space in it as `%20` and an unmatched parenthesis escaped, tabs and breaks as spaces and line breaks, tracked changes as if accepted, a deleted table row with them, `mc:AlternateContent` as its first choice and ruby as its base text, then footnotes and endnotes as `[^1]: …`. Headers, footers and comments are left out, as Docling's markdown leaves them. The numbers agree with Word 16's ListString on all 1,876 numbered paragraphs of 41 reference files, where Docling's agree on 31%: it restarts each `w:num`, ignores `startOverride` and numbering a style inherits along `w:basedOn`, and writes letters and roman numerals as decimals. [`test_word_numbering.py`](../../surfsense_local/backend/tests/unit/worker/test_word_numbering.py) rebuilds a fixture for each rule and expects the ListString Word gave. A counter and a `w:lvlText` come from the file, so a letter, numeral or dagger repeated past 32 times is written in decimal, a start past a 32-bit integer is ignored, and a marker is cut at 255 characters; the longest in the reference files is 10. On the 791 reference Word files Docling reads, the reader keeps a median 100% of Docling's words, and 100% at the 5th percentile, with numbers compared apart and a merged cell counted once. `SURFSENSE_LOCAL_DOCX_READER=docling` turns it off. A file it cannot read, one with native charts, whose data Docling writes as tables, and one where it finds no text go to Docling as before, so a blank Word or Excel file is still ready with no text. While Docling reads a `.docx`, for ingest and the figures-only pass alike, python-docx's style lookups are remembered for that one conversion, per document part and per style id and type ([`docx_style_cache.py`](../../surfsense_local/backend/worker/ingestion/docx_style_cache.py)): Docling asks for a paragraph's style once per run, and python-docx scans every style in the file for each answer, which was 80% of reading a long contract, 25.8 s against 5.4 s with the cache, idle. The Markdown and the whole `DoclingDocument` came out identical on 804 Word files; a file that defines few styles gains little. Only the converting thread reads the cache, and python-docx's own `get_style` is put back once the last conversion running ends, failed or not, so `revise_document` and Studio's writers never see a remembered style. OCR and table structure are switched on, so a scanned PDF is readable and a table survives as a table. RapidOCR's per-line orientation classifier is off: on noisy scans it turned upright lines 180 degrees and they read back as garbage, and dropping it cut the character error rate on generated scans about eightfold. An image is first turned upright by its EXIF orientation, which Docling ignores, and given a DPI that reads it as one A4 page, never finer than the 216 dpi OCR renders at nor coarser than its own DPI ([`image_page.py`](../../surfsense_local/backend/worker/ingestion/image_page.py)); with none, Docling takes it as 72 dpi and OCR upsampled a photo threefold. Images and PDFs share one set of options, so they share one pipeline and its models. Inference gets one thread per physical core, at most 8, as audio.cpp does, instead of Docling's 4: on an 8-core CPU that parsed generated scans, photos and PDFs 1.4 times faster. The converter is built once per process and its models load on the first conversion, so the first document a worker gives Docling pays for the import and the load, 11 to 17 s on the measuring machine; a worker given only notes, text files, text-layer PDFs and Word files imports it only once a figures-only pass runs. `HF_HOME` defaults to the models directory, because Docling would otherwise write weights into `site-packages`, which is read-only in a frozen bundle. When the bundled parser pack is present Docling uses it with RapidOCR, and `HF_HUB_OFFLINE=1` is set before Docling is imported, because huggingface_hub reads it only then, so a PDF or an image converts with networking off. The markdown is kept only in `content`. Beside the original, ingest writes only the pictures Docling found, as [figures](#figures); a PDF read from its text layer, wholly or but for some pages, gets its figures from the figures-only pass, and a `.docx` read without Docling drops those an earlier reading kept and leaves its pictures to that pass. +2. **Parse** ([`parsing.py`](../../surfsense_local/backend/worker/ingestion/parsing.py)). A `NOTE` or `ARTIFACT` already has its markdown in `content`. A recording, a `FILE` whose `mime_type` is `audio/*`, is transcribed before Docling is reached, which must never read one ([Recordings](#recordings)). A `FILE` with a text extension (`.md`, `.markdown`, `.txt`, `.text`) is read as UTF-8. A `.pdf` is read from its own pdfium text layer when every page passes the triage, and Docling reads only the pages that fail it when they are at most half the file ([PDFs with a text layer](#pdfs-with-a-text-layer)); a scanned page and an uploaded image are read by OCR alone, and Docling reads only a page OCR could not ([Scans and photos](#scans-and-photos)); a `.docx` is read without Docling (below); anything else goes to Docling, a PDF longer than 8 pages a batch of pages at a time ([Long Docling readings](#long-docling-readings)). A `.pptx` with no slides, such as a template, skips Docling, which refuses a deck with no pages: its text is `PowerPoint template with no slides.`, its slide size and its slide layout names in order, one line each, read with python-pptx ([`slideless_deck.py`](../../surfsense_local/backend/worker/ingestion/slideless_deck.py)), so search finds it and the agent can start a deck from it as a template ([studio](studio.md)). A `.docx` is read as markdown with python-docx and lxml ([`word_markdown/`](../../surfsense_local/backend/worker/ingestion/word_markdown/)): 0.13 s for the longest reference contract, which Docling reads in 5.1 s warm, and 0.05 s for a fresh worker's first Word file, which costs 4.4 s through Docling, its import mostly. The body comes in order: headings as Docling decides them (a style id or name with "heading", or the style's own `w:outlineLvl`; Title is `#`, Heading N is N+1 `#`s), list items and numbered headings with the numbers Word renders from `numbering.xml`, tables one compact line per row with a merged cell's text once and a nested table flattened into its cell, links as markdown links to their address, a space in it as `%20` and an unmatched parenthesis escaped, tabs and breaks as spaces and line breaks, tracked changes as if accepted, a deleted table row with them, `mc:AlternateContent` as its first choice and ruby as its base text, then footnotes and endnotes as `[^1]: …`. Headers, footers and comments are left out, as Docling's markdown leaves them. The numbers agree with Word 16's ListString on all 1,876 numbered paragraphs of 41 reference files, where Docling's agree on 31%: it restarts each `w:num`, ignores `startOverride` and numbering a style inherits along `w:basedOn`, and writes letters and roman numerals as decimals. [`test_word_numbering.py`](../../surfsense_local/backend/tests/unit/worker/test_word_numbering.py) rebuilds a fixture for each rule and expects the ListString Word gave. A counter and a `w:lvlText` come from the file, so a letter, numeral or dagger repeated past 32 times is written in decimal, a start past a 32-bit integer is ignored, and a marker is cut at 255 characters; the longest in the reference files is 10. On the 791 reference Word files Docling reads, the reader keeps a median 100% of Docling's words, and 100% at the 5th percentile, with numbers compared apart and a merged cell counted once. `SURFSENSE_LOCAL_DOCX_READER=docling` turns it off. A file it cannot read, one with native charts, whose data Docling writes as tables, and one where it finds no text go to Docling as before, so a blank Word or Excel file is still ready with no text. While Docling reads a `.docx`, for ingest and the figures-only pass alike, python-docx's style lookups are remembered for that one conversion, per document part and per style id and type ([`docx_style_cache.py`](../../surfsense_local/backend/worker/ingestion/docx_style_cache.py)): Docling asks for a paragraph's style once per run, and python-docx scans every style in the file for each answer, which was 80% of reading a long contract, 25.8 s against 5.4 s with the cache, idle. The Markdown and the whole `DoclingDocument` came out identical on 804 Word files; a file that defines few styles gains little. Only the converting thread reads the cache, and python-docx's own `get_style` is put back once the last conversion running ends, failed or not, so `revise_document` and Studio's writers never see a remembered style. OCR and table structure are switched on, so a scanned PDF is readable and a table survives as a table. RapidOCR's per-line orientation classifier is off: on noisy scans it turned upright lines 180 degrees and they read back as garbage, and dropping it cut the character error rate on generated scans about eightfold. An image is first turned upright by its EXIF orientation, which Docling ignores, and given a DPI that reads it as one A4 page, never finer than the 216 dpi OCR renders at nor coarser than its own DPI ([`image_page.py`](../../surfsense_local/backend/worker/ingestion/image_page.py)); with none, Docling takes it as 72 dpi and OCR upsampled a photo threefold. Images and PDFs share one set of options, so they share one pipeline and its models. Inference gets one thread per physical core, at most 8, as audio.cpp does, instead of Docling's 4: on an 8-core CPU that parsed generated scans, photos and PDFs 1.4 times faster. The converter is built once per process and its models load on the first conversion, so the first document a worker gives Docling pays for the import and the load, 11 to 17 s on the measuring machine; a worker given only notes, text files, text-layer PDFs and Word files imports it only once a figures-only pass runs. `HF_HOME` defaults to the models directory, because Docling would otherwise write weights into `site-packages`, which is read-only in a frozen bundle. When the bundled parser pack is present Docling uses it with RapidOCR, and `HF_HUB_OFFLINE=1` is set before Docling is imported, because huggingface_hub reads it only then, so a PDF or an image converts with networking off. The markdown is kept only in `content`. Beside the original, ingest writes only the pictures Docling found, as [figures](#figures); a PDF read from its text layer, wholly or but for some pages, gets its figures from the figures-only pass, and a `.docx` read without Docling drops those an earlier reading kept and leaves its pictures to that pass. 3. **Chunk** ([`chunking.py`](../../surfsense_local/backend/worker/ingestion/chunking.py)). Chonkie's `RecursiveChunker` splits at the coarsest boundary that fits: a heading, a paragraph, a line, a sentence, a word, then a bare split so an over-long line still ends. Passages are at most 480 tokens by bge-small's tokenizer, under its 512-token limit with room for the two tokens it adds. That tokenizer is pinned to the chunker rather than taken from the active embedder, so changing the embedder would not re-cut documents and move every chunk id. Pieces under 24 characters are merged into a neighbour while splitting, so only a document shorter than that yields a shorter passage. The tokenizer is passed as an object, because one named by a string would reach the network even offline. Each passage records the lines its first and last characters fall on. 4. **Embed** ([`encoder.py`](../../surfsense_local/backend/modules/embedding/encoder.py)). The active index's model, as its spec says, on onnxruntime's CPU provider: the document prefix, tokenize (truncated at the spec's `max_tokens`), pool, normalise, in batches of 32. For bge-small that is no prefix, 512 tokens, CLS pooling and L2 normalisation, and on x86-64 its FP16 file runs upcast to FP32 in memory, about six times as fast ([embedding](embedding.md#precision)). No network and no model server. A vector whose width is not the spec's is refused rather than stored. 5. **Index** ([`indexing.py`](../../surfsense_local/backend/worker/ingestion/indexing.py)). The document's old chunks are deleted, which clears both index tables through the delete trigger, and the new chunks are inserted with their float32 embeddings, followed by one row per chunk in the active index's vector table, and the document records that index in `embedding_index_id`. The keyword index follows by trigger; the vector cannot, because only ingest holds it. Editing a note therefore stops its old text being findable. @@ -138,7 +138,7 @@ A born-digital PDF already holds its text, and pdfium reads it in milliseconds a - **Every page passes:** the file is its text layer. - **Some pages fail, at most half of them:** Docling reads only the runs of consecutive failed pages, each run as one conversion with `page_range` and the usual options (a run longer than 8 pages in batches, [below](#long-docling-readings)), and the text layer the rest. A run's Markdown takes its pages' place, a blank line apart, as the text layer's pages are ([`spliced_text.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/spliced_text.py)). Such a PDF keeps no figures at ingest, as a text-read one. The log names the pages and the first three reasons, as `Docling reads pages 5, 9 of mixed.pdf: page 5 is ocr:no-text; page 9 is turned 90 degrees`. -- **Docling reads the whole file**, as before and keeping its figures as it reads, when more than half the pages fail (a scan), when a form field holds an answer, when pdfium cannot read the file, and when no page that passes is `ok:text`, since covers and stamps alone leave the content in pixels ([`docling_pages.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/docling_pages.py)). It logs `Docling reads : `. +- **Docling reads the whole file**, as before and keeping its figures as it reads, when more than half the pages fail (a scan) and OCR-first is off or reads no page as text ([Scans and photos](#scans-and-photos)), when a form field holds an answer, when pdfium cannot read the file, and when no page that passes is `ok:text`, since covers and stamps alone leave the content in pixels ([`docling_pages.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/docling_pages.py)). It logs `Docling reads : `. - **Forms.** Before any page is read, a form field holding an answer sends the file to Docling. That is a text, combo or list field with a value that is not blank, or a ticked box. The answer lives in the field, which the text layer never shows, while Docling reads it off the page ([`filled_fields.py`](../../surfsense_local/backend/worker/ingestion/pdf_text_layer/filled_fields.py)). A form nobody filled in is read from its text layer; 11 of the 19 Boston budget PDFs carry one. - **Turned pages.** A page with a `/Rotate` fails, because pdfium returns a turned page's lines last to first. @@ -178,6 +178,24 @@ A born-digital PDF already holds its text, and pdfium reads it in milliseconds a The two volumes' text was exactly the text layer's pages 2 to 12, paragraph breaks included, after Docling's Markdown of page 1 read alone, and the report's its text-layer pages with Docling's Markdown of pages 5 and 9 in their places. In the report, the turned table page is the expensive part. +### Scans and photos + +A scanned page and an uploaded image are read by OCR alone before Docling is asked ([ADR 0060](../adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md)). On 499 real pages, scanned forms, receipts and DocVQA pages, phone photos, book scans and public scanned PDFs, Docling took 4.9 s and 39 CPU-s a page; RapidOCR, the OCR it runs inside, read a whole page alone in about 2 s and 8 CPU-s, and found more of it, since Docling's layout step keeps only the text of the regions it classes. [`scan_ocr/`](../../surfsense_local/backend/worker/ingestion/scan_ocr/) holds the readers and the page verdict, and [`pdf_reading/ocr_first.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py) the PDF planner. + +- **Which pages.** In a PDF, after the text layer's triage ([above](#pdfs-with-a-text-layer)): every page with no text layer (`ocr:no-text`), a scanner's invisible OCR layer (`ocr-layer:scanner`) or text pdfium cannot map (`ocr:garbled`); a turned page that is a scan, which pdfium renders upright; and, when no page is running text, the cover and picture pages (`mixed:big-image`, `ok:sparse`), whose render includes their title. Pages of running text keep their text layer. Every uploaded image, frame by frame. +- **The reader.** `SURFSENSE_LOCAL_SCAN_OCR` chooses: `auto`, the default, is Apple's Vision on a Mac and RapidOCR alone elsewhere; `rapidocr` is RapidOCR on a Mac too; `off` sends every scan and photo to Docling, as before, and so does a value it cannot read. + - **RapidOCR** ([`rapid.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py)) loads the parser pack's PP-OCRv6 small detection and recognition files, or the identical ones the rapidocr wheel carries, with the line classifier off as Docling has it. One engine per process reads up to 4 pages at once, at most half the physical cores, at one onnxruntime thread each; a sideways page's turned reads share that cap. A page costs about 8 CPU-s however it is split, and one thread a page spends the least: on 100 real pages, one page at a time on 8 threads took 192 s and 1,016 CPU-s, 4 at once on one thread each 209 s and 824 CPU-s, and 8 at once 184 s and 1,436 CPU-s. A larger recognition batch was slower. The engine reads pages up to 2000 px long, which RapidOCR shrinks anything longer to anyway. + - **Vision** ([`vision.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/vision.py)), at its accurate level with language correction and automatic language detection (macOS 13), through PyObjC, not ocrmac, which would switch Docling's own OCR on a Mac with no parser pack. Its fast level found 0.27 of all words and is not used. If Vision cannot start, a Mac uses RapidOCR. + - **Reading order** ([`reading_order.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py)). Both readers give boxes of text; a visual row of boxes reads left to right as one line, so a receipt's item and its price, and a table's label and its figures, stay together, and two columns of prose read one after the other. +- **Render.** A PDF page greyscale at 216 dpi, Docling's OCR resolution, at most 2000 px on its long side for RapidOCR, which shrinks anything longer to that anyway (about 170 dpi for A4), holding `shared.pdfium.lock` only while it renders, so a page renders while earlier ones are read. The renders are closed on the ingest thread whatever stops the reading. An image upright by its EXIF and scaled as an A4 page at 216 dpi, never below its own pixels, as Docling's image page is sized: FUNSD's 754x1000 forms gave Windows OCR 0.49 of their words at their own size and 0.61 scaled, and Vision the same either way. +- **The verdict** ([`page.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/page.py)). A page's OCR text stands unless it is garbled: under 30% of its characters letters and digits, RapidOCR's own confidence under 0.8 (character-weighted; 0.85 or more on every upright real page, 0.56 to 0.79 on sideways photos), or, for Vision, under 30% of its characters in words of three letters or more. A page under 20 characters is blank paper, a photo or a stamp: it keeps what it read, or a cover's own title when the layer had one. Tables stay with OCR: Haiku 5.5, given each page's text whole, answered 0.79 of 374 DocVQA questions from RapidOCR's text and 0.60 from Docling's Markdown. +- **Sideways pages** ([`upright.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/upright.py)). A page whose text stands mostly in boxes on end, which RapidOCR reads confidently but across one row, one the engine was unsure of, or, with no confidence, one or two characters a token, is read again turned 90, then 270, then 180 degrees, one turn at a time, and the first turn that reads as upright text replaces it; with none, the page keeps its verdict. All 20 camera captures of printed pages in the corpus came sideways with no EXIF turn. +- **Docling.** A garbled page goes to Docling in its place, through the splice a mixed PDF's flagged pages take, in batches that give way ([below](#long-docling-readings)). A PDF with no page that reads as text, and an image with a garbled frame, go to Docling whole, as before. What OCR read of a PDF whose garbled pages Docling then reads is kept in `ocr/` in the document's folder until the file is ready, so a Docling reading that gives the queue up resumes without reading every page with OCR again. The log says `RapidOCR reads 17 of 20 pages of scan.pdf in 31.2 s; Docling reads 5, 9-10: page 5 is garbled; ...`. +- **Figures.** Ingest keeps none for a PDF read this way. A scan's figures wait until the agent or Studio asks for them, as a PDF over 40 pages does: the figures-only pass reads every scanned page with Docling in full, about 39 CPU-s a page, the cost this path removes. A PDF with born-digital pages beside its scanned ones still gets the pass queued at idle, as a text-read PDF does. An image is still its own figure. +- **Cancel.** A PDF's reading checks every 8 pages; an image's frames are read without a check. A long reading does not give the queue up as a long Docling reading does. +- **The release build** runs `worker --check-scan-ocr` in the frozen worker on every system: each reader reads a rendered line, and an import the spec dropped fails the build. +- **Measured** on the Ryzen 7 5800X, Windows 11, through the upload route and a real `worker.py ingest`: PENDING. + ## Figures Ingest keeps the pictures in each source, so the agent can place a source's chart or logo in a document it makes ([agent](agent.md#documents-the-agent-makes)); a Word file read without Docling gets them from the figures-only pass, which its ingest queues as it ends. Chat and search do not use them. @@ -307,3 +325,4 @@ The upload is accepted. Finding no speech model, the job fails without retry wit - Recordings take WAV, MP3, FLAC and Ogg Vorbis only; Opus, M4A, AAC and audio-only WebM wait for a decode-only LGPL FFmpeg ([proposal](../proposals/local-speech-to-text.md#order-of-work), step 9). - A recording cannot be played, nor a citation of one opened at its time; `GET .../original` does not commit before streaming, which a player would need ([proposal](../proposals/local-speech-to-text.md#order-of-work), step 10). - OCR reads one script family per converter: PP-OCRv6 covers Latin, Chinese and Japanese, so scanned Korean, Cyrillic, Devanagari or Arabic text is misread. +- A scan read by OCR alone has no Markdown headings or pipe tables; its tables are rows of cells on one line each. A long OCR-first reading does not give the queue up to other uploads, as a long Docling reading does: a 100-page scan holds it for about 3 minutes. A camera capture shot at an angle is read upright, but RapidOCR's rows can mix lines its perspective skews; Vision read the corpus's well. Vision's speed was measured only on a virtual Mac with no Neural Engine. diff --git a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py index 1c79db8b5d..8a37ad634e 100644 --- a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py +++ b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py @@ -12,8 +12,16 @@ from modules.documents.models import DocumentStatus from modules.documents.tasks import PRIORITY_IDLE from shared.queue import ingest_queue +from tests import pdf_samples +from tests.integration.worker.docling_stand_in import reading from tests.integration.worker.paged_docling_stand_in import PagedDocling -from tests.integration.worker.test_mixed_pdfs import folder_of, scans, session, upload +from tests.integration.worker.test_mixed_pdfs import ( + REPORT, + folder_of, + scans, + session, + upload, +) from worker.ingestion import run from worker.ingestion.pdf_reading import ocr_first from worker.ingestion.scan_ocr import images @@ -100,14 +108,34 @@ def test_a_garbled_page_of_a_scan_is_read_by_docling_in_its_place( assert "Scanned page 3 line 7" in parts[-1] -def test_a_short_scan_still_gets_its_figures_after_it_is_ready( +def test_a_scans_figures_wait_to_be_asked_for( session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr ) -> None: - """From the figures-only pass, queued at idle, as for a text-read PDF.""" + """The pass would read every scanned page with Docling in full.""" source = upload(session, scans(2)) run(source.id) + assert ingest_queue.pending() == [] + assert not (folder_of(source) / "figures" / "figures.json").exists() + + +def test_a_mixed_pdf_read_with_ocr_still_gets_its_idle_figures_pass( + session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr +) -> None: + """Its born-digital pages are read by layout alone, as before.""" + pdf = pdf_samples.joined( + pdf_samples.text_pages(REPORT[:3]), + scans(1), + pdf_samples.text_pages(REPORT[3:6]), + ) + source = upload(session, pdf) + + run(source.id) + + session.expire_all() + assert paged_docling.asked == [] + assert "Scanned page 2 line 0" in (source.content or "") assert [(t.name, t.args, t.priority) for t in ingest_queue.pending()] == [ ("extract_figures", (source.id,), PRIORITY_IDLE) ] @@ -147,3 +175,19 @@ def test_a_photo_is_read_by_ocr_and_is_its_own_figure( assert paged_docling.asked == [] assert "Scanned page 1 line 0" in (source.content or "") assert (folder_of(source) / "figures" / "figures.json").is_file() + + +def test_a_photo_ocr_cannot_read_is_read_by_docling_whole( + session: Session, stub_model: None, docling: list, ocr: StandInOcr +) -> None: + """A garbled frame: Docling reads the image, as before.""" + ocr.garbled = {1} + docling.append(reading()) + source = upload(session, photo(), name="receipt.jpg") + + run(source.id) + + session.expire_all() + assert source.status is DocumentStatus.READY + assert source.content == "Quarterly results." + assert docling == [] diff --git a/surfsense_local/backend/tests/integration/worker/test_vision_engine.py b/surfsense_local/backend/tests/integration/worker/test_vision_engine.py new file mode 100644 index 0000000000..04f3cf0e69 --- /dev/null +++ b/surfsense_local/backend/tests/integration/worker/test_vision_engine.py @@ -0,0 +1,42 @@ +"""Apple's Vision, on a Mac only: it reads a rendered page through PyObjC, and is +the reader a Mac picks first.""" + +import sys + +import pytest + +from tests.integration.worker.test_rapidocr_engine import a_page +from worker.ingestion.scan_ocr import reader +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import PageVerdict + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif(sys.platform != "darwin", reason="Vision is macOS's"), +] + + +def test_vision_reads_a_page_and_keeps_a_rows_cells_together() -> None: + """Its line boxes go through the same reading order as RapidOCR's.""" + from worker.ingestion.scan_ocr.vision import VisionOcr + + vision = VisionOcr.create() + assert vision is not None + + [(_, page)] = list(vision.read([Frame(1, a_page(), 0.05)])) + text = page.text() + + assert page.verdict() is PageVerdict.TEXT + assert "harbour board met on Tuesday" in text + assert any("Wages" in line and "8,400" in line for line in text.splitlines()) + + +def test_a_mac_reads_scans_with_vision_first(monkeypatch: pytest.MonkeyPatch) -> None: + """auto picks Vision on a Mac; rapidocr still picks RapidOCR.""" + monkeypatch.setenv("SURFSENSE_LOCAL_SCAN_OCR", "auto") + chosen = reader.page_reader() + assert chosen is not None and chosen.name == "Vision" + + monkeypatch.setenv("SURFSENSE_LOCAL_SCAN_OCR", "rapidocr") + chosen = reader.page_reader() + assert chosen is not None and chosen.name == "RapidOCR" diff --git a/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py b/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py index 330ee6c074..4c935f6263 100644 --- a/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py +++ b/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py @@ -330,3 +330,100 @@ def test_a_form_or_an_unreadable_pdf_is_left_to_docling( assert ( ocr_first.ocr_reading(Path("form.pdf"), TextLayer(None, "a form field")) is None ) + + +def test_a_turned_reading_must_read_as_text_to_replace_the_page() -> None: + """A turn that reads as blank paper would drop the page's text unread.""" + readings = iter( + [sideways(), page(line("ab", 300), ink=0.001), sideways(), sideways()] + ) + reader = StubReader({}) + reader.read = lambda frames: ((f, next(readings)) for f in frames) # type: ignore[method-assign] + frame = Frame(1, Image.new("L", (40, 20), 0), 0.005) + + [(kept, read)] = list(read_upright(reader, [frame])) + + assert read.verdict() is PageVerdict.GARBLED + assert kept.image.size == (40, 20) + + +def test_lines_standing_on_end_mean_a_page_read_sideways() -> None: + """RapidOCR reads a quarter-turned page confidently, every line in one row.""" + assert page(*prose(), confidence=0.97).looks_sideways() is False + tall = OcrPage(WIDTH, HEIGHT, tuple(prose()), 0.08, "en-US", 0.97, vertical=0.9) + assert tall.looks_sideways() + + +def test_a_cover_keeps_its_own_title_over_ocrs_scraps(ocr: dict[int, OcrPage]) -> None: + """With no page of running text, picture pages are read too; a reading of a + few characters must not replace the title the layer had.""" + cover = LayerPage("Annual Report of the Harbour Board", None, PageKind.BIG_IMAGE, 1) + ocr.update({1: page(line("Ann Rep", 300), ink=0.3), 2: page(*prose())}) + + reading = ocr_first.ocr_reading(Path("cover.pdf"), layer_of(cover, scanned(2))) + + assert reading is not None + assert reading.layer.pages[0].text == "Annual Report of the Harbour Board" + + +def test_blank_pages_alone_are_not_a_reading(ocr: dict[int, OcrPage]) -> None: + """Every other page garbled: Docling reads the file whole, as before.""" + ocr.update({1: page(ink=0.001), 2: page(*prose(), confidence=0.5)}) + + assert ( + ocr_first.ocr_reading(Path("scan.pdf"), layer_of(scanned(1), scanned(2))) + is None + ) + + +def test_the_pdf_closes_when_ocr_fails_midway( + ocr: dict[int, OcrPage], monkeypatch: pytest.MonkeyPatch +) -> None: + """Closed on the ingest thread, not by the garbage collector under pdfium's lock.""" + closed: list[bool] = [] + + def frames(_original: Path, numbers: list[int], _max_side: int) -> Iterator[Frame]: + try: + for number in numbers: + yield Frame(number, Image.new("L", (10, 10), 255), 0.0) + finally: + closed.append(True) + + class Failing(StubReader): + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + for frame in frames: + if frame.number == 2: + raise RuntimeError("onnxruntime could not run the detector") + yield frame, page(*prose()) + + monkeypatch.setattr(ocr_first, "pdf_frames", frames) + monkeypatch.setattr(ocr_first, "page_reader", lambda: Failing({})) + + assert ( + ocr_first.ocr_reading(Path("scan.pdf"), layer_of(*map(scanned, (1, 2, 3)))) + is None + ) + assert closed == [True] + + +def test_a_resumed_reading_keeps_ocrs_pages( + ocr: dict[int, OcrPage], tmp_path: Path +) -> None: + """When Docling's part gives the queue up, the run that resumes does not read + every page with OCR again; the same bytes and reader only.""" + original = tmp_path / "scan.pdf" + original.write_bytes(b"%PDF scan") + ocr.update({1: page(*prose()), 2: page(*prose(), confidence=0.5)}) + layer = layer_of(scanned(1), scanned(2)) + first = ocr_first.ocr_reading(original, layer) + assert first is not None and not first.resumed + + ocr.clear() # any read now would fail + second = ocr_first.ocr_reading(original, layer) + + assert second is not None and second.resumed + assert second.docling == first.docling == {2: "page 2 is garbled"} + assert second.layer.pages[0].text == first.layer.pages[0].text + # Other bytes: nothing kept applies, and the stand-in, asked again, fails. + original.write_bytes(b"%PDF another scan") + assert ocr_first.ocr_reading(original, layer) is None diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py index 912e544f95..d0d7c9b1d4 100644 --- a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py @@ -8,6 +8,7 @@ import logging import time +from contextlib import closing from dataclasses import dataclass, replace from pathlib import Path @@ -16,6 +17,7 @@ from modules.documents.models import Document from worker.ingestion.docling_batches.queue_turn import reading_turn from worker.ingestion.figures.scanned_pages import scanned_pages +from worker.ingestion.pdf_reading.ocr_kept import keep_reading, kept_reading from worker.ingestion.pdf_text_layer import LayerPage, TextLayer from worker.ingestion.pdf_text_layer.page_triage import PageKind from worker.ingestion.scan_ocr.frames import pdf_frames @@ -46,6 +48,8 @@ class OcrReading: read: int reader: str seconds: float + # Kept from a run that gave the queue up to Docling's reading. + resumed: bool = False def ocr_reading( @@ -98,37 +102,61 @@ def _read( document: Document | None, ) -> OcrReading | None: started = time.perf_counter() - # The ingest job's own session; a reading outside one has nothing to check. - session = object_session(document) if document is not None else None - turn = reading_turn(session, document) if session and document else None + kept = kept_reading(original, reader.name) + if kept is not None: + texts = {int(n): text for n, text in kept["texts"].items()} + docling = {int(n): why for n, why in kept["docling"].items()} + else: + texts, docling, stood = _ocr(reader, original, layer, candidates, document) + if not stood: + # No page read as text: Docling reads the file whole, as before. + return None + if docling: + keep_reading(original, reader.name, {"texts": texts, "docling": docling}) pages: list[LayerPage] = list(layer.pages) - docling: dict[int, str] = {} - read = 0 - frames = pdf_frames(original, candidates, reader.max_side) - for count, (frame, page) in enumerate(read_upright(reader, frames), start=1): - if turn is not None and count % CANCEL_EVERY == 0: - turn.check_cancelled() - verdict = page.verdict() - if verdict in KEPT: - # A blank render keeps what little the layer had, a cover's title. - own = pages[frame.number - 1].text or "" - text = own if verdict is PageVerdict.BLANK else page.text() - pages[frame.number - 1] = replace( - pages[frame.number - 1], text=text, refused_because=None - ) - read += 1 - else: - docling[frame.number] = f"page {frame.number} is {verdict}" - if not read: - # Docling would read every page of it anyway; let it read the file whole. - return None + for n, text in texts.items(): + pages[n - 1] = replace(pages[n - 1], text=text, refused_because=None) for n, page in enumerate(pages, start=1): if page.text is None and n not in docling: docling[n] = page.refused_because or f"page {n} is for Docling" return OcrReading( replace(layer, pages=tuple(pages)), docling, - read, + len(texts), reader.name, time.perf_counter() - started, + resumed=kept is not None, ) + + +def _ocr( + reader: PageReader, + original: Path, + layer: TextLayer, + candidates: list[int], + document: Document | None, +) -> tuple[dict[int, str], dict[int, str], int]: + """Each candidate page's text, why the others go to Docling, and how many read + as text.""" + # The ingest job's own session; a reading outside one has nothing to check. + session = object_session(document) if document is not None else None + turn = reading_turn(session, document) if session and document else None + texts: dict[int, str] = {} + docling: dict[int, str] = {} + stood = 0 + # Closed here, whatever stops the reading: its PDF closes under pdfium's lock, + # which the garbage collector could otherwise take while this thread holds it. + with closing(pdf_frames(original, candidates, reader.max_side)) as frames: + for count, (frame, page) in enumerate(read_upright(reader, frames), start=1): + if turn is not None and count % CANCEL_EVERY == 0: + turn.check_cancelled() + n, verdict = frame.number, page.verdict() + if verdict not in KEPT: + docling[n] = f"page {n} is {verdict}" + elif verdict is PageVerdict.TEXT: + texts[n] = page.text() + stood += 1 + else: + # Little or nothing read: a cover's own title, if it has one, wins. + texts[n] = layer.pages[n - 1].text or page.text() + return texts, docling, stood diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py new file mode 100644 index 0000000000..c3506534c4 --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py @@ -0,0 +1,61 @@ +"""What OCR read of a scan whose garbled pages Docling still reads: a Docling +reading that gives the queue up resumes from here instead of reading every page +with OCR again. + +In ocr/ in the document's own folder: a folder, so original_file.py never takes it +for the original, and outside docling/, which a new Docling reading empties. +""" + +import hashlib +import json +import logging +import shutil +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +FOLDER = "ocr" +_FILE = "pages.json" +_FORMAT = 1 + + +def kept_reading(original: Path, reader: str) -> dict[str, Any] | None: + """The pages kept for these bytes by this reader, or None.""" + try: + kept = json.loads((original.parent / FOLDER / _FILE).read_text("utf-8")) + except (OSError, ValueError): + return None + try: + stamp = {"format": _FORMAT, "reader": reader, "original": _digest(original)} + except OSError: + return None + if not isinstance(kept, dict) or kept.get("stamp") != stamp: + return None + return kept + + +def keep_reading(original: Path, reader: str, reading: dict[str, Any]) -> None: + """Never raises: without it a resume only reads the pages again.""" + folder = original.parent / FOLDER + try: + stamp = {"format": _FORMAT, "reader": reader, "original": _digest(original)} + # Not parents: a source deleted meanwhile must not get its folder back. + folder.mkdir(exist_ok=True) + partial = folder / f"{_FILE}.partial" + partial.write_text(json.dumps({"stamp": stamp, **reading}), "utf-8") + partial.replace(folder / _FILE) + except OSError as error: + logger.warning("could not keep the OCR of %s: %s", original.name, error) + + +def forget_reading(original: Path) -> None: + shutil.rmtree(original.parent / FOLDER, ignore_errors=True) + + +def _digest(original: Path) -> str: + digest = hashlib.sha256() + with original.open("rb") as file: + while block := file.read(1 << 20): + digest.update(block) + return digest.hexdigest() diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/route.py b/surfsense_local/backend/worker/ingestion/pdf_reading/route.py index 233999e6da..9a406b105b 100644 --- a/surfsense_local/backend/worker/ingestion/pdf_reading/route.py +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/route.py @@ -16,9 +16,11 @@ from worker.ingestion.figures.idle_pass import queue_idle_figures_pass from worker.ingestion.pdf_reading.docling_pages import pages_for_docling from worker.ingestion.pdf_reading.ocr_first import OcrReading, ocr_reading +from worker.ingestion.pdf_reading.ocr_kept import forget_reading from worker.ingestion.pdf_reading.spliced_text import spliced from worker.ingestion.pdf_reading.whole_reading import whole_markdown from worker.ingestion.pdf_text_layer import TextLayer, read_text_layer +from worker.ingestion.pdf_text_layer.page_triage import PageKind from worker.ingestion.pdf_text_layer.reading import PAGE_BREAK logger = logging.getLogger(__name__) @@ -57,8 +59,9 @@ def _ocr_first_markdown(document: Document, original: Path, reading: OcrReading) """OCR's pages with Docling's markdown in place of the pages it reads.""" runs = runs_of(sorted(reading.docling)) logger.info( - "%s reads %d of %d pages of %s in %.1f s%s", + "%s %s %d of %d pages of %s in %.1f s%s", reading.reader, + "kept" if reading.resumed else "reads", reading.read, len(reading.layer.pages), original.name, @@ -71,8 +74,12 @@ def _ocr_first_markdown(document: Document, original: Path, reading: OcrReading) markdown = spliced(reading.layer, _docling_markdown(document, original, runs)) else: markdown = PAGE_BREAK.join(page.text or "" for page in reading.layer.pages) - # A scan's figures come from the figures-only pass too, after it is ready. - queue_idle_figures_pass(document, original, reading.layer) + forget_reading(original) + # A scan's figures wait to be asked for: the pass would read every scanned + # page with Docling in full, 39 CPU-s a page. Born-digital pages beside them + # are read by layout alone, so a mixed PDF still gets its idle pass. + if any(page.kind is PageKind.TEXT for page in reading.layer.pages): + queue_idle_figures_pass(document, original, reading.layer) return markdown diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py b/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py index 334f82dc14..bd772b35e8 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/frames.py @@ -54,8 +54,8 @@ def pdf_frames(original: Path, pages: Sequence[int], max_side: int) -> Iterator[ def image_frames(path: Path, max_side: int) -> Iterator[Frame]: """Each frame upright by its EXIF and sized as an A4 page at 216 dpi, never - below its own pixels: real 754x1000 forms read 0.52 of their words at their - own size and 0.67 at A4 size.""" + below its own pixels, as Docling's image page is: real 754x1000 forms gave + Windows OCR 0.49 of their words at their own size and 0.61 at A4 size.""" with Image.open(path) as image: for index in range(getattr(image, "n_frames", 1)): image.seek(index) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/images.py b/surfsense_local/backend/worker/ingestion/scan_ocr/images.py index b804408d6e..7ad3f3a48c 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/images.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/images.py @@ -6,6 +6,7 @@ import logging import time +from contextlib import closing from pathlib import Path from worker.ingestion.pdf_text_layer.reading import PAGE_BREAK @@ -26,15 +27,19 @@ def image_markdown(path: Path) -> str | None: started = time.perf_counter() texts: list[str] = [] try: - for frame, page in read_upright(reader, image_frames(path, reader.max_side)): - verdict = page.verdict() - if verdict not in KEPT: - logger.info( - "Docling reads %s: frame %d is %s", path.name, frame.number, verdict - ) - return None - if verdict is not PageVerdict.BLANK: - texts.append(page.text()) + with closing(image_frames(path, reader.max_side)) as frames: + for frame, page in read_upright(reader, frames): + verdict = page.verdict() + if verdict not in KEPT: + logger.info( + "Docling reads %s: frame %d is %s", + path.name, + frame.number, + verdict, + ) + return None + if verdict is not PageVerdict.BLANK: + texts.append(page.text()) except Exception: logger.warning( "%s could not read %s; Docling reads it", diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/page.py b/surfsense_local/backend/worker/ingestion/scan_ocr/page.py index 8c7b28b7bb..c9ea1b2651 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/page.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/page.py @@ -34,6 +34,9 @@ # Characters a token, on average, with no confidence: 2.8 to 5.5 on real pages, # 1.0 to 1.8 on a page read sideways, its letters' strokes one by one. SIDEWAYS_TOKEN = 2.2 +# Most of its text in boxes standing on end: a page turned a quarter round, which +# an engine may read confidently, every line across one row. +SIDEWAYS_VERTICAL = 0.5 _WORD = re.compile(r"[^\W\d_]{3,}") @@ -85,6 +88,8 @@ class OcrPage: language: str | None = None # The engine's own, when it gives one: RapidOCR's; Vision's is not used. confidence: float | None = None + # Share of its characters in boxes standing on end. + vertical: float = 0.0 @property def chars(self) -> int: @@ -129,12 +134,15 @@ def word_share(self) -> float: return sum(len(m.group()) for m in _WORD.finditer(text)) / max(1, visible) def looks_sideways(self) -> bool: - """Read turned a quarter or half round: the engine unsure of its text, - or, with no confidence given, one or two characters a token.""" + """Read turned a quarter or half round: its lines standing on end, the + engine unsure of its text, or, with no confidence given, one or two + characters a token.""" tokens = " ".join(line.text for line in self.lines).split() if self.chars < MIN_CHARS or not tokens: # Little text says nothing of which way up: a photo, a signature. return False + if self.vertical > SIDEWAYS_VERTICAL: + return True if self.confidence is not None: return self.confidence < LOW_CONFIDENCE return sum(map(len, tokens)) / len(tokens) < SIDEWAYS_TOKEN diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/pages_at_once.py b/surfsense_local/backend/worker/ingestion/scan_ocr/pages_at_once.py new file mode 100644 index 0000000000..d4e9d07e6e --- /dev/null +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/pages_at_once.py @@ -0,0 +1,53 @@ +"""Several pages read at once, in order, within one cap per reader. + +At most 4, and half the physical cores, as chat may need the rest. The cap is a +semaphore every reading of the reader shares, a sideways page's turned reads +included, so no mix of pages and turns goes past it. +""" + +import threading +from collections import deque +from collections.abc import Callable, Iterable, Iterator +from concurrent.futures import Future, ThreadPoolExecutor + +import psutil + +from worker.ingestion.scan_ocr.frames import Frame +from worker.ingestion.scan_ocr.page import OcrPage + +MAX_AT_ONCE = 4 + + +def at_once() -> int: + return max(1, min(MAX_AT_ONCE, (psutil.cpu_count(logical=False) or 2) // 2)) + + +class PagesAtOnce: + def __init__(self, recognise: Callable[[Frame], OcrPage], name: str) -> None: + self._recognise = recognise + self._name = name + self._slots = threading.BoundedSemaphore(at_once()) + + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + """Each frame's page in order; the next frame renders while earlier ones + are read.""" + width = at_once() + with ThreadPoolExecutor(width, thread_name_prefix=self._name) as pool: + pending: deque[tuple[Frame, Future[OcrPage]]] = deque() + try: + for frame in frames: + if len(pending) >= width: + done, future = pending.popleft() + yield done, future.result() + pending.append((frame, pool.submit(self._within_cap, frame))) + while pending: + done, future = pending.popleft() + yield done, future.result() + finally: + # Pages not yet started are not read once the caller stops. + for _, future in pending: + future.cancel() + + def _within_cap(self, frame: Frame) -> OcrPage: + with self._slots: + return self._recognise(frame) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py b/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py index 0fa54f089e..2354a4bbda 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/rapid.py @@ -8,18 +8,15 @@ """ import logging -from collections import deque from collections.abc import Iterable, Iterator -from concurrent.futures import Future, ThreadPoolExecutor from pathlib import Path from typing import Any -import psutil - from worker.ingestion.parser_pack import parser_dir from worker.ingestion.scan_ocr.frames import Frame from worker.ingestion.scan_ocr.page import OcrPage -from worker.ingestion.scan_ocr.reading_order import Box, lines_in_order +from worker.ingestion.scan_ocr.pages_at_once import PagesAtOnce +from worker.ingestion.scan_ocr.reading_order import Box, lines_in_order, vertical_share log = logging.getLogger(__name__) @@ -29,10 +26,10 @@ # RapidOCR shrinks anything longer to this before reading; rendering larger only # costs time. MAX_SIDE = 2000 -# Pages read at once, one onnxruntime thread each: its CPU-s a page grow with its -# threads (14.9 at 1, 36.6 at 8), so several pages at one thread read more pages -# a CPU-second. At most 4, and half the physical cores, as chat may need the rest. -MAX_IN_FLIGHT = 4 +# One onnxruntime thread a page, several pages at once: a page costs about 8 CPU-s +# however it is split, and one thread a page spends the least (824 CPU-s for 100 +# pages against 1,016 at 8 threads a page, at about the same wall time). +THREADS_A_PAGE = 1 class RapidOcr: @@ -42,6 +39,9 @@ class RapidOcr: def __init__(self, engine: Any) -> None: self._engine = engine + # One engine, several pages at once: its only per-call state is harmless + # (rapidocr 3.9.2). + self._pages = PagesAtOnce(self._recognise, "rapidocr") @classmethod def create(cls) -> "RapidOcr | None": @@ -52,14 +52,13 @@ def create(cls) -> "RapidOcr | None": try: from rapidocr import RapidOCR - _no_spinning() engine = RapidOCR( params={ # What Docling passes for the app: RapidOcrOptions(use_cls=False). "Global.text_score": 0.5, "Global.use_cls": False, "Global.log_level": "error", - "EngineConfig.onnxruntime.intra_op_num_threads": 1, + "EngineConfig.onnxruntime.intra_op_num_threads": THREADS_A_PAGE, "Det.model_path": str(models / DET), "Rec.model_path": str(models / REC), # Built even when off; pointed at the pack so nothing downloads. @@ -72,21 +71,7 @@ def create(cls) -> "RapidOcr | None": return cls(engine) def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: - """Each frame's page, in order, several read at once on one engine: its - only per-call state is harmless (rapidocr 3.9.2).""" - in_flight = max( - 1, min(MAX_IN_FLIGHT, (psutil.cpu_count(logical=False) or 2) // 2) - ) - with ThreadPoolExecutor(in_flight, thread_name_prefix="rapidocr") as pool: - pending: deque[tuple[Frame, Future[OcrPage]]] = deque() - for frame in frames: - if len(pending) >= in_flight: - done, future = pending.popleft() - yield done, future.result() - pending.append((frame, pool.submit(self._recognise, frame))) - while pending: - done, future = pending.popleft() - yield done, future.result() + return self._pages.read(frames) def _recognise(self, frame: Frame) -> OcrPage: import numpy @@ -115,6 +100,7 @@ def _recognise(self, frame: Frame) -> OcrPage: frame.ink, self.language, confidence=weighed / chars if chars else None, + vertical=vertical_share(boxes), ) @@ -132,25 +118,3 @@ def _models() -> Path | None: if all((folder / name).is_file() for name in (DET, REC, CLS)): return folder return None - - -def _no_spinning() -> None: - """onnxruntime's idle threads spin, burning CPU-seconds a page never used: - 13 CPU-s a page at 8 threads in the corpus run. RapidOCR has no switch for it.""" - try: - from rapidocr.inference_engine.onnxruntime import main as ort_main - - build = ort_main.OrtInferSession._init_sess_opts - if getattr(build, "_no_spin", False): - return - - def quiet(config: Any) -> Any: - options = build(config) - options.add_session_config_entry("session.intra_op.allow_spinning", "0") - options.add_session_config_entry("session.inter_op.allow_spinning", "0") - return options - - quiet._no_spin = True # type: ignore[attr-defined] - ort_main.OrtInferSession._init_sess_opts = staticmethod(quiet) - except Exception: - log.debug("onnxruntime's threads keep spinning", exc_info=True) diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py b/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py index b53ceaa53a..3dc220de9f 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/reading_order.py @@ -44,6 +44,22 @@ def h(self) -> float: return statistics.median(box.h for box in self.boxes) +# A box this much taller than wide holds a line of text turned on its side. +_TALL = 1.5 + + +def vertical_share(boxes: list[Box]) -> float: + """The share of characters in boxes standing on end: a page turned a quarter + round, which RapidOCR still reads with confidence, its lines across one row.""" + chars = sum(len(box.text) for box in boxes) + tall = sum( + len(box.text) + for box in boxes + if len(box.text) > 1 and box.y1 - box.y0 > _TALL * (box.x1 - box.x0) + ) + return tall / chars if chars else 0.0 + + def lines_in_order(boxes: list[Box]) -> tuple[Line, ...]: if not boxes: return () diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py b/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py index f531e4b23f..84f0e0b552 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/upright.py @@ -1,8 +1,9 @@ -"""A page the system's OCR read sideways, read again turned upright. +"""A page OCR read sideways, read again turned upright. -The engines read upright text only. A phone photo with no EXIF turn, or a scan -fed in sideways, came back as "O 00 o c": all 20 real camera captures in the -corpus did. Turning costs a reading or three, and only on such pages. +The readers read upright text only. A phone photo with no EXIF turn, or a scan +fed in sideways, comes back as letters' strokes, as text the engine is unsure +of, or as lines standing on end: all 20 real camera captures in the corpus did. +Turning costs a reading or three, and only on such pages. """ from collections.abc import Iterable, Iterator @@ -18,17 +19,17 @@ def read_upright( reader: PageReader, frames: Iterable[Frame] ) -> Iterator[tuple[Frame, OcrPage]]: - """Each frame's page, turned to whichever way reads best when it looks sideways.""" + """Each frame's page; one that looks sideways turned the first way that reads + as upright text, or as it was when none does, for its verdict to decide.""" for frame, page in reader.read(frames): - if not page.looks_sideways(): - yield frame, page - continue - turned = [ - Frame(frame.number, frame.image.rotate(angle, expand=True), frame.ink) - for angle in TURNS - ] - best = max(reader.read(turned), key=lambda read: read[1].word_share()) - if best[1].verdict() in (PageVerdict.GARBLED, PageVerdict.TOO_LITTLE): - yield frame, page - else: - yield best + yield _upright(reader, frame, page) if page.looks_sideways() else (frame, page) + + +def _upright(reader: PageReader, frame: Frame, page: OcrPage) -> tuple[Frame, OcrPage]: + for angle in TURNS: + turned = Frame(frame.number, frame.image.rotate(angle, expand=True), frame.ink) + # One turn at a time: each is read within the reader's own cap. + [(_, read)] = list(reader.read([turned])) + if read.verdict() is PageVerdict.TEXT and not read.looks_sideways(): + return turned, read + return frame, page diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py b/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py index f25d9d5fe6..f1014013b1 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/vision.py @@ -11,21 +11,16 @@ """ import logging -from collections import deque from collections.abc import Iterable, Iterator -from concurrent.futures import Future, ThreadPoolExecutor from typing import Any -import psutil - from worker.ingestion.scan_ocr.frames import Frame from worker.ingestion.scan_ocr.page import OcrPage -from worker.ingestion.scan_ocr.reading_order import Box, lines_in_order +from worker.ingestion.scan_ocr.pages_at_once import PagesAtOnce +from worker.ingestion.scan_ocr.reading_order import Box, lines_in_order, vertical_share log = logging.getLogger(__name__) -# Throughput stops growing past the performance cores; leave room for chat. -MAX_IN_FLIGHT = 4 # Vision takes far larger images than a 216 dpi page; this only bounds memory. MAX_SIDE = 10_000 @@ -35,6 +30,11 @@ class VisionOcr: language = "auto" max_side = MAX_SIDE + def __init__(self) -> None: + # PyObjC lets go of the GIL inside every Objective-C call, so pages in + # flight do overlap. + self._pages = PagesAtOnce(_recognise, "vision-ocr") + @classmethod def create(cls) -> "VisionOcr | None": try: @@ -47,21 +47,7 @@ def create(cls) -> "VisionOcr | None": return cls() def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: - """Each frame's page, in order. PyObjC lets go of the GIL inside every - Objective-C call, so pages in flight do overlap.""" - in_flight = max( - 1, min(MAX_IN_FLIGHT, (psutil.cpu_count(logical=False) or 2) // 2) - ) - with ThreadPoolExecutor(in_flight, thread_name_prefix="vision-ocr") as pool: - pending: deque[tuple[Frame, Future[OcrPage]]] = deque() - for frame in frames: - if len(pending) >= in_flight: - done, future = pending.popleft() - yield done, future.result() - pending.append((frame, pool.submit(_recognise, frame))) - while pending: - done, future = pending.popleft() - yield done, future.result() + return self._pages.read(frames) def _recognise(frame: Frame) -> OcrPage: @@ -109,7 +95,14 @@ def _recognise(frame: Frame) -> OcrPage: ] # No confidence: Vision's was never measured against the corpus, and the # page verdict's threshold is RapidOCR's. - return OcrPage(width, height, lines_in_order(boxes), frame.ink, "auto") + return OcrPage( + width, + height, + lines_in_order(boxes), + frame.ink, + "auto", + vertical=vertical_share(boxes), + ) def _box(observation: Any, width: int, height: int) -> Box | None: From 02f88cc121b6efb55f10fca093493041ee0b2fa3 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 17 Oct 2026 02:57:07 -0700 Subject: [PATCH 3/6] docs(local): ADR 0060 and documents.md with the end-to-end and Linux timings --- docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md | 4 ++-- docs/architecture/documents.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md b/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md index b2586d6146..46fd362675 100644 --- a/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md +++ b/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md @@ -39,13 +39,13 @@ Answers followed: Haiku 5.5, given each page's text whole, answered 0.79 of the ## Consequences - A scan's pages no longer import Docling unless one is garbled, so a worker that sees only scans skips Docling's cold start and its 1 GB, until a figures-only pass runs. -- PENDING: time to ready through the upload route. +- Through the upload route and a real `worker.py ingest`, the six public scanned PDFs and the three book scans, 199 pages in one upload, were all ready in 509 s against 829 s through Docling, a file in 332 s on average against 598 s, for 1,752 worker CPU-s against 7,343. No page went to Docling. A 25-page court filing was ready 47 s after upload, against 614 s as Docling's long readings gave way to each other; its job took 46 s against 130 s. 40 receipt photos, receipt scans and forms in one upload were all ready in 113 s against 202 s, an image in 65 s on average against 129 s, for 131 worker CPU-s against 1,416; none went to Docling. RapidOCR reads about 2 s a page on this machine, so a scan is ready in seconds a page, not in seconds: Windows' own OCR was ten times as fast and read too little. - Docling's headings and pipe tables are gone from a scan's text; ADR 0059 found the same trade for text-layer PDFs to the app's benefit. - An image's text is RapidOCR's or Vision's whole, or Docling's whole: Docling cannot be given one frame of a multi-frame TIFF. ## Where the code stands -- Speed is measured on one Windows machine; RapidOCR on Linux runs the same onnxruntime CPU code, and in CI's tests on Ubuntu, macOS and Windows. Vision's speed is from a virtual Mac with no Neural Engine, so a real Mac should read faster; its accuracy is from that runner. +- Speed is measured on one machine. On it, RapidOCR under Linux (WSL's Ubuntu 24.04) read the same 100 pages in 174 s and 684 CPU-s, against 209 s and 824 under Windows, with the same text; CI runs its tests on Ubuntu, macOS and Windows. Vision's speed is from a virtual Mac with no Neural Engine, so a real Mac should read faster; its accuracy is from that runner. - A long OCR-first reading checks for a cancel every 8 pages but does not give way to other uploads, as a long Docling reading does: a 100-page scan holds the queue for about 3 minutes. When its garbled pages go to a Docling reading that gives way, OCR's pages are kept, so the resumed run does not read them again. - Camera captures shot at an angle are read upright, but RapidOCR's rows can still mix lines a perspective skews; Vision read them well. - PP-OCRv6 covers Latin, Chinese and Japanese, as Docling's did, so scanned Korean, Cyrillic, Devanagari or Arabic text is misread, and Vision's languages are the system's. diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index a8c7fb6c16..b8c78f60b1 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -194,7 +194,7 @@ A scanned page and an uploaded image are read by OCR alone before Docling is ask - **Figures.** Ingest keeps none for a PDF read this way. A scan's figures wait until the agent or Studio asks for them, as a PDF over 40 pages does: the figures-only pass reads every scanned page with Docling in full, about 39 CPU-s a page, the cost this path removes. A PDF with born-digital pages beside its scanned ones still gets the pass queued at idle, as a text-read PDF does. An image is still its own figure. - **Cancel.** A PDF's reading checks every 8 pages; an image's frames are read without a check. A long reading does not give the queue up as a long Docling reading does. - **The release build** runs `worker --check-scan-ocr` in the frozen worker on every system: each reader reads a rendered line, and an import the spec dropped fails the build. -- **Measured** on the Ryzen 7 5800X, Windows 11, through the upload route and a real `worker.py ingest`: PENDING. +- **Measured** on the Ryzen 7 5800X, Windows 11, through the upload route and a real `worker.py ingest`, one upload at a time: the six public scanned PDFs and three book scans, 199 pages, were all ready in 509 s against 829 s through Docling, a file in 332 s on average against 598 s, at 1,752 worker CPU-s against 7,343; no page went to Docling. 40 receipt photos, receipt scans and forms were all ready in 113 s against 202 s, an image in 65 s on average against 129 s, at 131 worker CPU-s against 1,416. ## Figures From 52556ca2ec1a0e3b7dcc91454e726e73190bc39d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 17 Oct 2026 03:47:02 -0700 Subject: [PATCH 4/6] fix(local): the scan OCR check needs the sample's words, and an image with no frame of text goes to Docling --- ...can-is-read-by-ocr-alone-before-docling.md | 2 +- docs/architecture/documents.md | 2 +- .../integration/worker/test_ocr_first.py | 21 +++++++++++++++++++ .../worker/ingestion/scan_ocr/check.py | 7 ++++++- .../worker/ingestion/scan_ocr/images.py | 9 +++++--- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md b/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md index 46fd362675..767af751e3 100644 --- a/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md +++ b/docs/adr/0060-a-scan-is-read-by-ocr-alone-before-docling.md @@ -29,7 +29,7 @@ Answers followed: Haiku 5.5, given each page's text whole, answered 0.79 of the - **What OCR reads first.** A PDF's pages with no text layer, with a scanner's invisible OCR layer, with text pdfium cannot map, a turned page that is a scan, and, when no page is running text, its cover and picture pages; and every uploaded image. They are rendered greyscale at 216 dpi, an image as an A4 page as before, and read by the reader below. The code is [`scan_ocr/`](../../surfsense_local/backend/worker/ingestion/scan_ocr/) and [`pdf_reading/ocr_first.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py). - **The reader.** RapidOCR alone on Windows and Linux: the parser pack's own PP-OCRv6 small files, or the identical ones in the rapidocr wheel, with the line classifier off as Docling has it, 4 pages at a time at one onnxruntime thread each, which spends the least CPU a page. On a Mac, Apple's Vision at its accurate level, through PyObjC; RapidOCR if Vision cannot start. Vision's fast level found 0.27 of all words and is not used. Each reader's boxes are put in reading order the same way: a visual row reads left to right as one line, so a table's label and amount stay together, and two columns of prose read one after the other. -- **Which pages stand.** A page stands unless its OCR is garbled: mostly symbols, RapidOCR's own confidence under 0.8 (0.85 or more on every upright real page, 0.56 to 0.79 on sideways photos), or, for Vision, which gives none used here, under 30% of its characters in words of three letters. A page that looks read sideways, its text unsure or its lines standing on end, is read again turned a quarter, three quarters and half round, one turn at a time, and the first turn that reads as upright text replaces it; all 20 camera captures were sideways. Docling reads a garbled page, in its place, through [ADR 0059](0059-a-born-digital-pdf-is-read-from-its-text-layer.md)'s splice; a scan with no page that stands goes to Docling whole, as before, and an image with a garbled frame too. Tables stay with OCR. +- **Which pages stand.** A page stands unless its OCR is garbled: mostly symbols, RapidOCR's own confidence under 0.8 (0.85 or more on every upright real page, 0.56 to 0.79 on sideways photos), or, for Vision, which gives none used here, under 30% of its characters in words of three letters. A page that looks read sideways, its text unsure or its lines standing on end, is read again turned a quarter, three quarters and half round, one turn at a time, and the first turn that reads as upright text replaces it; all 20 camera captures were sideways. Docling reads a garbled page, in its place, through [ADR 0059](0059-a-born-digital-pdf-is-read-from-its-text-layer.md)'s splice; a scan with no page that reads as text goes to Docling whole, as before, and so does an image with a garbled frame or no frame of text. Tables stay with OCR. - **Figures.** Ingest keeps none for a scan read this way, and a scan's figures wait until the agent or Studio asks, as a PDF over 40 pages does: the figures-only pass reads scanned pages with Docling in full, about 39 CPU-s a page, which would give back the CPU this saves. A PDF with born-digital pages beside its scans still gets the pass queued at idle. An image is still its own figure. - `SURFSENSE_LOCAL_SCAN_OCR=off` reads every scan and photo with Docling, as before; `rapidocr` uses RapidOCR on a Mac too. A value it cannot read also means off. `worker --check-scan-ocr` reads a rendered line with each reader in the frozen worker on every release build. - **Rejected: Windows' own OCR** (Windows.Media.Ocr, through pywinrt). It read a page in 0.13 to 0.24 s, and worked frozen, but found 0.61 of a form's words and 0.43 of a receipt photo's, writes a table column by column, and nothing in its output told its good pages from its bad ones: keeping only its cleanest-looking pages still lost 7 to 10 points of words against RapidOCR. diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index b8c78f60b1..49eaafeb5e 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -190,7 +190,7 @@ A scanned page and an uploaded image are read by OCR alone before Docling is ask - **Render.** A PDF page greyscale at 216 dpi, Docling's OCR resolution, at most 2000 px on its long side for RapidOCR, which shrinks anything longer to that anyway (about 170 dpi for A4), holding `shared.pdfium.lock` only while it renders, so a page renders while earlier ones are read. The renders are closed on the ingest thread whatever stops the reading. An image upright by its EXIF and scaled as an A4 page at 216 dpi, never below its own pixels, as Docling's image page is sized: FUNSD's 754x1000 forms gave Windows OCR 0.49 of their words at their own size and 0.61 scaled, and Vision the same either way. - **The verdict** ([`page.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/page.py)). A page's OCR text stands unless it is garbled: under 30% of its characters letters and digits, RapidOCR's own confidence under 0.8 (character-weighted; 0.85 or more on every upright real page, 0.56 to 0.79 on sideways photos), or, for Vision, under 30% of its characters in words of three letters or more. A page under 20 characters is blank paper, a photo or a stamp: it keeps what it read, or a cover's own title when the layer had one. Tables stay with OCR: Haiku 5.5, given each page's text whole, answered 0.79 of 374 DocVQA questions from RapidOCR's text and 0.60 from Docling's Markdown. - **Sideways pages** ([`upright.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/upright.py)). A page whose text stands mostly in boxes on end, which RapidOCR reads confidently but across one row, one the engine was unsure of, or, with no confidence, one or two characters a token, is read again turned 90, then 270, then 180 degrees, one turn at a time, and the first turn that reads as upright text replaces it; with none, the page keeps its verdict. All 20 camera captures of printed pages in the corpus came sideways with no EXIF turn. -- **Docling.** A garbled page goes to Docling in its place, through the splice a mixed PDF's flagged pages take, in batches that give way ([below](#long-docling-readings)). A PDF with no page that reads as text, and an image with a garbled frame, go to Docling whole, as before. What OCR read of a PDF whose garbled pages Docling then reads is kept in `ocr/` in the document's folder until the file is ready, so a Docling reading that gives the queue up resumes without reading every page with OCR again. The log says `RapidOCR reads 17 of 20 pages of scan.pdf in 31.2 s; Docling reads 5, 9-10: page 5 is garbled; ...`. +- **Docling.** A garbled page goes to Docling in its place, through the splice a mixed PDF's flagged pages take, in batches that give way ([below](#long-docling-readings)). A PDF with no page that reads as text, and an image with a garbled frame or no frame of text, go to Docling whole, as before. What OCR read of a PDF whose garbled pages Docling then reads is kept in `ocr/` in the document's folder until the file is ready, so a Docling reading that gives the queue up resumes without reading every page with OCR again. The log says `RapidOCR reads 17 of 20 pages of scan.pdf in 31.2 s; Docling reads 5, 9-10: page 5 is garbled; ...`. - **Figures.** Ingest keeps none for a PDF read this way. A scan's figures wait until the agent or Studio asks for them, as a PDF over 40 pages does: the figures-only pass reads every scanned page with Docling in full, about 39 CPU-s a page, the cost this path removes. A PDF with born-digital pages beside its scanned ones still gets the pass queued at idle, as a text-read PDF does. An image is still its own figure. - **Cancel.** A PDF's reading checks every 8 pages; an image's frames are read without a check. A long reading does not give the queue up as a long Docling reading does. - **The release build** runs `worker --check-scan-ocr` in the frozen worker on every system: each reader reads a rendered line, and an import the spec dropped fails the build. diff --git a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py index 8a37ad634e..ba2ab4e77d 100644 --- a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py +++ b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py @@ -191,3 +191,24 @@ def test_a_photo_ocr_cannot_read_is_read_by_docling_whole( assert source.status is DocumentStatus.READY assert source.content == "Quarterly results." assert docling == [] + + +def test_a_photo_with_no_frame_of_text_is_read_by_docling( + session: Session, stub_model: None, docling: list, monkeypatch: pytest.MonkeyPatch +) -> None: + """Too little read: Docling reads it whole, so nothing is indexed empty.""" + + class Little(StandInOcr): + def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: + for frame in frames: + yield frame, OcrPage(1786, 2526, (), 0.2, "en-US", confidence=0.9) + + reader = Little() + monkeypatch.setattr(images, "page_reader", lambda: reader) + docling.append(reading()) + source = upload(session, photo(), name="receipt.jpg") + + run(source.id) + + session.expire_all() + assert source.content == "Quarterly results." diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/check.py b/surfsense_local/backend/worker/ingestion/scan_ocr/check.py index 0fed6c7e8c..51bb3567af 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/check.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/check.py @@ -32,8 +32,13 @@ def check_scan_ocr() -> str: def _read(reader: PageReader) -> str: + """Raises unless every word of the sample came back.""" [(_, page)] = list(reader.read([Frame(1, _sample(), 0.05)])) - return f"{reader.name} read {page.text()!r}" + text = page.text() + missing = [word for word in SAMPLE.split() if word.lower() not in text.lower()] + if missing: + raise RuntimeError(f"{reader.name} read {text!r}, missing {missing}") + return f"{reader.name} read {text!r}" def _sample() -> Image.Image: diff --git a/surfsense_local/backend/worker/ingestion/scan_ocr/images.py b/surfsense_local/backend/worker/ingestion/scan_ocr/images.py index 7ad3f3a48c..e93e2e7888 100644 --- a/surfsense_local/backend/worker/ingestion/scan_ocr/images.py +++ b/surfsense_local/backend/worker/ingestion/scan_ocr/images.py @@ -19,13 +19,14 @@ def image_markdown(path: Path) -> str | None: - """None when Docling should read it: no OCR here, a garbled frame, nothing - but blank frames, or a failure. Never raises.""" + """None when Docling should read it: no OCR here, a garbled frame, no frame + that reads as text, or a failure. Never raises.""" reader = page_reader() if reader is None: return None started = time.perf_counter() texts: list[str] = [] + stood = False try: with closing(image_frames(path, reader.max_side)) as frames: for frame, page in read_upright(reader, frames): @@ -40,6 +41,7 @@ def image_markdown(path: Path) -> str | None: return None if verdict is not PageVerdict.BLANK: texts.append(page.text()) + stood = stood or verdict is PageVerdict.TEXT except Exception: logger.warning( "%s could not read %s; Docling reads it", @@ -48,7 +50,8 @@ def image_markdown(path: Path) -> str | None: exc_info=True, ) return None - if not texts: + if not stood: + # As for a PDF: with no frame of text, Docling reads it whole. return None logger.info( "%s reads %s in %.1f s", reader.name, path.name, time.perf_counter() - started From f3524e786eeb1ac41b27bee123bc805f75c8b5bf Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 17 Oct 2026 15:29:58 -0700 Subject: [PATCH 5/6] fix(local): a long OCR reading gives the queue up as a long Docling reading does Every 8 pages, once the reading has run its minute and an upload or note waits at its priority, OCR keeps the pages read so far in ocr/pages.json and gives way; the run that resumes reads only the pages left. It gives way only with the pages on disk, and never after its last page. A waiting source with ocr/ counts as a reading that gave way itself, so two long scans do not take turns. ocr/ goes with a cancel or a failure, in OCR's part or in Docling's after it, when Docling takes the file over, and when a cancel lands while it waits in the queue, so a retried source is never taken for one that gave way. --- ...can-is-read-by-ocr-alone-before-docling.md | 2 +- docs/architecture/data-model.md | 1 + docs/architecture/documents.md | 12 +- .../docling_batches/ended_in_queue.py | 8 +- .../documents/docling_batches/layout.py | 12 +- .../integration/worker/test_ocr_first.py | 111 ++++++++++++- .../tests/unit/worker/test_scan_ocr.py | 155 +++++++++++++++++- .../ingestion/docling_batches/waiting_work.py | 21 ++- .../worker/ingestion/pdf_reading/ocr_first.py | 136 ++++++++++----- .../worker/ingestion/pdf_reading/ocr_kept.py | 29 ++-- .../worker/ingestion/pdf_reading/route.py | 20 ++- 11 files changed, 428 insertions(+), 79 deletions(-) diff --git a/docs/adr/0062-a-scan-is-read-by-ocr-alone-before-docling.md b/docs/adr/0062-a-scan-is-read-by-ocr-alone-before-docling.md index 16ca65da02..6794198aee 100644 --- a/docs/adr/0062-a-scan-is-read-by-ocr-alone-before-docling.md +++ b/docs/adr/0062-a-scan-is-read-by-ocr-alone-before-docling.md @@ -46,6 +46,6 @@ Answers followed: Haiku 5.5, given each page's text whole, answered 0.79 of the ## Where the code stands - Speed is measured on one machine. On it, RapidOCR under Linux (WSL's Ubuntu 24.04) read the same 100 pages in 174 s and 684 CPU-s, against 209 s and 824 under Windows, with the same text; CI runs its tests on Ubuntu, macOS and Windows. Vision's speed is from a virtual Mac with no Neural Engine, so a real Mac should read faster; its accuracy is from that runner. -- A long OCR-first reading checks for a cancel every 8 pages but does not give way to other uploads, as a long Docling reading does: a 100-page scan holds the queue for about 3 minutes. When its garbled pages go to a Docling reading that gives way, OCR's pages are kept, so the resumed run does not read them again. +- A long OCR-first reading checks for a cancel every 8 pages and, at the same points, gives the queue up as a long Docling reading does, keeping the pages it has read in `ocr/` so the run that resumes reads only the rest; it gives way only once they are on disk. A reading does not give way to another that gave way itself, Docling's or OCR's. When its garbled pages go to a Docling reading that gives way, OCR's pages are kept the same way. A worker killed mid-reading keeps only what an earlier turn kept. - Camera captures shot at an angle are read upright, but RapidOCR's rows can still mix lines a perspective skews; Vision read them well. - PP-OCRv6 covers Latin, Chinese and Japanese, as Docling's did, so scanned Korean, Cyrillic, Devanagari or Arabic text is misread, and Vision's languages are the system's. diff --git a/docs/architecture/data-model.md b/docs/architecture/data-model.md index 12b28e9411..ceba18ec4b 100644 --- a/docs/architecture/data-model.md +++ b/docs/architecture/data-model.md @@ -190,6 +190,7 @@ One row per run of a plugin's action. What the run produced is not here: the plu │ ├── figures/ the pictures ingest kept, and figures.json │ │ └── work/ the runs a figures-only pass that gave way resumes from │ ├── docling/work/ a long Docling reading's batches, until it ends + │ ├── ocr/ a long OCR reading's pages.json, until it ends │ └── transcript/ a recording's: segments.json once transcribed; │ └── work/ the decoded audio, plan and windows a job resumes from ├── chats// diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index cb288f0100..58070deff7 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -113,7 +113,7 @@ Sources sit in folders of the workspace's Library ([data model](data-model.md#so ## The original file -No filename is stored in a row: the document's directory is the record. The original is the one file in it, so not the `figures/` folder, a recording's `transcript/` or a long Docling reading's `docling/`, that is not hidden, not `Thumbs.db` or `desktop.ini`, and not `extracted.md`. Any other count, an empty or absent directory included, means the file is missing. Directories written before uploads kept their names hold `original.` beside an `extracted.md` nothing reads. They are read as they are and deliberately never rewritten: nothing in the index depends on a file's name, so renaming would only change what the file manager shows, and a bulk rename can fail part way on a file that is open or already deleted. Because `extracted.md` is never taken for the original, a legacy directory whose original was deleted reads as missing rather than serving Docling's text as the user's file. +No filename is stored in a row: the document's directory is the record. The original is the one file in it, so not the `figures/` folder, a recording's `transcript/`, a long Docling reading's `docling/` or a long OCR reading's `ocr/`, that is not hidden, not `Thumbs.db` or `desktop.ini`, and not `extracted.md`. Any other count, an empty or absent directory included, means the file is missing. Directories written before uploads kept their names hold `original.` beside an `extracted.md` nothing reads. They are read as they are and deliberately never rewritten: nothing in the index depends on a file's name, so renaming would only change what the file manager shows, and a bulk rename can fail part way on a file that is open or already deleted. Because `extracted.md` is never taken for the original, a legacy directory whose original was deleted reads as missing rather than serving Docling's text as the user's file. `GET .../original` serves the stored bytes with the document's title as the filename and the stored MIME type, always as an attachment: an HTML or SVG file rendered inline would run its script against the app's origin. It answers `404` when there is no file behind the document, as for a note. @@ -130,7 +130,7 @@ The desktop app previews an original in place of the left sidebar when its MIME 5. **Index** ([`indexing.py`](../../surfsense_local/backend/worker/ingestion/indexing.py)). The document's old chunks are deleted, which clears both index tables through the delete trigger, and the new chunks are inserted with their float32 embeddings, followed by one row per chunk in the active index's vector table, and the document records that index in `embedding_index_id`. The keyword index follows by trigger; the vector cannot, because only ingest holds it. Editing a note therefore stops its old text being findable. 6. **Finish.** `finish_job` writes `ready` unless a cancel won the race, and beside it only what the job produced: a file's extracted markdown on `documents.content`. A note's content is the user's and is only read, and ingest never writes `title`, so a rename or a note edit committed while the job ran stands, and the ingest that edit enqueued indexes the new text. -Cancellation is checked after parsing and after embedding, between a recording's windows and between a long Docling reading's batches. A recording or a long Docling reading that gives the queue up rolls back and goes back to `pending` (`pause_job`, unless a cancel won), queued again at `PRIORITY_BULK` for a recording and at the priority it came with for a reading: neither a failure nor a retry. On any other failure the worker rolls back and re-reads the row. If the document was deleted or cancelled meanwhile it stops; otherwise it writes `failed` with `error_message` set to the exception type and message, cut to 500 characters (a recording's failures are sentences written for the person, stored as they are), notifies, and re-raises so Huey runs the job again, up to two more times. A file whose bytes Docling cannot load (a `ConversionError` caused by a `DocumentLoadError`, such as a damaged Office file) is not run again, because the same bytes are refused the same way: parsing raises `UnreadableFileError` and the task turns it into Huey's `CancelExecution(retry=False)`. Docling raises `ConversionError` too for a file it cannot open yet, such as one another program holds, and for a model that fails on every page, such as running out of memory; those are tried again. A recording that would fail the same way again is not run again either ([Recordings](#recordings)). A later success clears the message. The sources panel shows it and offers Retry. +Cancellation is checked after parsing and after embedding, between a recording's windows, between a long Docling reading's batches and every 8 pages of an OCR reading. A recording or a long Docling or OCR reading that gives the queue up rolls back and goes back to `pending` (`pause_job`, unless a cancel won), queued again at `PRIORITY_BULK` for a recording and at the priority it came with for a reading: neither a failure nor a retry. On any other failure the worker rolls back and re-reads the row. If the document was deleted or cancelled meanwhile it stops; otherwise it writes `failed` with `error_message` set to the exception type and message, cut to 500 characters (a recording's failures are sentences written for the person, stored as they are), notifies, and re-raises so Huey runs the job again, up to two more times. A file whose bytes Docling cannot load (a `ConversionError` caused by a `DocumentLoadError`, such as a damaged Office file) is not run again, because the same bytes are refused the same way: parsing raises `UnreadableFileError` and the task turns it into Huey's `CancelExecution(retry=False)`. Docling raises `ConversionError` too for a file it cannot open yet, such as one another program holds, and for a model that fails on every page, such as running out of memory; those are tried again. A recording that would fail the same way again is not run again either ([Recordings](#recordings)). A later success clears the message. The sources panel shows it and offers Retry. ### PDFs with a text layer @@ -192,7 +192,7 @@ A scanned page and an uploaded image are read by OCR alone before Docling is ask - **Sideways pages** ([`upright.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/upright.py)). A page whose text stands mostly in boxes on end, which RapidOCR reads confidently but across one row, one the engine was unsure of, or, with no confidence, one or two characters a token, is read again turned 90, then 270, then 180 degrees, one turn at a time, and the first turn that reads as upright text replaces it; with none, the page keeps its verdict. All 20 camera captures of printed pages in the corpus came sideways with no EXIF turn. - **Docling.** A garbled page goes to Docling in its place, through the splice a mixed PDF's flagged pages take, in batches that give way ([below](#long-docling-readings)). A PDF with no page that reads as text, and an image with a garbled frame or no frame of text, go to Docling whole, as before. What OCR read of a PDF whose garbled pages Docling then reads is kept in `ocr/` in the document's folder until the file is ready, so a Docling reading that gives the queue up resumes without reading every page with OCR again. The log says `RapidOCR reads 17 of 20 pages of scan.pdf in 31.2 s; Docling reads 5, 9-10: page 5 is garbled; ...`. - **Figures.** Ingest keeps none for a PDF read this way. A scan's figures wait until the agent or Studio asks for them, as a PDF over 40 pages does: the figures-only pass reads every scanned page with Docling in full, about 39 CPU-s a page, the cost this path removes. A PDF with born-digital pages beside its scanned ones still gets the pass queued at idle, as a text-read PDF does. An image is still its own figure. -- **Cancel.** A PDF's reading checks every 8 pages; an image's frames are read without a check. A long reading does not give the queue up as a long Docling reading does. +- **Cancel and giving way.** Every 8 pages, about 17 s of RapidOCR's, a PDF's reading checks for a cancel and gives the queue up as a long Docling reading does, to the same sources ([below](#long-docling-readings)). Before it gives way it keeps the pages read so far in `ocr/pages.json` in the document's folder, stamped with the original's SHA-256 and the reader's name ([`ocr_kept.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py)), and it gives way only once they are on disk, never after its last page. The run that resumes reads only the pages left; the few in flight as it gave way are read again. A reading whose garbled pages go to Docling keeps its pages there too, for a resume should Docling's part give way. The file goes when the source is ready; when Docling reads the file whole instead, because no page read as text, OCR failed or OCR is off; when the job is cancelled or fails, in OCR's part or in Docling's after it; and when a cancel lands while it waits in the queue after giving way ([`ended_in_queue.py`](../../surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py)). A worker killed mid-reading keeps only what an earlier turn kept, and Retry resumes from it. An image's frames are read without a check. - **The release build** runs `worker --check-scan-ocr` in the frozen worker on every system: each reader reads a rendered line, and an import the spec dropped fails the build. - **Measured** on the Ryzen 7 5800X, Windows 11, through the upload route and a real `worker.py ingest`, one upload at a time: the six public scanned PDFs and three book scans, 199 pages, were all ready in 509 s against 829 s through Docling, a file in 332 s on average against 598 s, at 1,752 worker CPU-s against 7,343; no page went to Docling. 40 receipt photos, receipt scans and forms were all ready in 113 s against 202 s, an image in 65 s on average against 129 s, at 131 worker CPU-s against 1,416. @@ -234,7 +234,7 @@ Ingest keeps the pictures in each source, so the agent can place a source's char - The queue survives restarts. Electron starts the workers beside the API and only the API migrates, so each consumer waits until the database is at the latest revision before taking a job ([`wait_for_schema.py`](../../surfsense_local/backend/worker/wait_for_schema.py)); a job queued before an update would otherwise run against the old schema and fail. A job carries only its task's name, so each consumer imports every task before it starts (`import_tasks()`), and [`tests/integration/test_registration.py`](../../surfsense_local/backend/tests/integration/test_registration.py) fails when that list falls behind the task modules on disk. - Jobs are enqueued by upload, note creation, a note content edit and retry, and by import ([`import.md`](import.md)). The queue takes the highest priority first: interactive work at 100, a bulk copy and an import at 10. Equal priorities run in the order they were queued, since Huey's SQLite storage takes `order by priority desc, id`. The figures-only pass, `extract_figures`, shares the queue: at 30 when someone asks for figures, behind every note and upload but ahead of an import; at 10 when a Word file's ingest queues it as it ends; and at -10 when ingest queues it for a PDF read from its text layer ([Figures](#figures)). After 20 s it gives way to a source whose ingest is queued at its priority or above. - A source the app quit in the middle of ingesting is failed with `interrupted when the app closed` when the ingest worker next starts, before it takes a job ([`interrupted_documents.py`](../../surfsense_local/backend/worker/interrupted_documents.py)). Huey drops a job as it starts it, so nothing else would end it, and Retry accepts only a failed or cancelled document. A source still `pending` is still queued and is left alone. -- A recording's job gives the queue up between windows once it has run a minute and a source that is not a recording is pending ([Recordings](#recordings)), and a long Docling reading between batches once an upload or note is queued at its own priority or above ([Long Docling readings](#long-docling-readings)), so a long file never holds every upload and note behind it. The ingest task is handed its own Huey task (`context=True`), so the job knows the priority it was queued at. +- A recording's job gives the queue up between windows once it has run a minute and a source that is not a recording is pending ([Recordings](#recordings)), and a long Docling reading between batches, or a long OCR reading every 8 pages ([Scans and photos](#scans-and-photos)), once an upload or note is queued at its own priority or above ([Long Docling readings](#long-docling-readings)), so a long file never holds every upload and note behind it. The ingest task is handed its own Huey task (`context=True`), so the job knows the priority it was queued at. - Each transition the ingest worker makes, to `processing`, `ready` or `failed`, sends a `documents` event keyed by document id, and a recording's job sends one per transcribed window carrying a transient `progress` `{done_s, total_s}`, never stored, and so does each change the API makes itself: a note written, an upload, a rename or edit, a retry, a cancel, and a delete, whose status is `deleted`. The sources panel reloads its list on each event and whenever a dropped stream is back, except for a transcribed window's event, whose `progress` it keeps on the row when the row already has the event's status ([the sources tree](#the-sources-tree)), so a note a plugin writes shows as it is written; it still refetches every 10 seconds while any row is `pending` or `processing`, in case a notice was lost ([`overview.md`](overview.md#freshness)). ## Long Docling readings @@ -243,7 +243,7 @@ A PDF Docling reads whole, and a run of failed pages, longer than 8 pages is con - **Joined.** The batches join into one `DoclingDocument` with `DoclingDocument.concatenate`, after a paragraph that runs on past a batch's last page is joined again by Docling's own rule: an open end, then a word going on, past headers, footers, tables, pictures, captions and footnotes ([`edge_paragraphs.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/edge_paragraphs.py)). Without it a budget volume read in 7-page batches split a paragraph at page 7. A whole-file reading keeps its figures from the joined document. Joining numbers each batch's pages on from where the last ended, so a page Docling could not read is first put back into its batch as an empty page, as one conversion of the file lacks it; otherwise every later picture would land on the page before its own ([`lost_pages.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/lost_pages.py)). - **Checkpoint.** Each batch read is kept as its `DoclingDocument`'s JSON, `pages--.json`, replaced whole, in `docling/work/` in the document's folder: a folder, so it is never taken for the original. A stamp names the format, the original's SHA-256, Docling's version and whether the parser pack is there, since the converter reads OCR without RapidOCR's line classifier only with it; a checkpoint whose stamp differs is dropped ([`checkpoint.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/checkpoint.py)). Hashing costs 0.26 s per 100 MiB, once a run. A source deleted meanwhile does not get its folder back. -- **Giving way** ([`docling_batches/queue_turn.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/queue_turn.py)). Before each batch but the first this job reads, it checks for a cancel and, once the reading has run 60 s, gives the queue up as a recording does, `JobYieldedError` and `pause_job`, and is queued again at the priority it came with, behind the source it gave way to. It gives way to a pending file or note whose ingest is queued at its own priority or above ([`queued_ingests.py`](../../surfsense_local/backend/worker/ingestion/queued_ingests.py)), so an upload's scan is not sent behind a folder copy started after it, and a pending row with no job queued holds nothing up. That source must not be a recording, and must have no `docling/work/` of its own: long readings do not take turns, which would leave all of them waiting until the last batch ([`waiting_work.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/waiting_work.py)). It gives way only when every batch it read is on disk. +- **Giving way** ([`docling_batches/queue_turn.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/queue_turn.py)). Before each batch but the first this job reads, it checks for a cancel and, once the reading has run 60 s, gives the queue up as a recording does, `JobYieldedError` and `pause_job`, and is queued again at the priority it came with, behind the source it gave way to. It gives way to a pending file or note whose ingest is queued at its own priority or above ([`queued_ingests.py`](../../surfsense_local/backend/worker/ingestion/queued_ingests.py)), so an upload's scan is not sent behind a folder copy started after it, and a pending row with no job queued holds nothing up. That source must not be a recording, and must have no `docling/work/` or `ocr/` of its own: long readings, Docling's or OCR's, do not take turns, which would leave all of them waiting until the last batch ([`waiting_work.py`](../../surfsense_local/backend/worker/ingestion/docling_batches/waiting_work.py)). It gives way only when every batch it read is on disk. - **Resume.** The next run reads only the batches not kept. A reading that gave way is `pending`, so the startup sweep leaves it alone and its queued copy carries on after a restart. - **End.** Read to the last batch, failed or cancelled, the job removes `docling/`. A cancel of a reading waiting in the queue after giving way removes it in the route, the document's or its folder's ([`ended_in_queue.py`](../../surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py)), and so does the job when a cancel lands as it gives way; a delete removes it with the folder. A worker killed mid-reading leaves it on purpose and the sweep fails the row, so Retry carries on from the batches kept; it stays until the source is retried or deleted. Huey's own retry after a failure starts over. - **Proved** with real Docling: on a generated 20-page scan and on Education, read in batches straight through and resumed after every batch, the Markdown, `figures.json` and every PNG were identical to one conversion's. @@ -325,4 +325,4 @@ The upload is accepted. Finding no speech model, the job fails without retry wit - Recordings take WAV, MP3, FLAC and Ogg Vorbis only; Opus, M4A, AAC and audio-only WebM wait for a decode-only LGPL FFmpeg ([proposal](../proposals/local-speech-to-text.md#order-of-work), step 9). - A recording cannot be played, nor a citation of one opened at its time; `GET .../original` does not commit before streaming, which a player would need ([proposal](../proposals/local-speech-to-text.md#order-of-work), step 10). - OCR reads one script family per converter: PP-OCRv6 covers Latin, Chinese and Japanese, so scanned Korean, Cyrillic, Devanagari or Arabic text is misread. -- A scan read by OCR alone has no Markdown headings or pipe tables; its tables are rows of cells on one line each. A long OCR-first reading does not give the queue up to other uploads, as a long Docling reading does: a 100-page scan holds it for about 3 minutes. A camera capture shot at an angle is read upright, but RapidOCR's rows can mix lines its perspective skews; Vision read the corpus's well. Vision's speed was measured only on a virtual Mac with no Neural Engine. +- A scan read by OCR alone has no Markdown headings or pipe tables; its tables are rows of cells on one line each. A camera capture shot at an angle is read upright, but RapidOCR's rows can mix lines its perspective skews; Vision read the corpus's well. Vision's speed was measured only on a virtual Mac with no Neural Engine. diff --git a/surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py b/surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py index ca9274fe8f..2998008488 100644 --- a/surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py +++ b/surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py @@ -1,9 +1,10 @@ -"""The batches of a reading that gave the queue up and was then cancelled: no run -is left to clear them, as a running one does when it stops.""" +"""What a reading that gave the queue up and was then cancelled kept, Docling's +batches and OCR's pages: no run is left to clear them, as a running one does +when it stops.""" import shutil -from modules.documents.docling_batches.layout import docling_dir +from modules.documents.docling_batches.layout import docling_dir, ocr_dir from modules.documents.models import Document, DocumentStatus, DocumentType from shared.config import get_storage_settings @@ -19,3 +20,4 @@ def drop_batches(workspace_id: int, document_id: int) -> None: """Of a file cancelled while it waited; nothing to do for most.""" folder = get_storage_settings().document_dir(workspace_id, document_id) shutil.rmtree(docling_dir(folder), ignore_errors=True) + shutil.rmtree(ocr_dir(folder), ignore_errors=True) diff --git a/surfsense_local/backend/modules/documents/docling_batches/layout.py b/surfsense_local/backend/modules/documents/docling_batches/layout.py index 96296fa008..3c917eca97 100644 --- a/surfsense_local/backend/modules/documents/docling_batches/layout.py +++ b/surfsense_local/backend/modules/documents/docling_batches/layout.py @@ -1,11 +1,13 @@ -"""Where a long Docling reading keeps the batches it has read: docling/work/ in -the document's folder. A folder, so original_file.py never takes it for the -original, and inside the document's own, so a delete removes it with the rest.""" +"""Where a long reading keeps what it has read when it gives the queue up: +Docling's batches in docling/work/ in the document's folder, OCR's pages in ocr/. +Folders, so original_file.py never takes one for the original, and inside the +document's own, so a delete removes them with the rest.""" from pathlib import Path DOCLING_FOLDER = "docling" WORK_FOLDER = "work" +OCR_FOLDER = "ocr" def docling_dir(document_dir: Path) -> Path: @@ -14,3 +16,7 @@ def docling_dir(document_dir: Path) -> Path: def batches_dir(document_dir: Path) -> Path: return docling_dir(document_dir) / WORK_FOLDER + + +def ocr_dir(document_dir: Path) -> Path: + return document_dir / OCR_FOLDER diff --git a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py index ba2ab4e77d..99082e2212 100644 --- a/surfsense_local/backend/tests/integration/worker/test_ocr_first.py +++ b/surfsense_local/backend/tests/integration/worker/test_ocr_first.py @@ -9,12 +9,15 @@ from PIL import Image from sqlalchemy.orm import Session -from modules.documents.models import DocumentStatus +from modules.documents.docling_batches.layout import docling_dir, ocr_dir +from modules.documents.models import Document, DocumentStatus from modules.documents.tasks import PRIORITY_IDLE from shared.queue import ingest_queue from tests import pdf_samples +from tests.integration.worker import test_long_docling_readings as long_readings from tests.integration.worker.docling_stand_in import reading from tests.integration.worker.paged_docling_stand_in import PagedDocling +from tests.integration.worker.test_long_docling_readings import no_minute from tests.integration.worker.test_mixed_pdfs import ( REPORT, folder_of, @@ -22,7 +25,7 @@ session, upload, ) -from worker.ingestion import run +from worker.ingestion import pipeline, run from worker.ingestion.pdf_reading import ocr_first from worker.ingestion.scan_ocr import images from worker.ingestion.scan_ocr.frames import Frame @@ -30,7 +33,7 @@ pytestmark = pytest.mark.integration -__all__ = ["session"] +__all__ = ["no_minute", "session"] def prose(page: int) -> OcrPage: @@ -212,3 +215,105 @@ def read(self, frames: Iterable[Frame]) -> Iterator[tuple[Frame, OcrPage]]: session.expire_all() assert source.content == "Quarterly results." + + +def test_a_long_scan_gives_way_to_a_note_and_reads_only_the_pages_left( + session: Session, + stub_model: None, + paged_docling: PagedDocling, + ocr: StandInOcr, + no_minute: None, +) -> None: + """Its pages are kept in ocr/ when it gives the queue up, as a long Docling + reading keeps its batches; each page is read once.""" + source = long_readings.upload(session, scans(20)) + + note = long_readings.give_way_once(session, source) + + session.expire_all() + assert source.status is DocumentStatus.PENDING + assert ocr.read_pages == list(range(1, 9)) + assert (ocr_dir(folder_of(source)) / "pages.json").is_file() + session.commit() # the read above holds SQLite's lock the queue's jobs need + + long_readings.drain(session) + + session.expire_all() + written = session.get(Document, note.id) + assert written is not None + assert (written.status, source.status) == (DocumentStatus.READY,) * 2 + assert ocr.read_pages == list(range(1, 21)) + assert paged_docling.asked == [] + content = source.content or "" + assert "Scanned page 1 line 0" in content and "Scanned page 20 line 7" in content + assert not ocr_dir(folder_of(source)).exists() + + +def test_a_cancel_as_an_ocr_reading_gives_way_drops_its_pages( + session: Session, + stub_model: None, + paged_docling: PagedDocling, + ocr: StandInOcr, + no_minute: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It lands after the reading's last check: no run is left to clear them.""" + source = long_readings.upload(session, scans(20)) + pause = pipeline.pause_job + + def cancelled_first(job: Session, document: Document) -> bool: + long_readings.set_status(session, source, DocumentStatus.CANCELLED) + return pause(job, document) + + monkeypatch.setattr(pipeline, "pause_job", cancelled_first) + + long_readings.give_way_once(session, source) + + session.expire_all() + assert source.status is DocumentStatus.CANCELLED + assert not ocr_dir(folder_of(source)).exists() + + +def test_an_ocr_reading_does_not_give_way_to_one_that_gave_way_itself( + session: Session, + stub_model: None, + paged_docling: PagedDocling, + ocr: StandInOcr, + no_minute: None, +) -> None: + """Two long scans: the first gives way to the second once, which then reads + to the end; taking turns would leave both waiting until the last page.""" + first = long_readings.upload(session, scans(20), name="first.pdf") + second = long_readings.upload(session, scans(20), name="second.pdf") + + outcomes = long_readings.drain(session) + + assert outcomes == [ + ("ingest_document", first.id, "pending"), + ("ingest_document", second.id, "ready"), + ("ingest_document", first.id, "ready"), + ] + assert ocr.read_pages == [*range(1, 9), *range(1, 21), *range(9, 21)] + + +def test_a_cancel_in_the_docling_part_drops_ocrs_pages_too( + session: Session, stub_model: None, paged_docling: PagedDocling, ocr: StandInOcr +) -> None: + """OCR kept its pages for the Docling part's resume; a cancel there ends the + reading, and a source left with ocr/ would look like one that gave way.""" + ocr.garbled = set(range(1, 18)) + source = long_readings.upload(session, scans(20)) + + def cancel(pages: tuple[int, int] | None) -> None: + if pages == (1, 8): + long_readings.set_status(session, source, DocumentStatus.CANCELLED) + + paged_docling.during = cancel + + long_readings.drain(session) + + session.expire_all() + assert source.status is DocumentStatus.CANCELLED + assert paged_docling.asked == [(1, 8)] + assert not ocr_dir(folder_of(source)).exists() + assert not docling_dir(folder_of(source)).exists() diff --git a/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py b/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py index 4c935f6263..09bb82362e 100644 --- a/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py +++ b/surfsense_local/backend/tests/unit/worker/test_scan_ocr.py @@ -7,13 +7,17 @@ import pytest from PIL import Image +from modules.documents.models import Document +from modules.documents.tasks import PRIORITY_INTERACTIVE from worker.ingestion.pdf_reading import ocr_first +from worker.ingestion.pdf_reading.ocr_kept import forget_reading from worker.ingestion.pdf_text_layer import LayerPage, TextLayer from worker.ingestion.pdf_text_layer.page_triage import PageKind from worker.ingestion.scan_ocr import switch from worker.ingestion.scan_ocr.frames import Frame from worker.ingestion.scan_ocr.page import Line, OcrPage, PageVerdict, Word from worker.ingestion.scan_ocr.upright import read_upright +from worker.jobs import JobCancelledError, JobYieldedError pytestmark = pytest.mark.unit @@ -416,14 +420,161 @@ def test_a_resumed_reading_keeps_ocrs_pages( ocr.update({1: page(*prose()), 2: page(*prose(), confidence=0.5)}) layer = layer_of(scanned(1), scanned(2)) first = ocr_first.ocr_reading(original, layer) - assert first is not None and not first.resumed + assert first is not None and first.kept == 0 ocr.clear() # any read now would fail second = ocr_first.ocr_reading(original, layer) - assert second is not None and second.resumed + assert second is not None and second.kept == 2 assert second.docling == first.docling == {2: "page 2 is garbled"} assert second.layer.pages[0].text == first.layer.pages[0].text # Other bytes: nothing kept applies, and the stand-in, asked again, fails. original.write_bytes(b"%PDF another scan") assert ocr_first.ocr_reading(original, layer) is None + + +class Turn: + """The ingest job's turn on the queue, past its minute with an upload + waiting: it gives way at every check it is asked.""" + + def __init__(self) -> None: + self.gives_way = True + self.cancelled = False + + def check_cancelled(self) -> None: + if self.cancelled: + raise JobCancelledError + + def give_way(self) -> None: + if self.gives_way: + raise JobYieldedError(PRIORITY_INTERACTIVE) + + +@pytest.fixture +def turn(monkeypatch: pytest.MonkeyPatch) -> Turn: + """A reading inside an ingest job, whose turn this is.""" + made = Turn() + monkeypatch.setattr(ocr_first, "object_session", lambda _document: object()) + monkeypatch.setattr(ocr_first, "reading_turn", lambda _session, _document: made) + return made + + +def long_scan( + folder: Path, ocr: dict[int, OcrPage], pages: int = 20 +) -> tuple[Path, TextLayer]: + """A scan of running text on disk, and its text layer.""" + original = folder / "scan.pdf" + original.write_bytes(b"%PDF a long scan") + ocr.update({n: page(*prose()) for n in range(1, pages + 1)}) + return original, layer_of(*map(scanned, range(1, pages + 1))) + + +def asked() -> list[int]: + """The pages the stand-in reader has been asked for, in order.""" + reader = ocr_first.page_reader() + assert isinstance(reader, StubReader) + return reader.asked + + +def test_a_long_reading_gives_the_queue_up_with_its_pages_kept( + ocr: dict[int, OcrPage], turn: Turn, tmp_path: Path +) -> None: + """The run that resumes reads only the pages left, and comes to what one + run would.""" + original, layer = long_scan(tmp_path, ocr) + + with pytest.raises(JobYieldedError): + ocr_first.ocr_reading(original, layer, Document()) + assert asked() == list(range(1, 9)) + + turn.gives_way = False + resumed = ocr_first.ocr_reading(original, layer, Document()) + + assert asked() == list(range(1, 21)) + assert resumed is not None and (resumed.kept, resumed.read) == (8, 20) + forget_reading(original) + straight = ocr_first.ocr_reading(original, layer) + assert straight is not None and straight.kept == 0 + assert (resumed.layer, resumed.docling) == (straight.layer, straight.docling) + + +def test_a_reading_does_not_give_way_after_its_last_page( + ocr: dict[int, OcrPage], turn: Turn, tmp_path: Path +) -> None: + """Nothing is left to read; Docling's part, if any, gives way on its own.""" + original, layer = long_scan(tmp_path, ocr, pages=8) + + reading = ocr_first.ocr_reading(original, layer, Document()) + + assert reading is not None and reading.read == 8 + + +def test_a_reading_that_cannot_keep_its_pages_reads_on( + ocr: dict[int, OcrPage], turn: Turn, tmp_path: Path +) -> None: + """Giving way would lose every page read so far.""" + original, layer = long_scan(tmp_path, ocr) + (tmp_path / "ocr").write_text("a file in the folder's place", "utf-8") + + reading = ocr_first.ocr_reading(original, layer, Document()) + + assert reading is not None and reading.read == 20 + assert asked() == list(range(1, 21)) + + +def test_a_cancelled_reading_forgets_the_pages_it_kept( + ocr: dict[int, OcrPage], turn: Turn, tmp_path: Path +) -> None: + """As a cancelled Docling reading drops its batches.""" + original, layer = long_scan(tmp_path, ocr) + with pytest.raises(JobYieldedError): + ocr_first.ocr_reading(original, layer, Document()) + assert (tmp_path / "ocr" / "pages.json").is_file() + + turn.cancelled = True + with pytest.raises(JobCancelledError): + ocr_first.ocr_reading(original, layer, Document()) + + assert not (tmp_path / "ocr").exists() + + +def test_a_resumed_reading_with_no_page_of_text_is_docling_s_whole( + ocr: dict[int, OcrPage], turn: Turn, tmp_path: Path +) -> None: + """The pages kept before it gave way count, and are forgotten with the rest.""" + original, layer = long_scan(tmp_path, ocr) + ocr.update({n: page(line("Fig.", 300), ink=0.2) for n in range(1, 9)}) + ocr.update({n: page(*prose(), confidence=0.5) for n in range(9, 21)}) + with pytest.raises(JobYieldedError): + ocr_first.ocr_reading(original, layer, Document()) + + turn.gives_way = False + assert ocr_first.ocr_reading(original, layer, Document()) is None + + # An unsure page is read again turned, so pages 9 on are asked for more than once. + assert asked()[:8] == list(range(1, 9)) + assert sorted(set(asked()[8:])) == list(range(9, 21)) + assert not (tmp_path / "ocr").exists() + + +@pytest.mark.parametrize("then", ["fails", "is turned off"]) +def test_pages_kept_for_a_reading_docling_takes_over_are_forgotten( + ocr: dict[int, OcrPage], + turn: Turn, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + then: str, +) -> None: + """Docling reads every page; a source left with ocr/ would look like a + reading that gave way, and be given way to by no other.""" + original, layer = long_scan(tmp_path, ocr) + with pytest.raises(JobYieldedError): + ocr_first.ocr_reading(original, layer, Document()) + if then == "fails": + ocr.clear() # any read now fails + else: + monkeypatch.setattr(ocr_first, "page_reader", lambda: None) + + assert ocr_first.ocr_reading(original, layer, Document()) is None + + assert not (tmp_path / "ocr").exists() diff --git a/surfsense_local/backend/worker/ingestion/docling_batches/waiting_work.py b/surfsense_local/backend/worker/ingestion/docling_batches/waiting_work.py index 8b61b50301..b8a60d3c8a 100644 --- a/surfsense_local/backend/worker/ingestion/docling_batches/waiting_work.py +++ b/surfsense_local/backend/worker/ingestion/docling_batches/waiting_work.py @@ -1,13 +1,15 @@ -"""What a long Docling reading gives the queue up to: a file or note whose ingest -waits at the reading's own priority or above, so a bulk copy started after it -does not finish first. Not a recording, as for a recording's job, and not -another reading that gave the queue up itself, or long readings would take turns -until all ended together.""" +"""What a long reading, Docling's or OCR's, gives the queue up to: a file or note +whose ingest waits at the reading's own priority or above, so a bulk copy started +after it does not finish first. Not a recording, as for a recording's job, and +not another reading that gave the queue up itself, or long readings would take +turns until all ended together.""" + +from pathlib import Path from sqlalchemy import or_, select from sqlalchemy.orm import Session -from modules.documents.docling_batches.layout import batches_dir +from modules.documents.docling_batches.layout import batches_dir, ocr_dir from modules.documents.models import Document, DocumentStatus, DocumentType from shared.config import get_storage_settings from worker.ingestion.audio.waiting_work import IS_RECORDING @@ -32,6 +34,11 @@ def sources_to_give_way_to(session: Session, priority: int) -> bool: # Matched here, not in SQL: a bulk copy queues more ids than SQLite binds. return any( document_id in queued - and not batches_dir(storage.document_dir(workspace_id, document_id)).is_dir() + and not _gave_way(storage.document_dir(workspace_id, document_id)) for workspace_id, document_id in waiting ) + + +def _gave_way(folder: Path) -> bool: + """A reading that gave the queue up keeps what it read here for its next run.""" + return batches_dir(folder).is_dir() or ocr_dir(folder).is_dir() diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py index d0d7c9b1d4..9e1592c6a5 100644 --- a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_first.py @@ -8,23 +8,30 @@ import logging import time +from collections.abc import Callable from contextlib import closing -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from pathlib import Path +from typing import Any from sqlalchemy.orm import object_session from modules.documents.models import Document +from worker.ingestion.audio.queue_turn import QueueTurn from worker.ingestion.docling_batches.queue_turn import reading_turn from worker.ingestion.figures.scanned_pages import scanned_pages -from worker.ingestion.pdf_reading.ocr_kept import keep_reading, kept_reading +from worker.ingestion.pdf_reading.ocr_kept import ( + forget_reading, + keep_reading, + kept_reading, +) from worker.ingestion.pdf_text_layer import LayerPage, TextLayer from worker.ingestion.pdf_text_layer.page_triage import PageKind from worker.ingestion.scan_ocr.frames import pdf_frames from worker.ingestion.scan_ocr.page import KEPT, PageVerdict from worker.ingestion.scan_ocr.reader import PageReader, page_reader from worker.ingestion.scan_ocr.upright import read_upright -from worker.jobs import JobCancelledError +from worker.jobs import JobCancelledError, JobYieldedError logger = logging.getLogger(__name__) @@ -34,8 +41,9 @@ # With no page of running text, a cover's title is all the layer has: the # pixels of these pages hold the rest. _PICTURE_KINDS = frozenset({PageKind.BIG_IMAGE, PageKind.SPARSE}) -# Pages between checks for a cancel, as a Docling batch is 8 pages. -CANCEL_EVERY = 8 +# Pages between checks for a cancel and for giving the queue up, as a Docling +# batch is 8 pages: some 17 s of RapidOCR's. +CHECK_EVERY = 8 @dataclass(frozen=True) @@ -48,26 +56,33 @@ class OcrReading: read: int reader: str seconds: float - # Kept from a run that gave the queue up to Docling's reading. - resumed: bool = False + # Pages kept from a run that gave the queue up, OCR's own or Docling's after it. + kept: int = 0 def ocr_reading( original: Path, layer: TextLayer, document: Document | None = None ) -> OcrReading | None: """None when there is nothing for OCR here, no OCR on this computer, or it - failed: Docling then reads as before. Raises only JobCancelledError.""" + failed: Docling then reads as before. Raises JobYieldedError when a long + reading gives the queue up with its pages kept, and JobCancelledError.""" if not layer.pages: return None reader = page_reader() if reader is None: + # Turned off since an earlier turn kept pages: Docling reads them all. + forget_reading(original) return None try: candidates = _candidates(original, layer) if not candidates: return None return _read(reader, original, layer, candidates, document) + except JobYieldedError: + raise except JobCancelledError: + # As a cancelled Docling reading drops its batches. + forget_reading(original) raise except Exception: logger.warning( @@ -76,6 +91,8 @@ def ocr_reading( original.name, exc_info=True, ) + # Docling reads every page, so the pages an earlier turn kept are no use. + forget_reading(original) return None @@ -94,6 +111,38 @@ def _candidates(original: Path, layer: TextLayer) -> list[int]: return sorted(picked) +@dataclass +class _Pages: + """OCR's pages so far: each page's text, why the others go to Docling, and + which read as text.""" + + texts: dict[int, str] = field(default_factory=dict) + docling: dict[int, str] = field(default_factory=dict) + stood: set[int] = field(default_factory=set) + + @classmethod + def kept(cls, kept: dict[str, Any] | None, candidates: list[int]) -> "_Pages": + """What an earlier run kept, of the pages OCR reads now.""" + if kept is None: + return cls() + wanted = set(candidates) + return cls( + {int(n): text for n, text in kept["texts"].items() if int(n) in wanted}, + {int(n): why for n, why in kept["docling"].items() if int(n) in wanted}, + {int(n) for n in kept["stood"] if int(n) in wanted}, + ) + + def __contains__(self, n: int) -> bool: + return n in self.texts or n in self.docling + + def to_keep(self) -> dict[str, Any]: + return { + "texts": self.texts, + "docling": self.docling, + "stood": sorted(self.stood), + } + + def _read( reader: PageReader, original: Path, @@ -102,19 +151,21 @@ def _read( document: Document | None, ) -> OcrReading | None: started = time.perf_counter() - kept = kept_reading(original, reader.name) - if kept is not None: - texts = {int(n): text for n, text in kept["texts"].items()} - docling = {int(n): why for n, why in kept["docling"].items()} - else: - texts, docling, stood = _ocr(reader, original, layer, candidates, document) - if not stood: - # No page read as text: Docling reads the file whole, as before. - return None - if docling: - keep_reading(original, reader.name, {"texts": texts, "docling": docling}) + read = _Pages.kept(kept_reading(original, reader.name), candidates) + kept = len(read.texts) + len(read.docling) + left = [n for n in candidates if n not in read] + if left: + _ocr(reader, original, layer, left, document, read) + if not read.stood: + # No page read as text: Docling reads the file whole, as before. + forget_reading(original) + return None + if left and read.docling: + # For the run that resumes should Docling's part give the queue up. + keep_reading(original, reader.name, read.to_keep()) + docling = dict(read.docling) pages: list[LayerPage] = list(layer.pages) - for n, text in texts.items(): + for n, text in read.texts.items(): pages[n - 1] = replace(pages[n - 1], text=text, refused_because=None) for n, page in enumerate(pages, start=1): if page.text is None and n not in docling: @@ -122,10 +173,10 @@ def _read( return OcrReading( replace(layer, pages=tuple(pages)), docling, - len(texts), + len(read.texts), reader.name, time.perf_counter() - started, - resumed=kept is not None, + kept=kept, ) @@ -133,30 +184,41 @@ def _ocr( reader: PageReader, original: Path, layer: TextLayer, - candidates: list[int], + left: list[int], document: Document | None, -) -> tuple[dict[int, str], dict[int, str], int]: - """Each candidate page's text, why the others go to Docling, and how many read - as text.""" + read: _Pages, +) -> None: + """Reads the pages `left` into `read`.""" # The ingest job's own session; a reading outside one has nothing to check. session = object_session(document) if document is not None else None turn = reading_turn(session, document) if session and document else None - texts: dict[int, str] = {} - docling: dict[int, str] = {} - stood = 0 # Closed here, whatever stops the reading: its PDF closes under pdfium's lock, # which the garbage collector could otherwise take while this thread holds it. - with closing(pdf_frames(original, candidates, reader.max_side)) as frames: + with closing(pdf_frames(original, left, reader.max_side)) as frames: for count, (frame, page) in enumerate(read_upright(reader, frames), start=1): - if turn is not None and count % CANCEL_EVERY == 0: - turn.check_cancelled() n, verdict = frame.number, page.verdict() if verdict not in KEPT: - docling[n] = f"page {n} is {verdict}" + read.docling[n] = f"page {n} is {verdict}" elif verdict is PageVerdict.TEXT: - texts[n] = page.text() - stood += 1 + read.texts[n] = page.text() + read.stood.add(n) else: # Little or nothing read: a cover's own title, if it has one, wins. - texts[n] = layer.pages[n - 1].text or page.text() - return texts, docling, stood + read.texts[n] = layer.pages[n - 1].text or page.text() + if turn is not None and count % CHECK_EVERY == 0: + turn.check_cancelled() + if count < len(left): + _give_way( + turn, + lambda: keep_reading(original, reader.name, read.to_keep()), + ) + + +def _give_way(turn: QueueTurn, keep: Callable[[], bool]) -> None: + """Gives the queue up only with OCR's pages kept, so the run that resumes + reads none of them again.""" + try: + turn.give_way() + except JobYieldedError: + if keep(): + raise diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py index c3506534c4..878f96214d 100644 --- a/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py @@ -1,9 +1,9 @@ -"""What OCR read of a scan whose garbled pages Docling still reads: a Docling -reading that gives the queue up resumes from here instead of reading every page -with OCR again. +"""What OCR has read of a scan, kept when the reading gives the queue up, its own +or the Docling part's after it: the run that resumes reads only the pages left. -In ocr/ in the document's own folder: a folder, so original_file.py never takes it -for the original, and outside docling/, which a new Docling reading empties. +In ocr/ in the document's own folder (modules/documents/docling_batches/ +layout.py): a folder, so original_file.py never takes it for the original, and +outside docling/, which a new Docling reading empties. """ import hashlib @@ -13,17 +13,19 @@ from pathlib import Path from typing import Any +from modules.documents.docling_batches.layout import ocr_dir + logger = logging.getLogger(__name__) -FOLDER = "ocr" _FILE = "pages.json" -_FORMAT = 1 +# 2: kept partway too, with the pages that read as text. +_FORMAT = 2 def kept_reading(original: Path, reader: str) -> dict[str, Any] | None: """The pages kept for these bytes by this reader, or None.""" try: - kept = json.loads((original.parent / FOLDER / _FILE).read_text("utf-8")) + kept = json.loads((ocr_dir(original.parent) / _FILE).read_text("utf-8")) except (OSError, ValueError): return None try: @@ -35,9 +37,10 @@ def kept_reading(original: Path, reader: str) -> dict[str, Any] | None: return kept -def keep_reading(original: Path, reader: str, reading: dict[str, Any]) -> None: - """Never raises: without it a resume only reads the pages again.""" - folder = original.parent / FOLDER +def keep_reading(original: Path, reader: str, reading: dict[str, Any]) -> bool: + """False when it could not be kept; never raises: without it a resume only + reads the pages again.""" + folder = ocr_dir(original.parent) try: stamp = {"format": _FORMAT, "reader": reader, "original": _digest(original)} # Not parents: a source deleted meanwhile must not get its folder back. @@ -47,10 +50,12 @@ def keep_reading(original: Path, reader: str, reading: dict[str, Any]) -> None: partial.replace(folder / _FILE) except OSError as error: logger.warning("could not keep the OCR of %s: %s", original.name, error) + return False + return True def forget_reading(original: Path) -> None: - shutil.rmtree(original.parent / FOLDER, ignore_errors=True) + shutil.rmtree(ocr_dir(original.parent), ignore_errors=True) def _digest(original: Path) -> str: diff --git a/surfsense_local/backend/worker/ingestion/pdf_reading/route.py b/surfsense_local/backend/worker/ingestion/pdf_reading/route.py index 9a406b105b..6dd3edf25e 100644 --- a/surfsense_local/backend/worker/ingestion/pdf_reading/route.py +++ b/surfsense_local/backend/worker/ingestion/pdf_reading/route.py @@ -22,6 +22,7 @@ from worker.ingestion.pdf_text_layer import TextLayer, read_text_layer from worker.ingestion.pdf_text_layer.page_triage import PageKind from worker.ingestion.pdf_text_layer.reading import PAGE_BREAK +from worker.jobs import JobYieldedError logger = logging.getLogger(__name__) @@ -30,8 +31,8 @@ def pdf_markdown(document: Document, original: Path) -> str: - """Raises JobYieldedError when a long Docling reading gives the queue up, and - JobCancelledError when a reading is cancelled.""" + """Raises JobYieldedError when a long reading, Docling's or OCR's, gives the + queue up, and JobCancelledError when a reading is cancelled.""" layer = read_text_layer(original) if layer.text is not None: queue_idle_figures_pass(document, original, layer) @@ -59,19 +60,28 @@ def _ocr_first_markdown(document: Document, original: Path, reading: OcrReading) """OCR's pages with Docling's markdown in place of the pages it reads.""" runs = runs_of(sorted(reading.docling)) logger.info( - "%s %s %d of %d pages of %s in %.1f s%s", + "%s reads %d of %d pages of %s in %.1f s%s%s", reading.reader, - "kept" if reading.resumed else "reads", reading.read, len(reading.layer.pages), original.name, reading.seconds, + f", {reading.kept} kept from an earlier run" if reading.kept else "", f"; Docling reads {described(runs)}: {_shown(list(reading.docling.values()))}" if runs else "", ) if runs: - markdown = spliced(reading.layer, _docling_markdown(document, original, runs)) + try: + docling = _docling_markdown(document, original, runs) + except JobYieldedError: + raise + except Exception: + # As Docling's batches go: a cancel or a failure starts over, and a + # source left with ocr/ would look like a reading that gave way. + forget_reading(original) + raise + markdown = spliced(reading.layer, docling) else: markdown = PAGE_BREAK.join(page.text or "" for page in reading.layer.pages) forget_reading(original) From 6d5be0a1b73c364cb83bf3264475fd0c25f86971 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 17 Oct 2026 16:09:29 -0700 Subject: [PATCH 6/6] docs(local): ocr/ goes once a scan's text is built, not when the source is ready --- docs/architecture/documents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index 58070deff7..43503786d9 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -192,7 +192,7 @@ A scanned page and an uploaded image are read by OCR alone before Docling is ask - **Sideways pages** ([`upright.py`](../../surfsense_local/backend/worker/ingestion/scan_ocr/upright.py)). A page whose text stands mostly in boxes on end, which RapidOCR reads confidently but across one row, one the engine was unsure of, or, with no confidence, one or two characters a token, is read again turned 90, then 270, then 180 degrees, one turn at a time, and the first turn that reads as upright text replaces it; with none, the page keeps its verdict. All 20 camera captures of printed pages in the corpus came sideways with no EXIF turn. - **Docling.** A garbled page goes to Docling in its place, through the splice a mixed PDF's flagged pages take, in batches that give way ([below](#long-docling-readings)). A PDF with no page that reads as text, and an image with a garbled frame or no frame of text, go to Docling whole, as before. What OCR read of a PDF whose garbled pages Docling then reads is kept in `ocr/` in the document's folder until the file is ready, so a Docling reading that gives the queue up resumes without reading every page with OCR again. The log says `RapidOCR reads 17 of 20 pages of scan.pdf in 31.2 s; Docling reads 5, 9-10: page 5 is garbled; ...`. - **Figures.** Ingest keeps none for a PDF read this way. A scan's figures wait until the agent or Studio asks for them, as a PDF over 40 pages does: the figures-only pass reads every scanned page with Docling in full, about 39 CPU-s a page, the cost this path removes. A PDF with born-digital pages beside its scanned ones still gets the pass queued at idle, as a text-read PDF does. An image is still its own figure. -- **Cancel and giving way.** Every 8 pages, about 17 s of RapidOCR's, a PDF's reading checks for a cancel and gives the queue up as a long Docling reading does, to the same sources ([below](#long-docling-readings)). Before it gives way it keeps the pages read so far in `ocr/pages.json` in the document's folder, stamped with the original's SHA-256 and the reader's name ([`ocr_kept.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py)), and it gives way only once they are on disk, never after its last page. The run that resumes reads only the pages left; the few in flight as it gave way are read again. A reading whose garbled pages go to Docling keeps its pages there too, for a resume should Docling's part give way. The file goes when the source is ready; when Docling reads the file whole instead, because no page read as text, OCR failed or OCR is off; when the job is cancelled or fails, in OCR's part or in Docling's after it; and when a cancel lands while it waits in the queue after giving way ([`ended_in_queue.py`](../../surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py)). A worker killed mid-reading keeps only what an earlier turn kept, and Retry resumes from it. An image's frames are read without a check. +- **Cancel and giving way.** Every 8 pages, about 17 s of RapidOCR's, a PDF's reading checks for a cancel and gives the queue up as a long Docling reading does, to the same sources ([below](#long-docling-readings)). Before it gives way it keeps the pages read so far in `ocr/pages.json` in the document's folder, stamped with the original's SHA-256 and the reader's name ([`ocr_kept.py`](../../surfsense_local/backend/worker/ingestion/pdf_reading/ocr_kept.py)), and it gives way only once they are on disk, never after its last page. The run that resumes reads only the pages left; the few in flight as it gave way are read again. A reading whose garbled pages go to Docling keeps its pages there too, for a resume should Docling's part give way. The file goes once the source's text is built, before it is cut into passages, so a failure after that reads every page again on Huey's retry; when Docling reads the file whole instead, because no page read as text, OCR failed or OCR is off; when the job is cancelled or fails, in OCR's part or in Docling's after it; and when a cancel lands while it waits in the queue after giving way ([`ended_in_queue.py`](../../surfsense_local/backend/modules/documents/docling_batches/ended_in_queue.py)). A worker killed mid-reading keeps only what an earlier turn kept, and Retry resumes from it. An image's frames are read without a check. - **The release build** runs `worker --check-scan-ocr` in the frozen worker on every system: each reader reads a rendered line, and an import the spec dropped fails the build. - **Measured** on the Ryzen 7 5800X, Windows 11, through the upload route and a real `worker.py ingest`, one upload at a time: the six public scanned PDFs and three book scans, 199 pages, were all ready in 509 s against 829 s through Docling, a file in 332 s on average against 598 s, at 1,752 worker CPU-s against 7,343; no page went to Docling. 40 receipt photos, receipt scans and forms were all ready in 113 s against 202 s, an image in 65 s on average against 129 s, at 131 worker CPU-s against 1,416.