diff --git a/docs/architecture/local-models/catalog.md b/docs/architecture/local-models/catalog.md index f9cb723f5f..2f3b01d15a 100644 --- a/docs/architecture/local-models/catalog.md +++ b/docs/architecture/local-models/catalog.md @@ -452,7 +452,7 @@ Event types: ```text queued "Waiting for the download ahead of it" only while another install runs; it starts when that one ends starting "Checking the model" every install; only a searched build's headers are read -error the reason, and the job ends when the exact check or the disk refuses +error the reason, and the job ends when the exact check, a curated build's fit, or the disk refuses starting "Preparing download" downloading completed / total, repeated across every file of the build verifying "Checking the model" each file with a hash was checked as it landed @@ -475,8 +475,12 @@ only when whole and verified, a cancelled download resumes with a `Range` request, and a file already where it lands with its pinned hash is not fetched again. One install runs at a time: a second job starts as `queued` and waits rather than failing, so a model chosen while another downloads still comes. -Before any byte moves, the files not yet on disk plus 1 GiB must fit in the -disk's free space, or the job ends with how much room the download needs. +A curated chat build is priced first, from the shape its entry commits and +through the same estimate its row shows, and one that is `TOO_BIG` here ends the +job with the refusal a searched build gets; a build that only spills installs +with no confirmation, as the screen's own rule says. Before any byte moves, the +files not yet on disk plus 1 GiB must fit in the disk's free space, or the job +ends with how much room the download needs. Then the install record is written, `reprice()` rewrites the preset, the job waits for the router to list the model and forwards its load progress, and with `select` the model becomes the @@ -640,7 +644,6 @@ and the screen in `download-chat-models.test.tsx`, `install-view.test.tsx` and t - An install that fails after the weights landed but before the projector did writes no install record. The curated row then shows the build installed, matched by file name, and it loads as text only. - Only the three audio defaults are validated; `validated` is empty on every other build. - `sampling`, `template.system_role` and llama.cpp's `run.args` are committed but nothing reads them, so chat does not use the publisher's sampling yet. sd.cpp's `image` defaults and `run.args` reach sd-server as launch flags. `template.tools` and `template.reasoning` reach a row's support, which the screen does not show. -- `POST /llm/installs` does not refuse a curated build that will not fit; only the screen's disabled Download does. - A gated repo is marked "Needs an account", but the app sends no Hugging Face credential, so installing one of its builds fails with the generic install error. - The API does not cache search and nothing debounces typing: once the query has two characters, every keystroke sends a request, unless the renderer's 300 s cache holds that exact query. - A curated file that can no longer be fetched at its pinned commit, because the repo was deleted, gated or made private, gets the generic install error, and so does a checksum mismatch; nothing says which. diff --git a/surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/engine.py b/surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/engine.py index 3faa84debc..e464a8352b 100644 --- a/surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/engine.py +++ b/surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/engine.py @@ -16,6 +16,7 @@ DownloadedModel, scan, ) +from modules.llm.catalog.local.engines.llamacpp.pricing import price from modules.llm.catalog.local.engines.llamacpp.rows.catalog import local_catalog from modules.llm.catalog.local.engines.llamacpp.search.exact_check import check_build from modules.llm.catalog.local.engines.llamacpp.search.hits import ( @@ -33,6 +34,8 @@ from modules.llm.model_type import ModelType from modules.llm.providers.llamacpp import PROVIDER +_TOO_BIG = "This build is too big for this computer. Pick a smaller one." + class LlamaCppEngine: name = ENGINE @@ -98,6 +101,19 @@ async def check(self, plan: InstallPlan) -> InstallPlan: """Read a searched build's headers and refuse what cannot run, before any bytes move. Returns the plan with a failed projector dropped.""" if not plan.needs_check: + # A curated build's header was read at refresh time; whether it fits + # this machine was not. Priced as its row is, and only TOO_BIG + # refuses: a build that spills runs, slower, and stays the user's + # to choose. + projector = plan.build.projector + fit = price( + plan.shape, + plan.build.weights_bytes, + projector.size_bytes if projector else 0, + self._budget(BudgetMode.CAPACITY), + ) + if fit.state is FitState.TOO_BIG: + raise InstallRefusedError(_TOO_BIG) return plan async with httpx.AsyncClient(follow_redirects=True) as client: checked = await check_build( @@ -110,9 +126,7 @@ async def check(self, plan: InstallPlan) -> InstallPlan: checked.classification.reason or "SurfSense cannot run this model." ) if checked.fit.state is FitState.TOO_BIG: - raise InstallRefusedError( - "This build is too big for this computer. Pick a smaller one." - ) + raise InstallRefusedError(_TOO_BIG) return InstallPlan( plan.model_id, checked.build, self.name, pipeline_tag=plan.pipeline_tag ) diff --git a/surfsense_local/backend/modules/llm/catalog/local/install/plan.py b/surfsense_local/backend/modules/llm/catalog/local/install/plan.py index 2c3ce9b060..8e1d24d136 100644 --- a/surfsense_local/backend/modules/llm/catalog/local/install/plan.py +++ b/surfsense_local/backend/modules/llm/catalog/local/install/plan.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from modules.llm.catalog.local.build import Build +from modules.llm.fit import ModelShape class InstallRefusedError(Exception): @@ -19,3 +20,6 @@ class InstallPlan: # read when the manifest was refreshed. needs_check: bool = False pipeline_tag: str | None = None + # A curated chat build's committed shape, so whether it fits here is priced + # from the manifest, as its row is, with no header read. + shape: ModelShape | None = None diff --git a/surfsense_local/backend/modules/llm/catalog/local/service.py b/surfsense_local/backend/modules/llm/catalog/local/service.py index 8fd1e8a917..b0b95652f3 100644 --- a/surfsense_local/backend/modules/llm/catalog/local/service.py +++ b/surfsense_local/backend/modules/llm/catalog/local/service.py @@ -22,7 +22,7 @@ from modules.llm.catalog.local.installs import forget_install from modules.llm.catalog.local.manifest import LocalManifest from modules.llm.catalog.local.rows import LocalRow -from modules.llm.fit import HardwareBudget +from modules.llm.fit import HardwareBudget, ModelShape from modules.llm.hardware import ( BudgetMode, Device, @@ -75,6 +75,13 @@ def __init__( self._tickets = TicketStore() self._curated_ids: dict[str, tuple[Build, str]] = {} self._curated_tokens: dict[tuple[str, str], str] = {} + # The shape each curated build was committed with, by the key an id is + # minted on, so an install can be priced where the manifest is known. + self._curated_shapes: dict[tuple[str, str], ModelShape | None] = { + (build.weights.repo, build.weights.path): model.model_shape + for model in manifest.models + for build in model.as_builds() + } # One pull at a time: two downloads compete for one disk and one bar. self._install_lock = asyncio.Lock() self._install_jobs = InstallJobs(self._install_lock) @@ -182,7 +189,8 @@ def resolve_install(self, catalog_id: str) -> InstallPlan | None: curated = self._curated_ids.get(catalog_id) if curated is not None: build, engine = curated - return InstallPlan(build.runtime_name, build, engine) + shape = self._curated_shapes.get((build.weights.repo, build.weights.path)) + return InstallPlan(build.runtime_name, build, engine, shape=shape) ticket = self._tickets.resolve(catalog_id) if ticket is None: return None diff --git a/surfsense_local/backend/tests/unit/llm/catalog/local/test_install_steps.py b/surfsense_local/backend/tests/unit/llm/catalog/local/test_install_steps.py new file mode 100644 index 0000000000..a4311d9491 --- /dev/null +++ b/surfsense_local/backend/tests/unit/llm/catalog/local/test_install_steps.py @@ -0,0 +1,43 @@ +"""What one install does when the check refuses it.""" + +import pytest + +from modules.llm.catalog.local.build import Build, BuildFile, FileRole +from modules.llm.catalog.local.install.plan import InstallPlan, InstallRefusedError +from modules.llm.catalog.local.install_jobs.steps import install_steps + +pytestmark = [pytest.mark.unit, pytest.mark.asyncio] + +REFUSAL = "This build is too big for this computer. Pick a smaller one." + + +class RefusingService: + """A catalog whose check refuses, and whose downloader must never run.""" + + async def check(self, plan: InstallPlan) -> InstallPlan: + raise InstallRefusedError(REFUSAL) + + def install(self, plan: InstallPlan): + raise AssertionError("no bytes may move after a refusal") + + +async def test_a_refused_plan_ends_the_job_with_its_reason_and_moves_no_bytes() -> None: + """A refusal is the job's only event: the reason a person reads, then the end.""" + build = Build( + "Q4_K_M", + (BuildFile(FileRole.WEIGHTS, "m-Q4_K_M.gguf", 1_000, "a" * 64, "r/m", "s"),), + ) + plan = InstallPlan("m-Q4_K_M", build, "llamacpp") + + events = [ + event + async for event in install_steps( + RefusingService(), # type: ignore[arg-type] + plan, + select=False, + model_type=None, + session_factory=lambda: None, # type: ignore[arg-type,return-value] + ) + ] + + assert events == [{"type": "error", "message": REFUSAL}] diff --git a/surfsense_local/backend/tests/unit/llm/catalog/local/test_service.py b/surfsense_local/backend/tests/unit/llm/catalog/local/test_service.py index 3fc30826eb..862feef415 100644 --- a/surfsense_local/backend/tests/unit/llm/catalog/local/test_service.py +++ b/surfsense_local/backend/tests/unit/llm/catalog/local/test_service.py @@ -8,6 +8,7 @@ import httpx import pytest +from modules.llm.catalog.local import service as service_module from modules.llm.catalog.local.build import Build, BuildFile, FileRole from modules.llm.catalog.local.install import download as download_module from modules.llm.catalog.local.install.plan import InstallPlan, InstallRefusedError @@ -408,6 +409,43 @@ def test_a_curated_id_resolves_to_its_pinned_build_without_a_check(service) -> N assert service.resolve_install("never-minted") is None +MIB = 1024**2 + + +def curated_plan(service: LocalCatalogService, row_id: str) -> InstallPlan: + """The row's leading build, resolved the way the install route does it.""" + row = next(r for r in service.catalog().rows if r.id == row_id) + plan = service.resolve_install(row.builds[0].catalog_id) + assert plan is not None + return plan + + +async def test_a_curated_build_that_will_not_fit_is_refused_before_download( + service, monkeypatch, fake_hub +) -> None: + """The screen's disabled Download is not the only thing in the way: a plan + that reaches the route from anywhere else is priced from its committed + shape and refused as a searched build is, before any bytes move.""" + monkeypatch.setattr(service_module, "available_bytes", lambda: 256 * MIB) + plan = curated_plan(service, "qwen3-32b") + + with pytest.raises(InstallRefusedError, match="too big"): + await service.check(plan) + + assert fake_hub == [] + + +async def test_a_curated_build_that_merely_spills_installs_with_no_confirmation( + service, monkeypatch +) -> None: + """Only TOO_BIG blocks. A build that runs slower is the user's to choose, + and refusing it would make a working model unreachable.""" + monkeypatch.setattr(service_module, "available_bytes", lambda: 64 * 1024 * MIB) + plan = curated_plan(service, "qwen3-0.6b") + + assert await service.check(plan) == plan + + def header(architecture: str = "qwen3", *, too_big: bool = False) -> bytes: """A chat model header, or one too big for any machine.""" blocks = 4000 if too_big else 28