Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions docs/architecture/local-models/catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,7 @@ Event types:
```text
queued "Waiting for the download ahead of it" only while another install runs; it starts when that one ends
starting "Checking the model" every install; only a searched build's headers are read
error the reason, and the job ends when the exact check or the disk refuses
error the reason, and the job ends when the exact check, a curated build's fit, or the disk refuses
starting "Preparing download"
downloading completed / total, repeated across every file of the build
verifying "Checking the model" each file with a hash was checked as it landed
Expand All @@ -475,8 +475,12 @@ only when whole and verified, a cancelled download resumes with a `Range`
request, and a file already where it lands with its pinned hash is not fetched
again. One install runs at a time: a second job starts as `queued` and waits
rather than failing, so a model chosen while another downloads still comes.
Before any byte moves, the files not yet on disk plus 1 GiB must fit in the
disk's free space, or the job ends with how much room the download needs.
A curated chat build is priced first, from the shape its entry commits and
through the same estimate its row shows, and one that is `TOO_BIG` here ends the
job with the refusal a searched build gets; a build that only spills installs
with no confirmation, as the screen's own rule says. Before any byte moves, the
files not yet on disk plus 1 GiB must fit in the disk's free space, or the job
ends with how much room the download needs.
Then the install record is written,
`reprice()` rewrites the preset, the job waits for the router to list the
model and forwards its load progress, and with `select` the model becomes the
Expand Down Expand Up @@ -640,7 +644,6 @@ and the screen in `download-chat-models.test.tsx`, `install-view.test.tsx` and t
- An install that fails after the weights landed but before the projector did writes no install record. The curated row then shows the build installed, matched by file name, and it loads as text only.
- Only the three audio defaults are validated; `validated` is empty on every other build.
- `sampling`, `template.system_role` and llama.cpp's `run.args` are committed but nothing reads them, so chat does not use the publisher's sampling yet. sd.cpp's `image` defaults and `run.args` reach sd-server as launch flags. `template.tools` and `template.reasoning` reach a row's support, which the screen does not show.
- `POST /llm/installs` does not refuse a curated build that will not fit; only the screen's disabled Download does.
- A gated repo is marked "Needs an account", but the app sends no Hugging Face credential, so installing one of its builds fails with the generic install error.
- The API does not cache search and nothing debounces typing: once the query has two characters, every keystroke sends a request, unless the renderer's 300 s cache holds that exact query.
- A curated file that can no longer be fetched at its pinned commit, because the repo was deleted, gated or made private, gets the generic install error, and so does a checksum mismatch; nothing says which.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
DownloadedModel,
scan,
)
from modules.llm.catalog.local.engines.llamacpp.pricing import price
from modules.llm.catalog.local.engines.llamacpp.rows.catalog import local_catalog
from modules.llm.catalog.local.engines.llamacpp.search.exact_check import check_build
from modules.llm.catalog.local.engines.llamacpp.search.hits import (
Expand All @@ -33,6 +34,8 @@
from modules.llm.model_type import ModelType
from modules.llm.providers.llamacpp import PROVIDER

_TOO_BIG = "This build is too big for this computer. Pick a smaller one."


class LlamaCppEngine:
name = ENGINE
Expand Down Expand Up @@ -98,6 +101,19 @@ async def check(self, plan: InstallPlan) -> InstallPlan:
"""Read a searched build's headers and refuse what cannot run, before any
bytes move. Returns the plan with a failed projector dropped."""
if not plan.needs_check:
# A curated build's header was read at refresh time; whether it fits
# this machine was not. Priced as its row is, and only TOO_BIG
# refuses: a build that spills runs, slower, and stays the user's
# to choose.
projector = plan.build.projector
fit = price(
plan.shape,
plan.build.weights_bytes,
projector.size_bytes if projector else 0,
self._budget(BudgetMode.CAPACITY),
)
if fit.state is FitState.TOO_BIG:
raise InstallRefusedError(_TOO_BIG)
return plan
async with httpx.AsyncClient(follow_redirects=True) as client:
checked = await check_build(
Expand All @@ -110,9 +126,7 @@ async def check(self, plan: InstallPlan) -> InstallPlan:
checked.classification.reason or "SurfSense cannot run this model."
)
if checked.fit.state is FitState.TOO_BIG:
raise InstallRefusedError(
"This build is too big for this computer. Pick a smaller one."
)
raise InstallRefusedError(_TOO_BIG)
return InstallPlan(
plan.model_id, checked.build, self.name, pipeline_tag=plan.pipeline_tag
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from dataclasses import dataclass

from modules.llm.catalog.local.build import Build
from modules.llm.fit import ModelShape


class InstallRefusedError(Exception):
Expand All @@ -19,3 +20,6 @@ class InstallPlan:
# read when the manifest was refreshed.
needs_check: bool = False
pipeline_tag: str | None = None
# A curated chat build's committed shape, so whether it fits here is priced
# from the manifest, as its row is, with no header read.
shape: ModelShape | None = None
12 changes: 10 additions & 2 deletions surfsense_local/backend/modules/llm/catalog/local/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from modules.llm.catalog.local.installs import forget_install
from modules.llm.catalog.local.manifest import LocalManifest
from modules.llm.catalog.local.rows import LocalRow
from modules.llm.fit import HardwareBudget
from modules.llm.fit import HardwareBudget, ModelShape
from modules.llm.hardware import (
BudgetMode,
Device,
Expand Down Expand Up @@ -75,6 +75,13 @@ def __init__(
self._tickets = TicketStore()
self._curated_ids: dict[str, tuple[Build, str]] = {}
self._curated_tokens: dict[tuple[str, str], str] = {}
# The shape each curated build was committed with, by the key an id is
# minted on, so an install can be priced where the manifest is known.
self._curated_shapes: dict[tuple[str, str], ModelShape | None] = {
(build.weights.repo, build.weights.path): model.model_shape
for model in manifest.models
for build in model.as_builds()
}
# One pull at a time: two downloads compete for one disk and one bar.
self._install_lock = asyncio.Lock()
self._install_jobs = InstallJobs(self._install_lock)
Expand Down Expand Up @@ -182,7 +189,8 @@ def resolve_install(self, catalog_id: str) -> InstallPlan | None:
curated = self._curated_ids.get(catalog_id)
if curated is not None:
build, engine = curated
return InstallPlan(build.runtime_name, build, engine)
shape = self._curated_shapes.get((build.weights.repo, build.weights.path))
return InstallPlan(build.runtime_name, build, engine, shape=shape)
ticket = self._tickets.resolve(catalog_id)
if ticket is None:
return None
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
"""What one install does when the check refuses it."""

import pytest

from modules.llm.catalog.local.build import Build, BuildFile, FileRole
from modules.llm.catalog.local.install.plan import InstallPlan, InstallRefusedError
from modules.llm.catalog.local.install_jobs.steps import install_steps

pytestmark = [pytest.mark.unit, pytest.mark.asyncio]

REFUSAL = "This build is too big for this computer. Pick a smaller one."


class RefusingService:
"""A catalog whose check refuses, and whose downloader must never run."""

async def check(self, plan: InstallPlan) -> InstallPlan:
raise InstallRefusedError(REFUSAL)

def install(self, plan: InstallPlan):
raise AssertionError("no bytes may move after a refusal")


async def test_a_refused_plan_ends_the_job_with_its_reason_and_moves_no_bytes() -> None:
"""A refusal is the job's only event: the reason a person reads, then the end."""
build = Build(
"Q4_K_M",
(BuildFile(FileRole.WEIGHTS, "m-Q4_K_M.gguf", 1_000, "a" * 64, "r/m", "s"),),
)
plan = InstallPlan("m-Q4_K_M", build, "llamacpp")

events = [
event
async for event in install_steps(
RefusingService(), # type: ignore[arg-type]
plan,
select=False,
model_type=None,
session_factory=lambda: None, # type: ignore[arg-type,return-value]
)
]

assert events == [{"type": "error", "message": REFUSAL}]
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import httpx
import pytest

from modules.llm.catalog.local import service as service_module
from modules.llm.catalog.local.build import Build, BuildFile, FileRole
from modules.llm.catalog.local.install import download as download_module
from modules.llm.catalog.local.install.plan import InstallPlan, InstallRefusedError
Expand Down Expand Up @@ -408,6 +409,43 @@ def test_a_curated_id_resolves_to_its_pinned_build_without_a_check(service) -> N
assert service.resolve_install("never-minted") is None


MIB = 1024**2


def curated_plan(service: LocalCatalogService, row_id: str) -> InstallPlan:
"""The row's leading build, resolved the way the install route does it."""
row = next(r for r in service.catalog().rows if r.id == row_id)
plan = service.resolve_install(row.builds[0].catalog_id)
assert plan is not None
return plan


async def test_a_curated_build_that_will_not_fit_is_refused_before_download(
service, monkeypatch, fake_hub
) -> None:
"""The screen's disabled Download is not the only thing in the way: a plan
that reaches the route from anywhere else is priced from its committed
shape and refused as a searched build is, before any bytes move."""
monkeypatch.setattr(service_module, "available_bytes", lambda: 256 * MIB)
plan = curated_plan(service, "qwen3-32b")

with pytest.raises(InstallRefusedError, match="too big"):
await service.check(plan)

assert fake_hub == []


async def test_a_curated_build_that_merely_spills_installs_with_no_confirmation(
service, monkeypatch
) -> None:
"""Only TOO_BIG blocks. A build that runs slower is the user's to choose,
and refusing it would make a working model unreachable."""
monkeypatch.setattr(service_module, "available_bytes", lambda: 64 * 1024 * MIB)
plan = curated_plan(service, "qwen3-0.6b")

assert await service.check(plan) == plan


def header(architecture: str = "qwen3", *, too_big: bool = False) -> bytes:
"""A chat model header, or one too big for any machine."""
blocks = 4000 if too_big else 28
Expand Down
Loading