diff --git a/.github/workflows/build-sdcpp.yml b/.github/workflows/build-sdcpp.yml index c8f50b12a1..9b20fdc094 100644 --- a/.github/workflows/build-sdcpp.yml +++ b/.github/workflows/build-sdcpp.yml @@ -1,7 +1,8 @@ # Compile stable-diffusion.cpp's sd-server for Linux and macOS with the scripts # in surfsense_local/electron/scripts/sdcpp/, check each against the app's -# floors, and hand the staged folder to the desktop release as an artifact. A -# pull request that changes the scripts runs it too. Windows takes upstream's +# floors and on generating an image, and hand the staged folder to the desktop +# release as an artifact. A pull request that changes the scripts, the image +# engine or that test runs it too. Windows takes upstream's # archive, a pinned download, which the release stages itself. # # TODO: ask upstream (leejet/stable-diffusion.cpp) to publish a Linux archive @@ -15,6 +16,10 @@ on: paths: - surfsense_local/electron/scripts/sdcpp/** - surfsense_local/electron/scripts/not-staged/** + - surfsense_local/backend/modules/llm/providers/sdcpp/** + - surfsense_local/backend/modules/llm/catalog/local/engines/sdcpp/** + - surfsense_local/backend/modules/llm/catalog/local/manifest/models.json + - surfsense_local/backend/tests/packaging/test_sdcpp_generation.py - .github/workflows/build-sdcpp.yml permissions: @@ -106,6 +111,45 @@ jobs: done exit $bad + - name: Setup uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + python-version: "3.12" + + # The staged server, started with the sidecar's flags on the cheapest + # curated model, asked for one image through the app's own client; the + # reply is checked as a picture. Run on a cached folder too: the manifest + # and the client can change without the scripts changing. + - name: Gate on a curated model generating an image + working-directory: surfsense_local/backend + env: + SURFSENSE_TEST_SDCPP_DIR: ${{ github.workspace }}/surfsense_local/electron/sdcpp + SURFSENSE_TEST_IMAGE_MODELS: ${{ runner.temp }}/image-models + # The hosted macOS runner's graphics card is virtual: Metal's buffers + # come back nil on it and the server dies on its first image. + SURFSENSE_TEST_SDCPP_BACKEND: ${{ runner.os == 'macOS' && 'cpu' || '' }} + run: | + uv sync + uv run python - <<'EOF' + import os + import urllib.request + from pathlib import Path + + from modules.llm.catalog.local.manifest import load_local_manifest + + folder = Path(os.environ["SURFSENSE_TEST_IMAGE_MODELS"]) + folder.mkdir(parents=True, exist_ok=True) + (model,) = [m for m in load_local_manifest().models if m.id == "stable-diffusion-1.5"] + pinned = model.builds[0].files[0] + url = f"https://huggingface.co/{pinned.repo}/resolve/{pinned.revision}/{pinned.path}" + print(url, flush=True) + urllib.request.urlretrieve(url, folder / pinned.path.rsplit("/", 1)[-1]) + EOF + # addopts pins -m 'not packaging'; the later -m on the command line wins. + # -s: the test prints how long the image took. + uv run pytest tests/packaging/test_sdcpp_generation.py -m packaging -s + # An artifact drops symlinks and file modes; a tar keeps both. - name: Pack run: tar -cf sdcpp.tar sdcpp diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a9665efcb0..4eeb6ab49b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -4,7 +4,7 @@ What the maintainers are working on, grouped by when. This page names the initia ## Now -- **Release and CI health.** v2.0.3 was the first release to carry the llama.cpp, sd.cpp and audio.cpp runtimes, built and packaged on all three runners, and [`desktop-tests.yml`](../.github/workflows/desktop-tests.yml) runs the desktop backend, frontend and Electron tests on pull requests. What is left is the CI that proves the runtimes: no job generates an image with the staged `sd-server`, and the release workflow still accepts a prerelease version. See [packaging](architecture/packaging.md) and [updates](architecture/updates.md). +- **Release and CI health.** v2.0.3 was the first release to carry the llama.cpp, sd.cpp and audio.cpp runtimes, built and packaged on all three runners, and [`desktop-tests.yml`](../.github/workflows/desktop-tests.yml) runs the desktop backend, frontend and Electron tests on pull requests. What is left is the CI that proves the runtimes: the release workflow still accepts a prerelease version. See [packaging](architecture/packaging.md) and [updates](architecture/updates.md). - **Egress gaps.** The follow-up fetch for a remote image URL leaves without a consent decision, and some stored grants no longer match what Settings › Network shows. See [egress](architecture/egress.md). - **Import from cloud.** The hosted export window closes on 18 Oct 2026. Imports need a progress summary, and a re-run must bring back the chat threads an interrupted import missed. See [import](architecture/import.md). - **Scraper API and MCP license mode.** PATs are purged on 18 Oct 2026; after that, license mode is how scraper API and MCP users keep access. See [contract 2](contracts/02-scraper-api-auth.md). diff --git a/docs/architecture/packaging.md b/docs/architecture/packaging.md index 7d4b400b18..6e422fe4fd 100644 --- a/docs/architecture/packaging.md +++ b/docs/architecture/packaging.md @@ -101,7 +101,7 @@ It is a stopgap. Once upstream publishes archives that meet both floors, the app ## Building sd.cpp -[`build-sdcpp.yml`](../../.github/workflows/build-sdcpp.yml) compiles the Linux and macOS builds with `stage.mjs --strict`, checks each against the app's floors, and hands each staged folder on as an artifact, as `build-audiocpp.yml` does. A pull request that changes `scripts/sdcpp/` or `scripts/not-staged/` runs it too, so a compile that a pin bump or a new runner image breaks fails in review rather than in a release. It caches the staged folder by the scripts' contents. Windows is a pinned download, so the release job stages it itself, as it does llama.cpp. +[`build-sdcpp.yml`](../../.github/workflows/build-sdcpp.yml) compiles the Linux and macOS builds with `stage.mjs --strict`, checks each against the app's floors, runs `test_sdcpp_generation.py` on the staged folder, cached or not, with Stable Diffusion 1.5's pinned file downloaded from the URL its manifest entry names ([Packaging tests](#packaging-tests)), and hands each staged folder on as an artifact, as `build-audiocpp.yml` does. A pull request that changes `scripts/sdcpp/`, `scripts/not-staged/`, the image engine, the local manifest or that test runs it too, so a compile that a pin bump or a new runner image breaks, or a server that starts and cannot diffuse, fails in review rather than in a release. It caches the staged folder by the scripts' contents. Windows is a pinned download, so the release job stages it itself, as it does llama.cpp. Why it compiles on Linux and macOS, in plain words: upstream's ready-made programs start only on the newest systems there, Ubuntu 24.04 (glibc 2.38) and macOS 26.0, while the app runs on Ubuntu 22.04, RHEL 9 and macOS 13.3, the floor its llama.cpp and audio.cpp builds already set. Upstream's Windows program runs on any x64 processor, since its AVX-512 code sits only in the per-processor ggml libraries ggml picks at start, so Windows takes it as built. @@ -135,7 +135,7 @@ The installer's version is the one the workflow resolved, passed as `-c.extraMet ## Packaging tests -All six carry the `packaging` marker, which `pyproject.toml` excludes by default because three of them freeze a binary and one runs audio.cpp's server on real models: +All seven carry the `packaging` marker, which `pyproject.toml` excludes by default because three of them freeze a binary, one runs audio.cpp's server on real models and one runs sd.cpp's on a 3 GB one: | Test | Proves | |---|---| @@ -145,12 +145,12 @@ All six carry the `packaging` marker, which `pyproject.toml` excludes by default | `test_spec_data_files.py` | every literal `datas` path in the specs exists, so a renamed file cannot ship missing, and both `api.spec` and `worker.spec` bundle the local model manifest | | `test_license_key.py` | the compiled license keys exclude the fixture key | | `test_audiocpp_voicing.py` | the staged audio.cpp server, started with the sidecar's flags from the `server.json` the app writes, voices two turns of each curated model through the app's adapter, at the sample rate its entry names, then holds no model loaded; it runs only when `SURFSENSE_TEST_AUDIO_MODELS` names a folder of the pinned files, each checked against its sha256, and then fails without a staged server; `build-audiocpp.yml` runs it on both builds | +| `test_sdcpp_generation.py` | the staged sd.cpp server, started with the sidecar's flags on Stable Diffusion 1.5 as the engine reads an installed build, makes one image of a red apple through the app's image client, and the reply is checked for what such a picture has, not for the apple itself: it decodes, is the 512 pixels square the entry asks for, has more than a thousand colours, is red on average, and its middle is redder than its border, which neither a flat frame nor noise is; it runs at 4 steps and on seed 42, sd-server's default written down, the two flags the app does not pass (and `--backend cpu` on the hosted macOS runner, whose virtual graphics card Metal cannot use, so Metal itself is not exercised in CI), and only when `SURFSENSE_TEST_IMAGE_MODELS` names a folder holding the pinned file, checked against its sha256, and then fails without a staged server | -Two run in CI: `test_license_key.py` inside the release workflow, and `test_audiocpp_voicing.py` inside `build-audiocpp.yml`. +Three run in CI: `test_license_key.py` inside the release workflow, `test_audiocpp_voicing.py` inside `build-audiocpp.yml`, and `test_sdcpp_generation.py` inside `build-sdcpp.yml`. Windows' `sd-server` is a pinned download the release job stages, so that gate does not cover it. ## Known gaps -- No CI job generates an image with the staged `sd-server`; its gates are `--help` and the platform floors, not a picture. - No issue on audio.cpp asks for archives that meet the app's floors yet, so `build-audiocpp.yml` has no end date. - No packaging test checks that the frozen API carries the agent's `SKILL.md`; `api.spec` collects it, and it was checked by hand. - What matplotlib adds to the installer has not been measured against its size gate. A scratch freeze of the worker put it at about 15 MB unpacked under `_internal/matplotlib`, plus about 0.6 MB for contourpy and kiwisolver. diff --git a/surfsense_local/backend/tests/packaging/test_sdcpp_generation.py b/surfsense_local/backend/tests/packaging/test_sdcpp_generation.py new file mode 100644 index 0000000000..c2eea86505 --- /dev/null +++ b/surfsense_local/backend/tests/packaging/test_sdcpp_generation.py @@ -0,0 +1,216 @@ +"""The staged sd.cpp server makes a picture: started the way the app starts it, +on a curated model, asked through the app's own image client. + +`--help` and the platform floors show the server starts and links only what it +may. They do not show it can diffuse: a backend library pruned one file too +far, or a pin bump that renamed a tensor, passes both. + +Opt-in, like the other packaging tests, and skipped unless +SURFSENSE_TEST_IMAGE_MODELS names a folder holding the curated model's weights +under their published name, checked against the manifest's sha256. Once it +does, a missing server (`pnpm build:sdcpp`, or SURFSENSE_TEST_SDCPP_DIR) fails +the run: CI sets the folder, and a skip there would pass unseen. +SURFSENSE_TEST_SDCPP_BACKEND forces a backend, for a machine whose graphics card +the server cannot use. +""" + +import asyncio +import hashlib +import io +import os +import shutil +import socket +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path + +import httpx +import pytest +from PIL import Image, ImageStat + +from modules.llm.catalog.local.engines.sdcpp.engine import SdCppEngine +from modules.llm.catalog.local.engines.sdcpp.images_folder.installed import ( + InstalledImage, +) +from modules.llm.catalog.local.installs import InstalledBuild, record_install +from modules.llm.catalog.local.manifest import load_local_manifest +from modules.llm.providers.openai_compatible import image as image_client +from modules.llm.providers.openai_compatible.image import ( + OpenAICompatibleImageProvider, +) +from modules.llm.providers.sdcpp.serving import wait_until_serving + +pytestmark = pytest.mark.packaging + +ELECTRON = Path(__file__).resolve().parents[3] / "electron" +STAGED = Path(os.environ.get("SURFSENSE_TEST_SDCPP_DIR", ELECTRON / "sdcpp")).resolve() +MODELS_DIR = os.environ.get("SURFSENSE_TEST_IMAGE_MODELS") +SERVER = STAGED / ("sd-server.exe" if sys.platform == "win32" else "sd-server") + +# The cheapest curated image model: one 3 GB file that carries its own VAE and +# text encoder. +MODEL_ID = "stable-diffusion-1.5" +# Set to `cpu` where the graphics card cannot be used: a hosted macOS runner's +# is virtual, and Metal's buffers come back nil on it. Unset, the server picks +# its backend as it does in the app. +BACKEND = os.environ.get("SURFSENSE_TEST_SDCPP_BACKEND") +# Not the entry's: enough steps for a picture, few enough for a runner with no +# graphics card. With the seed, the only flags the app does not pass. +STEPS = 4 + +if MODELS_DIR is None: + pytest.skip("needs SURFSENSE_TEST_IMAGE_MODELS", allow_module_level=True) + +(CURATED,) = [m for m in load_local_manifest().models if m.id == MODEL_ID] + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +@pytest.fixture(scope="module") +def installed(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, InstalledImage]: + """The pinned build linked into an images folder and recorded, as an + install leaves it, and read back the way the API reads it for Electron.""" + images = tmp_path_factory.mktemp("images") + build = CURATED.builds[0] + pinned = build.files[0] + name = pinned.path.rsplit("/", 1)[-1] + source = (Path(MODELS_DIR) / name).resolve() # type: ignore[arg-type] + assert _sha256(source) == pinned.sha256, f"{name} is not the pinned file" + try: + (images / name).symlink_to(source) + except OSError: # Windows without the symlink privilege + shutil.copyfile(source, images / name) + model_id = name.removesuffix(".gguf") + record_install( + images, + InstalledBuild( + model_id=model_id, + repo=pinned.repo, + revision=pinned.revision, + quantization=build.quantization, + weights=(name,), + ), + ) + image = SdCppEngine(images, [CURATED]).installed_image(model_id) + assert image is not None, "the engine does not see the build as installed" + return images, image + + +@pytest.fixture(scope="module") +def server_log(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Where the server writes; a test that fails shows its tail.""" + return tmp_path_factory.mktemp("server") / "sdcpp.log" + + +@pytest.fixture(scope="module") +def server(installed: tuple[Path, InstalledImage], server_log: Path) -> Iterator[str]: + """The server as Electron's sidecar starts it (`sdcppSpec()` in + `sidecars/sdcpp.ts`): each file on its flag, the port, `--diffusion-fa`, + the entry's own arguments, from the staged folder.""" + images, image = installed + assert SERVER.exists(), f"no staged sd.cpp server at {SERVER}" + log = server_log.open("wb") + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + files = [part for flag, path in image.files for part in (flag, str(images / path))] + process = subprocess.Popen( + [ + str(SERVER), + *files, + "--listen-port", + str(port), + "--diffusion-fa", + *image.args, + "--steps", + str(STEPS), + # sd-server's own default, written down: the picture checks below + # are calibrated on this seed's apple, and must not move with it. + "--seed", + "42", + *(["--backend", BACKEND] if BACKEND else []), + ], + cwd=STAGED, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + yield f"http://127.0.0.1:{port}" + finally: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + log.close() + + +async def _refuse_any_host(url: str) -> None: + raise AssertionError(f"sd-server pointed at {url} for its image") + + +def test_the_staged_server_generates_a_picture( + installed: tuple[Path, InstalledImage], + server: str, + server_log: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An image of the size the entry asks for, with the prompt's subject in it.""" + _images, image = installed + # The app waits 180 s for an image, on a machine with a graphics card. A + # hosted runner has none and took 134 s; the gate is on the picture, not + # on the runner's speed. + monkeypatch.setattr(image_client, "TIMEOUT", httpx.Timeout(600.0, connect=5.0)) + client = OpenAICompatibleImageProvider( + 0, f"{server}/v1", allow_url_host=_refuse_any_host + ) + + async def generate() -> bytes: + await wait_until_serving(server, image.served_file, timeout=180.0) + made = await client.generate( + image.model_id, + "a red apple on a wooden table, photograph", + ) + return made.content + + started = time.monotonic() + try: + content = asyncio.run(generate()) + except Exception: + tail = server_log.read_text(errors="replace").splitlines()[-60:] + sys.stderr.write("\n".join(["sd-server's log, last lines:", *tail])) + raise + # The run's cost, for whoever weighs this gate next (shown with `-s`). + took = time.monotonic() - started + sys.stderr.write(f"sd-server generated in {took:.0f} s at {STEPS} steps\n") + + assert CURATED.image is not None + side = CURATED.image.resolution + picture = Image.open(io.BytesIO(content)) + picture.load() # decodes every pixel, not only the header + assert picture.size == (side, side) + # A blank canvas decodes and has the right size too, and so does noise. The + # prompt is a red apple, so ask for that much of it: the frame is red, and + # its middle is brighter than its border, which is an object on a + # background. Measured at 4 steps on Metal and on the CPU: the middle's red + # is 218, the border's 143, and green and blue stay under 10. An unfinished + # frame of noise or of one flat wash has neither property. + rgb = picture.convert("RGB") + colours = rgb.getcolors(maxcolors=side * side) + assert colours is not None + assert len(colours) > 1000, f"only {len(colours)} colours" + red, green, blue = ImageStat.Stat(rgb).mean + assert red > 100 and red > 3 * max(green, blue), (red, green, blue) + quarter = side // 4 + middle = ImageStat.Stat(rgb.crop((quarter, quarter, 3 * quarter, 3 * quarter))) + # The middle is a quarter of the frame, so the border's mean follows. + border_red = (4 * red - middle.mean[0]) / 3 + assert middle.mean[0] - border_red > 40, (middle.mean[0], border_red)