Recipes, tools/bench.py and a KernelForge task: tuned settings as data, every number measured

CMakeLists.txt

@@ -58,7 +58,15 @@ FetchContent_Declare(httplib
     GIT_TAG fe332fa06bac76a1c6d402c08f414052999347da)  # v0.47.0
 FetchContent_MakeAvailable(json httplib)
 
-add_executable(onebit app/main.cpp app/serve.cpp)
+add_executable(onebit app/main.cpp app/serve.cpp app/recipes.cpp)
+# config/recipes.json (tuned llama-server settings per model and route, docs/recipes.md) is
+# compiled in as serve's default recipes; --recipes replaces them at run time.
+file(READ ${CMAKE_SOURCE_DIR}/config/recipes.json ONEBIT_RECIPES_JSON)
+file(WRITE ${CMAKE_BINARY_DIR}/generated/recipes_default.h
+     "// generated from config/recipes.json by CMakeLists.txt\n#pragma once\n"
+     "static const char kDefaultRecipesJson[] = R\"ONEBITRECIPES(${ONEBIT_RECIPES_JSON})ONEBITRECIPES\";\n")
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/config/recipes.json)
+target_include_directories(onebit PRIVATE ${CMAKE_BINARY_DIR}/generated)
 set_target_properties(onebit PROPERTIES OUTPUT_NAME 1bit)
 target_link_libraries(onebit PRIVATE nlohmann_json::nlohmann_json httplib::httplib Threads::Threads)
 # backends are compiled in by build-tree path; a package keeps that layout (app/built_path.h)
@@ -67,6 +75,8 @@ target_compile_definitions(onebit PRIVATE ONEBIT_BUILD_DIR="${CMAKE_BINARY_DIR}"
 # Runs everywhere, CI included: serve's proxy against a fake backend.
 add_test(NAME smoke_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/smoke_serve.sh $<TARGET_FILE:onebit>)
 add_test(NAME hadamard_route COMMAND ${CMAKE_SOURCE_DIR}/tests/hadamard_route.sh $<TARGET_FILE:onebit>)
+add_test(NAME recipes_route COMMAND ${CMAKE_SOURCE_DIR}/tests/recipes_route.sh $<TARGET_FILE:onebit>)
+add_test(NAME bench_selftest COMMAND ${CMAKE_SOURCE_DIR}/tests/bench_selftest.sh $<TARGET_FILE:onebit>)
 add_test(NAME long_route COMMAND ${CMAKE_SOURCE_DIR}/tests/long_route.sh $<TARGET_FILE:onebit>)
 add_test(NAME laya_gguf_route COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_gguf_route.sh $<TARGET_FILE:onebit>)
 # The NPU route's <think> split into reasoning_content (app/think_split.h); header only, runs in CI.

README.md

@@ -36,6 +36,11 @@ serves each model behind an OpenAI-compatible API (`1bit serve`), whatever devic
 - ComfyUI.cpp: ComfyUI workflows (Stable Diffusion 1.5 text-to-image and image-to-image) in C++, matching ComfyUI's output to 50 dB ([docs/comfyui.md](docs/comfyui.md))
 - every Hugging Face model architecture, kept current by a daily census
 
+Every tuned setting `1bit serve` gives a backend is a recipe with its measurement attached
+([docs/recipes.md](docs/recipes.md)), and the serve numbers in these docs come from
+`tools/bench.py`, which A/B-measures configurations against a baseline in the same run
+([docs/bench.md](docs/bench.md)).
+
 Packages ship every Sunday, rebuilt at that week's upstream pins: Linux, Lemonade with the engine,
 Windows and 1bit OS ([docs/releases.md](docs/releases.md)).
 

app/recipes.cpp

@@ -0,0 +1,148 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "recipes.h"
+
+#include "recipes_default.h"
+
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
+#include <cctype>
+#include <cstdlib>
+#include <set>
+
+namespace onebit {
+
+namespace {
+
+bool any_of(const std::vector<std::string>& allowed, const std::string& v) {
+    return allowed.empty() || std::find(allowed.begin(), allowed.end(), v) != allowed.end();
+}
+
+// A token that starts a flag: "-x" or "--xy", not a negative number.
+bool is_flag(const std::string& t) {
+    return t.size() > 1 && t[0] == '-' && !std::isdigit(static_cast<unsigned char>(t[1])) && t[1] != '.';
+}
+
+std::vector<std::string> strings(const nlohmann::json& j, const char* key, const std::string& id) {
+    if (!j.is_array()) throw std::runtime_error("recipe " + id + ": " + key + " is not a list");
+    std::vector<std::string> out;
+    for (const auto& v : j) {
+        if (!v.is_string()) throw std::runtime_error("recipe " + id + ": " + key + " holds a non-string");
+        out.push_back(v.get<std::string>());
+    }
+    return out;
+}
+
+}  // namespace
+
+bool Recipe::matches(const RecipeFacts& f) const {
+    if (!any_of(device, f.device) || !any_of(architecture, f.architecture) || !any_of(drafter, f.drafter))
+        return false;
+    if (moe >= 0 && (moe == 1) != f.moe) return false;
+    if (hadamard_q4_0 >= 0 && (hadamard_q4_0 == 1) != f.hadamard_q4_0) return false;
+    return true;
+}
+
+bool Recipes::parse(const std::string& json_text, std::string& err) {
+    list.clear();
+    try {
+        const auto doc = nlohmann::json::parse(json_text);
+        if (!doc.is_object() || !doc.contains("recipes") || !doc["recipes"].is_array())
+            throw std::runtime_error("expected an object with a \"recipes\" list");
+        std::set<std::string> ids;
+        for (const auto& r : doc["recipes"]) {
+            if (!r.is_object() || !r.contains("id") || !r["id"].is_string())
+                throw std::runtime_error("every recipe needs a string \"id\"");
+            Recipe rec;
+            rec.id = r["id"].get<std::string>();
+            if (!ids.insert(rec.id).second) throw std::runtime_error("recipe " + rec.id + " is listed twice");
+            for (const auto& [key, v] : r.items()) {
+                if (key == "id") continue;
+                else if (key == "why" || key == "measured" || key == "source") {
+                    if (!v.is_string()) throw std::runtime_error("recipe " + rec.id + ": " + key + " is not a string");
+                    (key == "why" ? rec.why : key == "measured" ? rec.measured : rec.source) = v.get<std::string>();
+                } else if (key == "args") {
+                    rec.args = strings(v, "args", rec.id);
+                } else if (key == "env") {
+                    if (!v.is_object()) throw std::runtime_error("recipe " + rec.id + ": env is not an object");
+                    for (const auto& [name, value] : v.items()) {
+                        if (!value.is_string()) throw std::runtime_error("recipe " + rec.id + ": env " + name + " is not a string");
+                        rec.env.emplace_back(name, value.get<std::string>());
+                    }
+                } else if (key == "match") {
+                    if (!v.is_object()) throw std::runtime_error("recipe " + rec.id + ": match is not an object");
+                    for (const auto& [mk, mv] : v.items()) {
+                        if (mk == "device") rec.device = strings(mv, "match.device", rec.id);
+                        else if (mk == "architecture") rec.architecture = strings(mv, "match.architecture", rec.id);
+                        else if (mk == "drafter") rec.drafter = strings(mv, "match.drafter", rec.id);
+                        else if (mk == "moe" || mk == "hadamard_q4_0") {
+                            if (!mv.is_boolean()) throw std::runtime_error("recipe " + rec.id + ": match." + mk + " is not true/false");
+                            (mk == "moe" ? rec.moe : rec.hadamard_q4_0) = mv.get<bool>() ? 1 : 0;
+                        } else {
+                            throw std::runtime_error("recipe " + rec.id + ": unknown match key \"" + mk + "\"");
+                        }
+                    }
+                } else {
+                    throw std::runtime_error("recipe " + rec.id + ": unknown key \"" + key + "\"");
+                }
+            }
+            if (rec.args.empty() && rec.env.empty()) throw std::runtime_error("recipe " + rec.id + " adds nothing");
+            if (!rec.args.empty() && !is_flag(rec.args.front()))
+                throw std::runtime_error("recipe " + rec.id + ": args must start with a flag");
+            if (rec.measured.empty()) throw std::runtime_error("recipe " + rec.id + " has no \"measured\"");
+            list.push_back(std::move(rec));
+        }
+    } catch (const std::exception& e) {
+        err = e.what();
+        list.clear();
+        return false;
+    }
+    return true;
+}
+
+std::vector<std::string> Recipes::apply(const RecipeFacts& f, std::vector<std::string>& argv,
+                                        std::vector<std::string>& env) const {
+    std::vector<std::string> applied;
+    for (const auto& r : list) {
+        if (!r.matches(f)) continue;
+        std::string added;
+        for (size_t i = 0; i < r.args.size();) {
+            size_t j = i + 1;
+            while (j < r.args.size() && !is_flag(r.args[j])) ++j;
+            if (std::find(argv.begin(), argv.end(), r.args[i]) == argv.end()) {
+                for (size_t k = i; k < j; ++k) {
+                    argv.push_back(r.args[k]);
+                    added += (added.empty() ? "" : " ") + r.args[k];
+                }
+            }
+            i = j;
+        }
+        for (const auto& [name, value] : r.env) {
+            const bool set = std::getenv(name.c_str()) != nullptr ||
+                             std::any_of(env.begin(), env.end(), [&](const std::string& e) { return e.rfind(name + "=", 0) == 0; });
+            if (set) continue;
+            env.push_back(name + "=" + value);
+            added += (added.empty() ? "" : " ") + name + "=" + value;
+        }
+        if (!added.empty()) applied.push_back(r.id + ": " + added);
+    }
+    return applied;
+}
+
+const char* default_recipes_json() { return kDefaultRecipesJson; }
+
+}  // namespace onebit

app/recipes.h

@@ -0,0 +1,64 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// app/recipes.h — tuned llama-server settings per model and route, as data (docs/recipes.md).
+// config/recipes.json is compiled in as the default; `serve --recipes FILE` replaces it and
+// `serve --no-recipes` turns recipes off. A recipe names what it matches, the flags and
+// environment it adds, why, and the measurement behind it. It never overrides a flag already on
+// the backend's command line or a variable already in the environment.
+#pragma once
+
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace onebit {
+
+// What a recipe can match on, for one backend launch.
+struct RecipeFacts {
+    std::string device;             // vulkan, hrx, rocm
+    std::string architecture;       // general.architecture of the -m file
+    bool moe = false;               // <architecture>.expert_count > 0
+    bool hadamard_q4_0 = false;     // stamped by tools/hadamard_q4_0.py
+    std::string drafter = "none";   // none, mtp, dflash
+};
+
+struct Recipe {
+    std::string id, why, measured, source;
+    std::vector<std::string> device, architecture, drafter;  // empty: any
+    int moe = -1, hadamard_q4_0 = -1;                         // -1: any, else 0 / 1
+    std::vector<std::string> args;                            // llama-server flags and values
+    std::vector<std::pair<std::string, std::string>> env;     // backend environment
+
+    bool matches(const RecipeFacts& f) const;
+};
+
+struct Recipes {
+    std::vector<Recipe> list;
+
+    // Parses recipe JSON; on failure returns false and leaves the reason in `err`.
+    bool parse(const std::string& json_text, std::string& err);
+
+    // Adds each matching recipe's flags (a flag with the values after it) unless `argv` already
+    // has that flag, and its variables unless `env` or the process environment already sets
+    // them. Returns "id: what it added" for every recipe that added something.
+    std::vector<std::string> apply(const RecipeFacts& f, std::vector<std::string>& argv,
+                                   std::vector<std::string>& env) const;
+};
+
+// config/recipes.json as built into this binary.
+const char* default_recipes_json();
+
+}  // namespace onebit

app/serve.cpp

@@ -69,6 +69,7 @@
 #include "built_path.h"
 
 #include "gguf_meta.h"
+#include "recipes.h"
 
 #ifdef ONEBIT_MOE
 #include "gguf_index.h"
@@ -163,6 +164,8 @@ struct Options {
     std::string laya_model;              // --laya-model DIR: route each request by the Laya scorer (docs/laya.md)
     bool laya_auto = false;              // --laya: the same with the installed checkpoint (laya_checkpoint())
     std::string route_policy;            // --route-policy FILE: request class -> device (default: built in)
+    std::string recipes;                 // --recipes FILE: tuned backend settings (default: built in, docs/recipes.md)
+    bool no_recipes = false;             // --no-recipes: the backend gets serve's own flags only
     std::string long_model;              // --long-model FILE: a Hadamard-rotated Q4_0 for long prompts (W4A4 on ROCm)
     int long_from = 2048;                // --long-from N: a conversation whose first prompt has N tokens or more
 };
@@ -662,12 +665,7 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) {
             if (device != "rocm") throw std::runtime_error(o.model + " is Hadamard-rotated: it runs on --device rocm only");
             if (!std::getenv("GGML_Q4_0_HADAMARD")) env.push_back("GGML_Q4_0_HADAMARD=1");
             if (!std::getenv("GGML_W4A4_TENSORS")) env.push_back("GGML_W4A4_TENSORS=all");
-            // MoE files: 1024-token micro-batches, prompt processing +3-8% over the default 512
-            // (Qwen3-Coder-30B-A3B, pp2048 2,001 -> 2,158, pp16384 1,227 -> 1,283; 2048 is no
-            // better), decode unchanged. A dense file keeps 512: Qwen3.8-27B-H32 serves a 1,838-token
-            // prompt at 491-495 tok/s with 512 and 471-473 with 1024.
-            if (gguf_int(o.model, gguf_architecture(o.model) + ".expert_count") > 0)
-                argv.insert(argv.end(), {"-ub", "1024"});
+            // micro-batch size: recipe rotated-moe-ub1024 (config/recipes.json)
         }
     } else if (device == "vulkan" || device == "hrx") {
         // --prefill-device hrx: the HRX build (it has both devices and the shared-KV split), flash
@@ -759,17 +757,37 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) {
             const long long block = gguf_int(o.dflash, gguf_architecture(o.dflash) + ".block_size");
             argv.insert(argv.end(), {"--spec-draft-n-max", block > 1 ? std::to_string(block - 1) : "16"});
         }
+        // without --mtp-p-min, recipe dflash-p-min-0 keeps DFlash blocks whole (config/recipes.json)
         if (!o.mtp_p_min.empty()) { argv.push_back("--spec-draft-p-min"); argv.push_back(o.mtp_p_min); }
-        // a DFlash block is kept whole unless --mtp-p-min says otherwise: upstream's default p-min
-        // is 0, the ROCm tree's is 0.75, which cuts DFlash2 blocks from 6.7 to 5.4 tokens a step
-        else if (!o.dflash.empty()) argv.insert(argv.end(), {"--spec-draft-p-min", "0"});
     }
     if (!o.mmproj.empty()) {
         if (device == "zinc" || device == "mlx" || device == "ds4") throw std::runtime_error("--mmproj works on the llama.cpp devices (vulkan, hrx, rocm)");
         // an image is decoded as one ubatch: models that attend to it bidirectionally (ZAYA1-VL,
         // Gemma 3) need all of it in one, and Qwen2.5-VL towers cap an image at 4096 tokens
         argv.insert(argv.end(), {"--mmproj", o.mmproj, "-b", "4096", "-ub", "4096"});
     }
+    if ((device == "vulkan" || device == "hrx" || device == "rocm") && !o.no_recipes &&
+        std::filesystem::path(o.model).extension() == ".gguf") {
+        // tuned settings for this model and route (config/recipes.json, docs/recipes.md); what
+        // serve set above wins over a recipe
+        Recipes recipes;
+        std::string text = default_recipes_json(), err;
+        if (!o.recipes.empty()) {
+            std::ifstream in(o.recipes, std::ios::binary);
+            if (!in) throw std::runtime_error("--recipes: cannot read " + o.recipes);
+            text.assign(std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>());
+        }
+        if (!recipes.parse(text, err))
+            throw std::runtime_error((o.recipes.empty() ? std::string("built-in recipes") : o.recipes) + ": " + err);
+        RecipeFacts facts;
+        facts.device = device;
+        facts.architecture = gguf_architecture(o.model);
+        facts.moe = gguf_int(o.model, facts.architecture + ".expert_count") > 0;
+        facts.hadamard_q4_0 = hadamard_q4_0(o.model);
+        facts.drafter = !o.dflash.empty() ? "dflash" : !o.mtp.empty() ? "mtp" : "none";
+        for (const auto& line : recipes.apply(facts, argv, env))
+            std::fprintf(stderr, "1bit serve: recipe %s\n", line.c_str());
+    }
     return l;
 }
 
@@ -1738,6 +1756,8 @@ void usage(FILE* out) {
                  "                  [--npu-opt KEY=VALUE ...]   an option for a private NPU route (docs/npu.md)\n"
                  "                  [--laya | --laya-model DIR] [--route-policy FILE]   with --device auto, Laya classifies\n"
                  "                  each conversation and the route policy picks its device (docs/laya.md)\n"
+                 "                  [--recipes FILE | --no-recipes]   tuned llama-server settings per model and route\n"
+                 "                  (default: config/recipes.json, built in; docs/recipes.md)\n"
                  "  <model>: an NPU model directory (model.q4nx + npu/, or one a private NPU route serves),\n"
                  "           an ONNX Runtime GenAI directory (genai_config.json: --device onnx),\n"
                  "           a .gguf file, or with --device mlx a Hugging Face id (mlx-community/...)\n");
@@ -1793,6 +1813,8 @@ int run_serve(int argc, char** argv) {
         else if (a == "--laya-model") o.laya_model = next();
         else if (a == "--laya") o.laya_auto = true;
         else if (a == "--route-policy") o.route_policy = next();
+        else if (a == "--recipes") o.recipes = next();
+        else if (a == "--no-recipes") o.no_recipes = true;
         else if (a == "-h" || a == "--help") { usage(stdout); return 0; }
         else throw std::runtime_error("unknown option " + a);
     }

config/recipes.json

@@ -0,0 +1,21 @@
+{
+  "about": "Tuned llama-server settings per model and route, applied by `1bit serve` (docs/recipes.md). A recipe never overrides a flag already on the backend's command line or a variable already set. Every recipe carries the measurement behind it; re-measure with tools/bench.py before changing one.",
+  "recipes": [
+    {
+      "id": "rotated-moe-ub1024",
+      "match": {"device": ["rocm"], "hadamard_q4_0": true, "moe": true},
+      "args": ["-ub", "1024"],
+      "why": "W4A4 expert matmuls fill the matrix units better with 1024-token micro-batches; 2048 is no better.",
+      "measured": "Qwen3-Coder-30B-A3B-Q4_0-H32, llama-bench pp2048 2,001 -> 2,158, pp16384 1,227 -> 1,283; decode unchanged. Dense Qwen3.8-27B-H32 is slower with it (1,838-token prompt 491-495 -> 471-473 tok/s), so dense files keep 512.",
+      "source": "engine #205, #229"
+    },
+    {
+      "id": "dflash-p-min-0",
+      "match": {"drafter": ["dflash"]},
+      "args": ["--spec-draft-p-min", "0"],
+      "why": "A DFlash block is only worth drafting whole; the ROCm tree's default p-min of 0.75 cuts it short (upstream's default is 0).",
+      "measured": "Qwen3.8-27B-H32 + DFlash2 on the lean ROCm route: mean accepted block 5.4 -> 6.7 tokens on code.",
+      "source": "engine #194"
+    }
+  ]
+}