Laya scorer on HRX: ggmlc's laya on AMD's backend instead of Vulkan

.gitmodules

@@ -89,9 +89,10 @@
 	url = https://github.com/1bit-MONSTER/ds4.git
 	branch = 1bit/main
 	shallow = true
-# ggmlc (MIT, monatis/ggmlc): its `laya` runs the Laya GGUFs (mys/laya-*-GGUF) on GGML's
-# Vulkan backend, the router's fast scorer (docs/laya.md). Built with -DONEBIT_LAYA_GGML=ON.
+# ggmlc (MIT, monatis/ggmlc), our fork: its `laya` runs the Laya GGUFs (mys/laya-*-GGUF) on HRX0,
+# the router's fast scorer (docs/laya.md). 1bit/main = upstream + our HRX build. -DONEBIT_LAYA_GGML=ON.
 [submodule "third_party/ggmlc"]
 	path = third_party/ggmlc
-	url = https://github.com/monatis/ggmlc.git
+	url = https://github.com/1bit-MONSTER/ggmlc.git
+	branch = 1bit/main
 	shallow = true

CMakeLists.txt

@@ -303,9 +303,9 @@ target_link_libraries(onebit_laya PUBLIC onebit_npu nlohmann_json::nlohmann_json
 # The engine binary gets `1bit route` and per-request routing through the scorer.
 target_link_libraries(onebit PRIVATE onebit_laya)
 target_compile_definitions(onebit PRIVATE ONEBIT_LAYA)
-# Off by default (it needs the Vulkan SDK): ggmlc's laya runs the pinned Laya GGUF on Vulkan,
-# 22 ms a routing decision instead of about 380 ms on the CPU scorer, same accuracy (docs/laya.md).
-option(ONEBIT_LAYA_GGML "Build ggmlc's laya (Vulkan) as the router's scorer" OFF)
+# Off by default (it needs TheRock and the HRX submodules): ggmlc's laya runs the pinned Laya GGUF
+# on HRX0, 25 ms a routing decision (15 at a steady length) instead of about 540 ms on the CPU scorer, same accuracy (docs/laya.md).
+option(ONEBIT_LAYA_GGML "Build ggmlc's laya (HRX) as the router's scorer" OFF)
 if(ONEBIT_LAYA_GGML)
     include(cmake/laya-ggml.cmake)
     add_dependencies(onebit laya_ggml)

NOTICE

@@ -143,11 +143,13 @@ convaiinnovations/laya, pinned in config/laya.json)
   Copyright Convai Innovations
   License: Apache-2.0
 
-ggmlc (third_party/ggmlc; its laya runtime is the router's GGUF scorer)
+ggmlc (third_party/ggmlc = our fork 1bit-MONSTER/ggmlc; its laya runtime is the router's
+GGUF scorer, built on third_party/llama.cpp's ggml with HRX)
   https://github.com/monatis/ggmlc
   Copyright the ggmlc authors (monatis)
   License: MIT (declared in its pyproject.toml and README; the repository has no LICENSE file)
-  Vendors GGML (third_party/ggmlc/third_party/ggml), Copyright the ggml authors, License: MIT
+  Vendors GGML (third_party/ggmlc/third_party/ggml, not built by the engine), Copyright the ggml
+  authors, License: MIT
 
 Laya typed-decisions GGUF (Hugging Face mys/laya-typed-decisions-GGUF, pinned in
 config/laya.json; compiled by ggmlc from convaiinnovations/laya-typed-decisions)

app/serve.cpp

@@ -833,11 +833,12 @@ std::string laya_checkpoint(const std::string& given) {
     return dir;
 }
 
-// The router's fast scorer: ggmlc's laya (third_party/ggmlc, -DONEBIT_LAYA_GGML=ON) on GGML's
-// Vulkan backend with the typed-decisions checkpoint compiled to a GGUF (mys/laya-typed-decisions-
-// GGUF, pinned in config/laya.json, fetched by scripts/fetch-laya.sh into <laya dir>/gguf/).
-// Q8_0 classifies tests/laya_route_cases.json as well as the C++ scorer on the safetensors (95.5%)
-// in 22 ms a decision instead of about 380 ms. Used when the binary and the GGUF are both there.
+// The router's fast scorer: ggmlc's laya (our fork, third_party/ggmlc, -DONEBIT_LAYA_GGML=ON) on
+// HRX0, AMD's backend, built against third_party/llama.cpp's ggml (its HRX kernels for ModernBERT,
+// fork #31), with the typed-decisions checkpoint compiled to a GGUF (mys/laya-typed-decisions-GGUF,
+// pinned in config/laya.json, fetched by scripts/fetch-laya.sh into <laya dir>/gguf/). Q8_0
+// classifies tests/laya_route_cases.json as well as the C++ scorer on the safetensors (95.5%) in
+// 25 ms a decision (15 at a steady 64 tokens) instead of about 540 ms. Used when the binary and the GGUF are both there.
 constexpr const char* kLayaGguf = "laya_typed_decisions_q8_0.gguf";
 
 std::string laya_ggml_bin() {
@@ -866,7 +867,8 @@ std::string laya_gguf(const std::string& given) {
 // log goes to our stderr). Pipes rather than its `serve`, which listens on every interface.
 class LayaDaemon {
 public:
-    explicit LayaDaemon(const std::vector<std::string>& argv) {
+    // env: NAME=VALUE entries set in the daemon's environment (one the user already set wins)
+    explicit LayaDaemon(const std::vector<std::string>& argv, const std::vector<std::string>& env = {}) {
         int in[2], out[2];
         if (::pipe2(in, O_CLOEXEC) != 0 || ::pipe2(out, O_CLOEXEC) != 0)
             throw std::runtime_error(std::string("laya: pipe: ") + std::strerror(errno));
@@ -880,6 +882,10 @@ class LayaDaemon {
         if (pid_ == 0) {
             ::prctl(PR_SET_PDEATHSIG, SIGTERM);
             if (::getppid() != parent) ::_exit(127);
+            for (const auto& e : env) {
+                const size_t eq = e.find('=');
+                if (eq != std::string::npos) ::setenv(e.substr(0, eq).c_str(), e.c_str() + eq + 1, 0);
+            }
             ::dup2(in[0], 0);
             ::dup2(out[1], 1);
             ::execvp(args[0], args.data());
@@ -1078,10 +1084,13 @@ int serve_child(const Options& given) {
         const std::string gguf = laya_gguf(o.laya_model), bin = laya_ggml_bin();
 #if defined(__linux__)
         if (!gguf.empty() && !bin.empty()) {
-            // the GGUF scorer on Vulkan (ONEBIT_LAYA_DEVICE picks another: cpu, vulkan:1, ...)
+            // the GGUF scorer on HRX0 (ONEBIT_LAYA_DEVICE picks another: cpu), with the HSA runtime HRX
+            // needs (docs/hrx.md)
             const char* dev = std::getenv("ONEBIT_LAYA_DEVICE");
+            std::vector<std::string> env;
+            if (const std::string hsa = hrx_libhsa(o.hrx_libhsa); !hsa.empty()) env.push_back("IREE_HAL_AMDGPU_LIBHSA_PATH=" + hsa);
             laya_daemon = std::make_unique<LayaDaemon>(std::vector<std::string>{
-                bin, "daemon", gguf, "--family", "typed-decisions", "--device", dev && *dev ? dev : "vulkan"});
+                bin, "daemon", gguf, "--family", "typed-decisions", "--device", dev && *dev ? dev : "hrx"}, env);
             std::fprintf(stderr, "1bit serve: laya: %s through %s, policy %s\n", gguf.c_str(), bin.c_str(),
                          o.route_policy.empty() ? "built in" : o.route_policy.c_str());
         }

cmake/laya-ggml.cmake

@@ -13,23 +13,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# The router's fast scorer (docs/laya.md): ggmlc's `laya` (third_party/ggmlc, MIT) runs the
-# Laya GGUFs on GGML's Vulkan backend. `1bit serve --laya` starts it as a child when this
-# binary and the pinned GGUF (scripts/fetch-laya.sh) are both there, else it uses the C++
-# scorer on the safetensors.
+# The router's fast scorer (docs/laya.md): ggmlc's `laya` (third_party/ggmlc, our fork, MIT) runs
+# the Laya GGUFs on HRX0, AMD's backend: it builds third_party/llama.cpp's ggml (ggml-hrx, with the
+# kernels ModernBERT needs) with TheRock's amdclang, like cmake/hrx.cmake. `1bit serve --laya`
+# starts it as a child when this binary and the pinned GGUF (scripts/fetch-laya.sh) are both there,
+# else it uses the C++ scorer on the safetensors.
 include(ExternalProject)
 
-if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/ggmlc/CMakeLists.txt")
-    message(FATAL_ERROR "ONEBIT_LAYA_GGML needs third_party/ggmlc: git submodule update --init third_party/ggmlc")
-endif()
+foreach(_sub ggmlc llama.cpp hrx-system)
+    if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/${_sub}/CMakeLists.txt")
+        message(FATAL_ERROR "ONEBIT_LAYA_GGML needs third_party/${_sub}: git submodule update --init third_party/${_sub}")
+    endif()
+endforeach()
+set(ONEBIT_HRX_TOOLCHAIN "/opt/rocm-therock" CACHE PATH "ROCm/TheRock root: its amdclang builds llama.cpp and HRX")
 
 set(ONEBIT_LAYA_GGML_BIN "${CMAKE_BINARY_DIR}/laya-ggml/examples/laya/laya")
 ExternalProject_Add(laya_ggml
     SOURCE_DIR ${CMAKE_SOURCE_DIR}/third_party/ggmlc
     BINARY_DIR ${CMAKE_BINARY_DIR}/laya-ggml
     DOWNLOAD_COMMAND ""
     CMAKE_GENERATOR Ninja
-    CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DGGMLC_ENABLE_VULKAN=ON -DGGMLC_BUILD_EXAMPLES=ON
+    CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_C_COMPILER=${ONEBIT_HRX_TOOLCHAIN}/bin/amdclang
+        -DCMAKE_CXX_COMPILER=${ONEBIT_HRX_TOOLCHAIN}/bin/amdclang++
+        -DGGMLC_ENABLE_HRX=ON -DGGMLC_GGML_DIR=${CMAKE_SOURCE_DIR}/third_party/llama.cpp/ggml
+        -DHRX_SOURCE_DIR=${CMAKE_SOURCE_DIR}/third_party/hrx-system
+        -DGGML_NATIVE=ON -DGGMLC_BUILD_EXAMPLES=ON
     BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target laya
     BUILD_BYPRODUCTS ${ONEBIT_LAYA_GGML_BIN}
     INSTALL_COMMAND ""

docs/laya.md

@@ -159,42 +159,66 @@ single-threaded scorer. Those figures are for the old device question on a short
 The request-class decision takes 0.4-0.6 s (the table above), and `serve` pays it once per
 conversation.
 
-## The GGUF scorer: ggmlc on Vulkan
+## The GGUF scorer: ggmlc on HRX
 
 [ggmlc](https://github.com/monatis/ggmlc) compiles Laya's checkpoints to GGUFs that its own
 `laya` runtime runs on GGML (they are not llama.cpp GGUFs): English
 ([mys/laya-GGUF](https://huggingface.co/mys/laya-GGUF), ModernBERT-large, 421M parameters,
 512-token context), multilingual ([mys/laya-multilingual-GGUF](https://huggingface.co/mys/laya-multilingual-GGUF),
 100+ languages) and typed-decisions
 ([mys/laya-typed-decisions-GGUF](https://huggingface.co/mys/laya-typed-decisions-GGUF), for
-`choice` / `score` / `noul`). The engine pins ggmlc as `third_party/ggmlc` and the
-typed-decisions Q8_0 in `config/laya.json`; `scripts/fetch-laya.sh` fetches it into
-`<laya dir>/gguf/` and checks its sha256.
+`choice` / `score` / `noul`). The engine pins our fork of ggmlc as `third_party/ggmlc`
+(`1bit/main`: upstream plus an HRX build) and the typed-decisions Q8_0 in `config/laya.json`;
+`scripts/fetch-laya.sh` fetches it into `<laya dir>/gguf/` and checks its sha256.
+
+The scorer runs on HRX0, AMD's backend: ggmlc's `laya` is built against `third_party/llama.cpp`'s
+ggml, whose ggml-hrx carries the LOOM kernels a ModernBERT encoder needs and llama models did not
+(LayerNorm, strided and broadcast arithmetic, clamp, F32 to F16 copies, attention over
+one-block-per-head layouts, small and odd-K matmuls; 1bit-MONSTER/llama.cpp#31).
 
 ```sh
-cmake -B build -DONEBIT_LAYA_GGML=ON ...   # builds ggmlc's laya with GGML's Vulkan backend
+cmake -B build -DONEBIT_LAYA_GGML=ON ...   # builds ggmlc's laya on HRX with TheRock's amdclang
 scripts/fetch-laya.sh
 1bit serve -m MODEL.gguf --laya            # uses the GGUF scorer when both are there
 ```
 
 With the binary and the GGUF present, `serve --laya` starts `laya daemon <gguf> --family
-typed-decisions --device vulkan` and asks it serve's question over its stdin and stdout (not
-its `serve`, which listens on every interface with no key). Without them it uses the C++
-scorer on the safetensors as before. `--laya-model FILE.gguf` picks another GGUF,
-`ONEBIT_LAYA_GGML` another binary, `ONEBIT_LAYA_DEVICE` another device (`cpu`, `vulkan:1`).
+typed-decisions --device hrx` (with the HSA runtime HRX needs, as for `--device hrx`) and asks it
+serve's question over its stdin and stdout (not its `serve`, which listens on every interface
+with no key). Without them it uses the C++ scorer on the safetensors. `--laya-model FILE.gguf`
+picks another GGUF, `ONEBIT_LAYA_GGML` another binary, `ONEBIT_LAYA_DEVICE` another device (`cpu`).
 
 **Measured** (Strix Halo, the 200 labelled requests of `tests/laya_route_cases.json`, 150 of
-them asked of the model, the rest long_doc by size; through `1bit serve --laya` with fake
-backends, so the request time is the decision):
+them asked of the model, the rest long_doc by size):
 
 | Scorer | Class accuracy | Decision, p50 |
 |---|---|---|
-| C++ scorer, typed-decisions safetensors, CPU | 95.5% | 538 ms |
-| **ggmlc `laya`, typed-decisions Q8_0 GGUF, Vulkan** | **95.5%** | **24 ms** |
-
-Asked directly (`laya serve`, same question): typed-decisions F16 95.5% at 36 ms; the English
-GGUF 93.0% (Q8_0, 22 ms) and 91.5% (F16), so the typed-decisions checkpoint stays the router's.
-`tests/laya_gguf_route.sh` covers the daemon wiring without a GPU (a fake daemon).
+| C++ scorer, typed-decisions safetensors, CPU (through `serve`) | 95.5% | 538 ms |
+| **ggmlc `laya` on HRX0, typed-decisions Q8_0 GGUF** | **95.5%** | **25 ms** (15 ms at a steady 64 tokens, 25 at 128) |
+| ggmlc `laya` on Vulkan, same GGUF (upstream ggmlc, for comparison; not built by the engine) | 95.5% | 22 ms (12 ms at a steady length) |
+
+The p50 includes the first request of each length bucket, which compiles and records that
+shape once (about 150 ms). At a steady length HRX is within 3 ms of Vulkan at 64 tokens and level
+with it at 128. Three things got it there from 44 ms: ggmlc gives its graphs a uid, so HRX's
+graph-program cache and graph replay hit instead of re-importing and re-recording every call
+(25 -> 15 ms); a fused kernel for the rotate-half RoPE that ggmlc lowers to eight nodes, and one
+for GEGLU (1,170 launches a decision down to about 780); and attention that computes each score
+once. Next is the NPU.
+Answers match Vulkan's probabilities within about 1e-3. Asked directly on Vulkan: typed-decisions
+F16 95.5% at 36 ms; the English GGUF 93.0% (Q8_0) and 91.5% (F16), so the typed-decisions
+checkpoint stays the router's. `tests/laya_gguf_route.sh` covers the daemon wiring without a GPU.
+
+**Other languages.** The 150 model-decided cases machine-translated into Spanish, German,
+Japanese, Chinese, Hindi and Russian (Qwen3-Coder-30B-A3B, greedy; 682 kept after dropping replies
+that answered instead of translating), on HRX0:
+
+| Scorer | es | de | ja | zh | hi | ru | All |
+|---|---|---|---|---|---|---|---|
+| **typed-decisions for every language** | **92.5%** | **90.7%** | **89.0%** | **86.4%** | 60.0% | **83.7%** | **83.7%** |
+| typed-decisions for English, multilingual GGUF otherwise (ggmlc's language routing) | 73.3% | 71.3% | 65.3% | 83.1% | 60.0% | 68.1% | 69.1% |
+
+The multilingual checkpoint answers this question worse in every language, so the router keeps
+typed-decisions for all of them; Hindi is the weak spot either way.
 
 ggmlc declares the MIT licence (`pyproject.toml`, README) but its repository carries no LICENSE
 file yet; the GGUFs are Apache-2.0 like the checkpoints.

registry/architectures.json

@@ -2,7 +2,7 @@
  "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.",
  "sources": {
   "llama.cpp (vulkan)": "cecf3ee01d9d99378e98bfea95f51cac714b04b8",
-  "llama.cpp (hrx)": "fcd83eb2a4a86cdae5d65e114981470593a190aa",
+  "llama.cpp (hrx)": "22ceda3ad7c97f5358b99f4fe0db42f5cf9f3bae",
   "zinc": "29bc350ac4cbf9f110ec628b9e177ea04ac816fb"
  },
  "counts": {

tests/laya_gguf_route.sh

@@ -19,8 +19,9 @@
 # the pinned GGUF name under $ONEBIT_LAYA_MODEL/gguf/ is an empty file, and fake llama-servers
 # and zinc answer with their device (tests/route-policy-e2e.json: code -> hrx, prose -> zinc,
 # short -> vulkan). It checks that
-#   - serve starts the daemon (daemon <gguf> --family typed-decisions --device vulkan) and needs
-#     no safetensors checkpoint,
+#   - serve starts the daemon (daemon <gguf> --family typed-decisions --device hrx) with the HSA
+#     runtime HRX needs (IREE_HAL_AMDGPU_LIBHSA_PATH, from --hrx-libhsa) and needs no safetensors
+#     checkpoint,
 #   - each class reaches its device, with X-1bit-Route "<class> <confidence> <device>",
 #   - the daemon gets serve's question: the three non-long_doc classes, the request as state,
 #   - a request of 1024 characters or more is long_doc without asking the daemon,
@@ -44,7 +45,7 @@ cat > "$scratch/laya-daemon.py" <<'PY'
 #!/usr/bin/env python3
 import json, os, sys
 log = open(os.environ["LAYA_LOG"], "a")
-log.write(json.dumps({"argv": sys.argv[1:]}) + "\n"); log.flush()
+log.write(json.dumps({"argv": sys.argv[1:], "hsa": os.environ.get("IREE_HAL_AMDGPU_LIBHSA_PATH", "")}) + "\n"); log.flush()
 print(json.dumps({"status": "ready", "model": "laya"}), flush=True)
 for line in sys.stdin:
     req = json.loads(line)
@@ -69,13 +70,14 @@ check "--laya-model FILE.gguf without ggmlc's laya says how to build it" '[[ "$n
 LAYA_LOG="$scratch/daemon.log" ONEBIT_LAYA_GGML="$scratch/laya-daemon.py" ONEBIT_LAYA_MODEL="$scratch/laya" \
     "$bin" serve -m "$scratch/tiny.gguf" --device auto --laya --route-policy "$here/route-policy-e2e.json" \
     --port "$port" --llama-server "$here/fake_backend_route.py" --zinc "$here/fake_backend_route.py" \
-    >"$scratch/serve.log" 2>&1 &
+    --hrx-libhsa "$scratch/libhsa-runtime64.so.1" >"$scratch/serve.log" 2>&1 &
 pid=$!
 for _ in $(seq 1 300); do curl -sf -o /dev/null "$api/health" && break; sleep 0.1; done
 
 argv=$(head -1 "$scratch/daemon.log" 2>/dev/null)
-check "serve starts the daemon on the pinned GGUF, typed-decisions, Vulkan" \
-    '[[ "$argv" == *"\"daemon\", \"$scratch/laya/gguf/laya_typed_decisions_q8_0.gguf\", \"--family\", \"typed-decisions\", \"--device\", \"vulkan\""* ]]'
+check "serve starts the daemon on the pinned GGUF, typed-decisions, HRX" \
+    '[[ "$argv" == *"\"daemon\", \"$scratch/laya/gguf/laya_typed_decisions_q8_0.gguf\", \"--family\", \"typed-decisions\", \"--device\", \"hrx\""* ]]'
+check "  with the HSA runtime HRX needs" '[[ "$argv" == *"\"hsa\": \"$scratch/libhsa-runtime64.so.1\""* ]]'
 
 ask() {  # <text>: the replying device, then the X-1bit-Route header
     local body