diff --git a/.gitmodules b/.gitmodules index cd0046c0..4cdf5176 100644 --- a/.gitmodules +++ b/.gitmodules @@ -89,9 +89,10 @@ url = https://github.com/1bit-MONSTER/ds4.git branch = 1bit/main shallow = true -# ggmlc (MIT, monatis/ggmlc): its `laya` runs the Laya GGUFs (mys/laya-*-GGUF) on GGML's -# Vulkan backend, the router's fast scorer (docs/laya.md). Built with -DONEBIT_LAYA_GGML=ON. +# ggmlc (MIT, monatis/ggmlc), our fork: its `laya` runs the Laya GGUFs (mys/laya-*-GGUF) on HRX0, +# the router's fast scorer (docs/laya.md). 1bit/main = upstream + our HRX build. -DONEBIT_LAYA_GGML=ON. [submodule "third_party/ggmlc"] path = third_party/ggmlc - url = https://github.com/monatis/ggmlc.git + url = https://github.com/1bit-MONSTER/ggmlc.git + branch = 1bit/main shallow = true diff --git a/CMakeLists.txt b/CMakeLists.txt index 413783aa..2e51d9ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -303,9 +303,9 @@ target_link_libraries(onebit_laya PUBLIC onebit_npu nlohmann_json::nlohmann_json # The engine binary gets `1bit route` and per-request routing through the scorer. target_link_libraries(onebit PRIVATE onebit_laya) target_compile_definitions(onebit PRIVATE ONEBIT_LAYA) -# Off by default (it needs the Vulkan SDK): ggmlc's laya runs the pinned Laya GGUF on Vulkan, -# 22 ms a routing decision instead of about 380 ms on the CPU scorer, same accuracy (docs/laya.md). -option(ONEBIT_LAYA_GGML "Build ggmlc's laya (Vulkan) as the router's scorer" OFF) +# Off by default (it needs TheRock and the HRX submodules): ggmlc's laya runs the pinned Laya GGUF +# on HRX0, 25 ms a routing decision (15 at a steady length) instead of about 540 ms on the CPU scorer, same accuracy (docs/laya.md). +option(ONEBIT_LAYA_GGML "Build ggmlc's laya (HRX) as the router's scorer" OFF) if(ONEBIT_LAYA_GGML) include(cmake/laya-ggml.cmake) add_dependencies(onebit laya_ggml) diff --git a/NOTICE b/NOTICE index 889a1663..8a4256eb 100644 --- a/NOTICE +++ b/NOTICE @@ -143,11 +143,13 @@ convaiinnovations/laya, pinned in config/laya.json) Copyright Convai Innovations License: Apache-2.0 -ggmlc (third_party/ggmlc; its laya runtime is the router's GGUF scorer) +ggmlc (third_party/ggmlc = our fork 1bit-MONSTER/ggmlc; its laya runtime is the router's +GGUF scorer, built on third_party/llama.cpp's ggml with HRX) https://github.com/monatis/ggmlc Copyright the ggmlc authors (monatis) License: MIT (declared in its pyproject.toml and README; the repository has no LICENSE file) - Vendors GGML (third_party/ggmlc/third_party/ggml), Copyright the ggml authors, License: MIT + Vendors GGML (third_party/ggmlc/third_party/ggml, not built by the engine), Copyright the ggml + authors, License: MIT Laya typed-decisions GGUF (Hugging Face mys/laya-typed-decisions-GGUF, pinned in config/laya.json; compiled by ggmlc from convaiinnovations/laya-typed-decisions) diff --git a/app/serve.cpp b/app/serve.cpp index 80a70dbd..f31c07b2 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -833,11 +833,12 @@ std::string laya_checkpoint(const std::string& given) { return dir; } -// The router's fast scorer: ggmlc's laya (third_party/ggmlc, -DONEBIT_LAYA_GGML=ON) on GGML's -// Vulkan backend with the typed-decisions checkpoint compiled to a GGUF (mys/laya-typed-decisions- -// GGUF, pinned in config/laya.json, fetched by scripts/fetch-laya.sh into /gguf/). -// Q8_0 classifies tests/laya_route_cases.json as well as the C++ scorer on the safetensors (95.5%) -// in 22 ms a decision instead of about 380 ms. Used when the binary and the GGUF are both there. +// The router's fast scorer: ggmlc's laya (our fork, third_party/ggmlc, -DONEBIT_LAYA_GGML=ON) on +// HRX0, AMD's backend, built against third_party/llama.cpp's ggml (its HRX kernels for ModernBERT, +// fork #31), with the typed-decisions checkpoint compiled to a GGUF (mys/laya-typed-decisions-GGUF, +// pinned in config/laya.json, fetched by scripts/fetch-laya.sh into /gguf/). Q8_0 +// classifies tests/laya_route_cases.json as well as the C++ scorer on the safetensors (95.5%) in +// 25 ms a decision (15 at a steady 64 tokens) instead of about 540 ms. Used when the binary and the GGUF are both there. constexpr const char* kLayaGguf = "laya_typed_decisions_q8_0.gguf"; std::string laya_ggml_bin() { @@ -866,7 +867,8 @@ std::string laya_gguf(const std::string& given) { // log goes to our stderr). Pipes rather than its `serve`, which listens on every interface. class LayaDaemon { public: - explicit LayaDaemon(const std::vector& argv) { + // env: NAME=VALUE entries set in the daemon's environment (one the user already set wins) + explicit LayaDaemon(const std::vector& argv, const std::vector& env = {}) { int in[2], out[2]; if (::pipe2(in, O_CLOEXEC) != 0 || ::pipe2(out, O_CLOEXEC) != 0) throw std::runtime_error(std::string("laya: pipe: ") + std::strerror(errno)); @@ -880,6 +882,10 @@ class LayaDaemon { if (pid_ == 0) { ::prctl(PR_SET_PDEATHSIG, SIGTERM); if (::getppid() != parent) ::_exit(127); + for (const auto& e : env) { + const size_t eq = e.find('='); + if (eq != std::string::npos) ::setenv(e.substr(0, eq).c_str(), e.c_str() + eq + 1, 0); + } ::dup2(in[0], 0); ::dup2(out[1], 1); ::execvp(args[0], args.data()); @@ -1078,10 +1084,13 @@ int serve_child(const Options& given) { const std::string gguf = laya_gguf(o.laya_model), bin = laya_ggml_bin(); #if defined(__linux__) if (!gguf.empty() && !bin.empty()) { - // the GGUF scorer on Vulkan (ONEBIT_LAYA_DEVICE picks another: cpu, vulkan:1, ...) + // the GGUF scorer on HRX0 (ONEBIT_LAYA_DEVICE picks another: cpu), with the HSA runtime HRX + // needs (docs/hrx.md) const char* dev = std::getenv("ONEBIT_LAYA_DEVICE"); + std::vector env; + if (const std::string hsa = hrx_libhsa(o.hrx_libhsa); !hsa.empty()) env.push_back("IREE_HAL_AMDGPU_LIBHSA_PATH=" + hsa); laya_daemon = std::make_unique(std::vector{ - bin, "daemon", gguf, "--family", "typed-decisions", "--device", dev && *dev ? dev : "vulkan"}); + bin, "daemon", gguf, "--family", "typed-decisions", "--device", dev && *dev ? dev : "hrx"}, env); std::fprintf(stderr, "1bit serve: laya: %s through %s, policy %s\n", gguf.c_str(), bin.c_str(), o.route_policy.empty() ? "built in" : o.route_policy.c_str()); } diff --git a/cmake/laya-ggml.cmake b/cmake/laya-ggml.cmake index 4d25df69..a836359c 100644 --- a/cmake/laya-ggml.cmake +++ b/cmake/laya-ggml.cmake @@ -13,15 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The router's fast scorer (docs/laya.md): ggmlc's `laya` (third_party/ggmlc, MIT) runs the -# Laya GGUFs on GGML's Vulkan backend. `1bit serve --laya` starts it as a child when this -# binary and the pinned GGUF (scripts/fetch-laya.sh) are both there, else it uses the C++ -# scorer on the safetensors. +# The router's fast scorer (docs/laya.md): ggmlc's `laya` (third_party/ggmlc, our fork, MIT) runs +# the Laya GGUFs on HRX0, AMD's backend: it builds third_party/llama.cpp's ggml (ggml-hrx, with the +# kernels ModernBERT needs) with TheRock's amdclang, like cmake/hrx.cmake. `1bit serve --laya` +# starts it as a child when this binary and the pinned GGUF (scripts/fetch-laya.sh) are both there, +# else it uses the C++ scorer on the safetensors. include(ExternalProject) -if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/ggmlc/CMakeLists.txt") - message(FATAL_ERROR "ONEBIT_LAYA_GGML needs third_party/ggmlc: git submodule update --init third_party/ggmlc") -endif() +foreach(_sub ggmlc llama.cpp hrx-system) + if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/${_sub}/CMakeLists.txt") + message(FATAL_ERROR "ONEBIT_LAYA_GGML needs third_party/${_sub}: git submodule update --init third_party/${_sub}") + endif() +endforeach() +set(ONEBIT_HRX_TOOLCHAIN "/opt/rocm-therock" CACHE PATH "ROCm/TheRock root: its amdclang builds llama.cpp and HRX") set(ONEBIT_LAYA_GGML_BIN "${CMAKE_BINARY_DIR}/laya-ggml/examples/laya/laya") ExternalProject_Add(laya_ggml @@ -29,7 +33,12 @@ ExternalProject_Add(laya_ggml BINARY_DIR ${CMAKE_BINARY_DIR}/laya-ggml DOWNLOAD_COMMAND "" CMAKE_GENERATOR Ninja - CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DGGMLC_ENABLE_VULKAN=ON -DGGMLC_BUILD_EXAMPLES=ON + CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=${ONEBIT_HRX_TOOLCHAIN}/bin/amdclang + -DCMAKE_CXX_COMPILER=${ONEBIT_HRX_TOOLCHAIN}/bin/amdclang++ + -DGGMLC_ENABLE_HRX=ON -DGGMLC_GGML_DIR=${CMAKE_SOURCE_DIR}/third_party/llama.cpp/ggml + -DHRX_SOURCE_DIR=${CMAKE_SOURCE_DIR}/third_party/hrx-system + -DGGML_NATIVE=ON -DGGMLC_BUILD_EXAMPLES=ON BUILD_COMMAND ${CMAKE_COMMAND} --build --target laya BUILD_BYPRODUCTS ${ONEBIT_LAYA_GGML_BIN} INSTALL_COMMAND "" diff --git a/docs/laya.md b/docs/laya.md index ab1e89b7..2710258c 100644 --- a/docs/laya.md +++ b/docs/laya.md @@ -159,7 +159,7 @@ single-threaded scorer. Those figures are for the old device question on a short The request-class decision takes 0.4-0.6 s (the table above), and `serve` pays it once per conversation. -## The GGUF scorer: ggmlc on Vulkan +## The GGUF scorer: ggmlc on HRX [ggmlc](https://github.com/monatis/ggmlc) compiles Laya's checkpoints to GGUFs that its own `laya` runtime runs on GGML (they are not llama.cpp GGUFs): English @@ -167,34 +167,58 @@ conversation. 512-token context), multilingual ([mys/laya-multilingual-GGUF](https://huggingface.co/mys/laya-multilingual-GGUF), 100+ languages) and typed-decisions ([mys/laya-typed-decisions-GGUF](https://huggingface.co/mys/laya-typed-decisions-GGUF), for -`choice` / `score` / `noul`). The engine pins ggmlc as `third_party/ggmlc` and the -typed-decisions Q8_0 in `config/laya.json`; `scripts/fetch-laya.sh` fetches it into -`/gguf/` and checks its sha256. +`choice` / `score` / `noul`). The engine pins our fork of ggmlc as `third_party/ggmlc` +(`1bit/main`: upstream plus an HRX build) and the typed-decisions Q8_0 in `config/laya.json`; +`scripts/fetch-laya.sh` fetches it into `/gguf/` and checks its sha256. + +The scorer runs on HRX0, AMD's backend: ggmlc's `laya` is built against `third_party/llama.cpp`'s +ggml, whose ggml-hrx carries the LOOM kernels a ModernBERT encoder needs and llama models did not +(LayerNorm, strided and broadcast arithmetic, clamp, F32 to F16 copies, attention over +one-block-per-head layouts, small and odd-K matmuls; 1bit-MONSTER/llama.cpp#31). ```sh -cmake -B build -DONEBIT_LAYA_GGML=ON ... # builds ggmlc's laya with GGML's Vulkan backend +cmake -B build -DONEBIT_LAYA_GGML=ON ... # builds ggmlc's laya on HRX with TheRock's amdclang scripts/fetch-laya.sh 1bit serve -m MODEL.gguf --laya # uses the GGUF scorer when both are there ``` With the binary and the GGUF present, `serve --laya` starts `laya daemon --family -typed-decisions --device vulkan` and asks it serve's question over its stdin and stdout (not -its `serve`, which listens on every interface with no key). Without them it uses the C++ -scorer on the safetensors as before. `--laya-model FILE.gguf` picks another GGUF, -`ONEBIT_LAYA_GGML` another binary, `ONEBIT_LAYA_DEVICE` another device (`cpu`, `vulkan:1`). +typed-decisions --device hrx` (with the HSA runtime HRX needs, as for `--device hrx`) and asks it +serve's question over its stdin and stdout (not its `serve`, which listens on every interface +with no key). Without them it uses the C++ scorer on the safetensors. `--laya-model FILE.gguf` +picks another GGUF, `ONEBIT_LAYA_GGML` another binary, `ONEBIT_LAYA_DEVICE` another device (`cpu`). **Measured** (Strix Halo, the 200 labelled requests of `tests/laya_route_cases.json`, 150 of -them asked of the model, the rest long_doc by size; through `1bit serve --laya` with fake -backends, so the request time is the decision): +them asked of the model, the rest long_doc by size): | Scorer | Class accuracy | Decision, p50 | |---|---|---| -| C++ scorer, typed-decisions safetensors, CPU | 95.5% | 538 ms | -| **ggmlc `laya`, typed-decisions Q8_0 GGUF, Vulkan** | **95.5%** | **24 ms** | - -Asked directly (`laya serve`, same question): typed-decisions F16 95.5% at 36 ms; the English -GGUF 93.0% (Q8_0, 22 ms) and 91.5% (F16), so the typed-decisions checkpoint stays the router's. -`tests/laya_gguf_route.sh` covers the daemon wiring without a GPU (a fake daemon). +| C++ scorer, typed-decisions safetensors, CPU (through `serve`) | 95.5% | 538 ms | +| **ggmlc `laya` on HRX0, typed-decisions Q8_0 GGUF** | **95.5%** | **25 ms** (15 ms at a steady 64 tokens, 25 at 128) | +| ggmlc `laya` on Vulkan, same GGUF (upstream ggmlc, for comparison; not built by the engine) | 95.5% | 22 ms (12 ms at a steady length) | + +The p50 includes the first request of each length bucket, which compiles and records that +shape once (about 150 ms). At a steady length HRX is within 3 ms of Vulkan at 64 tokens and level +with it at 128. Three things got it there from 44 ms: ggmlc gives its graphs a uid, so HRX's +graph-program cache and graph replay hit instead of re-importing and re-recording every call +(25 -> 15 ms); a fused kernel for the rotate-half RoPE that ggmlc lowers to eight nodes, and one +for GEGLU (1,170 launches a decision down to about 780); and attention that computes each score +once. Next is the NPU. +Answers match Vulkan's probabilities within about 1e-3. Asked directly on Vulkan: typed-decisions +F16 95.5% at 36 ms; the English GGUF 93.0% (Q8_0) and 91.5% (F16), so the typed-decisions +checkpoint stays the router's. `tests/laya_gguf_route.sh` covers the daemon wiring without a GPU. + +**Other languages.** The 150 model-decided cases machine-translated into Spanish, German, +Japanese, Chinese, Hindi and Russian (Qwen3-Coder-30B-A3B, greedy; 682 kept after dropping replies +that answered instead of translating), on HRX0: + +| Scorer | es | de | ja | zh | hi | ru | All | +|---|---|---|---|---|---|---|---| +| **typed-decisions for every language** | **92.5%** | **90.7%** | **89.0%** | **86.4%** | 60.0% | **83.7%** | **83.7%** | +| typed-decisions for English, multilingual GGUF otherwise (ggmlc's language routing) | 73.3% | 71.3% | 65.3% | 83.1% | 60.0% | 68.1% | 69.1% | + +The multilingual checkpoint answers this question worse in every language, so the router keeps +typed-decisions for all of them; Hindi is the weak spot either way. ggmlc declares the MIT licence (`pyproject.toml`, README) but its repository carries no LICENSE file yet; the GGUFs are Apache-2.0 like the checkpoints. diff --git a/registry/architectures.json b/registry/architectures.json index f4053f5f..4c10f96a 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -2,7 +2,7 @@ "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { "llama.cpp (vulkan)": "cecf3ee01d9d99378e98bfea95f51cac714b04b8", - "llama.cpp (hrx)": "fcd83eb2a4a86cdae5d65e114981470593a190aa", + "llama.cpp (hrx)": "22ceda3ad7c97f5358b99f4fe0db42f5cf9f3bae", "zinc": "29bc350ac4cbf9f110ec628b9e177ea04ac816fb" }, "counts": { diff --git a/tests/laya_gguf_route.sh b/tests/laya_gguf_route.sh index e21e37d7..b4e58007 100755 --- a/tests/laya_gguf_route.sh +++ b/tests/laya_gguf_route.sh @@ -19,8 +19,9 @@ # the pinned GGUF name under $ONEBIT_LAYA_MODEL/gguf/ is an empty file, and fake llama-servers # and zinc answer with their device (tests/route-policy-e2e.json: code -> hrx, prose -> zinc, # short -> vulkan). It checks that -# - serve starts the daemon (daemon --family typed-decisions --device vulkan) and needs -# no safetensors checkpoint, +# - serve starts the daemon (daemon --family typed-decisions --device hrx) with the HSA +# runtime HRX needs (IREE_HAL_AMDGPU_LIBHSA_PATH, from --hrx-libhsa) and needs no safetensors +# checkpoint, # - each class reaches its device, with X-1bit-Route " ", # - the daemon gets serve's question: the three non-long_doc classes, the request as state, # - a request of 1024 characters or more is long_doc without asking the daemon, @@ -44,7 +45,7 @@ cat > "$scratch/laya-daemon.py" <<'PY' #!/usr/bin/env python3 import json, os, sys log = open(os.environ["LAYA_LOG"], "a") -log.write(json.dumps({"argv": sys.argv[1:]}) + "\n"); log.flush() +log.write(json.dumps({"argv": sys.argv[1:], "hsa": os.environ.get("IREE_HAL_AMDGPU_LIBHSA_PATH", "")}) + "\n"); log.flush() print(json.dumps({"status": "ready", "model": "laya"}), flush=True) for line in sys.stdin: req = json.loads(line) @@ -69,13 +70,14 @@ check "--laya-model FILE.gguf without ggmlc's laya says how to build it" '[[ "$n LAYA_LOG="$scratch/daemon.log" ONEBIT_LAYA_GGML="$scratch/laya-daemon.py" ONEBIT_LAYA_MODEL="$scratch/laya" \ "$bin" serve -m "$scratch/tiny.gguf" --device auto --laya --route-policy "$here/route-policy-e2e.json" \ --port "$port" --llama-server "$here/fake_backend_route.py" --zinc "$here/fake_backend_route.py" \ - >"$scratch/serve.log" 2>&1 & + --hrx-libhsa "$scratch/libhsa-runtime64.so.1" >"$scratch/serve.log" 2>&1 & pid=$! for _ in $(seq 1 300); do curl -sf -o /dev/null "$api/health" && break; sleep 0.1; done argv=$(head -1 "$scratch/daemon.log" 2>/dev/null) -check "serve starts the daemon on the pinned GGUF, typed-decisions, Vulkan" \ - '[[ "$argv" == *"\"daemon\", \"$scratch/laya/gguf/laya_typed_decisions_q8_0.gguf\", \"--family\", \"typed-decisions\", \"--device\", \"vulkan\""* ]]' +check "serve starts the daemon on the pinned GGUF, typed-decisions, HRX" \ + '[[ "$argv" == *"\"daemon\", \"$scratch/laya/gguf/laya_typed_decisions_q8_0.gguf\", \"--family\", \"typed-decisions\", \"--device\", \"hrx\""* ]]' +check " with the HSA runtime HRX needs" '[[ "$argv" == *"\"hsa\": \"$scratch/libhsa-runtime64.so.1\""* ]]' ask() { # : the replying device, then the X-1bit-Route header local body diff --git a/third_party/ggmlc b/third_party/ggmlc index 1519ea42..df08d1b0 160000 --- a/third_party/ggmlc +++ b/third_party/ggmlc @@ -1 +1 @@ -Subproject commit 1519ea422ac31fb2e3e26947f69a0eb09217fbce +Subproject commit df08d1b064eb66e9a979f6a02142415898da588b diff --git a/third_party/llama.cpp b/third_party/llama.cpp index fcd83eb2..22ceda3a 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit fcd83eb2a4a86cdae5d65e114981470593a190aa +Subproject commit 22ceda3ad7c97f5358b99f4fe0db42f5cf9f3bae