tandem.patch

diff --git a/common/arg.cpp b/common/arg.cpp
index e6d6450f4..62d1fdf22 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -2786,15 +2786,33 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         }
     ).set_env("LLAMA_ARG_N_CPU_MOE"));
     add_opt(common_arg(
-        {"--moe-cache-mib"}, "N",
-        "GPU cache size in MiB for the MoE experts kept in the CPU. with multiple GPUs, it is split among them like the layers (--tensor-split) (default: 0, disabled)",
+        {"--moe-cache-mib"}, "N|auto",
+        "GPU cache size in MiB for the MoE experts kept in the CPU. with multiple GPUs, it is split among them like the layers (--tensor-split); "
+        "'auto': the device memory left after the model, the KV cache and the compute buffers, minus --moe-cache-margin-mib "
+        "and the draft model (default: 0, disabled)",
+        [](common_params & params, const std::string & value) {
+            if (value == "auto") {
+                params.moe_cache_size = SIZE_MAX;
+                return;
+            }
+            const int v = std::stoi(value);
+            if (v < 0) {
+                throw std::invalid_argument("invalid value");
+            }
+            params.moe_cache_size = (size_t) v*1024*1024;
+        }
+    ).set_env("LLAMA_ARG_MOE_CACHE_MIB"));
+    add_opt(common_arg(
+        {"--moe-cache-margin-mib"}, "N",
+        string_format("with --moe-cache-mib auto: device memory in MiB left free (default: %d; a draft model's file size is added)",
+            (int) (params.moe_cache_margin/(1024*1024))),
         [](common_params & params, int value) {
             if (value < 0) {
                 throw std::invalid_argument("invalid value");
             }
-            params.moe_cache_size = (size_t) value*1024*1024;
+            params.moe_cache_margin = (size_t) value*1024*1024;
         }
-    ).set_env("LLAMA_ARG_MOE_CACHE_MIB"));
+    ).set_env("LLAMA_ARG_MOE_CACHE_MARGIN_MIB"));
     add_opt(common_arg(
         {"-ncffn", "--n-cpu-ffn"}, "N",
         "keep the dense FFN weights of the first N layers in the CPU\n"
diff --git a/common/common.cpp b/common/common.cpp
index 221f0897c..4d4d22d0b 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -4,6 +4,7 @@
 
 #include "build-info.h"
 #include "common.h"
+#include "../src/llama-prof.h"
 
 #include "../src/llama-ext.h"
 
@@ -386,7 +387,53 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD
     return true;
 }
 
+// tandem.cpp release defaults: the tuned paths that are exact (or differ only in rounding, as any kernel choice) and that
+// the benchmarks use. Applied to the environment when a variable is unset, before anything reads it; VAR=0 turns one off.
+// Model-specific paths (Qwen3.8 GDN/conv/PLE/hc, MTP) only act on their architecture. Not set here: LLAMA_MTP_DRAFT_VOCAB
+// (a draft vocabulary subset whose size depends on the model).
+static const char * const lc_release_defaults[][2] = {
+    { "GGML_CUDA_GRAPH_KEY_SHAPE", "1"    }, // one CUDA graph per batch shape (speculative windows do not evict each other)
+    { "GGML_SCHED_BATCH_INPUTS",   "3"    }, // the graph's host inputs in one packed upload
+    { "GGML_CUDA_MMVF_MAX",        "8"    }, // vector kernel for float weights up to 8 columns
+    { "GGML_CUDA_Q8_CACHE",        "1"    }, // reuse the q8_1 activations of a matvec input across matvecs
+    { "LLAMA_GRAPH_POOL",          "6"    }, // keep 6 more allocated graphs for alternating small batch shapes
+    { "LLAMA_SRV_USER_BREAK",      "0"    }, // server: no prompt checkpoint at every user message
+    { "LLAMA_MOE_CPU",             "auto" }, // MoE cache: the CPU computes a measured share of the missed experts
+    { "LLAMA_MOE_PREFILL_CPU",     "auto" }, // ... and of the prompt batches' missed experts
+    { "LLAMA_MOE_CPU_RB",          "64"   }, // CPU expert job: 64 gate/up rows per work item
+    { "LLAMA_MOE_CPU_DOWN_PE",     "64"   }, // CPU expert job: down rows per expert, 64 per item
+    { "LLAMA_HC_FUSE",             "1"    }, // Qwen3.8 / DS4 hc: scale folded into the weights
+    { "GGML_CUDA_HC_MIX_FUSE",     "1"    }, // hc mix block in 2 kernels (different rounding)
+    { "LLAMA_GDN_DIRECT",          "1"    }, // Qwen3.8 GDN: state read from the cache by row
+    { "LLAMA_GDN_GATES",           "1"    }, // Qwen3.8 GDN: beta and alpha in one matvec
+    { "LLAMA_CONV_CONT",           "1"    }, // Qwen3.8 conv: contiguous token input
+    { "LLAMA_CONV_TAIL",           "1"    }, // Qwen3.8 conv: rollback slots from the strided tail
+    { "LLAMA_PLE_GPU",             "1"    }, // per-layer embeddings read by the GPU from pinned host memory
+    { "LLAMA_MTP_DEFER",           "1"    }, // MTP: catch-up decode deferred to the next draft
+};
+
+static void lc_apply_release_defaults() {
+    if (const char * off = getenv("LLAMA_MOE_NO_DEFAULTS"); off && atoi(off) != 0) {
+        return;
+    }
+    std::string set;
+    for (const auto & d : lc_release_defaults) {
+        if (getenv(d[0]) == nullptr) {
+#if defined(_WIN32)
+            _putenv_s(d[0], d[1]);
+#else
+            setenv(d[0], d[1], 0);
+#endif
+            set += string_format(" %s=%s", d[0], d[1]);
+        }
+    }
+    if (!set.empty()) {
+        LOG_INF("tandem.cpp defaults (unset variables; LLAMA_MOE_NO_DEFAULTS=1 for none):%s\n", set.c_str());
+    }
+}
+
 void common_init() {
+    lc_apply_release_defaults();
 #if defined(_WIN32)
     SetConsoleOutputCP(CP_UTF8);
     SetConsoleCP(CP_UTF8);
@@ -1772,7 +1819,38 @@ struct llama_context_params common_context_params_to_llama(const common_params &
     cparams.type_k = params.cache_type_k;
     cparams.type_v = params.cache_type_v;
 
-    cparams.moe_cache_size = params.moe_cache_size;
+    cparams.moe_cache_size   = params.moe_cache_size;
+    cparams.moe_cache_margin = params.moe_cache_margin;
+    if (params.moe_cache_size == SIZE_MAX && params.speculative.has_dft()) {
+        // the draft model is loaded after this context: keep its weights (plus its KV cache and graphs, ~512 MiB) free too
+        const auto & dft = params.speculative.draft;
+        std::error_code ec;
+        size_t sz = std::filesystem::file_size(dft.mparams.path, ec);
+        if (ec) {
+            sz = 0;
+        }
+        // its token_embd stays in host memory (the input layer) unless a draft override (-otd) places it
+        bool embd_override = false;
+        for (const auto & o : dft.tensor_buft_overrides) {
+            if (o.pattern != nullptr && std::regex_search("token_embd.weight", std::regex(o.pattern))) {
+                embd_override = true;
+            }
+        }
+        if (sz > 0 && !embd_override) {
+            gguf_init_params gp = { /* .no_alloc = */ true, /* .ctx = */ nullptr };
+            ggml_context * meta = nullptr;
+            gp.ctx = &meta;
+            gguf_context_ptr g(gguf_init_from_file(dft.mparams.path.c_str(), gp));
+            const ggml_tensor * te = meta ? ggml_get_tensor(meta, "token_embd.weight") : nullptr;
+            if (te != nullptr && ggml_nbytes(te) < sz) {
+                sz -= ggml_nbytes(te);
+            }
+            if (meta) {
+                ggml_free(meta);
+            }
+        }
+        cparams.moe_cache_margin += sz + (size_t) 512*1024*1024;
+    }
 
     return cparams;
 }
@@ -2410,6 +2488,38 @@ void common_prompt_checkpoint::update_pos(
     this->pos_max  = pos_max;
 }
 
+static std::mutex g_ckpt_pool_mu;
+static std::vector<std::vector<uint8_t>> g_ckpt_pool;
+
+void common_prompt_checkpoint_recycle(common_prompt_checkpoint & cur) {
+    static const bool on = !getenv("LLAMA_CKPT_POOL") || atoi(getenv("LLAMA_CKPT_POOL")) != 0;
+    if (!on) {
+        return;
+    }
+    std::lock_guard<std::mutex> lock(g_ckpt_pool_mu);
+    for (auto * v : { &cur.data_tgt, &cur.data_dft }) {
+        if (v->size() >= (1u << 20) && g_ckpt_pool.size() < 8) {
+            g_ckpt_pool.push_back(std::move(*v));
+            v->clear();
+        }
+    }
+}
+
+// a pooled buffer of exactly n bytes (its pages are already in memory), else dst is left as it is
+static void common_ckpt_take(std::vector<uint8_t> & dst, size_t n) {
+    if (dst.size() == n) {
+        return;
+    }
+    std::lock_guard<std::mutex> lock(g_ckpt_pool_mu);
+    for (auto it = g_ckpt_pool.begin(); it != g_ckpt_pool.end(); ++it) {
+        if (it->size() == n) {
+            dst.swap(*it);
+            g_ckpt_pool.erase(it);
+            return;
+        }
+    }
+}
+
 void common_prompt_checkpoint::update_tgt(
         llama_context * ctx,
         llama_seq_id seq_id,
@@ -2420,8 +2530,13 @@ void common_prompt_checkpoint::update_tgt(
 
     const size_t ckpt_size = llama_state_seq_get_size_ext(ctx, seq_id, flags);
 
+    {
+    llama_prof_scope ps("ckpt.tgt.resize");
+    common_ckpt_take(data_tgt, ckpt_size);
     data_tgt.resize(ckpt_size);
+    }
 
+    llama_prof_scope ps("ckpt.tgt.get");
     const size_t n = llama_state_seq_get_data_ext(ctx, data_tgt.data(), ckpt_size, seq_id, flags);
     if (n != ckpt_size) {
         GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", ckpt_size, n);
@@ -2438,8 +2553,13 @@ void common_prompt_checkpoint::update_dft(
 
     const size_t ckpt_size = llama_state_seq_get_size_ext(ctx, seq_id, flags);
 
+    {
+    llama_prof_scope ps("ckpt.dft.resize");
+    common_ckpt_take(data_dft, ckpt_size);
     data_dft.resize(ckpt_size);
+    }
 
+    llama_prof_scope ps("ckpt.dft.get");
     const size_t n = llama_state_seq_get_data_ext(ctx, data_dft.data(), ckpt_size, seq_id, flags);
     if (n != ckpt_size) {
         GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", ckpt_size, n);
diff --git a/common/common.h b/common/common.h
index cd89a27e9..5f25a34db 100644
--- a/common/common.h
+++ b/common/common.h
@@ -597,7 +597,8 @@ struct common_params {
     ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K
     ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V
 
-    size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers
+    size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers; SIZE_MAX: auto
+    size_t moe_cache_margin = (size_t) 1024*1024*1024; // auto size: device memory left free (+ the draft model's file size)
 
     common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO;
 
@@ -1340,3 +1341,7 @@ struct common_prompt_checkpoint {
     void clear_tgt();
     void clear_dft();
 };
+
+// lc-: hands the buffers of a checkpoint about to be erased to a small pool; update_tgt/update_dft take a pooled buffer of the
+// same size instead of a fresh one, which costs ~44 ms per 112 MiB to fault in and zero (asrok)
+void common_prompt_checkpoint_recycle(common_prompt_checkpoint & cur);
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 4305ac6ac..e13b31fa7 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -1,4 +1,5 @@
 #include "speculative.h"
+#include "../src/llama-prof.h"
 
 #include "common.h"
 #include "ggml.h"
@@ -350,6 +351,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
     }
 
     void draft(common_speculative_draft_params_vec & dparams) override {
+        llama_prof_lap("lap.d_entry");
         auto & ctx_dft = params.ctx_dft;
 
         batch.clear();
@@ -392,6 +394,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
         }
 
         int i = 0;
+        llama_prof_lap("lap.d_setup");
 
         while (n_drafting > 0) {
             int i_batch = 0;
@@ -1524,6 +1527,52 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
     std::vector<int>                i_last;
     std::vector<std::vector<float>> chain_h;
 
+    // LLAMA_MTP_DEFER=1: the catch-up decode of a processed batch waits for the next draft() (or process()), so after
+    // a verify it covers only the accepted tokens and shares one decode with the first draft step
+    struct mtp_stash {
+        std::vector<llama_token> tok;
+        std::vector<llama_pos>   pos;
+        std::vector<float>       h; // [n][n_embd], the target row each token pairs with
+    };
+    bool defer = false;
+    std::vector<mtp_stash> stash; // per seq
+
+    // decode whatever is stashed (no outputs), e.g. a prompt ubatch followed by another one
+    bool flush_stash() {
+        bool any = false;
+        for (const auto & st : stash) { any = any || !st.tok.empty(); }
+        if (!any) {
+            return true;
+        }
+        std::vector<mtp_stash> hold;
+        hold.swap(stash);
+        stash.assign(n_seq, {});
+        batch.clear();
+        for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
+            const auto & st = hold[seq_id];
+            // a sequence restarted while its catch-up was deferred (the server cleared both memories, then a stale stash was
+            // flushed into the draft's): the newest stash wins, the draft memory from its first position on is dropped
+            if (!st.tok.empty()) {
+                auto * mem = llama_get_memory(params.ctx_dft);
+                if (st.pos[0] <= llama_memory_seq_pos_max(mem, seq_id)) {
+                    SPC_WRN("seq %d: deferred batch starts at %d, draft memory ends at %d: dropping the stale part\n",
+                        (int) seq_id, (int) st.pos[0], (int) llama_memory_seq_pos_max(mem, seq_id));
+                    llama_memory_seq_rm(mem, seq_id, st.pos[0], -1);
+                }
+            }
+            for (size_t k = 0; k < st.tok.size(); ++k) {
+                const int32_t idx = batch.add(st.tok[k], st.pos[k], seq_id, false);
+                batch.set_embd_state(idx, { st.h.data() + k*n_embd, 1, (size_t) n_embd });
+            }
+        }
+        const int32_t rc = llama_process(params.ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch.get());
+        if (rc != 0) {
+            SPC_ERR("llama_process(ctx_dft) for the deferred batch failed rc=%d\n", (int) rc);
+        }
+        batch.clear();
+        return rc == 0;
+    }
+
     common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
         : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max)
         , params(params.draft)
@@ -1598,6 +1647,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
 
         verify_h.assign(n_seq, {});
         verify_h_rows.assign(n_seq, 0);
+
+        defer = getenv("LLAMA_MTP_DEFER") != nullptr && atoi(getenv("LLAMA_MTP_DEFER")) != 0 && !is_mem_shared && !chain_heads;
+        stash.assign(n_seq, {});
+        if (defer) {
+            SPC_TRC("%s", "- deferred catch-up decode (LLAMA_MTP_DEFER)\n");
+        }
     }
 
     ~common_speculative_impl_draft_mtp() override {
@@ -1617,6 +1672,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
     void begin(llama_seq_id seq_id, const llama_tokens & prompt) override {
         // reset here rather than per round, or two identical requests differ
         common_sampler_reset(smpls[seq_id].get());
+        flush_stash();
 
         const int32_t N = (int32_t) prompt.size();
         if (N <= 0) {
@@ -1666,8 +1722,34 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
 
         const size_t row_bytes = (size_t) n_embd * sizeof(float);
 
+        const bool use_defer = defer && !batch_in.has_embd();
+        if (use_defer) {
+            // an earlier stash (no draft since) is decoded first, in order
+            {
+                LC_PROF("mtp.proc.flush");
+                if (!flush_stash()) {
+                    return false;
+                }
+            }
+            {
+                LC_PROF("mtp.proc.sync_tgt");
+                llama_synchronize(ctx_tgt);
+            }
+            llama_prof_scope lc_prof_p("mtp.proc.h_tgt");
+            const float * h_tgt = llama_get_embeddings_nextn(ctx_tgt);
+            lc_prof_p.~llama_prof_scope(); new (&lc_prof_p) llama_prof_scope("mtp.proc.stash");
+            for (int k = 0; k < n_tokens; ++k) {
+                const auto & t = batch_in.tokens[k];
+                auto & st = stash[t.seq_id];
+                const float * h_row = k == i_batch_beg[t.seq_id] ? pending_h[t.seq_id].data() : h_tgt + (size_t) (k - 1) * n_embd;
+                st.tok.push_back(t.id);
+                st.pos.push_back(t.pos[0]);
+                st.h.insert(st.h.end(), h_row, h_row + n_embd);
+            }
+        }
+
         // if kv is shared with target (e.g Gemma4), then we can skip this catch-up decode
-        if (!is_mem_shared) {
+        if (!is_mem_shared && !use_defer) {
             batch.clear();
 
             // pair each token with the tgt embedding shifted right by one position, and
@@ -1727,6 +1809,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
         }
 
+        LC_PROF("mtp.proc.verify_h");
         for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
             if (i_batch_end[seq_id] < 0) {
                 continue;
@@ -1751,6 +1834,22 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
     void draft(common_speculative_draft_params_vec & dparams) override {
         auto & ctx_dft = params.ctx_dft;
 
+        // a stash that does not end right before the draft position, or would not fit one batch together with the
+        // draft tokens, is decoded on its own first
+        size_t n_stash = 0;
+        for (const auto & st : stash) { n_stash += st.tok.size(); }
+        bool flush = n_stash + n_seq > (size_t) llama_n_batch(ctx_dft);
+        for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq && !flush; ++seq_id) {
+            const auto & st = stash[seq_id];
+            flush = !st.tok.empty() && (!dparams[seq_id].drafting || st.pos.back() + 1 != dparams[seq_id].pos0);
+        }
+        if (flush) {
+            flush_stash();
+        }
+        std::vector<mtp_stash> hold; // keeps the stashed rows alive through the first decode
+        hold.swap(stash);
+        stash.assign(n_seq, {});
+
         batch.clear();
 
         // keep track of which sequences are still drafting
@@ -1781,6 +1880,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                 common_sampler_reset(smpls[seq_id].get());
             }
 
+            if (!hold.empty()) {
+                const auto & st = hold[seq_id];
+                for (size_t k = 0; k < st.tok.size(); ++k) {
+                    const int32_t j = batch.add(st.tok[k], st.pos[k], seq_id, false);
+                    batch.set_embd_state(j, { st.h.data() + k*n_embd, 1, (size_t) n_embd });
+                }
+            }
+
             const int32_t idx = batch.add(dp.id_last, dp.pos0, seq_id, true);
             batch.set_embd_state(idx, { pending_h[seq_id].data(), 1, (size_t) n_embd });
 
@@ -1791,6 +1898,44 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
         }
 
+        // LLAMA_MTP_CHAIN: one greedy sequence, no confidence cut: every step runs on the device back to back, the
+        // first one (catch-up plus the last token) included; the host launches them and reads the tokens once
+        if (n_drafting == 1 && params.p_min <= 0.0f && !chain_heads && !is_mem_shared && llama_mtp_chain_enabled(ctx_dft)) {
+            llama_seq_id sid = 0;
+            while (!drafting[sid]) { sid++; }
+            auto & dp = dparams[sid];
+            if (!dp.result_q) {
+                const int n_steps = std::min<int>(std::min(params.n_max, dp.n_max > 0 ? dp.n_max : params.n_max), 64);
+                llama_mtp_chain_begin(ctx_dft, dp.id_last, pending_h[sid].data());
+                int n_done = 0;
+                for (int k = 0; k < n_steps; ++k) {
+                    if (k > 0) {
+                        batch.clear();
+                        const int32_t idx = batch.add(dp.id_last, dp.pos0 + k, sid, true); // token and h: placeholders
+                        batch.set_embd_state(idx, { pending_h[sid].data(), 1, (size_t) n_embd });
+                    }
+                    LC_PROF("mtp.chain_decode");
+                    if (llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+                        SPC_ERR("chained llama_process[%d] failed\n", k);
+                        break;
+                    }
+                    llama_mtp_chain_record(ctx_dft, k);
+                    n_done++;
+                }
+                std::vector<llama_token> toks(n_done);
+                {
+                    LC_PROF("mtp.chain_end");
+                    llama_mtp_chain_end(ctx_dft, toks.data(), n_done);
+                }
+                dp.result->insert(dp.result->end(), toks.begin(), toks.end());
+                batch.clear();
+                if (dp.result->size() < (size_t) params.n_min) {
+                    dp.result->clear();
+                }
+                return;
+            }
+        }
+
         int i = 0;
 
         while (n_drafting > 0) {
@@ -1810,7 +1955,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                 llama_set_nextn_layer_offset(ctx_dft, i);
             }
 
-            int ret = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch.get());
+            int ret;
+            {
+                llama_prof_lap("lap.d_seqrm");
+                LC_PROF("mtp.draft_decode");
+                ret = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch.get());
+            }
             if (ret != 0) {
                 SPC_ERR("llama_process[%d] returned %d\n", i, ret);
                 break;
@@ -1828,8 +1978,15 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
 
                 auto * smpl = smpls[seq_id].get();
 
+                {
+                    LC_PROF("mtp.sync");
+                    llama_synchronize(ctx_dft);
+                }
+                llama_prof_scope lc_prof_s("mtp.sample");
                 const llama_token id_sampled = common_sampler_sample(smpl, ctx_dft, i_last[seq_id], true);
+                lc_prof_s.~llama_prof_scope(); new (&lc_prof_s) llama_prof_scope("mtp.h_row");
                 const float * h_row = llama_get_embeddings_nextn_ith(ctx_dft, i_last[seq_id]);
+                lc_prof_s.~llama_prof_scope(); new (&lc_prof_s) llama_prof_scope("mtp.candidates_rest");
 
                 const auto * cur_p = common_sampler_get_candidates(smpl, true);
 
@@ -1843,7 +2000,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                 const llama_token id = dparams.at(seq_id).result_q ? id_sampled : cur_p->data[0].id;
 
                 // only collect very high-confidence draft tokens
-                if (cur_p->data[0].p < params.p_min) {
+                // LLAMA_MTP_PMIN_FLOOR=k (hydev): the cut applies only once k drafts exist (Strata always verifies at least
+                // one draft); a round without drafts is a plain 1-token decode
+                static const size_t pmin_floor = getenv("LLAMA_MTP_PMIN_FLOOR") ? (size_t) atoi(getenv("LLAMA_MTP_PMIN_FLOOR")) : 0;
+                if (cur_p->data[0].p < params.p_min && dparams.at(seq_id).result->size() >= pmin_floor) {
                     drafting[seq_id] = false;
                     n_drafting--;
 
@@ -1861,7 +2021,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                     dp.result_q->emplace_back(cur_p->data, cur_p->data + cur_p->size);
                 }
 
-                if (params.n_max <= (int) result.size()) {
+                // the request's limit (server: context room, LLAMA_SPEC_AUTO's count) stops the drafting here instead of a
+                // truncation after the full draft
+                if (std::min(params.n_max, dp.n_max > 0 ? dp.n_max : params.n_max) <= (int) result.size()) {
                     drafting[seq_id] = false;
                     n_drafting--;
                     continue;
@@ -1884,6 +2046,34 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                     const int32_t idx = batch.add(id, dp.pos0, seq_id, true);
                     batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd });
                     i_last[seq_id] = idx;
+                } else if (n_drafting == 1 && !dp.result_q && params.p_min <= 0.0f && llama_mtp_chain_enabled(ctx_dft)) {
+                    // LLAMA_MTP_CHAIN: the remaining greedy steps run back to back on the device, each reading the
+                    // token and h the previous one wrote; the host only launches them and reads the tokens once
+                    const int n_rest = std::min<int>(params.n_max - (int) result.size(), 64);
+                    std::vector<llama_token> toks(n_rest);
+                    llama_mtp_chain_begin(ctx_dft, id, h_row);
+                    int n_done = 0;
+                    for (int k = 0; k < n_rest; ++k) {
+                        batch.clear();
+                        const int32_t idx = batch.add(id, dp.pos0 + i + 1 + k, seq_id, true); // token and h: placeholders
+                        batch.set_embd_state(idx, { pending_h[seq_id].data(), 1, (size_t) n_embd });
+                        LC_PROF("mtp.chain_decode");
+                        if (llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+                            SPC_ERR("chained llama_process[%d] failed\n", k);
+                            break;
+                        }
+                        llama_mtp_chain_record(ctx_dft, k);
+                        n_done++;
+                    }
+                    {
+                        LC_PROF("mtp.chain_end");
+                        llama_mtp_chain_end(ctx_dft, toks.data(), n_done);
+                    }
+                    result.insert(result.end(), toks.begin(), toks.begin() + n_done);
+                    batch.clear();
+                    drafting[seq_id] = false;
+                    n_drafting--;
+                    continue;
                 } else {
                     const int32_t idx = batch.add(id, dp.pos0 + i + 1, seq_id, true);
                     batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd });
@@ -1927,6 +2117,15 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
         const int32_t i_h = std::min<int32_t>(n_accepted, n_rows - 1);
         const size_t row_bytes = (size_t) n_embd * sizeof(float);
         std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes);
+
+        // the deferred catch-up only needs the sampled token and the accepted drafts
+        auto & st = stash[seq_id];
+        if (defer && !st.tok.empty()) {
+            const size_t keep = std::min(st.tok.size(), (size_t) n_accepted + 1);
+            st.tok.resize(keep);
+            st.pos.resize(keep);
+            st.h.resize(keep*n_embd);
+        }
     }
 };
 
diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
index 7a65d8121..f9afcc438 100644
--- a/ggml/include/ggml-backend.h
+++ b/ggml/include/ggml-backend.h
@@ -363,6 +363,8 @@ extern "C" {
     GGML_API enum ggml_status     ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
     GGML_API enum ggml_status     ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
     GGML_API void                 ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
+    // GGML_SCHED_BATCH_INPUTS=2: the user input copies are not waited for; call this before overwriting the inputs
+    GGML_API void                 ggml_backend_sched_wait_input_copies(ggml_backend_sched_t sched);
 
     // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.
     // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index d62a63104..a0f75accd 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -602,6 +602,10 @@ extern "C" {
 
         GGML_OP_GLU,
 
+        GGML_OP_HOST_ADD,
+        GGML_OP_MOE_DOORBELL,
+        GGML_OP_GET_ROWS_HOST,
+
         GGML_OP_COUNT,
     };
 
@@ -2678,6 +2682,24 @@ extern "C" {
             struct ggml_tensor  * state,
             int64_t               K);
 
+    // same, with two optional extensions:
+    //   ids (I32 [n_seqs]): the initial state of sequence i is states[ids[i]] (states: [S_v, S_v, H_v, n_rows]),
+    //       read straight from the recurrent cache instead of a gathered copy
+    //   dt, A (F32 [H_v]): g and beta are raw projections (scalar gate only, strided views allowed, same strides);
+    //       the op uses g = softplus(g + dt) * A and beta = sigmoid(beta)
+    GGML_API struct ggml_tensor * ggml_gated_delta_net_ext(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * q,
+            struct ggml_tensor  * k,
+            struct ggml_tensor  * v,
+            struct ggml_tensor  * g,
+            struct ggml_tensor  * beta,
+            struct ggml_tensor  * states,
+            struct ggml_tensor  * ids,
+            struct ggml_tensor  * dt,
+            struct ggml_tensor  * A,
+            int64_t               K);
+
     // DSA lightning indexer
     //
     // q:       [n_embd_idx, n_head_idx, n_batch, ne3 ]
@@ -2731,6 +2753,18 @@ extern "C" {
             struct ggml_tensor  * gate,
             float                 scale);
 
+    // hc_pre with the weights as gate logits (DeepSeek V4): weights[h, t] = sigmoid(mix[h, t]*scale[0] + base[h]) + eps,
+    // the same operations as mul -> add -> sigmoid -> scale_bias(1, eps), without contraction into an fma
+    //   mix [hc, n_tokens] (may be a view), scale [>= 1], base [hc]
+    //
+    GGML_API struct ggml_tensor * ggml_dsv4_hc_pre_affine(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * x,
+            struct ggml_tensor  * mix,
+            struct ggml_tensor  * scale,
+            struct ggml_tensor  * base,
+            float                 eps);
+
     // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens],
     //          post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens]
     //          -> [n_embd, hc, n_tokens]
@@ -2765,6 +2799,31 @@ extern "C" {
             struct ggml_tensor  * blocks,
             struct ggml_tensor  * extra);
 
+    // as ggml_dsv4_hc_post, with post given as gate logits (DeepSeek V4): the op uses
+    // scale_out * sigmoid(post[h, t]*scale[0] + base[h]), the same operations as mul -> add -> sigmoid -> scale
+    GGML_API struct ggml_tensor * ggml_dsv4_hc_post_affine(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * x,
+            struct ggml_tensor  * residual,
+            struct ggml_tensor  * post,
+            struct ggml_tensor  * comb,
+            struct ggml_tensor  * scale,
+            struct ggml_tensor  * base,
+            float                 scale_out);
+
+    // as ggml_dsv4_hc_post, with post given as logits: the op uses scale_out * sigmoid(scale_in * post).
+    // add/add_gate (optional, [n_embd, n_tokens] and [1, n_tokens]): x is replaced by x + add * sigmoid(add_gate)
+    GGML_API struct ggml_tensor * ggml_dsv4_hc_post_sig(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * x,
+            struct ggml_tensor  * residual,
+            struct ggml_tensor  * post,
+            struct ggml_tensor  * comb,
+            float                 scale_in,
+            float                 scale_out,
+            struct ggml_tensor  * add,
+            struct ggml_tensor  * add_gate);
+
     // custom operators
 
     typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
@@ -2870,6 +2929,48 @@ extern "C" {
             struct ggml_tensor  * v,
             struct ggml_tensor  * adamw_params); // parameters such as the learning rate
 
+    // a + data, where data is F32 in pinned host memory with the shape of a (contiguous), written by the host while the graph runs:
+    // the backend waits until flag[0] != flag[1] (the host increments flag[0] after writing data), then sets flag[1] = flag[0] and
+    // flag[2] = how long it waited (ns); flag[3] counts waits that timed out. data and flag must stay valid while the graph is used
+    GGML_API struct ggml_tensor * ggml_host_add(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * a,
+        const float         * data,
+        uint32_t            * flag);
+
+    // MoE cache doorbell: hands the selected experts of a layer to a host thread while the graph runs, and returns their slots
+    // db is a control block in pinned host memory, see struct ggml_moe_db: the backend writes ids (and x, w if not NULL) into it,
+    // sets req = ack + 1 and waits until the host sets ack = req, after it wrote the slots of ids (at ans_off) and the uploads
+    // (at ans_off + slots_cap*4: n_fill, pad, then n_fill ggml_moe_db_fill); the backend copies both to the device, then copies
+    // each upload from host to device memory itself (zero-copy reads)
+    // result: I32 [n_ids + GGML_MOE_DB_SCRATCH(max_fill)], the slots of ids first (the rest is scratch)
+    struct ggml_moe_db {
+        uint32_t req, ack;     // written by the backend / by the host
+        uint32_t n_tok, n_used, hyb;
+        uint32_t n_fill;       // unused
+        uint32_t wait_ns;      // how long the backend waited for ack, last call
+        uint32_t timeouts;     // waits that gave up after 2 s
+        uint32_t ids_off, x_off, w_off, ans_off, slots_cap, max_fill; // byte offsets from the block, capacities
+        uint32_t pad[2];
+    };
+    struct ggml_moe_db_fill { uint64_t src, dst, n; }; // host source, device destination, bytes
+    // scratch: from word (n_ids + 1) & ~1: n_fill, pad, then max_fill ggml_moe_db_fill
+    #define GGML_MOE_DB_SCRATCH(max_fill) (3 + 6*(max_fill))
+    GGML_API struct ggml_tensor * ggml_moe_doorbell(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * ids,   // I32 [n_tok*n_used], contiguous
+        struct ggml_tensor  * x,     // F32 [n_embd, n_tok] or NULL
+        struct ggml_tensor  * w,     // F32 [n_used*n_tok] or NULL
+        struct ggml_moe_db  * db,
+        int                   n_used);
+
+    // rows ids of w, read by the backend from host memory it can address (pinned and mapped, e.g. registered), as F32
+    // [w->ne[0], n_ids]; w is only described (type, row size, data pointer), it is not a source, so the op runs where ids is
+    GGML_API struct ggml_tensor * ggml_get_rows_host(
+        struct ggml_context * ctx,
+        const struct ggml_tensor * w,
+        struct ggml_tensor  * ids);
+
     // stochastic gradient descent step (with weight decay)
     GGML_API struct ggml_tensor * ggml_opt_step_sgd(
         struct ggml_context * ctx,
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 22e5fd9c7..8d335b1b9 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -924,6 +924,9 @@ struct ggml_backend_sched_split {
 };
 
 struct ggml_backend_sched {
+    // GGML_SCHED_BATCH_INPUTS=2: per backend, an event recorded after the last user input copies
+    ggml_backend_event_t input_copy_events[GGML_SCHED_MAX_BACKENDS] = {};
+    bool                 input_copy_pending[GGML_SCHED_MAX_BACKENDS] = {};
     bool is_reset; // true if the scheduler has been reset since the last graph split
     bool is_alloc;
 
@@ -1212,6 +1215,19 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru
 
 // assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
 void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
+    // tandem fix: the context buffer allocation at creation time is unchecked,
+    // so under host RAM pressure it can be NULL, which used to GGML_ABORT here
+    // (crashing the whole server on the first decode after a long prompt).
+    // Retry the allocation now and fail the graph build gracefully instead;
+    // callers report an alloc error and the server stays alive.
+    if (sched->context_buffer == NULL) {
+        sched->context_buffer = (char *) malloc(sched->context_buffer_size);
+        if (sched->context_buffer == NULL) {
+            GGML_LOG_ERROR("%s: failed to allocate %zu bytes of scheduler context (host RAM exhausted)\n", __func__, sched->context_buffer_size);
+            sched->ctx = NULL;
+            return;
+        }
+    }
     // reset splits
     sched->n_splits = 0;
     sched->n_graph_inputs = 0;
@@ -1227,7 +1243,8 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
 
     sched->ctx = ggml_init(params);
     if (sched->ctx == NULL) {
-        GGML_ABORT("%s: failed to initialize context\n", __func__);
+        GGML_LOG_ERROR("%s: failed to initialize context (host RAM exhausted)\n", __func__);
+        return;
     }
 
     graph->uid = ggml_graph_next_uid();
@@ -1837,14 +1854,24 @@ static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggm
         return;
     }
 
+    // without events, a copy callback that takes the input orders its copies on the split backend itself: the host does not wait
+    // for the split backend here, so it can launch the graph of the split while the copies run (GGML_SCHED_CALLBACK_SYNC=1: wait)
+    static const bool callback_sync = getenv("GGML_SCHED_CALLBACK_SYNC") != nullptr;
+    const bool callback = sched->callback_copy != NULL && ggml_backend_sched_is_host_weight(input);
+    const bool no_events = sched->events[split_backend_id][sched->cur_copy] == NULL;
+    if (callback && no_events && !callback_sync &&
+        sched->callback_copy(split_backend, input, input_cpy, &split->graph, sched->callback_copy_user_data)) {
+        return;
+    }
+
     // wait for the split backend to finish using the input before overwriting it
-    if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+    if (!no_events) {
         ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
     } else {
         ggml_backend_synchronize(split_backend);
     }
 
-    if (sched->callback_copy != NULL && ggml_backend_sched_is_host_weight(input) &&
+    if (callback && (!no_events || callback_sync) &&
         sched->callback_copy(split_backend, input, input_cpy, &split->graph, sched->callback_copy_user_data)) {
         return;
     }
@@ -1862,6 +1889,7 @@ static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggm
     }
 }
 
+#include "../../src/llama-prof.h"
 static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {
     GGML_ASSERT(sched);
     struct ggml_backend_sched_split * splits = sched->splits;
@@ -1872,6 +1900,9 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
         struct ggml_backend_sched_split * split = &splits[split_id];
         int split_backend_id = split->backend_id;
         ggml_backend_t split_backend = sched->backends[split_backend_id];
+        const bool lc_cpu = split_backend_id == sched->n_backends - 1;
+        const bool lc_big = sched->graph.n_nodes > 2000;
+        llama_prof_scope lc_prof_sp(lc_cpu ? (lc_big ? "sched.inputs.cpu.tgt" : "sched.inputs.cpu.dft") : (lc_big ? "sched.inputs.gpu.tgt" : "sched.inputs.gpu.dft"));
 
         // ensure the previous split's async work has completed before we start
         // this split, the allocator may have reused buffer regions across splits
@@ -1883,11 +1914,94 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
             }
         }
 
+        // GGML_SCHED_BATCH_INPUTS=1: the user inputs (graph inputs in host memory) are copied with one wait for the split
+        // backend before and one after, instead of a wait and a blocking copy each; same bytes, fewer host round trips
+        static const bool batch_inputs = getenv("GGML_SCHED_BATCH_INPUTS") != nullptr && atoi(getenv("GGML_SCHED_BATCH_INPUTS")) != 0;
+        static const bool no_wait      = batch_inputs && atoi(getenv("GGML_SCHED_BATCH_INPUTS")) >= 2;
+        // mode 3: as mode 2, but the backend packs the copies into one upload when it can (CUDA: one H2D copy and one
+        // scatter kernel instead of a cudaMemcpyAsync per input); the host data is staged at once, so no input wait
+        static const bool packed       = batch_inputs && atoi(getenv("GGML_SCHED_BATCH_INPUTS")) == 3;
+        // mode 1: the flagged user inputs; mode 2: every input in host memory (also views of inputs and the results of a
+        // CPU split), which the host writes only after ggml_backend_sched_wait_input_copies
+        auto batched = [&](const struct ggml_tensor * input) {
+            return input->buffer && ggml_backend_buffer_is_host(input->buffer) && !ggml_backend_sched_is_host_weight(input) &&
+                   (no_wait || (input->flags & GGML_TENSOR_FLAG_INPUT));
+        };
+        bool user_done = false;
+        if (batch_inputs && split_backend->iface.set_tensor_async != NULL) {
+            int n_user = 0;
+            for (int input_id = 0; input_id < split->n_inputs; input_id++) {
+                struct ggml_tensor * input = split->inputs[input_id];
+                n_user += batched(input);
+            }
+            if (no_wait && sched->input_copy_events[split_backend_id] == NULL) {
+                sched->input_copy_events[split_backend_id] = ggml_backend_event_new(ggml_backend_get_device(split_backend));
+            }
+            typedef bool (*set_packed_t)(ggml_backend_t, int, ggml_tensor * const *, const void * const *, const size_t *);
+            set_packed_t set_packed = nullptr;
+            if (packed && n_user > 1) {
+                ggml_backend_dev_t dev = ggml_backend_get_device(split_backend);
+                ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+                set_packed = reg ? (set_packed_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_tensors_packed") : nullptr;
+            }
+            if (set_packed) {
+                static thread_local std::vector<ggml_tensor *> dst;
+                static thread_local std::vector<const void *> src;
+                static thread_local std::vector<size_t> nb;
+                dst.clear(); src.clear(); nb.clear();
+                for (int input_id = 0; input_id < split->n_inputs; input_id++) {
+                    struct ggml_tensor * input = split->inputs[input_id];
+                    if (batched(input)) {
+                        dst.push_back(tensor_copy(input, split_backend_id, sched->cur_copy));
+                        src.push_back(input->data);
+                        nb.push_back(ggml_nbytes(input));
+                    }
+                }
+                user_done = set_packed(split_backend, (int) dst.size(), dst.data(), src.data(), nb.data());
+            }
+            if (user_done) {
+                // staged by the host already: nothing to wait for before the inputs are written again
+            } else if (n_user > 1 && no_wait && sched->input_copy_events[split_backend_id] != NULL) {
+                // the copies queue on the split backend after its previous work, so they cannot overwrite inputs a
+                // running graph reads; the host waits for the event before it writes the inputs again
+                for (int input_id = 0; input_id < split->n_inputs; input_id++) {
+                    struct ggml_tensor * input = split->inputs[input_id];
+                    if (batched(input)) {
+                        struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);
+                        ggml_backend_tensor_set_async(split_backend, input_cpy, input->data, 0, ggml_nbytes(input));
+                    }
+                }
+                ggml_backend_event_record(sched->input_copy_events[split_backend_id], split_backend);
+                sched->input_copy_pending[split_backend_id] = true;
+                user_done = true;
+            } else if (n_user > 1) {
+                if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+                    ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
+                } else {
+                    ggml_backend_synchronize(split_backend);
+                }
+                for (int input_id = 0; input_id < split->n_inputs; input_id++) {
+                    struct ggml_tensor * input = split->inputs[input_id];
+                    if (batched(input)) {
+                        struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);
+                        ggml_backend_tensor_set_async(split_backend, input_cpy, input->data, 0, ggml_nbytes(input));
+                    }
+                }
+                // the host may overwrite the inputs once this returns
+                ggml_backend_synchronize(split_backend);
+                user_done = true;
+            }
+        }
+
         // copy the input tensors to the split backend
         // the weights in host memory are copied last, so that the copy callback can read the other inputs of the split
         for (int input_id = 0; input_id < split->n_inputs; input_id++) {
-            if (!ggml_backend_sched_is_host_weight(split->inputs[input_id])) {
-                ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]);
+            struct ggml_tensor * input = split->inputs[input_id];
+            if (user_done && batched(input)) {
+                continue;
+            }
+            if (!ggml_backend_sched_is_host_weight(input)) {
+                ggml_backend_sched_copy_input(sched, split, input);
             }
         }
         for (int input_id = 0; input_id < split->n_inputs; input_id++) {
@@ -1896,6 +2010,8 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
             }
         }
 
+        llama_prof_lap("lap.d_sched_inputs");
+        lc_prof_sp.~llama_prof_scope(); new (&lc_prof_sp) llama_prof_scope(lc_cpu ? (lc_big ? "sched.compute.cpu.tgt" : "sched.compute.cpu.dft") : (lc_big ? "sched.compute.gpu.tgt" : "sched.compute.gpu.dft"));
         if (!sched->callback_eval) {
             enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);
             if (ec != GGML_STATUS_SUCCESS) {
@@ -1978,7 +2094,11 @@ ggml_backend_sched_t ggml_backend_sched_new(
     sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
     sched->hv_tensor_copies      = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
 
-    const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph
+    // Tandem (Windows low-RAM fix): the worst-case every-node-is-a-split reservation costs
+// max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(ggml_tensor) (~20 KB per node!), which is
+// ~487 MB of host RAM for a 24k-node graph (Qwen3.5-MoE). Real splits are bounded by backend
+// boundaries (measured 121 for a 35B MoE with the expert cache), so cap the reservation.
+const size_t ggml_sched_max_splits = graph_size < 4096 ? graph_size : (graph_size/8 > 4096 ? graph_size/8 : 4096);
     const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;
     sched->node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->node_backend_ids[0]));
     sched->leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));
@@ -2081,6 +2201,10 @@ bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph *
 
     ggml_backend_sched_split_graph(sched, measure_graph);
 
+    if (sched->ctx == NULL) {
+        return false;
+    }
+
     if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {
         return false;
     }
@@ -2100,6 +2224,12 @@ bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgra
 
     ggml_backend_sched_split_graph(sched, graph);
 
+    // tandem fix: a failed split leaves sched->ctx NULL; report an alloc
+    // error to the caller instead of running on a stale graph
+    if (sched->ctx == NULL) {
+        return false;
+    }
+
     if (!ggml_backend_sched_alloc_splits(sched)) {
         return false;
     }
@@ -2130,6 +2260,16 @@ enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sch
     return ggml_backend_sched_compute_splits(sched);
 }
 
+void ggml_backend_sched_wait_input_copies(ggml_backend_sched_t sched) {
+    GGML_ASSERT(sched);
+    for (int i = 0; i < sched->n_backends; i++) {
+        if (sched->input_copy_pending[i]) {
+            ggml_backend_event_synchronize(sched->input_copy_events[i]);
+            sched->input_copy_pending[i] = false;
+        }
+    }
+}
+
 void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {
     GGML_ASSERT(sched);
     for (int i = 0; i < sched->n_backends; i++) {
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 53d39d249..852fe3c9f 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -1742,6 +1742,38 @@ static void ggml_compute_forward_mul_mat_id(
 
 /////////////////////////////////
 
+// see ggml_host_add
+static void ggml_compute_forward_host_add(struct ggml_tensor * dst) {
+    uint64_t p[2];
+    memcpy(p, dst->op_params, sizeof(p));
+    const float       * data = (const float *) (uintptr_t) p[0];
+    volatile uint32_t * flag = (volatile uint32_t *) (uintptr_t) p[1];
+
+    const uint32_t seen = flag[1];
+    const int64_t  t0   = ggml_time_us();
+    uint32_t v;
+    while ((v = flag[0]) == seen) {
+        if (ggml_time_us() - t0 > 2000000) {
+            flag[3] = flag[3] + 1;
+            break;
+        }
+    }
+#if defined(_MSC_VER)
+    _ReadBarrier();
+#else
+    __atomic_thread_fence(__ATOMIC_ACQUIRE);
+#endif
+    flag[1] = v;
+    flag[2] = (uint32_t) ((ggml_time_us() - t0)*1000);
+
+    const float * a = (const float *) dst->src[0]->data;
+    float       * d = (float *) dst->data;
+    const int64_t n = ggml_nelements(dst);
+    for (int64_t i = 0; i < n; i++) {
+        d[i] = a[i] + data[i];
+    }
+}
+
 static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {
     GGML_ASSERT(params);
 
@@ -2154,6 +2186,21 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
                 ggml_compute_forward_opt_step_sgd(params, tensor);
             }
             break;
+        case GGML_OP_HOST_ADD:
+            {
+                if (params->ith == 0) {
+                    ggml_compute_forward_host_add(tensor);
+                }
+            }
+            break;
+        case GGML_OP_MOE_DOORBELL:
+            {
+                GGML_ABORT("MOE_DOORBELL is only implemented on CUDA");
+            }
+        case GGML_OP_GET_ROWS_HOST:
+            {
+                GGML_ABORT("GET_ROWS_HOST is only implemented on CUDA");
+            }
         case GGML_OP_NONE:
             {
                 // nop
@@ -2503,6 +2550,12 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
             {
                 n_tasks = n_threads;
             } break;
+        case GGML_OP_HOST_ADD:
+        case GGML_OP_MOE_DOORBELL:
+        case GGML_OP_GET_ROWS_HOST:
+            {
+                n_tasks = 1;
+            } break;
         case GGML_OP_NONE:
             {
                 n_tasks = 1;
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 85e17018c..86043305b 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -11117,8 +11117,8 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
     GGML_ASSERT(ggml_is_contiguous_rows(src_q));
     GGML_ASSERT(ggml_is_contiguous_rows(src_k));
     GGML_ASSERT(ggml_is_contiguous_rows(src_v));
-    GGML_ASSERT(ggml_is_contiguous(src_g));
-    GGML_ASSERT(ggml_is_contiguous(src_beta));
+    GGML_ASSERT(dst->src[7] || ggml_is_contiguous(src_g));
+    GGML_ASSERT(dst->src[7] || ggml_is_contiguous(src_beta));
     GGML_ASSERT(ggml_is_contiguous(src_state));
 
     GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v);
@@ -11186,7 +11186,8 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
 
         // copy input state into the working buffer and operate in-place
         // state layout [S_v, S_v, H, n_seqs]: seq iv3 starts at iv3 * state_seq_stride.
-        const float * s_in = state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v;
+        const int64_t s_row = dst->src[6] ? (int64_t) ((const int32_t *) dst->src[6]->data)[iv3] : iv3;
+        const float * s_in = state_in_base + s_row * state_seq_stride + iv1 * S_v * S_v;
         memcpy(s_out, s_in, S_v * S_v * sizeof(float));
 
         // attn output pointer for first token of this (head, seq)
@@ -11197,8 +11198,18 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
             const float * k_d = (const float *)((const char *)src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1);
             const float * v_d = (const float *)((const char *)src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1);
 
-            const float beta_val = *(const float *)((const char *)src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1);
+            float beta_val = *(const float *)((const char *)src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1);
             const float * g_d    =  (const float *)((const char *)src_g->data    + iv3 * nbg3 + t * nbg2 + iv1 * nbg1);
+            // raw gates (ggml_gated_delta_net_ext): the same float operations as add -> softplus -> mul and sigmoid
+            float g_raw;
+            if (dst->src[7]) {
+                beta_val = 1.0f / (1.0f + expf(-beta_val));
+                volatile float x = g_d[0] + ((const float *) dst->src[7]->data)[iv1];
+                const float sp = (x > 20.0f) ? (float) x : logf(1.0f + expf(x));
+                volatile float gl = sp * ((const float *) dst->src[8]->data)[iv1];
+                g_raw = gl;
+                g_d = &g_raw;
+            }
 
             // state is stored transposed: s_out[j*S_v + i] = S[i][j]
             // so row j of s_out = column j of S (contiguous access)
@@ -11465,6 +11476,9 @@ static void ggml_compute_forward_dsv4_hc_pre_f32(
 
     const float scale = ggml_get_op_params_f32(dst, 0);
     const bool  gated = ggml_get_op_params_i32(dst, 1) != 0;
+    const bool  affine  = ggml_get_op_params_i32(dst, 2) != 0; // weights are gate logits (ggml_dsv4_hc_pre_affine)
+    const float aff_eps = affine ? ggml_get_op_params_f32(dst, 3) : 0.0f;
+    const float aff_s   = affine ? *(const float *) dst->src[2]->data : 0.0f;
 
     GGML_ASSERT(dst->ne[0] == n_embd);
     GGML_ASSERT(dst->ne[1] == n_tokens);
@@ -11502,6 +11516,13 @@ static void ggml_compute_forward_dsv4_hc_pre_f32(
                 wv = 1.0f / (1.0f + expf(-gv));
             } else {
                 wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1);
+                if (affine) {
+                    // mul -> add -> sigmoid -> scale_bias(1, eps), kept separate as the unfused ops
+                    volatile float m = wv * aff_s;
+                    volatile float v = m + *(const float *) ((const char *) dst->src[3]->data + ih*dst->src[3]->nb[0]);
+                    volatile float g = 1.0f / (1.0f + expf(-v));
+                    wv = g + aff_eps;
+                }
             }
             sum += xv * wv;
         }
@@ -11581,13 +11602,33 @@ static void ggml_compute_forward_dsv4_hc_post_f32(
     const int64_t ir0 = dr * ith;
     const int64_t ir1 = MIN(ir0 + dr, nr);
 
+    const int   post_sig = ggml_get_op_params_i32(dst, 0); // post holds logits: post_out * sigmoid(post_in * post)
+    const float post_in  = ggml_get_op_params_f32(dst, 1);
+    const float post_out = ggml_get_op_params_f32(dst, 2);
+
     for (int64_t ir = ir0; ir < ir1; ++ir) {
         const int64_t i0     = ir % n_embd;
         const int64_t idst   = (ir / n_embd) % hc;
         const int64_t it     = ir / (n_embd * hc);
 
-        const float xv = *(const float *) ((const char *) x->data    + i0*nbx0 + it*nbx1);
-        const float pv = *(const float *) ((const char *) post->data + idst*nbp0 + it*nbp1);
+        float xv = *(const float *) ((const char *) x->data    + i0*nbx0 + it*nbx1);
+        if (dst->src[4]) {
+            const ggml_tensor * add = dst->src[4];
+            const ggml_tensor * ag  = dst->src[5];
+            const float g = 1.0f / (1.0f + expf(-*(const float *) ((const char *) ag->data + it*ag->nb[1])));
+            const float a = *(const float *) ((const char *) add->data + i0*add->nb[0] + it*add->nb[1]);
+            volatile float prod = a * g; // keep mul and add separate, as the unfused ops
+            xv = xv + prod;
+        }
+        float pv = *(const float *) ((const char *) post->data + idst*nbp0 + it*nbp1);
+        if (post_sig == 2) {
+            // gate logits (ggml_dsv4_hc_post_affine): mul -> add -> sigmoid -> scale, kept separate as the unfused ops
+            volatile float m = pv * *(const float *) dst->src[6]->data;
+            volatile float v = m + *(const float *) ((const char *) dst->src[7]->data + idst*dst->src[7]->nb[0]);
+            pv = post_out * (1.0f / (1.0f + expf(-v)));
+        } else if (post_sig) {
+            pv = post_out * (1.0f / (1.0f + expf(-(post_in * pv))));
+        }
 
         float sum = xv * pv;
         if (comb) {
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
index 6b36ca90b..5dd74c77d 100644
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -1275,6 +1275,8 @@ struct ggml_cuda_graph {
     bool disable_due_to_gpu_arch = false;
     bool warmup_complete = false;
     uint64_t uid = 0;
+    uint64_t compat_uid = 0;  // the ggml graph uid whose compatibility check gave compat_ok (a reused graph is not walked again)
+    bool     compat_ok  = false;
     int64_t last_used_time = 0;
     struct node_properties {
         ggml_tensor node;
@@ -1442,11 +1444,34 @@ struct ggml_cuda_stream_context {
     }
 };
 
+// GGML_CUDA_Q8_CACHE: q8_1 copies of matmul inputs, reused within one graph evaluation when several quantized matmuls
+// read the same input (q/k/v, qkv/gate, experts/shared expert)
+struct ggml_cuda_q8_cache_entry {
+    const ggml_tensor * base  = nullptr; // the tensor the input views, a graph node of this evaluation
+    const void *        data  = nullptr;
+    int64_t             ne[4] = {};
+    size_t              nb[4] = {};
+    uint64_t            epoch = 0;
+};
+static constexpr int    GGML_CUDA_Q8_CACHE_N    = 4;
+static constexpr size_t GGML_CUDA_Q8_CACHE_SIZE = 4u << 20; // per entry; larger inputs bypass the cache
+
 struct ggml_backend_cuda_context {
     int device;
     std::string name;
     cudaEvent_t copy_event = nullptr;
 
+    ggml_cuda_q8_cache_entry q8_cache[GGML_CUDA_Q8_CACHE_N];
+    char *   q8_cache_mem  = nullptr; // allocated once: captured CUDA graphs keep these addresses
+
+    // GGML_SCHED_BATCH_INPUTS=3: packed input uploads (ggml_backend_cuda_set_tensors_packed), two pinned/device slots
+    char *      pack_host[2] = {};
+    char *      pack_dev[2]  = {};
+    cudaEvent_t pack_ev[2]   = {};
+    int         pack_i       = 0;
+    uint64_t q8_cache_epoch = 1;
+    int      q8_cache_next  = 0;
+
     cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
     cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
     void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
diff --git a/ggml/src/ggml-cuda/dsv4-hc.cu b/ggml/src/ggml-cuda/dsv4-hc.cu
index 2d0b3cfea..c2ae4413b 100644
--- a/ggml/src/ggml-cuda/dsv4-hc.cu
+++ b/ggml/src/ggml-cuda/dsv4-hc.cu
@@ -116,7 +116,11 @@ static __global__ void dsv4_hc_pre_f32(
         int64_t sw2,
         int64_t sd0,
         int64_t sd1,
-        float   scale) {
+        float   scale,
+        const float * aff_scale,
+        const float * aff_base,
+        int64_t sb0,
+        float   aff_eps) {
     ggml_cuda_pdl_lc();
     const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
     const int64_t nr = n_embd * n_tokens;
@@ -138,6 +142,11 @@ static __global__ void dsv4_hc_pre_f32(
             wv = 1.0f / (1.0f + expf(-weights[i0*sw0 + ih*sw1 + it*sw2]));
         } else {
             wv = weights[ih*sw0 + it*sw1];
+            if (aff_scale) {
+                // the same operations as mul -> add -> sigmoid -> scale_bias(1, eps), without contraction into an fma
+                const float v = __fadd_rn(__fmul_rn(wv, aff_scale[0]), aff_base[ih*sb0]);
+                wv = __fadd_rn(1.0f / (1.0f + expf(-v)), aff_eps);
+            }
         }
         sum += xv * wv;
     }
@@ -170,7 +179,17 @@ static __global__ void dsv4_hc_post_f32(
         int64_t sd1,
         int64_t sd2,
         float gate_scale_in,
-        float gate_scale_out) {
+        float gate_scale_out,
+        int     post_sig,
+        float   post_in,
+        float   post_out,
+        const float * add,
+        const float * add_gate,
+        int64_t sa1,
+        int64_t sg1,
+        const float * aff_scale,
+        const float * aff_base,
+        int64_t sb0) {
     ggml_cuda_pdl_lc();
     const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
     const int64_t nr = n_embd * hc * n_tokens;
@@ -185,12 +204,25 @@ static __global__ void dsv4_hc_post_f32(
     const int64_t idst = (ir / n_embd) % hc;
     const int64_t it   = ir / (n_embd * hc);
 
-    float p = post[idst*sp0 + it*sp1];
+    float pv = post[idst*sp0 + it*sp1];
     if constexpr (gated) {
-        p = gate_scale_out * (1.0f / (1.0f + expf(-(gate_scale_in * p))));
+        pv = gate_scale_out * (1.0f / (1.0f + expf(-(gate_scale_in * pv))));
     }
-
-    float sum = x[i0*sx0 + it*sx1] * p;
+    if (post_sig == 2) {
+        // the same operations as mul -> add -> sigmoid -> scale, without contraction into an fma
+        const float v = __fadd_rn(__fmul_rn(pv, aff_scale[0]), aff_base[idst*sb0]);
+        pv = post_out * (1.0f / (1.0f + expf(-v)));
+    } else if (post_sig) {
+        // same operations as scale -> sigmoid -> scale
+        pv = post_out * (1.0f / (1.0f + expf(-(post_in * pv))));
+    }
+    float xv = x[i0*sx0 + it*sx1];
+    if (add) {
+        // same operations as sigmoid -> mul -> add, without contraction into an fma
+        const float g = 1.0f / (1.0f + expf(-add_gate[it*sg1]));
+        xv = __fadd_rn(xv, __fmul_rn(add[i0 + it*sa1], g));
+    }
+    float sum = xv * pv;
     if constexpr (has_comb) {
         for (int64_t isrc = 0; isrc < hc; ++isrc) {
             sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2];
@@ -270,6 +302,9 @@ void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst
     const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1);
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream());
 
+    const bool affine = ggml_get_op_params_i32(dst, 2) != 0;
+    GGML_ASSERT(!affine || (!gated && dst->src[2] && dst->src[3]));
+
     auto kernel = gated ? dsv4_hc_pre_f32<true> : dsv4_hc_pre_f32<false>;
     ggml_cuda_kernel_launch(kernel, launch_params,
             (const float *) x->data, (const float *) weights->data, (float *) dst->data,
@@ -277,7 +312,11 @@ void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst
             nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float),
             nbw0 / sizeof(float), nbw1 / sizeof(float), nbw2 / sizeof(float),
             nbd0 / sizeof(float), nbd1 / sizeof(float),
-            scale);
+            scale,
+            affine ? (const float *) dst->src[2]->data : nullptr,
+            affine ? (const float *) dst->src[3]->data : nullptr,
+            affine ? dst->src[3]->nb[0] / sizeof(float) : 0,
+            affine ? ggml_get_op_params_f32(dst, 3) : 0.0f);
 }
 
 void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
@@ -298,6 +337,8 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds
     GGML_ASSERT(post->type == GGML_TYPE_F32);
     GGML_ASSERT(comb == nullptr || comb->type == GGML_TYPE_F32);
     GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(dst->src[4] == nullptr || dst->src[4]->nb[0] == sizeof(float));
+    GGML_ASSERT(ggml_get_op_params_i32(dst, 0) != 2 || (dst->src[6] && dst->src[7]));
 
     GGML_TENSOR_LOCALS(size_t, nbx, x,        nb);
     GGML_TENSOR_LOCALS(size_t, nbr, residual, nb);
@@ -329,5 +370,13 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds
             nbp0 / sizeof(float), nbp1 / sizeof(float),
             nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float),
             nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float),
-            gate_in, gate_out);
+            gate_in, gate_out,
+            ggml_get_op_params_i32(dst, 0), ggml_get_op_params_f32(dst, 1), ggml_get_op_params_f32(dst, 2),
+            dst->src[4] ? (const float *) dst->src[4]->data : nullptr,
+            dst->src[5] ? (const float *) dst->src[5]->data : nullptr,
+            dst->src[4] ? dst->src[4]->nb[1] / sizeof(float) : 0,
+            dst->src[5] ? dst->src[5]->nb[1] / sizeof(float) : 0,
+            dst->src[6] ? (const float *) dst->src[6]->data : nullptr,
+            dst->src[7] ? (const float *) dst->src[7]->data : nullptr,
+            dst->src[7] ? dst->src[7]->nb[0] / sizeof(float) : 0);
 }
diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu
index ae2374cbc..19860deff 100644
--- a/ggml/src/ggml-cuda/gated_delta_net.cu
+++ b/ggml/src/ggml-cuda/gated_delta_net.cu
@@ -14,6 +14,9 @@ gated_delta_net_cuda(const float * q,
                                      const float * g,
                                      const float * beta,
                                      const float * curr_state,
+                                     const int32_t * state_ids,
+                                     const float * gate_dt,
+                                     const float * gate_a,
                                      float *       dst,
                                      float *       state,
                                      int64_t       H,
@@ -66,6 +69,10 @@ gated_delta_net_cuda(const float * q,
     // state is stored transposed: M[col][i] = S[i][col], row col is contiguous
 
     ggml_cuda_pdl_sync();
+    if (state_ids != nullptr) {
+        // the input is the whole cache and sequence i starts at row state_ids[i]
+        curr_state += ((int64_t) state_ids[sequence] - (int64_t) sequence) * H * S_v * S_v;
+    }
 #pragma unroll
     for (int r = 0; r < rows_per_lane; r++) {
         const int i = r * lanes_per_col + lane_in_col;
@@ -81,7 +88,8 @@ gated_delta_net_cuda(const float * q,
         const float * beta_t = beta + gb_offset;
         const float * g_t    = g    + gb_offset * (KDA ? S_v : 1);
 
-        const float beta_val = *beta_t;
+        // raw gates: the same float operations as add -> softplus -> mul and sigmoid
+        const float beta_val = gate_dt ? 1.0f / (1.0f + expf(-*beta_t)) : *beta_t;
 
         // Cache k and q in registers
         float k_reg[rows_per_lane];
@@ -94,7 +102,13 @@ gated_delta_net_cuda(const float * q,
         }
 
         if constexpr (!KDA) {
-            const float g_val = expf(*g_t);
+            float g_log = *g_t;
+            if (gate_dt) {
+                const float x  = __fadd_rn(g_log, gate_dt[h_idx]);
+                const float sp = (x > 20.0f) ? x : logf(1.0f + expf(x));
+                g_log = __fmul_rn(sp, gate_a[h_idx]);
+            }
+            const float g_val = expf(g_log);
 
             // kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i]
             float kv_shard = 0.0f;
@@ -181,7 +195,8 @@ gated_delta_net_cuda(const float * q,
 template <bool KDA, bool keep_rs_t>
 static void launch_gated_delta_net(
         const float * q_d, const float * k_d, const float * v_d,
-        const float * g_d, const float * b_d, const float * s_d,
+        const float * g_d, const float * b_d, const float * s_d, const int32_t * ids_d,
+        const float * dt_d, const float * a_d,
         float * dst_d, float * state_d,
         int64_t S_v,   int64_t H, int64_t n_tokens, int64_t n_seqs,
         int64_t sq1,   int64_t sq2, int64_t sq3,
@@ -210,26 +225,26 @@ static void launch_gated_delta_net(
     switch (S_v) {
         case 16:
             ggml_cuda_kernel_launch(gated_delta_net_cuda<16, KDA, keep_rs_t>, launch_params,
-                q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H,
+                q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d, H,
                 n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
             break;
         case 32:
             ggml_cuda_kernel_launch(gated_delta_net_cuda<32, KDA, keep_rs_t>, launch_params,
-                q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H,
+                q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d, H,
                 n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
             break;
         case 64: {
             ggml_cuda_kernel_launch(gated_delta_net_cuda<64, KDA, keep_rs_t>, launch_params,
-                q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H,
+                q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d, H,
                 n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
             break;
         }
         case 128: {
             ggml_cuda_kernel_launch(gated_delta_net_cuda<128, KDA, keep_rs_t>, launch_params,
-                q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H,
+                q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d, H,
                 n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
             break;
@@ -315,6 +330,11 @@ static void ggml_cuda_op_gated_delta_net_impl(
     const float * b_d = (const float *) src_beta->data;
 
     const float * s_d   = (const float *) src_state->data;
+    // optional row index per sequence into a whole-cache state (ggml_gated_delta_net_ids)
+    const int32_t * ids_d = dst->src[6] ? (const int32_t *) dst->src[6]->data : nullptr;
+    // optional raw gates (ggml_gated_delta_net_ext): softplus/sigmoid applied in the kernel
+    const float * dt_d = dst->src[7] ? (const float *) dst->src[7]->data : nullptr;
+    const float * a_d  = dst->src[8] ? (const float *) dst->src[8]->data : nullptr;
     float *       dst_d = (float *) dst->data;
 
     GGML_ASSERT(ggml_is_contiguous_rows(src_q));
@@ -322,8 +342,9 @@ static void ggml_cuda_op_gated_delta_net_impl(
     GGML_ASSERT(ggml_is_contiguous_rows(src_v));
     GGML_ASSERT(ggml_are_same_stride(src_q, src_k));
     GGML_ASSERT(src_g->ne[0] == 1 || kda);
-    GGML_ASSERT(ggml_is_contiguous(src_g));
-    GGML_ASSERT(ggml_is_contiguous(src_beta));
+    GGML_ASSERT(dt_d != nullptr || ggml_is_contiguous(src_g));
+    GGML_ASSERT(dt_d != nullptr || ggml_is_contiguous(src_beta));
+    GGML_ASSERT(dt_d == nullptr || (!kda && src_g->nb[2] == src_beta->nb[2] && src_g->nb[3] == src_beta->nb[3]));
     GGML_ASSERT(ggml_is_contiguous(src_state));
 
     // strides in floats (beta strides used for both g and beta offset computation)
@@ -362,21 +383,21 @@ static void ggml_cuda_op_gated_delta_net_impl(
     }
     if (kda) {
         if (keep_rs) {
-            launch_gated_delta_net<true, true>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d,
+            launch_gated_delta_net<true, true>(q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d,
                 S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream);
         } else {
-            launch_gated_delta_net<true, false>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d,
+            launch_gated_delta_net<true, false>(q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d,
                 S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream);
         }
     } else {
         if (keep_rs) {
-            launch_gated_delta_net<false, true>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d,
+            launch_gated_delta_net<false, true>(q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d,
                 S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream);
         } else {
-            launch_gated_delta_net<false, false>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d,
+            launch_gated_delta_net<false, false>(q_d, k_d, v_d, g_d, b_d, s_d, ids_d, dt_d, a_d, dst_d, state_d,
                 S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,
                 sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream);
         }
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 72e961d5c..5a2fe201f 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -62,6 +62,9 @@
 #include "ggml-cuda/gla.cuh"
 #include "ggml-cuda/gated_delta_net.cuh"
 #include "ggml-cuda/dsv4-hc.cuh"
+#include "ggml-cuda/hc-fused.cuh"
+#include "../../../src/llama-prof.h"
+static inline bool lc_big_graph_for_lap(const ggml_cgraph * g) { return g->n_nodes > 2000; }
 #include "ggml-cuda/set.cuh"
 #include "ggml-cuda/set-rows.cuh"
 #include "ggml-cuda/pad_reflect_1d.cuh"
@@ -387,7 +390,9 @@ static ggml_cuda_device_info ggml_cuda_init() {
         // Setting device scheduling strategy for iGPUs with cc121 to "spinning" to avoid delays in cuda synchronize calls.
         // TODO: Check for future drivers the default scheduling strategy and
         // remove this call again when cudaDeviceScheduleSpin is default.
-        if (prop.major == 12 && prop.minor == 1) {
+        // GGML_CUDA_SPIN=1: spin in synchronize calls on every device (as Strata does), no wake-up latency after the wait
+        static const bool spin_all = getenv("GGML_CUDA_SPIN") != nullptr && atoi(getenv("GGML_CUDA_SPIN")) != 0;
+        if ((prop.major == 12 && prop.minor == 1) || spin_all) {
             CUDA_CHECK(cudaSetDevice(physical_id));
             CUDA_CHECK(cudaSetDeviceFlags(cudaDeviceScheduleSpin));
         }
@@ -748,6 +753,11 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() {
     if (copy_event != nullptr) {
         CUDA_CHECK(cudaEventDestroy(copy_event));
     }
+    for (int i = 0; i < 2; ++i) {
+        if (pack_ev[i])   { CUDA_CHECK(cudaEventDestroy(pack_ev[i])); }
+        if (pack_host[i]) { CUDA_CHECK(cudaFreeHost(pack_host[i])); }
+        if (pack_dev[i])  { CUDA_CHECK(cudaFree(pack_dev[i])); }
+    }
     for (int i = 0; i < GGML_CUDA_MAX_DEVICES; ++i) {
         for (int j = 0; j < GGML_CUDA_MAX_STREAMS; ++j) {
             if (streams[i][j] != nullptr) {
@@ -2265,6 +2275,179 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
         nb1, nb2, nb3, stream);
 }
 
+// see ggml_host_add: one thread polls the flag in pinned host memory, then the add reads the data from host memory
+static __global__ void k_host_add_wait(volatile uint32_t * flag) {
+    const uint32_t seen = flag[1];
+    uint64_t t0, t;
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t0));
+    t = t0;
+    uint32_t v;
+    while ((v = flag[0]) == seen) {
+        __nanosleep(200);
+        asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+        if (t - t0 > 2000000000ull) {
+            flag[3] = flag[3] + 1;
+            break;
+        }
+    }
+    flag[1] = v;
+    flag[2] = (uint32_t) (t - t0);
+    __threadfence_system();
+}
+
+static __global__ void k_host_add(const float * a, const volatile float * data, float * dst, const int64_t n) {
+    const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x;
+    if (i < n) {
+        dst[i] = a[i] + data[i];
+    }
+}
+
+static void ggml_cuda_op_host_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+    uint64_t p[2];
+    memcpy(p, dst->op_params, sizeof(p));
+    const float * data = (const float *) (uintptr_t) p[0];
+    uint32_t    * flag = (uint32_t *) (uintptr_t) p[1];
+    const int64_t n = ggml_nelements(dst);
+    cudaStream_t stream = ctx.stream();
+    k_host_add_wait<<<1, 1, 0, stream>>>(flag);
+    k_host_add<<<(n + 255)/256, 256, 0, stream>>>((const float *) dst->src[0]->data, data, (float *) dst->data, n);
+    CUDA_CHECK(cudaGetLastError());
+}
+
+// see ggml_moe_doorbell. Kernel 1 (one block): writes ids, x, w into the pinned control block, rings (req = ack + 1), waits for
+// the host's answer, then copies it (the slots of ids and the upload list) to the device in one parallel read. Kernel 2 (many
+// blocks): the uploads, read from pinned host memory by the SMs (16-byte loads where aligned). The block's addresses are kernel
+// arguments: every read of host memory is a PCIe round trip
+static __global__ void k_moe_doorbell(volatile ggml_moe_db * vdb, int32_t * h_ids, float * h_x, float * h_w, const volatile uint32_t * h_ans,
+        const int32_t * ids, const float * x, const float * w, const int n_ids, const int n_x, const int n_tok, const int n_used,
+        const int slots_cap, const int max_fill, int32_t * dst) {
+    const int tid = threadIdx.x;
+    for (int i = tid; i < n_ids; i += blockDim.x) {
+        h_ids[i] = ids[i];
+    }
+    if (x != nullptr) {
+        if ((n_x & 3) == 0) {
+            for (int i = tid; i < n_x/4; i += blockDim.x) {
+                ((float4 *) h_x)[i] = ((const float4 *) x)[i];
+            }
+        } else {
+            for (int i = tid; i < n_x; i += blockDim.x) {
+                h_x[i] = x[i];
+            }
+        }
+        for (int i = tid; i < n_ids; i += blockDim.x) {
+            h_w[i] = w[i];
+        }
+    }
+    __threadfence_system();
+    __syncthreads();
+    __shared__ uint32_t ok;
+    if (tid == 0) {
+        const uint32_t r = vdb->ack + 1;
+        vdb->n_tok  = n_tok;
+        vdb->n_used = n_used;
+        vdb->hyb    = x != nullptr;
+        __threadfence_system();
+        vdb->req = r;
+        __threadfence_system();
+        uint64_t t0, t;
+        asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t0));
+        t = t0;
+        ok = 1;
+        while (vdb->ack != r) {
+            __nanosleep(64);
+            asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+            if (t - t0 > 2000000000ull) {
+                vdb->timeouts = vdb->timeouts + 1;
+                ok = 0;
+                break;
+            }
+        }
+        vdb->wait_ns = (uint32_t) (t - t0);
+        __threadfence();
+    }
+    __syncthreads();
+    // the answer: slots [slots_cap], then n_fill, pad and the uploads, all read at once
+    uint32_t * scratch = (uint32_t *) (dst + ((n_ids + 1) & ~1));
+    const int n_ans = slots_cap + 2 + 6*max_fill;
+    for (int i = tid; i < n_ans; i += blockDim.x) {
+        const uint32_t v = h_ans[i];
+        if (i < n_ids) {
+            dst[i] = ok ? (int32_t) v : 0;
+        } else if (i >= slots_cap) {
+            scratch[i - slots_cap] = ok || i != slots_cap ? v : 0;
+        }
+    }
+}
+
+static __global__ void k_moe_db_upload(const uint32_t * scratch) {
+    const uint32_t n_fill = scratch[0];
+    const ggml_moe_db_fill * fill = (const ggml_moe_db_fill *) (scratch + 2);
+    const uint64_t i0 = (uint64_t) blockIdx.x*blockDim.x + threadIdx.x;
+    const uint64_t stride = (uint64_t) gridDim.x*blockDim.x;
+    for (uint32_t f = 0; f < n_fill; ++f) {
+        const uint64_t src = fill[f].src, dst = fill[f].dst, n = fill[f].n;
+        if (((src | dst | n) & 15) == 0) {
+            const uint4 * s = (const uint4 *) src;
+            uint4 * d = (uint4 *) dst;
+            const uint64_t n16 = n/16;
+            uint64_t i = i0;
+            for (; i + 3*stride < n16; i += 4*stride) {
+                const uint4 a = s[i], b = s[i + stride], c = s[i + 2*stride], e = s[i + 3*stride];
+                d[i] = a; d[i + stride] = b; d[i + 2*stride] = c; d[i + 3*stride] = e;
+            }
+            for (; i < n16; i += stride) {
+                d[i] = s[i];
+            }
+        } else {
+            const uint8_t * s = (const uint8_t *) src;
+            uint8_t * d = (uint8_t *) dst;
+            for (uint64_t i = i0; i < n; i += stride) {
+                d[i] = s[i];
+            }
+        }
+    }
+}
+
+static void ggml_cuda_op_moe_doorbell(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+    uint64_t p;
+    memcpy(&p, dst->op_params, sizeof(p));
+    ggml_moe_db * db = (ggml_moe_db *) (uintptr_t) p;
+    char * base = (char *) db;
+    const int n_used = dst->op_params[2];
+    const ggml_tensor * ids = dst->src[0];
+    const ggml_tensor * x   = dst->src[1];
+    const ggml_tensor * w   = dst->src[2];
+    const int n_ids = (int) ggml_nelements(ids);
+    GGML_ASSERT(n_ids <= (int) db->slots_cap);
+    cudaStream_t stream = ctx.stream();
+    int32_t * d = (int32_t *) dst->data;
+    k_moe_doorbell<<<1, 256, 0, stream>>>(db, (int32_t *) (base + db->ids_off), (float *) (base + db->x_off), (float *) (base + db->w_off),
+        (const uint32_t *) (base + db->ans_off), (const int32_t *) ids->data, x ? (const float *) x->data : nullptr,
+        w ? (const float *) w->data : nullptr, n_ids, x ? (int) ggml_nelements(x) : 0, n_ids/n_used, n_used, db->slots_cap, db->max_fill, d);
+    static const int n_blocks = [] {
+        const char * s = getenv("GGML_CUDA_MOE_DB_BLOCKS");
+        return s ? std::max(1, atoi(s)) : 256;
+    }();
+    k_moe_db_upload<<<n_blocks, 256, 0, stream>>>((const uint32_t *) (d + ((n_ids + 1) & ~1)));
+    CUDA_CHECK(cudaGetLastError());
+}
+
+// see ggml_get_rows_host: one block per row, rows of IQ4_NL read from mapped host memory, dequantized like the CPU's
+// dequantize_row_iq4_nl (y[j] = d*kvalues[qs[j] & 15], y[j + 16] = d*kvalues[qs[j] >> 4]) for any row length (CUDA's
+// get_rows takes IQ4_NL rows only in multiples of 256)
+static __global__ void k_get_rows_host_iq4_nl(const char * w, const size_t nb1, const int64_t ne0, const int32_t * ids, float * dst, const size_t nb_dst1) {
+    const int64_t r = blockIdx.x;
+    const block_iq4_nl * row = (const block_iq4_nl *) (w + (size_t) ids[r]*nb1);
+    float * y = (float *) ((char *) dst + r*nb_dst1);
+    for (int64_t i = threadIdx.x; i < ne0; i += blockDim.x) {
+        const block_iq4_nl & b = row[i/QK4_NL];
+        const int j = i % QK4_NL;
+        const uint8_t q = b.qs[j % (QK4_NL/2)];
+        y[i] = __half2float(b.d)*kvalues_iq4nl[j < QK4_NL/2 ? (q & 0xf) : (q >> 4)];
+    }
+}
+
 static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) {
     switch (dst->op) {
         case GGML_OP_ARGMAX:
@@ -2607,6 +2790,31 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
         case GGML_OP_OPT_STEP_SGD:
             ggml_cuda_opt_step_sgd(ctx, dst);
             break;
+        case GGML_OP_HOST_ADD:
+            ggml_cuda_op_host_add(ctx, dst);
+            break;
+        case GGML_OP_MOE_DOORBELL:
+            ggml_cuda_op_moe_doorbell(ctx, dst);
+            break;
+        case GGML_OP_GET_ROWS_HOST:
+            {
+                uint64_t p, nb1, ne0;
+                memcpy(&p,   dst->op_params + 0, sizeof(p));
+                memcpy(&nb1, dst->op_params + 2, sizeof(nb1));
+                memcpy(&ne0, dst->op_params + 4, sizeof(ne0));
+                const ggml_tensor * ids = dst->src[0];
+                const int64_t n = ggml_nelements(ids);
+                if ((ggml_type) dst->op_params[6] == GGML_TYPE_IQ4_NL) {
+                    k_get_rows_host_iq4_nl<<<n, 128, 0, ctx.stream()>>>((const char *) (uintptr_t) p, (size_t) nb1, (int64_t) ne0,
+                        (const int32_t *) ids->data, (float *) dst->data, dst->nb[1]);
+                    break;
+                }
+                get_rows_cuda((const void *) (uintptr_t) p, (ggml_type) dst->op_params[6], (const int32_t *) ids->data, dst->data, GGML_TYPE_F32,
+                    (int64_t) ne0, (size_t) nb1, (size_t) nb1*n, (size_t) nb1*n,
+                    n, 1, 1, sizeof(int32_t), n*sizeof(int32_t), n*sizeof(int32_t),
+                    dst->nb[1], dst->nb[2], dst->nb[3], ctx.stream());
+            }
+            break;
         case GGML_OP_SOLVE_TRI:
             ggml_cuda_op_solve_tri(ctx, dst);
             break;
@@ -2658,6 +2866,68 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
     CUDA_CHECK(cudaMemcpyAsync((char *) tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
 }
 
+// packed upload of many small host inputs: the host packs them (with a descriptor table) into a pinned staging slot, one
+// H2D copy moves the slot, one kernel scatters it to the tensors. Returns false (nothing queued) when it does not apply.
+struct ggml_cuda_pack_desc { char * dst; uint32_t off; uint32_t n; };
+static constexpr size_t GGML_CUDA_PACK_CAP = 4u << 20;
+
+static __global__ void k_unpack_inputs(const char * __restrict__ stage, const ggml_cuda_pack_desc * __restrict__ desc) {
+    const ggml_cuda_pack_desc d = desc[blockIdx.x];
+    const char * src = stage + d.off;
+    uint32_t i = 0;
+    if ((((uintptr_t) d.dst) & 15) == 0) {
+        for (i = 16*threadIdx.x; i + 16 <= d.n; i += 16*blockDim.x) {
+            *(int4 *) (d.dst + i) = *(const int4 *) (src + i);
+        }
+        i = d.n & ~15u;
+    }
+    for (uint32_t j = i + threadIdx.x; j < d.n; j += blockDim.x) {
+        d.dst[j] = src[j];
+    }
+}
+
+static bool ggml_backend_cuda_set_tensors_packed(ggml_backend_t backend, int n, ggml_tensor * const * dst, const void * const * src,
+        const size_t * size) {
+    ggml_backend_cuda_context * ctx = (ggml_backend_cuda_context *) backend->context;
+    const size_t hdr = ((size_t) n*sizeof(ggml_cuda_pack_desc) + 15) & ~(size_t) 15;
+    size_t total = hdr;
+    for (int i = 0; i < n; ++i) {
+        ggml_backend_buffer_t buf = dst[i]->view_src ? dst[i]->view_src->buffer : dst[i]->buffer;
+        if (buf == nullptr || buf->buft != ggml_backend_cuda_buffer_type(ctx->device) || size[i] > UINT32_MAX) {
+            return false;
+        }
+        total += (size[i] + 15) & ~(size_t) 15;
+    }
+    if (n <= 0 || total > GGML_CUDA_PACK_CAP) {
+        return false;
+    }
+    ggml_cuda_set_device(ctx->device);
+    const int k = ctx->pack_i;
+    if (ctx->pack_host[k] == nullptr) {
+        CUDA_CHECK(cudaMallocHost((void **) &ctx->pack_host[k], GGML_CUDA_PACK_CAP));
+        CUDA_CHECK(cudaMalloc((void **) &ctx->pack_dev[k], GGML_CUDA_PACK_CAP));
+        CUDA_CHECK(cudaEventCreateWithFlags(&ctx->pack_ev[k], cudaEventDisableTiming));
+    } else {
+        // the slot's previous copy has to be done before the host writes it again (normally long ago)
+        CUDA_CHECK(cudaEventSynchronize(ctx->pack_ev[k]));
+    }
+    char * h = ctx->pack_host[k];
+    ggml_cuda_pack_desc * desc = (ggml_cuda_pack_desc *) h;
+    size_t off = hdr;
+    for (int i = 0; i < n; ++i) {
+        desc[i] = { (char *) dst[i]->data, (uint32_t) off, (uint32_t) size[i] };
+        memcpy(h + off, src[i], size[i]);
+        off += (size[i] + 15) & ~(size_t) 15;
+    }
+    cudaStream_t stream = ctx->stream();
+    CUDA_CHECK(cudaMemcpyAsync(ctx->pack_dev[k], h, total, cudaMemcpyHostToDevice, stream));
+    CUDA_CHECK(cudaEventRecord(ctx->pack_ev[k], stream));
+    k_unpack_inputs<<<n, 128, 0, stream>>>(ctx->pack_dev[k], (const ggml_cuda_pack_desc *) ctx->pack_dev[k]);
+    CUDA_CHECK(cudaGetLastError());
+    ctx->pack_i ^= 1;
+    return true;
+}
+
 static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
     ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
     ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
@@ -2801,7 +3071,16 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
 }
 
 static const void * ggml_cuda_graph_get_key(ggml_cgraph * cgraph) {
-    return cgraph->nodes[0];
+    // GGML_CUDA_GRAPH_KEY_SHAPE: one CUDA graph per graph shape (node count, output width), so graphs of different
+    // batch sizes (speculative verify windows) don't evict each other; the key is only a map key, never dereferenced
+    static const bool by_shape = getenv("GGML_CUDA_GRAPH_KEY_SHAPE") != nullptr && atoi(getenv("GGML_CUDA_GRAPH_KEY_SHAPE")) != 0;
+    if (!by_shape) {
+        return cgraph->nodes[0];
+    }
+    const ggml_tensor * last = cgraph->nodes[cgraph->n_nodes - 1];
+    uintptr_t k = (uintptr_t) cgraph->nodes[0];
+    k ^= ((uintptr_t) cgraph->n_nodes << 40) ^ ((uintptr_t) (last->ne[1] & 0xffff) << 24) ^ ((uintptr_t) (cgraph->nodes[0]->ne[1] & 0xffff) << 8);
+    return (const void *) k;
 }
 
 static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph) {
@@ -3657,6 +3936,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
         // the chain must feed hc_post->src[2]; a bias on either scale is not handled
         if (ggml_check_edges(cgraph, node_idx, {{1, 0, 0}, {2, 0, 1}, {3, 2, 2}}) &&
             ggml_get_unary_op(sigmoid) == GGML_UNARY_OP_SIGMOID &&
+            ggml_get_op_params_i32(cgraph->nodes[node_idx + 3], 0) == 0 && // not the post_sig / post_affine variants
             ggml_get_op_params_f32(scale_in, 1) == 0.0f && ggml_get_op_params_f32(scale_out, 1) == 0.0f &&
             scale_in->src[0]->type == GGML_TYPE_F32 && scale_out->type == GGML_TYPE_F32 &&
             ggml_are_same_shape(scale_in->src[0], scale_out)) {
@@ -4660,6 +4940,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
 }
 
 static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, const bool use_cuda_graph, const bool cuda_graph_update_required, const void * graph_key) {
+    // GGML_CUDA_Q8_CACHE: q8_1 inputs are reused only within one evaluation
+    cuda_ctx->q8_cache_epoch++;
+
     bool graph_evaluated_or_captured = false;
 
     // flag used to determine whether it is an integrated_gpu
@@ -4815,6 +5098,21 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud
                     continue;
                 }
 
+                if (node->view_src != nullptr) {
+                    // a node writing through a view (in place) changes its base: drop q8_1 copies of that base
+                    const ggml_tensor * vbase = node->view_src;
+                    while (vbase->view_src) { vbase = vbase->view_src; }
+                    for (auto & e : cuda_ctx->q8_cache) {
+                        if (e.base == vbase) { e.epoch = 0; }
+                    }
+                }
+                {
+                    const int hc_skip = ggml_cuda_hc_mix_fuse(*cuda_ctx, cgraph, i);
+                    if (hc_skip > 0) {
+                        i += hc_skip;
+                        continue;
+                    }
+                }
                 int nodes_to_skip = ggml_cuda_try_fuse(cuda_ctx, cgraph, i);
 
                 if (nodes_to_skip != 0) {
@@ -4908,6 +5206,18 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
 static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
     ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
 
+    // GGML_CUDA_PRINT_NODES=N: print the first N nodes of the first graph with more than 1000 nodes (debug)
+    static int print_nodes = [] { const char * e = getenv("GGML_CUDA_PRINT_NODES"); return e ? atoi(e) : 0; }();
+    if (print_nodes > 0 && cgraph->n_nodes > 1000) {
+        for (int i = 0; i < std::min(print_nodes, cgraph->n_nodes); ++i) {
+            const ggml_tensor * t = cgraph->nodes[i];
+            GGML_LOG_INFO("node %4d %-16s %-28s [%lld,%lld,%lld] %s src0 %s src1 %s\n", i, ggml_op_desc(t), t->name,
+                (long long) t->ne[0], (long long) t->ne[1], (long long) t->ne[2], ggml_type_name(t->type),
+                t->src[0] ? t->src[0]->name : "-", t->src[1] ? t->src[1]->name : "-");
+        }
+        print_nodes = 0;
+    }
+
     ggml_cuda_set_device(cuda_ctx->device);
 
     bool use_cuda_graph             = false;
@@ -4921,9 +5231,21 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
 
     ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);
     if (graph->is_enabled()) {
-        const bool graph_compatible = ggml_cuda_graph_check_compability(cgraph);
+        // the walk over every node costs ~47 us for a 7000-node graph; a reused ggml graph (same uid) gives the same answer
+        bool graph_compatible;
+        if (cgraph->uid != 0 && cgraph->uid == graph->compat_uid) {
+            graph_compatible = graph->compat_ok;
+        } else {
+            graph_compatible  = ggml_cuda_graph_check_compability(cgraph);
+            graph->compat_uid = cgraph->uid;
+            graph->compat_ok  = graph_compatible;
+        }
         if (graph_compatible) {
-            const bool properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph);
+            bool properties_changed;
+            {
+                LC_PROF(cgraph->n_nodes > 2000 ? "cuda.props_check.tgt" : "cuda.props_check.dft");
+                properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph);
+            }
 
             if (!graph->warmup_complete) {
                 // Warmup: need at least 2 calls with no property change on the 2nd call
@@ -4963,7 +5285,59 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
         CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed));
     }
 
+    // LLAMA_HOST_PROF: GPU time of each replayed or eager graph (events around it) and the GPU idle time before it
+    // (from the end of the previous timed graph on any stream), named by the graph kinds. Read back without blocking
+    // from a ring of 4 once the events are done; gaps > 20 ms (between requests) are left out. Capture passes are not
+    // timed and break the chain.
+    struct lc_gpu_ev { cudaEvent_t a = nullptr, b = nullptr; bool big = false, timed = false, done = true, chain = false; };
+    static lc_gpu_ev lc_ev[4];
+    static unsigned lc_ev_k = 0;
+    const bool lc_time = llama_prof_on() && !(use_cuda_graph && cuda_graph_update_required);
+    if (llama_prof_on()) {
+        for (unsigned j = lc_ev_k >= 3 ? lc_ev_k - 3 : 0; j < lc_ev_k; ++j) {
+            lc_gpu_ev & e = lc_ev[j % 4];
+            if (e.done || cudaEventQuery(e.b) != cudaSuccess) { continue; }
+            float ms = 0.0f;
+            CUDA_CHECK(cudaEventElapsedTime(&ms, e.a, e.b));
+            llama_prof_add(e.big ? "gpu.graph.tgt" : "gpu.graph.dft", ms * 1e3);
+            if (e.chain) {
+                const lc_gpu_ev & p = lc_ev[(j + 3) % 4];
+                CUDA_CHECK(cudaEventElapsedTime(&ms, p.b, e.a));
+                if (ms < 20.0f) {
+                    llama_prof_add(p.big ? (e.big ? "gpu.idle.tgt>tgt" : "gpu.idle.tgt>dft") : (e.big ? "gpu.idle.dft>tgt" : "gpu.idle.dft>dft"), ms * 1e3);
+                }
+            }
+            e.done = true;
+        }
+    }
+    lc_gpu_ev * lc_cur = nullptr;
+    if (lc_time) {
+        lc_cur = &lc_ev[lc_ev_k % 4];
+        if (!lc_cur->a) { CUDA_CHECK(cudaEventCreate(&lc_cur->a)); CUDA_CHECK(cudaEventCreate(&lc_cur->b)); }
+        const lc_gpu_ev & p = lc_ev[(lc_ev_k + 3) % 4];
+        // the previous slot must still hold the previous graph's events: timed, and read (or readable) before reuse
+        lc_cur->chain = lc_ev_k > 0 && p.timed;
+        if (!lc_cur->done) { lc_cur->chain = false; }
+        CUDA_CHECK(cudaEventRecord(lc_cur->a, cuda_ctx->stream()));
+        lc_cur->big = cgraph->n_nodes > 2000;
+    } else if (llama_prof_on()) {
+        lc_ev[(lc_ev_k + 3) % 4].timed = false;
+    }
+
+    {
+    const bool lc_big = cgraph->n_nodes > 2000;
+    LC_PROF(!use_cuda_graph ? (lc_big ? "cuda.eval_eager.tgt" : "cuda.eval_eager.dft") : cuda_graph_update_required ? (lc_big ? "cuda.eval_capture.tgt" : "cuda.eval_capture.dft") : (lc_big ? "cuda.launch.tgt" : "cuda.launch.dft"));
     ggml_cuda_graph_evaluate_and_capture(cuda_ctx, cgraph, use_cuda_graph, cuda_graph_update_required, graph_key);
+    }
+    if (!lc_big_graph_for_lap(cgraph)) {
+        llama_prof_lap("lap.draft_to_launch", true);
+    }
+    if (lc_cur) {
+        CUDA_CHECK(cudaEventRecord(lc_cur->b, cuda_ctx->stream()));
+        lc_cur->timed = true;
+        lc_cur->done = false;
+        lc_ev_k++;
+    }
 
     return GGML_STATUS_SUCCESS;
 }
@@ -5435,6 +5809,11 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
 
 #if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA) || defined(GGML_USE_HIP)
     cudaError_t err = cudaHostRegister(buffer, size, cudaHostRegisterPortable | cudaHostRegisterReadOnly);
+    if (err == cudaErrorNotSupported) {
+        // read-only registration is not available everywhere (asrok/rome: Linux 6.8, driver 615); writable memory still registers
+        (void)cudaGetLastError();
+        err = cudaHostRegister(buffer, size, cudaHostRegisterPortable);
+    }
     if (err != cudaSuccess) {
         // clear the error
         (void)cudaGetLastError();
@@ -6111,6 +6490,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
         case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
         case GGML_OP_OPT_STEP_ADAMW:
         case GGML_OP_OPT_STEP_SGD:
+        case GGML_OP_HOST_ADD:
+        case GGML_OP_MOE_DOORBELL:
+        case GGML_OP_GET_ROWS_HOST:
         case GGML_OP_FILL:
         case GGML_OP_CUMSUM:
         case GGML_OP_TRI:
@@ -6293,6 +6675,9 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con
     if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) {
         return (void *)ggml_backend_cuda_comm_allreduce_tensor;
     }
+    if (strcmp(name, "ggml_backend_set_tensors_packed") == 0) {
+        return (void *)ggml_backend_cuda_set_tensors_packed;
+    }
     if (strcmp(name, "ggml_backend_register_host_buffer") == 0) {
         return (void *)ggml_backend_cuda_register_host_buffer;
     }
diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index 752d28c4f..d970d4e73 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -805,6 +805,17 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const
         }
     }
 
+    // GGML_CUDA_MMVF_MAX=N: use the vector kernel up to batch N for float weights (small batches such as speculative
+    // verify windows, where the tile kernel loses on small matrices)
+    static const int mmvf_max = [] {
+        const char * env = getenv("GGML_CUDA_MMVF_MAX");
+        return env != nullptr ? atoi(env) : 0;
+    }();
+    if (mmvf_max > 0 && ne11 <= std::min(mmvf_max, MMVF_MAX_BATCH_SIZE) &&
+        (type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16)) {
+        return true;
+    }
+
     switch (type) {
         case GGML_TYPE_F32:
             if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 930f1adc4..681402099 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -1031,6 +1031,400 @@ static __global__ void mul_mat_vec_q_moe(
     }
 }
 
+// lc- expert-deduplicated MoE kernel (GGML_CUDA_MOE_DEDUP=1). Grid: (row blocks, n_used*ncols_dst pairs [+ shared expert]).
+// A pair's block takes its expert for every token that routes to it, so the weights of an expert used by several tokens
+// of the ubatch are read once per block; a pair whose expert an earlier pair already has exits at once. The shared
+// expert's block takes all tokens. Each warp takes c_rows_per_warp rows and runs, per token, the lane loop, dot, warp
+// reduction and epilogue of mul_mat_vec_q_moe in the same order: the output is bitwise the same.
+#define MMVQ_MOE_DEDUP_NWARPS 4
+#define MMVQ_MOE_DEDUP_MAX_PAIRS 256
+#ifndef MMVQ_MOE_DEDUP_MINB
+#define MMVQ_MOE_DEDUP_MINB 1 // minimum blocks per SM for __launch_bounds__: caps the registers
+#endif
+
+// Each dot of vecdotq.cuh split into load (everything that depends only on the weight: the signed grid words, the
+// integer scales, the fp16 scale as a float) and apply (the activation loads and the dp4a chain, integer scale step and
+// float expression of the dot, in the same order): apply(load(...), y) is bitwise the dot. Types without a split take
+// mul_mat_vec_q_moe.
+template <ggml_type type> struct moe_split { static constexpr bool ok = false; };
+
+template <> struct moe_split<GGML_TYPE_Q2_0> {
+    static constexpr bool ok = true;
+    struct W { int qx[4]; int qy[4]; float d2; };
+    static __device__ __forceinline__ W load(const void * __restrict__ vbq, const int kbx, const int iqs) {
+        const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx;
+        const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4;
+        W w;
+        w.d2 = bq2_0->d;
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            const int q  = qs[j];
+            const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);
+            const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);
+            w.qx[j] = __byte_perm(qe, qo, 0x5140);
+            w.qy[j] = __byte_perm(qe, qo, 0x7362);
+        }
+        return w;
+    }
+    static __device__ __forceinline__ float apply(const W & w, const block_q8_1 * __restrict__ bq8_1, const int iqs) {
+        const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;
+        int sumi = 0;
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            const int u = get_int_b4(bq8_1_chunk->qs, j*2+0);
+            const int v = get_int_b4(bq8_1_chunk->qs, j*2+1);
+            sumi = ggml_cuda_dp4a(u, w.qx[j], sumi);
+            sumi = ggml_cuda_dp4a(v, w.qy[j], sumi);
+        }
+        const float d8 = __low2float(bq8_1_chunk->ds);
+        return w.d2 * d8 * sumi;
+    }
+};
+
+template <> struct moe_split<GGML_TYPE_IQ2_S> {
+    static constexpr bool ok = true;
+    struct W { int gl[4]; int gh[4]; int ls0, ls1; float dw; };
+    static __device__ __forceinline__ W load(const void * __restrict__ vbq, const int kbx, const int iqs) {
+        const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;
+        const int       qs_packed = get_int_b2(bq2->qs, iqs/2);
+        const uint8_t * qs        = (const uint8_t *) &qs_packed;
+        const int qh = bq2->qh[iqs/2];
+        const int       signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);
+        const uint8_t * signs_packed_8  = (const uint8_t *) &signs_packed_32;
+        W w;
+        w.ls0 = bq2->scales[iqs/2] & 0x0F;
+        w.ls1 = bq2->scales[iqs/2] >> 4;
+#pragma unroll
+        for (int l0 = 0; l0 < 8; l0 += 2) {
+            const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));
+            const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
+            const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
+            w.gl[l0/2] = __vsub4(grid_pos[0] ^ signs0, signs0);
+            w.gh[l0/2] = __vsub4(grid_pos[1] ^ signs1, signs1);
+        }
+        w.dw = __half2float(bq2->d);
+        return w;
+    }
+    static __device__ __forceinline__ float apply(const W & w, const block_q8_1 * __restrict__ bq8_1, const int iqs) {
+        int sumi0 = 0;
+        int sumi1 = 0;
+#pragma unroll
+        for (int l0 = 0; l0 < 8; l0 += 2) {
+            const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+            const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+            if (l0 < 4) {
+                sumi0 = ggml_cuda_dp4a(w.gl[l0/2], u0, sumi0);
+                sumi0 = ggml_cuda_dp4a(w.gh[l0/2], u1, sumi0);
+            } else {
+                sumi1 = ggml_cuda_dp4a(w.gl[l0/2], u0, sumi1);
+                sumi1 = ggml_cuda_dp4a(w.gh[l0/2], u1, sumi1);
+            }
+        }
+        const int sumi = (sumi0*w.ls0 + sumi1*w.ls1 + (sumi0 + sumi1)/2)/4;
+        const float d = w.dw * __low2float(bq8_1[iqs/2].ds);
+        return d * sumi;
+    }
+};
+
+template <> struct moe_split<GGML_TYPE_IQ2_XXS> {
+    static constexpr bool ok = true;
+    struct W { int g[8]; int ls; float dw; };
+    static __device__ __forceinline__ W load(const void * __restrict__ vbq, const int kbx, const int iqs) {
+        const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;
+        const int q2 = get_int_b2(bq2->qs, iqs);
+        const uint8_t * aux8 = (const uint8_t *) &q2;
+        const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);
+        W w;
+#pragma unroll
+        for (int k0 = 0; k0 < 8; k0 += 2) {
+            const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[k0/2]];
+            const uint32_t signs = unpack_ksigns(aux32 >> (7 * k0 / 2));
+            const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+            w.g[k0 + 0] = __vsub4(grid_pos.x ^ signs0, signs0);
+            const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+            w.g[k0 + 1] = __vsub4(grid_pos.y ^ signs1, signs1);
+        }
+        w.ls = aux32 >> 27 | 1;
+        w.dw = __half2float(bq2->d);
+        return w;
+    }
+    static __device__ __forceinline__ float apply(const W & w, const block_q8_1 * __restrict__ bq8_1, const int iqs) {
+        int sumi = 0;
+#pragma unroll
+        for (int k0 = 0; k0 < 8; k0 += 2) {
+            const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);
+            sumi = ggml_cuda_dp4a(w.g[k0 + 0], u0, sumi);
+            const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);
+            sumi = ggml_cuda_dp4a(w.g[k0 + 1], u1, sumi);
+        }
+        sumi = sumi * w.ls / 8;
+        const float d = w.dw * __low2float(bq8_1[iqs/2].ds);
+        return d * sumi;
+    }
+};
+
+template <ggml_type type, int c_rows_per_warp, int c_ntok, bool has_fusion = false>
+__launch_bounds__(MMVQ_MOE_DEDUP_NWARPS*ggml_cuda_get_physical_warp_size(), MMVQ_MOE_DEDUP_MINB)
+static __global__ void mul_mat_vec_q_moe_dedup(
+        const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,
+        float * dst_ptr,
+        const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+        const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst,
+        const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
+        const uint32_t ncols_dst, const uint32_t ids_stride, const uint32_t n_used, const uint3 n_used_fd) {
+    const void    * GGML_CUDA_RESTRICT vx  = vx_ptr;
+    const void    * GGML_CUDA_RESTRICT vy  = vy_ptr;
+    const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+    float         * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+    constexpr int qk  = ggml_cuda_type_traits<type>::qk;
+    constexpr int qi  = ggml_cuda_type_traits<type>::qi;
+    constexpr int vdr = get_vdr_mmvq(type);
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    constexpr int max_tok   = c_ntok; // == ncols_dst
+
+    const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;
+    if (shared_expert) {
+        vx = fusion.shared_up;
+        dst = fusion.shared_dst;
+        stride_col_dst = fusion.shared_stride_col_dst;
+    }
+
+    bool use_gate = false;
+    const void  * vgate      = nullptr;
+    const float * x_bias     = nullptr;
+    const float * gate_bias  = nullptr;
+    const float * x_scale    = nullptr;
+    const float * gate_scale = nullptr;
+    ggml_glu_op   active_glu = GGML_GLU_OP_SWIGLU;
+    float         glu_limit  = 0.0f;
+
+    if constexpr (has_fusion) {
+        use_gate   = fusion.gate != nullptr;
+        vgate      = shared_expert ? fusion.shared_gate : fusion.gate;
+        x_bias     = (const float *) fusion.x_bias;
+        gate_bias  = (const float *) fusion.gate_bias;
+        active_glu = fusion.glu_op;
+        glu_limit  = fusion.glu_limit;
+    }
+
+    const int lane = threadIdx.x;
+    // a block loops over row tiles, so its id scan is paid once for many rows
+    const int row_first = c_rows_per_warp*(MMVQ_MOE_DEDUP_NWARPS*blockIdx.x + threadIdx.y);
+    const int row_step  = c_rows_per_warp*MMVQ_MOE_DEDUP_NWARPS*gridDim.x;
+    const int blocks_per_row_x = ncols_x / qk;
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+    ggml_cuda_pdl_sync();
+
+    // the tokens (and their slots) of this block's expert, ascending: warp 0 finds them, the block reads them from
+    // shared memory
+    __shared__ int      s_n;
+    __shared__ uint32_t s_channel_x;
+    __shared__ uint32_t s_tok[max_tok];
+    __shared__ uint32_t s_slot[max_tok];
+    if (shared_expert) {
+        if (threadIdx.y == 0 && lane < max_tok) {
+            s_tok[lane]  = lane;
+            s_slot[lane] = 0;
+        }
+        if (threadIdx.y == 0 && lane == 0) {
+            s_n = ncols_dst;
+            s_channel_x = 0;
+        }
+    } else if (threadIdx.y == 0) {
+        const uint32_t p  = blockIdx.y;
+        const uint32_t n_pairs = n_used*ncols_dst;
+        const uint2 pdm = fast_div_modulo(p, n_used_fd);
+        const uint32_t cx = ids[pdm.y + pdm.x*ids_stride];
+        // pairs with the same expert are taken max_tok at a time (an expert appears once per token, but the hybrid
+        // cache's zero slot can take many pairs): the block of the pair whose rank among them is a multiple of
+        // max_tok computes it and the next max_tok - 1, the others exit
+        // all id loads first (n_pairs <= MMVQ_MOE_DEDUP_MAX_PAIRS, checked by the launcher), then the ballots
+        constexpr int max_chunks = MMVQ_MOE_DEDUP_MAX_PAIRS / warp_size;
+        int32_t idq[max_chunks];
+#pragma unroll
+        for (int c = 0; c < max_chunks; ++c) {
+            const uint32_t q = c*warp_size + lane;
+            const uint2 qdm = fast_div_modulo(q, n_used_fd);
+            idq[c] = q < n_pairs ? ids[qdm.y + qdm.x*ids_stride] : -1;
+        }
+        int n = 0;
+        int n_before = 0;
+        bool leader = true;
+#pragma unroll
+        for (int c = 0; c < max_chunks; ++c) {
+            const uint32_t base = c*warp_size;
+            if (base >= n_pairs) {
+                break;
+            }
+            uint32_t mask = __ballot_sync(0xFFFFFFFF, (uint32_t) idq[c] == cx);
+            if (base < p) {
+                const uint32_t before = p - base >= warp_size ? 0xFFFFFFFF : ((1u << (p - base)) - 1);
+                n_before += __popc(mask & before);
+                mask &= ~before;
+            }
+            if (base <= p && p < base + warp_size && n_before % max_tok != 0) {
+                leader = false;
+                break;
+            }
+            // lane k writes the k-th new pair
+            const uint32_t below = mask & ((1u << lane) - 1);
+            const int k = n + __popc(below);
+            if ((mask >> lane & 1) && k < max_tok) {
+                const uint2 qdm = fast_div_modulo(base + lane, n_used_fd);
+                s_tok[k]  = qdm.x;
+                s_slot[k] = qdm.y;
+            }
+            n = min(n + __popc(mask), max_tok);
+        }
+        if (lane == 0) {
+            s_n = leader ? n : -1;
+            s_channel_x = cx;
+        }
+    }
+    __syncthreads();
+    const int n = s_n;
+    if (n < 0) {
+        return;
+    }
+    const uint32_t channel_x = s_channel_x;
+    uint32_t tok[max_tok];
+    uint32_t slot[max_tok];
+#pragma unroll
+    for (int j = 0; j < max_tok; ++j) {
+        tok[j]  = j < n ? s_tok[j]  : 0;
+        slot[j] = j < n ? s_slot[j] : 0;
+    }
+
+    const block_q8_1 * y[max_tok];
+#pragma unroll
+    for (int j = 0; j < max_tok; ++j) {
+        y[j] = ((const block_q8_1 *) vy) + fastmodulo(slot[j], nchannels_y)*stride_channel_y + tok[j]*stride_col_y;
+    }
+    for (int row0 = row_first; row0 < (int) nrows_x; row0 += row_step) {
+        const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x;
+
+        float tmp[c_rows_per_warp][max_tok];
+        float tmp_gate[c_rows_per_warp][max_tok];
+    #pragma unroll
+        for (int i = 0; i < c_rows_per_warp; ++i) {
+    #pragma unroll
+            for (int j = 0; j < max_tok; ++j) {
+                tmp[i][j] = 0.0f;
+                tmp_gate[i][j] = 0.0f;
+            }
+        }
+
+        for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+            const int kby = kbx * (qk/QK8_1);
+            const int kqs = vdr * (lane % (qi/vdr));
+
+    #pragma unroll
+            for (int i = 0; i < c_rows_per_warp; ++i) {
+                using S = moe_split<type>;
+                const typename S::W w = S::load(vx, kbx_offset + i*stride_row_x + kbx, kqs);
+    #pragma unroll
+                for (int j = 0; j < max_tok; ++j) {
+                    if (j < n) {
+                        tmp[i][j] += S::apply(w, &y[j][kby], kqs);
+                    }
+                }
+                if constexpr (has_fusion) {
+                    if (use_gate) {
+                        const typename S::W wg = S::load(vgate, kbx_offset + i*stride_row_x + kbx, kqs);
+    #pragma unroll
+                        for (int j = 0; j < max_tok; ++j) {
+                            if (j < n) {
+                                tmp_gate[i][j] += S::apply(wg, &y[j][kby], kqs);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    #pragma unroll
+        for (int j = 0; j < max_tok; ++j) {
+            if (j >= n) {
+                break;
+            }
+            float res[c_rows_per_warp];
+            float res_gate[c_rows_per_warp];
+    #pragma unroll
+            for (int i = 0; i < c_rows_per_warp; ++i) {
+                res[i] = warp_reduce_sum<warp_size>(tmp[i][j]);
+                res_gate[i] = 0.0f;
+                if constexpr (has_fusion) {
+                    if (use_gate) {
+                        res_gate[i] = warp_reduce_sum<warp_size>(tmp_gate[i][j]);
+                    }
+                }
+            }
+
+            if (lane < c_rows_per_warp && uint32_t(row0 + lane) < nrows_x) {
+                float result = res[0];
+                float gate_value = res_gate[0];
+    #pragma unroll
+                for (int i = 1; i < c_rows_per_warp; ++i) {
+                    if (lane == i) {
+                        result = res[i];
+                        gate_value = res_gate[i];
+                    }
+                }
+                if constexpr (has_fusion) {
+                    const uint32_t bias_idx = channel_x*stride_channel_dst + row0 + lane;
+
+                    if constexpr (type == GGML_TYPE_NVFP4) {
+                        if (x_scale) {
+                            result *= x_scale[channel_x];
+                        }
+                    }
+                    if (x_bias) {
+                        result += x_bias[bias_idx];
+                    }
+                    if (use_gate) {
+                        if constexpr (type == GGML_TYPE_NVFP4) {
+                            if (gate_scale) {
+                                gate_value *= gate_scale[channel_x];
+                            }
+                        }
+                        if (gate_bias) {
+                            gate_value += gate_bias[bias_idx];
+                        }
+                        switch (active_glu) {
+                            case GGML_GLU_OP_SWIGLU:
+                                result *= ggml_cuda_op_silu_single(gate_value);
+                                break;
+                            case GGML_GLU_OP_GEGLU:
+                                result *= ggml_cuda_op_gelu_single(gate_value);
+                                break;
+                            case GGML_GLU_OP_SWIGLU_OAI:
+                                result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
+                                break;
+                            case GGML_GLU_OP_SWIGLU_CLAMP:
+                                result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
+                                break;
+                            default:
+                                result = result * gate_value;
+                                break;
+                        }
+                    }
+                }
+                dst[slot[j]*stride_channel_dst + tok[j]*stride_col_dst + row0 + lane] = result;
+            }
+        }
+    }
+
+    ggml_cuda_pdl_lc();
+
+    if constexpr (!has_fusion) {
+        GGML_UNUSED_VARS(use_gate, vgate, x_bias, gate_bias, active_glu, glu_limit, x_scale, gate_scale);
+    } else if constexpr (type != GGML_TYPE_NVFP4) {
+        GGML_UNUSED_VARS(x_scale, gate_scale);
+    }
+}
+
 template<ggml_type type>
 static std::pair<dim3, dim3> calc_launch_params(
         const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,
@@ -1084,15 +1478,54 @@ static void mul_mat_vec_q_moe_launch(
         const uint32_t ncols_dst, const uint32_t ids_stride,
         const int warp_size, const int nchannels_dst, cudaStream_t stream) {
 
+    const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
+                            fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
+
+    static const bool dedup = getenv("GGML_CUDA_MOE_DEDUP") != nullptr && atoi(getenv("GGML_CUDA_MOE_DEDUP")) != 0;
+    if constexpr (moe_split<type>::ok) if (dedup && nchannels_dst*ncols_dst <= MMVQ_MOE_DEDUP_MAX_PAIRS) {
+        constexpr int rows_per_warp = 2;
+        constexpr int rows_per_block = rows_per_warp*MMVQ_MOE_DEDUP_NWARPS;
+        const uint32_t n_used = nchannels_dst;
+        static const int gx_max = getenv("GGML_CUDA_MOE_DEDUP_GX") ? atoi(getenv("GGML_CUDA_MOE_DEDUP_GX")) : 16;
+        const int gx = std::max(1, std::min<int>((nrows_x + rows_per_block - 1) / rows_per_block, gx_max));
+        const uint3 n_used_fd = init_fastdiv_values(n_used);
+        const dim3 block_nums(gx, n_used*ncols_dst + (fusion.shared_up != nullptr));
+        const dim3 block_dims(warp_size, MMVQ_MOE_DEDUP_NWARPS);
+        const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
+        auto launch = [&](auto ntok) {
+            constexpr int c_ntok = decltype(ntok)::value;
+            if (has_fusion) {
+                ggml_cuda_kernel_launch(mul_mat_vec_q_moe_dedup<type, rows_per_warp, c_ntok, true>, launch_params,
+                    vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
+                    stride_row_x, stride_col_y, stride_col_dst,
+                    stride_channel_x, stride_channel_y, stride_channel_dst,
+                    ncols_dst, ids_stride, n_used, n_used_fd);
+            } else {
+                ggml_cuda_kernel_launch(mul_mat_vec_q_moe_dedup<type, rows_per_warp, c_ntok, false>, launch_params,
+                    vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
+                    stride_row_x, stride_col_y, stride_col_dst,
+                    stride_channel_x, stride_channel_y, stride_channel_dst,
+                    ncols_dst, ids_stride, n_used, n_used_fd);
+            }
+        };
+        switch (ncols_dst) {
+            case 2: launch(std::integral_constant<int, 2>{}); break;
+            case 3: launch(std::integral_constant<int, 3>{}); break;
+            case 4: launch(std::integral_constant<int, 4>{}); break;
+            case 5: launch(std::integral_constant<int, 5>{}); break;
+            case 6: launch(std::integral_constant<int, 6>{}); break;
+            case 7: launch(std::integral_constant<int, 7>{}); break;
+            default: launch(std::integral_constant<int, 8>{}); break;
+        }
+        return;
+    }
+
     constexpr int rows_per_block = 2; // 2 gives best perf based on tuning
     const int64_t nblocks_rows = (nrows_x + rows_per_block - 1) / rows_per_block;
     const dim3 block_nums(nblocks_rows, nchannels_dst + (fusion.shared_up != nullptr));
     const dim3 block_dims(warp_size, ncols_dst);
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
 
-    const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
-                            fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
-
     if (has_fusion) {
         ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true>, launch_params,
             vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
@@ -1549,12 +1982,43 @@ void ggml_cuda_mul_mat_vec_q(
     }
 
     const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
-    ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
-    {
+    const size_t  q8_size     = ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1;
+    static const bool q8_cache = getenv("GGML_CUDA_Q8_CACHE") != nullptr && atoi(getenv("GGML_CUDA_Q8_CACHE")) != 0;
+    ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool());
+    char * q8_d = nullptr;
+    bool   q8_hit = false;
+    if (q8_cache && q8_size <= GGML_CUDA_Q8_CACHE_SIZE) {
+        if (ctx.q8_cache_mem == nullptr) {
+            CUDA_CHECK(cudaMalloc(&ctx.q8_cache_mem, GGML_CUDA_Q8_CACHE_N*GGML_CUDA_Q8_CACHE_SIZE));
+        }
+        const ggml_tensor * base = src1;
+        while (base->view_src) { base = base->view_src; }
+        int slot = -1;
+        for (int i = 0; i < GGML_CUDA_Q8_CACHE_N; ++i) {
+            const ggml_cuda_q8_cache_entry & e = ctx.q8_cache[i];
+            if (e.epoch == ctx.q8_cache_epoch && e.base == base && e.data == src1->data &&
+                    memcmp(e.ne, src1->ne, sizeof(e.ne)) == 0 && memcmp(e.nb, src1->nb, sizeof(e.nb)) == 0) {
+                slot = i;
+                q8_hit = true;
+                break;
+            }
+        }
+        if (slot < 0) {
+            slot = ctx.q8_cache_next;
+            ctx.q8_cache_next = (ctx.q8_cache_next + 1) % GGML_CUDA_Q8_CACHE_N;
+            ggml_cuda_q8_cache_entry & e = ctx.q8_cache[slot];
+            e.base = base; e.data = src1->data; e.epoch = ctx.q8_cache_epoch;
+            memcpy(e.ne, src1->ne, sizeof(e.ne)); memcpy(e.nb, src1->nb, sizeof(e.nb));
+        }
+        q8_d = ctx.q8_cache_mem + slot*GGML_CUDA_Q8_CACHE_SIZE;
+    } else {
+        q8_d = src1_q8_1.alloc(q8_size);
+    }
+    if (!q8_hit) {
         const int64_t s11 = src1->nb[1] / ts_src1;
         const int64_t s12 = src1->nb[2] / ts_src1;
         const int64_t s13 = src1->nb[3] / ts_src1;
-        quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+        quantize_row_q8_1_cuda(src1_d, nullptr, q8_d, src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
     }
 
     const int64_t s01 = src0->nb[1] / ts_src0;
@@ -1580,7 +2044,7 @@ void ggml_cuda_mul_mat_vec_q(
     const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
 
     mul_mat_vec_q_switch_type(
-        src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00,
+        src0->data, src0->type, q8_d, ids_d, fusion_local, dst_d, ne00,
         ne01,              ncols_dst,     s01, stride_col_y,     stride_col_dst,
         ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
         ne03,              ne3,           s03, s13,              s3,               ids_stride, stream);
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index 5d8449a88..65d1f32cd 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -1108,9 +1108,13 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
     "OPT_STEP_SGD",
 
     "GLU",
+
+    "HOST_ADD",
+    "MOE_DOORBELL",
+    "GET_ROWS_HOST",
 };
 
-static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT != 102");
+static_assert(GGML_OP_COUNT == 105, "GGML_OP_COUNT != 105");
 
 static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
     "none",
@@ -1224,9 +1228,13 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
     "sgd(x)",
 
     "glu(x)",
+
+    "host_add(x)",
+    "moe_doorbell(ids)",
+    "get_rows_host(x)",
 };
 
-static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT != 102");
+static_assert(GGML_OP_COUNT == 105, "GGML_OP_COUNT != 105");
 
 static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
 
@@ -6359,6 +6367,77 @@ struct ggml_tensor * ggml_opt_step_adamw(
 
 // opt_step_sgd
 
+struct ggml_tensor * ggml_host_add(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * a,
+        const float         * data,
+        uint32_t            * flag) {
+    GGML_ASSERT(a->type == GGML_TYPE_F32 && ggml_is_contiguous(a));
+
+    struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+
+    const uint64_t params[2] = { (uint64_t) (uintptr_t) data, (uint64_t) (uintptr_t) flag };
+    ggml_set_op_params(result, params, sizeof(params));
+
+    result->op     = GGML_OP_HOST_ADD;
+    result->src[0] = a;
+
+    return result;
+}
+
+struct ggml_tensor * ggml_moe_doorbell(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * ids,
+        struct ggml_tensor  * x,
+        struct ggml_tensor  * w,
+        struct ggml_moe_db  * db,
+        int                   n_used) {
+    GGML_ASSERT(ids->type == GGML_TYPE_I32 && ggml_is_contiguous(ids));
+    GGML_ASSERT((x == NULL) == (w == NULL));
+    GGML_ASSERT(x == NULL || (x->type == GGML_TYPE_F32 && w->type == GGML_TYPE_F32 && ggml_is_contiguous(x) && ggml_is_contiguous(w)));
+    GGML_ASSERT(n_used > 0 && ggml_nelements(ids) % n_used == 0);
+
+    struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ggml_nelements(ids) + GGML_MOE_DB_SCRATCH(db->max_fill));
+
+    const uint64_t p = (uint64_t) (uintptr_t) db;
+    int32_t params[3];
+    memcpy(params, &p, sizeof(p));
+    params[2] = n_used;
+    ggml_set_op_params(result, params, sizeof(params));
+
+    result->op     = GGML_OP_MOE_DOORBELL;
+    result->src[0] = ids;
+    result->src[1] = x;
+    result->src[2] = w;
+
+    return result;
+}
+
+struct ggml_tensor * ggml_get_rows_host(
+        struct ggml_context * ctx,
+        const struct ggml_tensor * w,
+        struct ggml_tensor  * ids) {
+    GGML_ASSERT(ids->type == GGML_TYPE_I32 && ggml_is_contiguous(ids) && ggml_is_matrix(w));
+
+    struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, w->ne[0], ggml_nelements(ids));
+
+    int32_t params[8];
+    const uint64_t p   = (uint64_t) (uintptr_t) w->data;
+    const uint64_t nb1 = (uint64_t) w->nb[1];
+    const uint64_t ne0 = (uint64_t) w->ne[0];
+    memcpy(params + 0, &p,   sizeof(p));
+    memcpy(params + 2, &nb1, sizeof(nb1));
+    memcpy(params + 4, &ne0, sizeof(ne0));
+    params[6] = (int32_t) w->type;
+    params[7] = 0;
+    ggml_set_op_params(result, params, sizeof(params));
+
+    result->op     = GGML_OP_GET_ROWS_HOST;
+    result->src[0] = ids;
+
+    return result;
+}
+
 struct ggml_tensor * ggml_opt_step_sgd(
         struct ggml_context * ctx,
         struct ggml_tensor  * a,
@@ -6471,6 +6550,54 @@ struct ggml_tensor * ggml_gated_delta_net(
     return result;
 }
 
+struct ggml_tensor * ggml_gated_delta_net_ext(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * q,
+        struct ggml_tensor  * k,
+        struct ggml_tensor  * v,
+        struct ggml_tensor  * g,
+        struct ggml_tensor  * beta,
+        struct ggml_tensor  * states,
+        struct ggml_tensor  * ids,
+        struct ggml_tensor  * dt,
+        struct ggml_tensor  * A,
+        int64_t               K) {
+    const int64_t H      = v->ne[1];
+    const int64_t n_seqs = v->ne[3];
+
+    // build with contiguous placeholders of the checked shapes, then swap in the real sources
+    struct ggml_tensor * s0 = states;
+    if (ids) {
+        GGML_ASSERT(ids->type == GGML_TYPE_I32);
+        GGML_ASSERT(ggml_is_contiguous(ids) && ids->ne[0] == n_seqs);
+        GGML_ASSERT(ggml_is_contiguous(states));
+        s0 = ggml_view_4d(ctx, states, states->ne[0], states->ne[1], states->ne[2], n_seqs,
+                states->nb[1], states->nb[2], states->nb[3], 0);
+    }
+    struct ggml_tensor * g0 = g;
+    struct ggml_tensor * b0 = beta;
+    if (dt) {
+        GGML_ASSERT(A != NULL);
+        GGML_ASSERT(dt->type == GGML_TYPE_F32 && A->type == GGML_TYPE_F32);
+        GGML_ASSERT(ggml_is_contiguous(dt) && ggml_nelements(dt) == H);
+        GGML_ASSERT(ggml_is_contiguous(A)  && ggml_nelements(A)  == H);
+        GGML_ASSERT(g->type == GGML_TYPE_F32 && beta->type == GGML_TYPE_F32);
+        GGML_ASSERT(g->ne[0] == 1 && ggml_are_same_shape(g, beta));
+        GGML_ASSERT(g->nb[1] == sizeof(float) && g->nb[1] == beta->nb[1] && g->nb[2] == beta->nb[2] && g->nb[3] == beta->nb[3]);
+        g0 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, g->ne);
+        b0 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, beta->ne);
+    }
+    struct ggml_tensor * result = ggml_gated_delta_net(ctx, q, k, v, g0, b0, s0, K);
+    result->src[3] = g;
+    result->src[4] = beta;
+    result->src[5] = states;
+    result->src[6] = ids;
+    result->src[7] = dt;
+    result->src[8] = A;
+
+    return result;
+}
+
 // ggml_lightning_indexer
 
 struct ggml_tensor * ggml_lightning_indexer(
@@ -6613,6 +6740,23 @@ struct ggml_tensor * ggml_dsv4_hc_pre_gated(
     return ggml_dsv4_hc_pre_impl(ctx, x, gate, scale, true);
 }
 
+struct ggml_tensor * ggml_dsv4_hc_pre_affine(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * x,
+        struct ggml_tensor  * mix,
+        struct ggml_tensor  * scale,
+        struct ggml_tensor  * base,
+        float                 eps) {
+    GGML_ASSERT(scale->type == GGML_TYPE_F32 && base->type == GGML_TYPE_F32);
+    GGML_ASSERT(ggml_nelements(scale) >= 1 && base->ne[0] == x->ne[1]);
+    struct ggml_tensor * result = ggml_dsv4_hc_pre_impl(ctx, x, mix, 1.0f, false);
+    ggml_set_op_params_i32(result, 2, 1);
+    ggml_set_op_params_f32(result, 3, eps);
+    result->src[2] = scale;
+    result->src[3] = base;
+    return result;
+}
+
 // ggml_dsv4_hc_post
 
 struct ggml_tensor * ggml_dsv4_hc_post(
@@ -6703,6 +6847,50 @@ struct ggml_tensor * ggml_top_k_mask(
     return result;
 }
 
+struct ggml_tensor * ggml_dsv4_hc_post_affine(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * x,
+        struct ggml_tensor  * residual,
+        struct ggml_tensor  * post,
+        struct ggml_tensor  * comb,
+        struct ggml_tensor  * scale,
+        struct ggml_tensor  * base,
+        float                 scale_out) {
+    GGML_ASSERT(scale->type == GGML_TYPE_F32 && base->type == GGML_TYPE_F32);
+    GGML_ASSERT(ggml_nelements(scale) >= 1 && base->ne[0] == residual->ne[1]);
+    struct ggml_tensor * result = ggml_dsv4_hc_post(ctx, x, residual, post, comb);
+    ggml_set_op_params_i32(result, 0, 2);
+    ggml_set_op_params_f32(result, 2, scale_out);
+    result->src[6] = scale;
+    result->src[7] = base;
+    return result;
+}
+
+struct ggml_tensor * ggml_dsv4_hc_post_sig(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * x,
+        struct ggml_tensor  * residual,
+        struct ggml_tensor  * post,
+        struct ggml_tensor  * comb,
+        float                 scale_in,
+        float                 scale_out,
+        struct ggml_tensor  * add,
+        struct ggml_tensor  * add_gate) {
+    struct ggml_tensor * result = ggml_dsv4_hc_post(ctx, x, residual, post, comb);
+    ggml_set_op_params_i32(result, 0, 1);
+    ggml_set_op_params_f32(result, 1, scale_in);
+    ggml_set_op_params_f32(result, 2, scale_out);
+    if (add) {
+        GGML_ASSERT(add_gate != NULL);
+        GGML_ASSERT(add->type == GGML_TYPE_F32 && add_gate->type == GGML_TYPE_F32);
+        GGML_ASSERT(add->ne[0] == x->ne[0] && add->ne[1] == x->ne[1]);
+        GGML_ASSERT(add_gate->ne[0] == 1 && add_gate->ne[1] == x->ne[1]);
+        result->src[4] = add;
+        result->src[5] = add_gate;
+    }
+    return result;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 struct ggml_hash_set ggml_hash_set_new(size_t size) {
diff --git a/include/llama.h b/include/llama.h
index ae54e661b..99fd18af8 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -396,7 +396,8 @@ extern "C" {
         enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]
         enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
 
-        size_t moe_cache_size; // device cache in bytes for the experts kept in host memory, split among the devices like the layers, 0 = disabled [EXPERIMENTAL]
+        size_t moe_cache_size; // device cache in bytes for the experts kept in host memory, split among the devices like the layers, 0 = disabled, SIZE_MAX = auto [EXPERIMENTAL]
+        size_t moe_cache_margin; // auto size: device memory in bytes left free after the compute buffers
 
         // Abort callback
         // if it returns true, execution of llama_decode() will be aborted
@@ -1436,6 +1437,14 @@ extern "C" {
     // note: prefer initializing the context with llama_context_params.samplers when possible
     LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl);
 
+    // LLAMA_MTP_CHAIN (hydev): greedy MTP draft steps chained on the device. begin() sets the first token and h
+    // and stops the 1-token graph input from overwriting the chain buffers; after each chained 1-token decode,
+    // record(step) keeps its greedy token; end() waits and returns the n recorded tokens
+    LLAMA_API bool llama_mtp_chain_enabled(const struct llama_context * ctx);
+    LLAMA_API void llama_mtp_chain_begin (struct llama_context * ctx, llama_token token, const float * h);
+    LLAMA_API void llama_mtp_chain_record(struct llama_context * ctx, int32_t step);
+    LLAMA_API void llama_mtp_chain_end   (struct llama_context * ctx, llama_token * out, int32_t n);
+
     // mirror of llama_sampler_i:
     LLAMA_API struct llama_sampler * llama_sampler_init  (      struct llama_sampler_i * iface, llama_sampler_context_t ctx);
     LLAMA_API const char *           llama_sampler_name  (const struct llama_sampler * smpl);
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 33ad7845f..af1e8f335 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -1,4 +1,5 @@
 #include "llama-context.h"
+#include "llama-prof.h"
 
 #include "ggml.h"
 #include "llama-arch.h"
@@ -368,7 +369,8 @@ llama_context::llama_context(
 
     cparams.op_offload     = params.op_offload;
     cparams.kv_unified     = params.kv_unified;
-    cparams.moe_cache_size = params.moe_cache_size;
+    cparams.moe_cache_size   = params.moe_cache_size;
+    cparams.moe_cache_margin = params.moe_cache_margin;
 
     // initialized later
     cparams.pipeline_parallel = false;
@@ -559,6 +561,31 @@ llama_context::llama_context(
             LLAMA_LOG_INFO("%s: pipeline parallelism enabled\n", __func__);
         }
 
+        if (cparams.moe_cache_size > 0 && model.split_mode() != LLAMA_SPLIT_MODE_NONE && model.split_mode() != LLAMA_SPLIT_MODE_LAYER) {
+            throw std::runtime_error("the MoE cache (--moe-cache-mib) supports --split-mode none and layer only");
+        }
+        if (cparams.moe_cache_size == SIZE_MAX) {
+            // auto: the compute buffers of the worst-case graphs first (without the cache: its graph differs only by views and
+            // the hybrid's small host tensors), then what is left of the devices' memory, minus the margin
+            sched_reserve();
+            size_t free_sum = 0;
+            for (const auto & d : model.devices) {
+                if (d.is_meta || ggml_backend_dev_type(d.dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
+                    continue;
+                }
+                size_t free = 0, total = 0;
+                ggml_backend_dev_memory(d.dev, &free, &total);
+                free_sum += free;
+            }
+            const size_t margin = cparams.moe_cache_margin;
+            cparams.moe_cache_size = free_sum > margin ? (free_sum - margin)/(1024*1024)*(1024*1024) : 0;
+            LLAMA_LOG_INFO("%s: MoE cache auto size: %zu MiB (%zu MiB free after the compute buffers, %zu MiB margin)\n", __func__,
+                cparams.moe_cache_size/(1024*1024), free_sum/(1024*1024), margin/(1024*1024));
+            if (cparams.moe_cache_size < (size_t) 256*1024*1024) {
+                LLAMA_LOG_WARN("%s: MoE cache auto size: less than 256 MiB free, the cache is disabled\n", __func__);
+                cparams.moe_cache_size = 0;
+            }
+        }
         if (cparams.moe_cache_size > 0) {
             moe_cache = std::make_unique<llama_moe_cache>(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size);
         }
@@ -587,6 +614,11 @@ llama_context::~llama_context() {
     // wait for any pending asynchronous copies into the output buffers before they are freed
     synchronize();
 
+    if (!graph_pool.empty()) {
+        LLAMA_LOG_INFO("%s: graph pool (%zu): %zu swaps reused a pooled graph, %zu rebuilt in a pooled scheduler\n", __func__,
+            graph_pool.size(), n_pool_hits, n_pool_misses);
+    }
+
     // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
     if (!model.no_alloc() && !opt_ctx) {
         for (size_t i = 0; i < backend_ptrs.size(); ++i) {
@@ -727,6 +759,7 @@ void llama_context::sched_new(size_t max_nodes, bool parallel) {
 
     ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
     ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this);
+    sched_main = sched.get();
 }
 
 void llama_context::sched_reserve() {
@@ -754,6 +787,7 @@ void llama_context::sched_reserve() {
     }
     gf_res_reserve.reset(new llm_graph_result(max_nodes));
     gf_res_prev_active = nullptr;
+    graph_pool_reset();
 
     sched_new(max_nodes, cparams.pipeline_parallel);
 
@@ -885,12 +919,56 @@ void llama_context::sched_reserve() {
             __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get()));
 }
 
+bool llama_context::mtp_chain_enabled() const {
+    return model.mtp_chain_tok != nullptr;
+}
+
+void llama_context::mtp_chain_begin(llama_token token, const float * h) {
+    GGML_ASSERT(model.mtp_chain_tok);
+    synchronize();
+    ggml_backend_tensor_set(model.mtp_chain_tok, &token, 0, sizeof(token));
+    ggml_backend_tensor_set(model.mtp_chain_h, h, 0, ggml_nbytes(model.mtp_chain_h));
+    model.mtp_chain_active = true;
+}
+
+void llama_context::mtp_chain_record(int32_t step) {
+    GGML_ASSERT(model.mtp_chain_tok && step >= 0 && step < (int32_t) model.mtp_chain_out_rows.size());
+    // on the stream of the device that holds the buffers, after the step's graph and before the next one
+    ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(model.mtp_chain_tok->buffer);
+    for (auto & b : backends) {
+        if (ggml_backend_supports_buft(b.get(), buft) && !ggml_backend_buft_is_host(buft)) {
+            ggml_backend_tensor_copy_async(b.get(), b.get(), model.mtp_chain_tok, model.mtp_chain_out_rows[step]);
+            return;
+        }
+    }
+    GGML_ABORT("no backend for the MTP chain buffers");
+}
+
+void llama_context::mtp_chain_end(llama_token * out, int32_t n) {
+    synchronize();
+    for (auto & b : backends) {
+        ggml_backend_synchronize(b.get());
+    }
+    if (n > 0) {
+        ggml_backend_tensor_get(model.mtp_chain_out, out, 0, n*sizeof(llama_token));
+    }
+    model.mtp_chain_active = false;
+}
+
 void llama_context::synchronize() {
+    LC_PROF("ctx.synchronize");
     if (!sched) {
         return;
     }
 
-    ggml_backend_sched_synchronize(sched.get());
+    {
+        const auto lc_t0 = std::chrono::steady_clock::now();
+        ggml_backend_sched_synchronize(sched.get());
+        // only the verify pass waits this long: start the verify end -> first draft laps here
+        if (llama_prof_on() && std::chrono::steady_clock::now() - lc_t0 > std::chrono::milliseconds(2)) {
+            llama_prof_lap_begin();
+        }
+    }
 
     // FIXME: if multiple single tokens are evaluated without a synchronization,
     // the stats will be added to the prompt evaluation stats
@@ -996,6 +1074,7 @@ bool llama_context::memory_update(bool optimize) {
             }
         }
         gf_res_prev_active = nullptr;
+        graph_pool_reset();
 
         if (!mctx->apply()) {
             LLAMA_LOG_ERROR("%s: failed to apply memory update\n", __func__);
@@ -1523,11 +1602,32 @@ bool llama_context::set_adapter_cvec(
 }
 
 llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) {
+    {
+    LC_PROF("pu.mctx_apply");
     if (mctx && !mctx->apply()) {
         LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__);
         ret = GGML_STATUS_FAILED;
         return nullptr;
     }
+    }
+
+    // the MoE cache's hybrid gate may have switched its CPU path on or off: the graph changes
+    const bool moe_graph_change = moe_cache && moe_cache->take_graph_change();
+
+    // LLAMA_GRAPH_POOL: a pooled graph of this shape (or a free scheduler) is swapped in first. With a MoE cache, a pooled
+    // graph is not reused after the gate switched, and its hybrid registrations (the tensors the per-layer callbacks read)
+    // are replayed
+    if (moe_graph_change) {
+        graph_pool_reset();
+    }
+    if (!graph_pool.empty() && ubatch.n_tokens > 8) {
+        graph_pool_main();
+    }
+    if (!graph_pool.empty() && ubatch.n_tokens <= 8 && !graph_reuse_disable && !moe_graph_change) {
+        if (graph_pool_swap(ubatch, mctx, gtype) && moe_cache) {
+            moe_replay_regs(get_gf_res_prev());
+        }
+    }
 
     auto * res = get_gf_res_prev();
     auto * gf  = res->get_gf();
@@ -1535,8 +1635,10 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
     // the new graph parameters
     // in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters
     const auto gparams = graph_params(res, ubatch, mctx, gtype);
+    llama_prof_lap("lap.d_ubatch_to_reuse");
+    llama_prof_scope lc_prof_reuse("pu.reuse_check_or_build");
 
-    if (!graph_reuse_disable && gf_res_prev_active == res && res->can_reuse(gparams)) {
+    if (!graph_reuse_disable && !moe_graph_change && gf_res_prev_active == res && res->can_reuse(gparams)) {
         //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__);
 
         // with pipeline parallelism, the previous graph_compute_async may still be running
@@ -1546,6 +1648,11 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
             ggml_backend_sched_synchronize(sched.get());
         }
 
+        // the MoE cache's prefill share registrations may have been replaced by another large-batch graph
+        if (moe_cache && ubatch.n_tokens > 8) {
+            moe_replay_regs(res);
+        }
+
         n_reused++;
     } else {
         gf_res_prev_active = nullptr;
@@ -1556,6 +1663,7 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
 
         //const auto t_start_us = ggml_time_us();
 
+        LC_PROF(ggml_graph_n_nodes(res->get_gf()) > 2000 || gparams.ubatch.n_tokens > 64 ? "pu.rebuild.tgt_or_big" : "pu.rebuild.small");
         gf = model.build_graph(gparams);
 
         //LLAMA_LOG_INFO("graph build time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0);
@@ -1566,6 +1674,7 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
             return nullptr;
         }
 
+        llama_prof_scope lc_prof_alloc(gparams.ubatch.n_tokens > 64 || ggml_graph_n_nodes(gf) > 2000 ? "pu.sched_alloc.tgt_or_big" : "pu.sched_alloc.small");
         if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) {
             LLAMA_LOG_ERROR("%s: failed to allocate graph\n", __func__);
             ret = GGML_STATUS_ALLOC_FAILED;
@@ -1580,11 +1689,16 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
         //const auto t_start_us = ggml_time_us();
 
         // FIXME this call causes a crash if any model inputs were not used in the graph and were therefore not allocated
+        llama_prof_lap("lap.d_reuse");
+        lc_prof_reuse.~llama_prof_scope(); new (&lc_prof_reuse) llama_prof_scope("pu.set_inputs");
+        ggml_backend_sched_wait_input_copies(sched.get()); // GGML_SCHED_BATCH_INPUTS=2
         res->set_inputs(&ubatch);
 
         //LLAMA_LOG_INFO("graph set inputs time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0);
     }
 
+    llama_prof_lap("lap.d_set_inputs");
+    lc_prof_reuse.~llama_prof_scope(); new (&lc_prof_reuse) llama_prof_scope(ggml_graph_n_nodes(res->get_gf()) > 2000 ? "pu.graph_compute.tgt" : "pu.graph_compute.dft");
     const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1);
     if (status != GGML_STATUS_SUCCESS) {
         LLAMA_LOG_ERROR("%s: failed to compute graph, compute status: %d\n", __func__, status);
@@ -1841,6 +1955,7 @@ static bool needs_raw_logits(const llama_ubatch & ubatch, const std::map<llama_s
 }
 
 int llama_context::decode(const llama_batch_ext & batch_inp) {
+    LC_PROF("ctx.decode");
     if (!memory) {
         LLAMA_LOG_DEBUG("%s: cannot decode batches with this context (calling encode() instead)\n", __func__);
         return encode(batch_inp);
@@ -1944,6 +2059,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
     llama_memory_context_ptr mctx;
 
     while (true) {
+        LC_PROF("ctx.decode.init_batch");
         mctx = memory->init_batch(*balloc, cparams.n_ubatch, embd_all);
         if (!mctx) {
             return -2;
@@ -2021,7 +2137,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
 
         ggml_status status;
 
-        const auto * res = process_ubatch(ubatch, ctx_type_to_graph_type(cparams.ctx_type), mctx.get(), status);
+        const llm_graph_result * res;
+        {
+            llama_prof_lap("lap.d_decode_to_ubatch");
+            LC_PROF("ctx.decode.process_ubatch");
+            res = process_ubatch(ubatch, ctx_type_to_graph_type(cparams.ctx_type), mctx.get(), status);
+        }
 
         if (!res) {
             // the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module
@@ -2572,6 +2693,99 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
     return res;
 }
 
+void llama_context::graph_pool_reset() {
+    if (graph_pool.empty()) {
+        static const int k = getenv("LLAMA_GRAPH_POOL") ? atoi(getenv("LLAMA_GRAPH_POOL")) : 0;
+        if (k <= 0) {
+            return;
+        }
+        graph_pool.resize(k);
+    }
+    graph_pool_main(); // a reserve after the reset must use the main scheduler
+    for (auto & e : graph_pool) {
+        if (e.res) {
+            e.res->reset();
+        }
+        if (e.sched) {
+            ggml_backend_sched_reset(e.sched.get());
+        }
+        e.active = false;
+    }
+}
+
+void llama_context::graph_pool_main() {
+    if (sched.get() == sched_main) {
+        return;
+    }
+    for (auto & e : graph_pool) {
+        if (e.sched.get() != sched_main) {
+            continue;
+        }
+        // as graph_pool_swap: the current pair goes to the pool, the main one comes back with its graph if still allocated
+        auto & cur = gf_res_prev[n_outputs > 0];
+        const bool cur_active = cur && gf_res_prev_active == cur.get();
+        if (!cur) {
+            cur.reset(new llm_graph_result(gf_res_reserve->get_max_nodes()));
+        }
+        const bool main_active = e.active;
+        std::swap(sched, e.sched);
+        std::swap(cur, e.res);
+        if (!cur) {
+            cur.reset(new llm_graph_result(gf_res_reserve->get_max_nodes()));
+        }
+        e.active = cur_active;
+        e.used   = ++graph_pool_clock;
+        gf_res_prev_active = main_active ? cur.get() : nullptr;
+        return;
+    }
+}
+
+bool llama_context::graph_pool_swap(const llama_ubatch & ubatch, const llama_memory_context_i * mctx, llm_graph_type gtype) {
+    auto & cur = gf_res_prev[n_outputs > 0];
+    if (cur && gf_res_prev_active == cur.get() && cur->can_reuse(graph_params(cur.get(), ubatch, mctx, gtype))) {
+        return true;
+    }
+    graph_pool_entry * slot = nullptr;
+    for (auto & e : graph_pool) {
+        if (e.active && e.res->can_reuse(graph_params(e.res.get(), ubatch, mctx, gtype))) {
+            slot = &e;
+            break;
+        }
+    }
+    const bool hit = slot != nullptr;
+    if (!hit) {
+        // least recently used, free ones first
+        for (auto & e : graph_pool) {
+            if (!slot || (!e.active && slot->active) || (e.active == slot->active && e.used < slot->used)) {
+                slot = &e;
+            }
+        }
+        if (!slot->sched) {
+            slot->sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(),
+                gf_res_reserve->get_max_nodes(), cparams.pipeline_parallel, cparams.op_offload));
+            ggml_backend_sched_set_copy_callback(slot->sched.get(), sched_copy_experts, this);
+            ggml_backend_sched_set_realloc_callback(slot->sched.get(), sched_graph_realloc, this);
+        }
+        if (!slot->res) {
+            slot->res.reset(new llm_graph_result(gf_res_reserve->get_max_nodes()));
+        }
+        slot->res->reset();
+        slot->active = false;
+    }
+    // the current pair goes to the pool, with its graph if that graph is the one allocated in its scheduler
+    const bool cur_active = cur && gf_res_prev_active == cur.get();
+    if (!cur) {
+        cur.reset(new llm_graph_result(gf_res_reserve->get_max_nodes()));
+    }
+    std::swap(sched, slot->sched);
+    std::swap(cur, slot->res);
+    slot->active = cur_active;
+    slot->used   = ++graph_pool_clock;
+    gf_res_prev_active = hit ? cur.get() : nullptr;
+    (hit ? n_pool_hits : n_pool_misses)++;
+    return hit;
+}
+
 llm_graph_result * llama_context::get_gf_res_reserve() const {
     return static_cast<llm_graph_result *>(gf_res_reserve.get());
 }
@@ -2660,6 +2874,7 @@ ggml_cgraph * llama_context::graph_reserve(
         }
     }
     gf_res_prev_active = nullptr;
+    graph_pool_reset();
 
     // store the n_outputs as it is, and restore it afterwards
     // TODO: not sure if needed, might simplify in the future by removing this
@@ -2782,6 +2997,17 @@ ggml_status llama_context::graph_compute(
     return status;
 }
 
+void llama_context::moe_replay_regs(llm_graph_result * res) {
+    for (const auto & r : res->moe_hyb_regs) {
+        if (r.ids != nullptr) {
+            moe_cache->set_prefill_layer(r.il, r.x, r.w, r.ids, (llama_moe_cache::act_type) r.act, r.limit);
+        } else {
+            moe_cache->set_hybrid_layer(r.il, r.x, r.w, (llama_moe_cache::act_type) r.act, r.limit);
+            moe_cache->set_lookahead_layer(r.il, r.la);
+        }
+    }
+}
+
 bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data) {
     auto * lctx = static_cast<llama_context *>(user_data);
 
@@ -2790,6 +3016,9 @@ bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor
         return true;
     }
 
+    // the scheduler does not wait for the backend before calling the callback (see ggml_backend_sched_copy_input)
+    ggml_backend_synchronize(backend);
+
     auto & st = lctx->copy_experts;
 
     // the ids must be computed before the split starts, so only the first node of the split is considered
@@ -2823,6 +3052,11 @@ bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor
             }
         }
 
+        // the MoE cache's prefill share: the CPU computes some of these experts, they are not uploaded
+        if (lctx->moe_cache) {
+            lctx->moe_cache->prefill_plan(backend, src, ids, st.ids_data, st.used);
+        }
+
         st.ids = ids;
     }
 
@@ -3920,7 +4154,8 @@ void llama_context::opt_epoch_iter(
             ggml_opt_prepare_alloc(opt_ctx, ctx_compute_opt, gf, res->get_inp_tokens(), res->get_logits());
             ggml_opt_alloc(opt_ctx, train);
 
-            res->set_inputs(&ubatch);
+            ggml_backend_sched_wait_input_copies(sched.get()); // GGML_SCHED_BATCH_INPUTS=2
+        res->set_inputs(&ubatch);
             {
                 struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);
                 GGML_ASSERT(labels->ne[1] == n_ubatch);
@@ -4025,6 +4260,7 @@ llama_context_params llama_context_default_params() {
         /*.type_k                      =*/ GGML_TYPE_F16,
         /*.type_v                      =*/ GGML_TYPE_F16,
         /*.moe_cache_size              =*/ 0,
+        /*.moe_cache_margin            =*/ (size_t) 1024*1024*1024,
         /*.abort_callback              =*/ nullptr,
         /*.abort_callback_data         =*/ nullptr,
         /*.embeddings                  =*/ false,
@@ -4318,6 +4554,22 @@ bool llama_set_sampler(llama_context * ctx, llama_seq_id seq_id, llama_sampler *
     return ctx->set_sampler(seq_id, smpl);
 }
 
+bool llama_mtp_chain_enabled(const llama_context * ctx) {
+    return ctx->mtp_chain_enabled();
+}
+
+void llama_mtp_chain_begin(llama_context * ctx, llama_token token, const float * h) {
+    ctx->mtp_chain_begin(token, h);
+}
+
+void llama_mtp_chain_record(llama_context * ctx, int32_t step) {
+    ctx->mtp_chain_record(step);
+}
+
+void llama_mtp_chain_end(llama_context * ctx, llama_token * out, int32_t n) {
+    ctx->mtp_chain_end(out, n);
+}
+
 llama_token llama_get_sampled_token_ith(llama_context * ctx, int32_t i) {
     ctx->synchronize();
 
diff --git a/src/llama-context.h b/src/llama-context.h
index 6938fb103..17de64bc9 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -60,6 +60,12 @@ struct llama_context {
 
     void synchronize();
 
+    // LLAMA_MTP_CHAIN, see llama_mtp_chain_*
+    bool mtp_chain_enabled() const;
+    void mtp_chain_begin(llama_token token, const float * h);
+    void mtp_chain_record(int32_t step);
+    void mtp_chain_end(llama_token * out, int32_t n);
+
     const llama_model   & get_model()   const;
     const llama_cparams & get_cparams() const;
 
@@ -278,6 +284,8 @@ private:
 
     // ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID and updates the MoE cache
     static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data);
+    // the MoE cache registrations of a graph that is reused (set_hybrid_layer, set_prefill_layer)
+    void moe_replay_regs(llm_graph_result * res);
 
     // ggml_backend_sched realloc callback, counts and traces the graph reallocations of this context
     static void sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * gf, bool backend_ids_changed, bool graph_size_changed, void * user_data);
@@ -417,6 +425,27 @@ private:
 
     llm_graph_result * gf_res_prev_active = nullptr;
 
+    // LLAMA_GRAPH_POOL=K (hydev): up to K more (scheduler, graph) pairs for ubatches of up to 8 tokens. Alternating batch shapes
+    // (an MTP draft's catch-up step and its 1-token steps, verify windows of varying size) then swap in their own allocated
+    // graph instead of rebuilding and reallocating it, and their CUDA graphs stay valid. Each scheduler allocates only what its
+    // graphs need
+    struct graph_pool_entry {
+        ggml_backend_sched_ptr sched;
+        llm_graph_result_ptr   res;
+        bool    active = false; // res holds the graph currently allocated in sched
+        int64_t used   = 0;
+    };
+    std::vector<graph_pool_entry> graph_pool;
+    int64_t graph_pool_clock = 0;
+    size_t  n_pool_hits = 0, n_pool_misses = 0;
+    void graph_pool_reset();
+    // the scheduler made by sched_new, the only one reserved for the worst-case graph: ubatches of more than 8 tokens run in
+    // it (a pooled scheduler that took such a graph would allocate its own prefill-sized buffers, one per pool entry)
+    ggml_backend_sched * sched_main = nullptr;
+    void graph_pool_main();
+    // swaps a pooled (scheduler, graph) pair for this ubatch in, returns true if its graph can be reused as is
+    bool graph_pool_swap(const llama_ubatch & ubatch, const llama_memory_context_i * mctx, llm_graph_type gtype);
+
     // host buffer for the model output (logits and embeddings)
     ggml_backend_buffer_ptr buf_output;
 
diff --git a/src/llama-cparams.h b/src/llama-cparams.h
index 0c260f7b6..c2efa0872 100644
--- a/src/llama-cparams.h
+++ b/src/llama-cparams.h
@@ -58,6 +58,7 @@ struct llama_cparams {
     bool training;           // set by llama_opt_init()
 
     size_t moe_cache_size;
+    size_t moe_cache_margin;
 
     std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer
 
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index cffd083d3..ea35c1953 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -156,6 +156,12 @@ void llm_graph_input_embd_h::set_input(const llama_ubatch * ubatch) {
     GGML_ASSERT(n_embd_state == h->ne[0]);
 
     ggml_backend_tensor_set(h, ubatch->embd_state, 0, n_tokens*n_embd_state*ggml_element_size(h));
+
+    if (chain_tok && !*chain_active) {
+        GGML_ASSERT(n_tokens == 1 && ubatch->token);
+        ggml_backend_tensor_set(chain_tok, ubatch->token, 0, sizeof(int32_t));
+        ggml_backend_tensor_set(chain_h, ubatch->embd_state, 0, n_embd_state*ggml_element_size(h));
+    }
 }
 
 bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) {
@@ -1424,6 +1430,7 @@ void llm_graph_result::reset() {
     inputs.clear();
     fused_nodes.clear();
     hdmd_inputs.clear();
+    moe_hyb_regs.clear();
 
     buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
 
@@ -2372,8 +2379,112 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
     //call early so that topk-moe can be used
     ggml_build_forward_expand(gf, weights);
 
+    // MoE cache hybrid mode: the CPU computes the missed experts that are not uploaded, from host copies of the expert input and
+    // the router weights read at the slot lookup, so both must be computed before it; (swi)glu experts, and gpt-oss's
+    // swiglu_oai with expert biases (the CPU kernel adds them; llama_moe_cache::cpu_can)
+    ggml_tensor * hyb_x = nullptr;
+    ggml_tensor * hyb_w = nullptr;
+    llama_moe_cache::act_type hyb_act = llama_moe_cache::ACT_NONE;
+    float hyb_limit = 0.0f;
+    // up to 8 tokens: the experts computed by the CPU share one zero slot, which MMVQ allows (it computes every token and
+    // slot pair on its own) but MMQ, used for more tokens, does not (a token's experts must be distinct there)
+    // larger ubatches on the large-batch path: the prefill share (see llama_moe_cache::set_prefill_layer)
+    const bool oai = type_op == LLM_FFN_SWIGLU_OAI_MOE;
+    const bool hyb_shape = moe_cache && il >= 0 && !weight_before_ffn && (type_op == LLM_FFN_SILU || oai) &&
+            (gate_exps || gate_up_exps) && (oai || (!up_exps_b && !gate_exps_b && !down_exps_b && !gate_up_exps_b)) &&
+            !up_exps_s && !gate_exps_s && !down_exps_s;
+    if (moe_cache && il >= 0 && !hyb_shape && moe_cache->hybrid_configured()) {
+        static bool warned = false;
+        if (!warned) {
+            warned = true;
+            LLAMA_LOG_WARN("%s: MoE cache hybrid mode: this model's experts (activation %d%s) are not supported by the CPU path; "
+                "every missed expert is uploaded instead\n", __func__, (int) type_op,
+                up_exps_s || gate_exps_s || down_exps_s ? ", expert scales" : weight_before_ffn ? ", weights before the FFN" : "");
+        }
+    }
+    const bool hyb_dec = hyb_shape && moe_cache->hybrid_enabled() && n_tokens <= 8;
+    const bool hyb_pf  = hyb_shape && moe_cache->prefill_enabled(n_tokens) && ggml_is_contiguous(weights);
+    if (hyb_dec || hyb_pf) {
+        bool rotated = false;
+        for (ggml_tensor * w : { up_exps, gate_exps, down_exps, gate_up_exps }) {
+            rotated = rotated || (w && hdmd && hdmd->rot.count(w));
+        }
+        const float limit = hparams.swiglu_clamp_exp[il];
+        if (rotated) {
+            hyb_act = llama_moe_cache::ACT_NONE;
+        } else if (oai) {
+            hyb_act   = llama_moe_cache::ACT_SWIGLU_OAI;
+            hyb_limit = 7.0f; // as the LLM_FFN_SWIGLU_OAI_MOE case below
+        } else if (limit <= 1e-6f || !gate_exps) {
+            hyb_act = llama_moe_cache::ACT_SWIGLU;
+        } else if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) {
+            hyb_act   = llama_moe_cache::ACT_SWIGLU_CLAMP;
+            hyb_limit = limit;
+        }
+        if (hyb_act != llama_moe_cache::ACT_NONE) {
+            hyb_x = ggml_is_contiguous(cur)     ? cur     : ggml_cont(ctx0, cur);
+            hyb_w = ggml_is_contiguous(weights) ? weights : ggml_cont(ctx0, weights);
+            ggml_build_forward_expand(gf, hyb_x);
+            ggml_build_forward_expand(gf, hyb_w);
+        }
+    }
+
+    // lookahead prefetch: the next layer's router applied to this layer's expert input predicts the next layer's experts; the
+    // slot lookup callback reads them and prefetches on a side stream (see llama_moe_cache::lookahead_router)
+    ggml_tensor * la_pred = nullptr;
+    if (hyb_dec && hyb_x != nullptr) {
+        ggml_tensor * nb = nullptr;
+        int la_k = 0;
+        if (ggml_tensor * gn = moe_cache->lookahead_router(il, &nb, &la_k)) {
+            ggml_tensor * lg = ggml_mul_mat(ctx0, gn, hyb_x); // [n_expert, n_tokens]
+            if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
+                lg = ggml_sqrt(ctx0, ggml_softplus(ctx0, lg));
+            } else if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID && nb) {
+                lg = ggml_sigmoid(ctx0, lg);
+            }
+            if (nb) {
+                lg = ggml_add(ctx0, lg, nb);
+            }
+            la_pred = ggml_cont(ctx0, ggml_argsort_top_k(ctx0, lg, la_k)); // [la_k, n_tokens]
+            cb(la_pred, "ffn_moe_lookahead", il);
+            ggml_build_forward_expand(gf, la_pred);
+        }
+    }
+
     // the experts of host-resident layers may be read from the MoE cache
-    ggml_tensor * slots = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, il);
+    ggml_tensor * slots = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, il, hyb_dec ? hyb_x : nullptr, hyb_dec ? hyb_w : nullptr);
+    if (slots != nullptr) {
+        // the cached experts are views of the cache banks: put the view nodes first, so that the gate, up and glu nodes
+        // are adjacent and the backend can fuse them (LLAMA_MOE_VIEWS_FIRST=0: the views where the matmuls use them)
+        static const bool views_first = !getenv("LLAMA_MOE_VIEWS_FIRST") || atoi(getenv("LLAMA_MOE_VIEWS_FIRST")) != 0;
+        if (views_first) {
+            for (ggml_tensor * w : { gate_up_exps, up_exps, gate_exps, down_exps }) {
+                if (w != nullptr && moe_cache->get_experts(w) != nullptr) {
+                    ggml_build_forward_expand(gf, moe_cache->get_experts(w));
+                }
+            }
+        }
+    }
+
+    // the large-batch path: every such graph registers (or clears) the layer's prefill share, w is the tensor the outputs are
+    // multiplied by (the copy callback zeroes the router weights of the CPU's pairs there)
+    ggml_tensor * pf_out = nullptr;
+    if (slots == nullptr && moe_cache && il >= 0 && n_tokens > 8) {
+        const bool pf_ok = hyb_pf && hyb_x != nullptr && hyb_w == weights;
+        pf_out = moe_cache->set_prefill_layer(il, pf_ok ? hyb_x : nullptr, hyb_w, selected_experts, hyb_act, hyb_limit);
+        res->moe_hyb_regs.push_back({ il, pf_out ? hyb_x : nullptr, hyb_w, (int) hyb_act, hyb_limit, selected_experts, nullptr });
+    }
+    if (!hyb_dec) {
+        hyb_x = hyb_w = nullptr;
+    }
+
+    // every graph that reads the cache sets (or clears) the layer's hybrid tensors
+    ggml_tensor * hyb_out = nullptr;
+    if (slots != nullptr && moe_cache->hybrid_configured()) {
+        hyb_out = moe_cache->set_hybrid_layer(il, hyb_x, hyb_w, hyb_x ? hyb_act : llama_moe_cache::ACT_NONE, hyb_limit);
+        moe_cache->set_lookahead_layer(il, la_pred);
+        res->moe_hyb_regs.push_back({ il, hyb_x, hyb_w, (int) (hyb_x ? hyb_act : llama_moe_cache::ACT_NONE), hyb_limit, nullptr, la_pred });
+    }
 
     cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
 
@@ -2595,6 +2706,18 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
         moe_out = ggml_cont(ctx0, moe_out);
     }
 
+    if (hyb_out != nullptr) {
+        // the experts computed by the CPU, added after the cached experts are queued
+        moe_out = moe_cache->hybrid_add(ctx0, il, moe_out);
+        ggml_backend_sched_set_tensor_backend(sched, moe_out, moe_cache->backend(il));
+        cb(moe_out, "ffn_moe_out_hybrid", il);
+    }
+    if (pf_out != nullptr) {
+        moe_out = moe_cache->prefill_add(ctx0, il, moe_out);
+        ggml_backend_sched_set_tensor_backend(sched, moe_out, moe_cache->backend(il));
+        cb(moe_out, "ffn_moe_out_prefill", il);
+    }
+
     cb(moe_out, "ffn_moe_out", il);
 
     return moe_out;
@@ -2606,7 +2729,9 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots(
          ggml_tensor * gate_exps,
          ggml_tensor * down_exps,
          ggml_tensor * gate_up_exps,
-                 int   il) const {
+                 int   il,
+         ggml_tensor * hyb_x,
+         ggml_tensor * hyb_w) const {
     if (moe_cache == nullptr) {
         return nullptr;
     }
@@ -2627,6 +2752,18 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots(
     }
     ids = ggml_reshape_1d(ctx0, ids, ggml_nelements(ids));
 
+    // doorbell mode: a device op hands the selected experts to the cache's host thread and returns their slots, no split
+    if (ggml_tensor * db = moe_cache->doorbell_slots(ctx0, il, ids, hyb_x,
+            hyb_w ? ggml_reshape_1d(ctx0, hyb_w, ggml_nelements(hyb_w)) : nullptr, selected_experts->ne[1], selected_experts->ne[0])) {
+        if (ggml_backend_supports_op(moe_cache->backend(il), db)) {
+            ggml_backend_sched_set_tensor_backend(sched, db, moe_cache->backend(il));
+            ggml_tensor * slots = ggml_view_1d(ctx0, db, ggml_nelements(ids), 0);
+            cb(slots, "ffn_moe_slots", il);
+            ggml_build_forward_expand(gf, slots);
+            return ggml_reshape_2d(ctx0, slots, selected_experts->ne[0], selected_experts->ne[1]);
+        }
+    }
+
     // the slot map is a host weight, so the scheduler starts a new split here and copies it with the copy callback
     // the callback reads the selected experts, uploads the missing ones and updates the slot map
     ggml_tensor * slots = ggml_get_rows(ctx0, slot_map, ids); // [1, n_expert_used*n_tokens]
diff --git a/src/llama-graph.h b/src/llama-graph.h
index c1a7d36ee..c95441ca1 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -166,6 +166,12 @@ public:
     ggml_tensor * embd   = nullptr; // F32 [n_embd, n_batch]
     ggml_tensor * h      = nullptr; // F32 [n_embd_state, n_batch]
 
+    // LLAMA_MTP_CHAIN (1-token graphs only): the graph reads these instead of tokens/h; set_input mirrors the host
+    // inputs into them unless a chain is active (then the previous step wrote them on the device)
+    ggml_tensor * chain_tok    = nullptr;
+    ggml_tensor * chain_h      = nullptr;
+    const bool  * chain_active = nullptr;
+
     const int64_t n_embd       = 0;
     const int64_t n_embd_state = 0;
 };
@@ -935,6 +941,12 @@ public:
 
     int64_t get_max_nodes() const;
 
+    // the MoE cache's hybrid registrations made while building this graph (set_hybrid_layer), replayed when a pooled graph is
+    // reused after another graph was built (LLAMA_GRAPH_POOL)
+    // ids non-null: a prefill share registration (set_prefill_layer), else a hybrid one (set_hybrid_layer)
+    struct moe_hyb_reg { int32_t il; ggml_tensor * x; ggml_tensor * w; int act; float limit; const ggml_tensor * ids; ggml_tensor * la; };
+    std::vector<moe_hyb_reg> moe_hyb_regs;
+
     void reset();
 
     void set_inputs(const llama_ubatch * ubatch);
@@ -1232,7 +1244,9 @@ struct llm_graph_context {
              ggml_tensor * gate_exps,
              ggml_tensor * down_exps,
              ggml_tensor * gate_up_exps,
-                     int   il) const;
+                     int   il,
+             ggml_tensor * hyb_x = nullptr,   // doorbell mode: the expert input and router weights for the CPU, see set_hybrid_layer
+             ggml_tensor * hyb_w = nullptr) const;
 
     //
     // inputs
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 81af8903f..2722532c9 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -25,13 +25,20 @@
 #include "ggml.h"
 #include "ggml-cpp.h"
 
+#ifdef __linux__
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
+
 #include <algorithm>
 #include <cassert>
 #include <cfloat>
 #include <cstdint>
+#include <cerrno>
 #include <cstring>
 #include <cmath>
 #include <functional>
+#include <thread>
 #include <map>
 #include <numeric>
 #include <regex>
@@ -2063,10 +2070,369 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
     }
 
     load_tensors_hadamard();
+    load_tensors_draft_vocab();
+    load_tensors_hc_fold();
+    load_tensors_gdn_gates();
+    load_tensors_mtp_chain();
+    load_tensors_ple_gpu();
+    load_tensors_exps_pin();
 
     return true;
 }
 
+void llama_model_base::load_tensors_ple_gpu() {
+    if (getenv("LLAMA_PLE_GPU") == nullptr || atoi(getenv("LLAMA_PLE_GPU")) == 0 || per_layer_tok_embd == nullptr ||
+            per_layer_tok_embd->buffer == nullptr || !ggml_backend_buffer_is_host(per_layer_tok_embd->buffer) ||
+            per_layer_tok_embd->data == nullptr) {
+        return;
+    }
+    // a CUDA host buffer (e.g. -lm none) is pinned and mapped already
+    if (strcmp(ggml_backend_buffer_name(per_layer_tok_embd->buffer), "CUDA_Host") == 0) {
+        ple_host_mapped = true;
+        LLAMA_LOG_INFO("%s: LLAMA_PLE_GPU: '%s' is in CUDA_Host memory, the GPU gathers its rows\n", __func__, per_layer_tok_embd->name);
+        return;
+    }
+    ggml_backend_reg_t reg = ggml_backend_reg_by_name("CUDA");
+    auto reg_fn = reg ? (bool (*)(void *, size_t)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_register_host_buffer") : nullptr;
+    if (reg_fn == nullptr) {
+        LLAMA_LOG_WARN("%s: LLAMA_PLE_GPU: no CUDA host registration, the table stays on the CPU path\n", __func__);
+        return;
+    }
+    // ggml-cuda registers host memory only with GGML_CUDA_REGISTER_HOST set (read once, at its first use; nothing else here uses it)
+#if defined(_WIN32)
+    _putenv_s("GGML_CUDA_REGISTER_HOST", "1");
+#else
+    setenv("GGML_CUDA_REGISTER_HOST", "1", 0);
+#endif
+    // whole pages around the table (inside the same mapping or allocation)
+    const size_t page = 4096;
+    const uintptr_t b = (uintptr_t) per_layer_tok_embd->data & ~(uintptr_t) (page - 1);
+    const uintptr_t e = ((uintptr_t) per_layer_tok_embd->data + ggml_nbytes(per_layer_tok_embd) + page - 1) & ~(uintptr_t) (page - 1);
+    const int64_t t0 = ggml_time_us();
+    ple_host_mapped = reg_fn((void *) b, e - b);
+    LLAMA_LOG_INFO("%s: LLAMA_PLE_GPU: %s %.2f GiB of '%s' (%s) in %.1f s\n", __func__, ple_host_mapped ? "registered" : "FAILED to register",
+        (e - b)/1073741824.0, per_layer_tok_embd->name, ggml_backend_buffer_name(per_layer_tok_embd->buffer), (ggml_time_us() - t0)/1e6);
+#ifdef __linux__
+    // file-backed pages cannot be pinned on newer kernels: register an anonymous copy (LLAMA_PLE_GPU_COPY=0 keeps the CPU path,
+    // =1 forces the copy). By default only when the RAM holds the whole model beside it: the pinned copy cannot be evicted, so on a
+    // smaller host the mmapped experts fall out of the page cache and are read from the disk again (64 GB host: 74 -> 27 tok/s)
+    const char * copy_env = getenv("LLAMA_PLE_GPU_COPY");
+    bool copy = copy_env == nullptr || atoi(copy_env) != 0;
+    if (!ple_host_mapped && copy && copy_env == nullptr) {
+        const double ram = (double) sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGE_SIZE);
+        const double need = (double) size() + 8.0*1073741824.0;
+        if (ram > 0 && ram < need) {
+            copy = false;
+            LLAMA_LOG_INFO("%s: LLAMA_PLE_GPU: no anonymous copy, %.1f GiB of RAM < %.1f GiB (the model + 8 GiB), the table stays on the "
+                "CPU path (LLAMA_PLE_GPU_COPY=1 forces it)\n", __func__, ram/1073741824.0, need/1073741824.0);
+        }
+    }
+    if (!ple_host_mapped && copy) {
+        auto unreg_fn = (void (*)(void *)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_unregister_host_buffer");
+        const size_t n = ggml_nbytes(per_layer_tok_embd);
+        const size_t na = (n + page - 1) & ~(page - 1);
+        const int64_t t1 = ggml_time_us();
+        void * dst = mmap(nullptr, na, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (dst == MAP_FAILED || unreg_fn == nullptr) {
+            LLAMA_LOG_WARN("%s: LLAMA_PLE_GPU: no anonymous copy (%s), the table stays on the CPU path\n", __func__,
+                dst == MAP_FAILED ? strerror(errno) : "no unregister function");
+            if (dst != MAP_FAILED) { munmap(dst, na); }
+            return;
+        }
+        madvise(dst, na, MADV_HUGEPAGE);
+        const int nt = std::max(1, std::min(16, (int) std::thread::hardware_concurrency()));
+        std::vector<std::thread> th;
+        const size_t chunk = ((n + nt - 1)/nt + page - 1) & ~(page - 1);
+        for (int i = 0; i < nt; ++i) {
+            const size_t o = i*chunk;
+            if (o >= n) { break; }
+            th.emplace_back([=]() { memcpy((char *) dst + o, (const char *) per_layer_tok_embd->data + o, std::min(chunk, n - o)); });
+        }
+        for (auto & t : th) { t.join(); }
+        if (!reg_fn(dst, na)) {
+            LLAMA_LOG_WARN("%s: LLAMA_PLE_GPU: FAILED to register the anonymous copy, the table stays on the CPU path\n", __func__);
+            munmap(dst, na);
+            return;
+        }
+        per_layer_tok_embd->data = dst;
+        ple_host_copy = std::shared_ptr<void>(dst, [na, unreg_fn](void * q) { unreg_fn(q); munmap(q, na); });
+        ple_host_mapped = true;
+        // the mapped file pages are no longer read: let the page cache drop them
+        madvise((void *) b, e - b, MADV_DONTNEED);
+        LLAMA_LOG_INFO("%s: LLAMA_PLE_GPU: copied %.2f GiB of '%s' to anonymous memory and registered it in %.1f s\n", __func__,
+            na/1073741824.0, per_layer_tok_embd->name, (ggml_time_us() - t1)/1e6);
+    }
+#endif
+}
+
+// mmap mode: the routed experts in mapped host memory are copied once into one anonymous allocation (huge pages) and
+// registered with CUDA, so the MoE cache uploads them from pinned memory, while the rest (e.g. Qwen3.8's 27 GiB per-layer
+// embedding table) stays mapped. On by default when the RAM holds the experts + 10 GiB (Qwen3.8 IQ3_S, 64 GB host, RTX 4090:
+// 81.2 -> 93.6 tok/s); LLAMA_MOE_PIN_EXPS=0 keeps them mapped.
+void llama_model_base::load_tensors_exps_pin() {
+#ifdef __linux__
+    if (getenv("LLAMA_MOE_PIN_EXPS") != nullptr && atoi(getenv("LLAMA_MOE_PIN_EXPS")) == 0) {
+        return;
+    }
+    std::vector<ggml_tensor *> ts;
+    size_t total = 0;
+    const size_t page = 4096;
+    for (const auto & [name, t] : tensors_by_name) {
+        if (name.find("_exps.weight") == std::string::npos || t->buffer == nullptr || t->data == nullptr ||
+                strcmp(ggml_backend_buffer_name(t->buffer), "CPU_Mapped") != 0) {
+            continue;
+        }
+        ts.push_back(t);
+        total += (ggml_nbytes(t) + page - 1) & ~(page - 1);
+    }
+    if (ts.empty()) {
+        return;
+    }
+    const double ram = (double) sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGE_SIZE);
+    if (ram > 0 && ram < (double) total + 10.0*1073741824.0) {
+        LLAMA_LOG_INFO("%s: LLAMA_MOE_PIN_EXPS: %.1f GiB of RAM < %.1f GiB of mapped experts + 10 GiB, they stay mapped\n", __func__,
+            ram/1073741824.0, total/1073741824.0);
+        return;
+    }
+    ggml_backend_reg_t reg = ggml_backend_reg_by_name("CUDA");
+    auto reg_fn   = reg ? (bool (*)(void *, size_t)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_register_host_buffer") : nullptr;
+    auto unreg_fn = reg ? (void (*)(void *)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_unregister_host_buffer") : nullptr;
+    if (reg_fn == nullptr || unreg_fn == nullptr) {
+        LLAMA_LOG_WARN("%s: LLAMA_MOE_PIN_EXPS: no CUDA host registration, the experts stay mapped\n", __func__);
+        return;
+    }
+    setenv("GGML_CUDA_REGISTER_HOST", "1", 0);
+    const int64_t t0 = ggml_time_us();
+    char * dst = (char *) mmap(nullptr, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (dst == MAP_FAILED) {
+        LLAMA_LOG_WARN("%s: LLAMA_MOE_PIN_EXPS: no anonymous memory (%s), the experts stay mapped\n", __func__, strerror(errno));
+        return;
+    }
+    madvise(dst, total, MADV_HUGEPAGE);
+    // tensor by tensor (each read once from the file, then its file pages dropped), the tensor split over the threads
+    const int nt = std::max(1, std::min(16, (int) std::thread::hardware_concurrency()));
+    std::vector<size_t> offs;
+    size_t o = 0;
+    for (ggml_tensor * t : ts) {
+        const size_t n = ggml_nbytes(t);
+        const size_t chunk = ((n + nt - 1)/nt + page - 1) & ~(page - 1);
+        std::vector<std::thread> th;
+        for (int i = 0; i < nt; ++i) {
+            const size_t c = i*chunk;
+            if (c >= n) { break; }
+            th.emplace_back([=]() { memcpy(dst + o + c, (const char *) t->data + c, std::min(chunk, n - c)); });
+        }
+        for (auto & x : th) { x.join(); }
+        const uintptr_t b = (uintptr_t) t->data & ~(uintptr_t) (page - 1);
+        const uintptr_t e = ((uintptr_t) t->data + n) & ~(uintptr_t) (page - 1);   // whole pages inside the tensor only
+        if (e > b + page) {
+            madvise((void *) (b + page), e - b - page, MADV_DONTNEED);
+        }
+        offs.push_back(o);
+        o += (n + page - 1) & ~(page - 1);
+    }
+    if (!reg_fn(dst, total)) {
+        LLAMA_LOG_WARN("%s: LLAMA_MOE_PIN_EXPS: FAILED to register %.2f GiB, the experts stay mapped\n", __func__, total/1073741824.0);
+        munmap(dst, total);
+        return;
+    }
+    for (size_t i = 0; i < ts.size(); ++i) {
+        ts[i]->data = dst + offs[i];
+    }
+    exps_host_copy = std::shared_ptr<void>(dst, [total, unreg_fn](void * q) { unreg_fn(q); munmap(q, total); });
+    LLAMA_LOG_INFO("%s: LLAMA_MOE_PIN_EXPS: copied %zu expert tensors (%.2f GiB) to pinned anonymous memory in %.1f s\n", __func__,
+        ts.size(), total/1073741824.0, (ggml_time_us() - t0)/1e6);
+#endif
+}
+
+void llama_model_base::load_tensors_mtp_chain() {
+    const uint32_t il = hparams.n_layer();
+    if (getenv("LLAMA_MTP_CHAIN") == nullptr || atoi(getenv("LLAMA_MTP_CHAIN")) == 0 || arch != LLM_ARCH_QWEN4EXP ||
+            output == nullptr || output->buffer == nullptr || hparams.n_layer_nextn == 0 || layers.size() <= il ||
+            layers[il].nextn.eh_proj == nullptr) {
+        return;
+    }
+    const int n_out = 64;
+    ggml_init_params ip = { (3 + n_out)*ggml_tensor_overhead(), nullptr, true };
+    ggml_context_ptr ctx { ggml_init(ip) };
+    mtp_chain_tok = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, 1);
+    mtp_chain_h   = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, hparams.n_embd_out());
+    mtp_chain_out = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, n_out);
+    ggml_set_name(mtp_chain_tok, "mtp_chain.tok");
+    ggml_set_name(mtp_chain_h,   "mtp_chain.h");
+    ggml_set_name(mtp_chain_out, "mtp_chain.out");
+    ggml_backend_buffer_ptr buf { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), ggml_backend_buffer_get_type(output->buffer)) };
+    if (!buf) {
+        throw std::runtime_error("LLAMA_MTP_CHAIN: cannot allocate the chain buffers");
+    }
+    for (int i = 0; i < n_out; ++i) {
+        ggml_tensor * v = ggml_view_1d(ctx.get(), mtp_chain_out, 1, i*sizeof(int32_t));
+        ggml_backend_view_init(v);
+        mtp_chain_out_rows.push_back(v);
+    }
+    LLAMA_LOG_INFO("%s: MTP draft chain buffers in %s\n", __func__, ggml_backend_buffer_name(buf.get()));
+    std::vector<ggml_backend_buffer_ptr> bufs;
+    bufs.emplace_back(std::move(buf));
+    pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(bufs));
+}
+
+// LLAMA_GDN_GATES: concatenate the GDN beta and alpha projections row-wise so one matvec computes both; each output
+// row is still its own dot product, so the values are unchanged. One zero pad row keeps the row count off the tile
+// sizes of the CUDA MMF kernel, so small batches stay on the vector kernel like the separate projections (a
+// different kernel would round differently). The originals stay for the unfused path
+void llama_model_base::load_tensors_gdn_gates() {
+    if (getenv("LLAMA_GDN_GATES") == nullptr || atoi(getenv("LLAMA_GDN_GATES")) == 0 || arch != LLM_ARCH_QWEN4EXP) {
+        return;
+    }
+    int n_done = 0;
+    size_t n_bytes = 0;
+    for (auto & layer : layers) {
+        ggml_tensor * b = layer.ssm_beta;
+        ggml_tensor * a = layer.ssm_alpha;
+        if (b == nullptr || a == nullptr || layer.ssm_beta_s || layer.ssm_alpha_s || b->type != a->type ||
+            b->ne[0] != a->ne[0] || b->ne[2] != 1 || a->ne[2] != 1 || !ggml_is_contiguous(b) || !ggml_is_contiguous(a) ||
+            b->buffer == nullptr || ggml_backend_buffer_get_type(b->buffer) != ggml_backend_buffer_get_type(a->buffer)) {
+            continue;
+        }
+        ggml_init_params ip = { ggml_tensor_overhead(), nullptr, true };
+        ggml_context_ptr ctx { ggml_init(ip) };
+        const int64_t n_rows = b->ne[1] + a->ne[1];
+        ggml_tensor * t = ggml_new_tensor_2d(ctx.get(), b->type, b->ne[0], n_rows + (n_rows % 2 == 0 ? 1 : 0));
+        ggml_format_name(t, "%s.cat", b->name);
+        ggml_backend_buffer_ptr buf { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), ggml_backend_buffer_get_type(b->buffer)) };
+        if (!buf) {
+            throw std::runtime_error("LLAMA_GDN_GATES: cannot allocate the merged gate projection");
+        }
+        std::vector<uint8_t> data(ggml_nbytes(t), 0);
+        ggml_backend_tensor_get(b, data.data(), 0, ggml_nbytes(b));
+        ggml_backend_tensor_get(a, data.data() + ggml_nbytes(b), 0, ggml_nbytes(a));
+        ggml_backend_tensor_set(t, data.data(), 0, data.size());
+        layer.ssm_ba_cat = t;
+        n_done++;
+        n_bytes += ggml_nbytes(t);
+        std::vector<ggml_backend_buffer_ptr> bufs;
+        bufs.emplace_back(std::move(buf));
+        pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(bufs));
+    }
+    LLAMA_LOG_INFO("%s: merged beta+alpha projections of %d layers (%.1f MiB)\n", __func__, n_done, n_bytes/1024.0/1024.0);
+}
+
+// LLAMA_HC_FUSE: fold the 1/hc scale that follows each hyper-connection down projection into its weights. Only
+// for BF16 and F32 weights with hc a power of two, where the scaling is exact (BF16 has the F32 exponent range),
+// so the products and their sums are the same as scaling the result
+void llama_model_base::load_tensors_hc_fold() {
+    if (getenv("LLAMA_HC_FUSE") == nullptr || atoi(getenv("LLAMA_HC_FUSE")) == 0 || arch != LLM_ARCH_QWEN4EXP || hparams.dsv4_hc_mult <= 1) { // only qwen4exp reads hc_scale_folded
+        return;
+    }
+    const uint32_t hc = hparams.dsv4_hc_mult;
+    if ((hc & (hc - 1)) != 0) {
+        return;
+    }
+    const float s = 1.0f / (float) hc;
+    std::vector<ggml_tensor *> ts = { hc_head_down };
+    for (auto & l : layers) {
+        ts.push_back(l.hc_attn_down);
+        ts.push_back(l.hc_ffn_down);
+        ts.push_back(l.hc_attn_down_inject); // merged by the converter: the inject rows take the same 1/hc
+        ts.push_back(l.hc_ffn_down_inject);
+        ts.push_back(l.nextn.hc_head_down);
+    }
+    size_t n_done = 0, n_skip = 0;
+    std::vector<uint8_t> buf;
+    for (ggml_tensor * t : ts) {
+        if (t == nullptr) {
+            continue;
+        }
+        if (t->type != GGML_TYPE_BF16 && t->type != GGML_TYPE_F32) {
+            n_skip++;
+            continue;
+        }
+        buf.resize(ggml_nbytes(t));
+        ggml_backend_tensor_get(t, buf.data(), 0, buf.size());
+        const int64_t n = ggml_nelements(t);
+        if (t->type == GGML_TYPE_F32) {
+            float * p = (float *) buf.data();
+            for (int64_t i = 0; i < n; ++i) { p[i] *= s; }
+        } else {
+            ggml_bf16_t * p = (ggml_bf16_t *) buf.data();
+            for (int64_t i = 0; i < n; ++i) {
+                const float v = ggml_bf16_to_fp32(p[i]);
+                const ggml_bf16_t r = ggml_fp32_to_bf16(v * s);
+                GGML_ASSERT(ggml_bf16_to_fp32(r) == v * s && "LLAMA_HC_FUSE: the folded scale is not exact");
+                p[i] = r;
+            }
+        }
+        ggml_backend_tensor_set(t, buf.data(), 0, buf.size());
+        hc_scale_folded.insert(t);
+        n_done++;
+    }
+    LLAMA_LOG_INFO("%s: folded 1/%u into %zu hyper-connection down projections (%zu left as they are: not BF16/F32)\n",
+        __func__, hc, n_done, n_skip);
+}
+
+// an MTP draft model may score only a subset of the vocabulary: its head reads N rows of output instead of n_vocab,
+// and a token outside the subset is never drafted (the target still verifies every token, so the output is the same)
+void llama_model_base::load_tensors_draft_vocab() {
+    const char * path = getenv("LLAMA_MTP_DRAFT_VOCAB");
+    const uint32_t il = hparams.n_layer();
+    if (path == nullptr || output == nullptr || hparams.n_layer_nextn == 0 || layers.size() <= il || layers[il].nextn.eh_proj == nullptr) {
+        return;
+    }
+    // a number N: the first N token ids (BPE ids come roughly in merge-frequency order) plus every control and
+    // user-defined token; otherwise a file of int32 token ids
+    std::vector<int32_t> ids;
+    char * end = nullptr;
+    const long n_first = strtol(path, &end, 10);
+    if (end != path && *end == '\0') {
+        for (int32_t id = 0; id < (int32_t) output->ne[1]; ++id) {
+            const auto attr = vocab.token_get_attr(id);
+            if (id < n_first || (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED))) {
+                ids.push_back(id);
+            }
+        }
+    } else {
+        FILE * f = fopen(path, "rb");
+        if (f == nullptr) {
+            throw std::runtime_error(format("LLAMA_MTP_DRAFT_VOCAB: cannot open %s", path));
+        }
+        int32_t id;
+        while (fread(&id, sizeof(id), 1, f) == 1) {
+            if (id < 0 || id >= output->ne[1]) {
+                fclose(f);
+                throw std::runtime_error(format("LLAMA_MTP_DRAFT_VOCAB: token id %d out of range", id));
+            }
+            ids.push_back(id);
+        }
+        fclose(f);
+    }
+    const int64_t n = (int64_t) ids.size();
+    GGML_ASSERT(n > 0);
+
+    ggml_init_params ip = { 2*ggml_tensor_overhead(), nullptr, true };
+    ggml_context_ptr ctx { ggml_init(ip) };
+    output_draft     = ggml_new_tensor_2d(ctx.get(), output->type, output->ne[0], n);
+    output_draft_ids = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, n);
+    ggml_set_name(output_draft, "output_draft.weight");
+    ggml_set_name(output_draft_ids, "output_draft.ids");
+    ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(output->buffer);
+    ggml_backend_buffer_ptr buf { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) };
+    if (!buf) {
+        throw std::runtime_error("LLAMA_MTP_DRAFT_VOCAB: cannot allocate the draft head");
+    }
+    const size_t row = output->nb[1];
+    std::vector<uint8_t> data(n*row);
+    for (int64_t i = 0; i < n; ++i) {
+        ggml_backend_tensor_get(output, data.data() + i*row, (size_t) ids[i]*row, row);
+    }
+    ggml_backend_tensor_set(output_draft, data.data(), 0, data.size());
+    ggml_backend_tensor_set(output_draft_ids, ids.data(), 0, n*sizeof(int32_t));
+    LLAMA_LOG_INFO("%s: MTP draft head over %lld of %lld tokens from %s (%.1f MiB in %s, full head %.1f MiB)\n", __func__,
+        (long long) n, (long long) output->ne[1], path, ggml_nbytes(output_draft)/1024.0/1024.0, ggml_backend_buffer_name(buf.get()), ggml_nbytes(output)/1024.0/1024.0);
+    std::vector<ggml_backend_buffer_ptr> bufs;
+    bufs.emplace_back(std::move(buf));
+    pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(bufs));
+}
+
 // read and check the prism.hadamard metadata, and record the folded weights (load_tensors_hadamard makes the tensors)
 void llama_model_base::load_hparams_hadamard(llama_model_loader & ml) {
     uint32_t hadamard_version = 0;
diff --git a/src/llama-model.h b/src/llama-model.h
index 222238efe..03d8717bc 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -435,6 +435,8 @@ struct llama_layer {
 
     // qwen3.5
     struct ggml_tensor * ssm_alpha = nullptr;
+    // LLAMA_GDN_GATES: rows of ssm_beta then ssm_alpha, one matvec for both (see load_tensors_gdn_gates)
+    struct ggml_tensor * ssm_ba_cat = nullptr;
 
     // rwkv
     struct ggml_tensor * time_mix_w1         = nullptr;
@@ -759,6 +761,29 @@ struct llama_model {
     struct ggml_tensor * nextn_proj_pre  = nullptr;
     struct ggml_tensor * nextn_proj_post = nullptr;
 
+    // MTP draft head over a vocabulary subset (LLAMA_MTP_DRAFT_VOCAB: N first ids, or a file of int32 token ids): rows of output, and their ids
+    struct ggml_tensor * output_draft     = nullptr;
+    struct ggml_tensor * output_draft_ids = nullptr;
+
+    // LLAMA_MTP_CHAIN: device buffers that chain the 1-token MTP draft steps without host round trips. The 1-token
+    // MTP graph reads its token and h from them and writes its argmax token and h back (see llama_mtp_chain_*)
+    struct ggml_tensor * mtp_chain_tok = nullptr;          // I32 [1]
+    struct ggml_tensor * mtp_chain_h   = nullptr;          // F32 [n_embd_out]
+    struct ggml_tensor * mtp_chain_out = nullptr;          // I32 [64], the token of each chained step
+    std::vector<struct ggml_tensor *> mtp_chain_out_rows;  // views of mtp_chain_out, one per step
+    mutable bool mtp_chain_active = false;                 // while true, the graph input leaves the buffers alone
+
+    // LLAMA_PLE_GPU=1: the per-layer embedding table in host memory is registered with the GPU (pinned and mapped), and the
+    // GPU gathers its rows itself (ggml_get_rows_host): no CPU split before every ubatch
+    bool ple_host_mapped = false;
+    // where the kernel cannot pin file-backed pages (Linux 6.8: cudaHostRegister "operation not supported"), an anonymous
+    // copy of the table is registered instead and the tensor points at it; freed (unregistered, unmapped) with the model
+    std::shared_ptr<void> ple_host_copy;
+    std::shared_ptr<void> exps_host_copy;  // LLAMA_MOE_PIN_EXPS: the pinned copy of the mapped experts
+
+    // LLAMA_HC_FUSE: hyper-connection down projections whose 1/hc scale was folded into the weights at load
+    std::unordered_set<const struct ggml_tensor *> hc_scale_folded;
+
     // hrm-text initial low-cycle state
     struct ggml_tensor * hrm_z_l_init = nullptr;
 
@@ -978,6 +1003,12 @@ struct llama_model_base : public llama_model {
 
     // helper: make the prism.hadamard rotation and sign tensors for the folded weights
     void load_tensors_hadamard();
+    void load_tensors_draft_vocab();
+    void load_tensors_hc_fold();
+    void load_tensors_gdn_gates();
+    void load_tensors_mtp_chain();
+    void load_tensors_ple_gpu();
+    void load_tensors_exps_pin();
 
     // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern
     void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);
diff --git a/src/llama-moe-cache.cpp b/src/llama-moe-cache.cpp
index 6a343a760..a4ca8a75c 100644
--- a/src/llama-moe-cache.cpp
+++ b/src/llama-moe-cache.cpp
@@ -1,33 +1,268 @@
 #include "llama-moe-cache.h"
+#include "llama-moe-iq.h"
 
 #include "llama-impl.h"
-#include "llama-memory.h"
 #include "llama-model.h"
 
 #include "ggml-cpp.h"
+#include "ggml-cpu.h"
 
 #include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <functional>
+#include <mutex>
+#include <thread>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
 #include <stdexcept>
 #include <unordered_map>
 #include <vector>
+#include <ctime>
+#include <cstdio>
+#include <fstream>
+#include <map>
+#include <set>
+#include <string>
+#ifdef __linux__
+#include <pthread.h>
+#include <sched.h>
+#endif
+
+#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__))
+#include <immintrin.h>
+#define LLAMA_MOE_Q2_X86 1
+#endif
+
+// hybrid mode, Q2_0 experts: a faster dot product than ggml's for one Q2_0 row against Q8_0 activations. The activations are
+// permuted once (per token, or per pair for the down projection) into the weights' bit-plane order: byte b of a 64-weight block
+// holds weights 4b+k at bits 2k, so plane k of the 16 bytes is (q >> 2k) & 3 and needs no shuffle. Same integer products and the
+// same scales as ggml's kernel, only the float summation order differs.
+namespace {
+struct q2_block { ggml_fp16_t d; uint8_t qs[16]; };
+struct q8_block { ggml_fp16_t d; int8_t  qs[32]; };
+static_assert(sizeof(q2_block) == 18 && sizeof(q8_block) == 34, "unexpected Q2_0 / Q8_0 block size");
+
+// one 64-element block of prepared activations: planes [0|1] and [2|3] (each 16 elements of 2 Q8_0 halves), the scale of each
+// int32 lane after the dot products (dwords 0,1,4,5 come from the first Q8_0 block, 2,3,6,7 from the second), and the offset
+// c = sum over the block of y (the weights are v - 1)
+struct q2_act { int8_t q[64]; float dyv[8]; float c; };
+
+void q2_prepare(const q8_block * y, int64_t n, q2_act * a) {
+    for (int64_t i = 0; i < n/64; ++i) {
+        const q8_block & y0 = y[2*i];
+        const q8_block & y1 = y[2*i + 1];
+        const float d0 = ggml_fp16_to_fp32(y0.d), d1 = ggml_fp16_to_fp32(y1.d);
+        int s0 = 0, s1 = 0;
+        for (int b = 0; b < 16; ++b) {
+            for (int k = 0; k < 4; ++k) {
+                const int e = 4*b + k;
+                a[i].q[k*16 + b] = e < 32 ? y0.qs[e] : y1.qs[e - 32];
+            }
+        }
+        for (int k = 0; k < 32; ++k) {
+            s0 += y0.qs[k];
+            s1 += y1.qs[k];
+        }
+        const float dyv[8] = { d0, d0, d1, d1, d0, d0, d1, d1 };
+        memcpy(a[i].dyv, dyv, sizeof(dyv));
+        a[i].c = d0*s0 + d1*s1;
+    }
+}
+
+#ifdef LLAMA_MOE_Q2_X86
+// the same body for each target (an always_inline template cannot be compiled for a target its callers add)
+#define LLAMA_MOE_Q2_DOT_BODY(DOT)                                                                                         \
+    __m256 acc = _mm256_setzero_ps();                                                                                      \
+    float corr = 0.0f;                                                                                                     \
+    const __m256i sh0 = _mm256_setr_epi32(0, 0, 0, 0, 2, 2, 2, 2);                                                         \
+    const __m256i sh1 = _mm256_setr_epi32(4, 4, 4, 4, 6, 6, 6, 6);                                                         \
+    const __m256i m3  = _mm256_set1_epi8(3);                                                                               \
+    for (int64_t i = 0; i < n/64; ++i) {                                                                                   \
+        const __m256i q  = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *) x[i].qs));                        \
+        const __m256i vA = _mm256_and_si256(_mm256_srlv_epi32(q, sh0), m3);                                                \
+        const __m256i vB = _mm256_and_si256(_mm256_srlv_epi32(q, sh1), m3);                                                \
+        const __m256i yA = _mm256_loadu_si256((const __m256i *) (a[i].q));                                                 \
+        const __m256i yB = _mm256_loadu_si256((const __m256i *) (a[i].q + 32));                                            \
+        const __m256i s  = DOT;                                                                                            \
+        const float dx = _cvtsh_ss(x[i].d);                                                                                \
+        acc = _mm256_fmadd_ps(_mm256_mul_ps(_mm256_set1_ps(dx), _mm256_loadu_ps(a[i].dyv)), _mm256_cvtepi32_ps(s), acc);  \
+        corr += dx*a[i].c;                                                                                                 \
+    }                                                                                                                      \
+    __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));                                     \
+    r = _mm_hadd_ps(r, r);                                                                                                 \
+    r = _mm_hadd_ps(r, r);                                                                                                 \
+    return _mm_cvtss_f32(r) - corr;
+
+__attribute__((target("avx2,fma,f16c")))
+float q2_dot_avx2(int64_t n, const q2_block * x, const q2_act * a) {
+    LLAMA_MOE_Q2_DOT_BODY(_mm256_madd_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(vA, yA), _mm256_maddubs_epi16(vB, yB)), _mm256_set1_epi16(1)))
+}
+
+__attribute__((target("avx2,fma,f16c,avxvnni")))
+float q2_dot_avxvnni(int64_t n, const q2_block * x, const q2_act * a) {
+    LLAMA_MOE_Q2_DOT_BODY(_mm256_dpbusd_avx_epi32(_mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), vA, yA), vB, yB))
+}
+#undef LLAMA_MOE_Q2_DOT_BODY
+
+// 4 rows against one activation: the activation loads are shared, each row's arithmetic is q2_dot's (same results)
+#define LLAMA_MOE_Q2_DOT4_BODY(DOT)                                                                                        \
+    __m256 acc[4] = { _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps() };               \
+    float corr[4] = { 0.0f, 0.0f, 0.0f, 0.0f };                                                                            \
+    const __m256i sh0 = _mm256_setr_epi32(0, 0, 0, 0, 2, 2, 2, 2);                                                         \
+    const __m256i sh1 = _mm256_setr_epi32(4, 4, 4, 4, 6, 6, 6, 6);                                                         \
+    const __m256i m3  = _mm256_set1_epi8(3);                                                                               \
+    for (int64_t i = 0; i < n/64; ++i) {                                                                                   \
+        const __m256i yA  = _mm256_loadu_si256((const __m256i *) (a[i].q));                                                \
+        const __m256i yB  = _mm256_loadu_si256((const __m256i *) (a[i].q + 32));                                           \
+        const __m256  dyv = _mm256_loadu_ps(a[i].dyv);                                                                     \
+        const float   c   = a[i].c;                                                                                        \
+        _Pragma("GCC unroll 4")                                                                                            \
+        for (int r = 0; r < 4; ++r) {                                                                                      \
+            const q2_block & xb = *(const q2_block *) ((const char *) x + r*row_stride + i*sizeof(q2_block));            \
+            const __m256i q  = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *) xb.qs));                     \
+            const __m256i vA = _mm256_and_si256(_mm256_srlv_epi32(q, sh0), m3);                                            \
+            const __m256i vB = _mm256_and_si256(_mm256_srlv_epi32(q, sh1), m3);                                            \
+            const __m256i s  = DOT;                                                                                        \
+            const float dx = _cvtsh_ss(xb.d);                                                                              \
+            acc[r] = _mm256_fmadd_ps(_mm256_mul_ps(_mm256_set1_ps(dx), dyv), _mm256_cvtepi32_ps(s), acc[r]);               \
+            corr[r] += dx*c;                                                                                               \
+        }                                                                                                                  \
+    }                                                                                                                      \
+    for (int r = 0; r < 4; ++r) {                                                                                          \
+        __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc[r]), _mm256_extractf128_ps(acc[r], 1));                           \
+        v = _mm_hadd_ps(v, v);                                                                                             \
+        v = _mm_hadd_ps(v, v);                                                                                             \
+        out[r] = _mm_cvtss_f32(v) - corr[r];                                                                               \
+    }
+
+__attribute__((target("avx2,fma,f16c")))
+void q2_dot4_avx2(int64_t n, const q2_block * x, size_t row_stride, const q2_act * a, float * out) {
+    LLAMA_MOE_Q2_DOT4_BODY(_mm256_madd_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(vA, yA), _mm256_maddubs_epi16(vB, yB)), _mm256_set1_epi16(1)))
+}
+
+__attribute__((target("avx2,fma,f16c,avxvnni")))
+void q2_dot4_avxvnni(int64_t n, const q2_block * x, size_t row_stride, const q2_act * a, float * out) {
+    LLAMA_MOE_Q2_DOT4_BODY(_mm256_dpbusd_avx_epi32(_mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), vA, yA), vB, yB))
+}
+#undef LLAMA_MOE_Q2_DOT4_BODY
+#endif
+
+using q2_dot_t = float (*)(int64_t, const q2_block *, const q2_act *);
+// rows x, x + row_stride, x + 2*row_stride, x + 3*row_stride (bytes) against one activation, results in out[4]
+using q2_dot4_t = void (*)(int64_t, const q2_block *, size_t, const q2_act *, float *);
+
+q2_dot4_t q2_dot4_select() {
+    if (const char * e = getenv("LLAMA_MOE_CPU_Q2"); e && atoi(e) == 0) {
+        return nullptr;
+    }
+    if (const char * e = getenv("LLAMA_MOE_CPU_DOT4"); e && atoi(e) == 0) {
+        return nullptr;
+    }
+#ifdef LLAMA_MOE_Q2_X86
+    __builtin_cpu_init();
+    if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma") && __builtin_cpu_supports("f16c")) {
+        if (__builtin_cpu_supports("avxvnni")) {
+            return q2_dot4_avxvnni;
+        }
+        return q2_dot4_avx2;
+    }
+#endif
+    return nullptr;
+}
+const char * q2_dot_name();
+
+// the fast Q2_0 dot product this CPU supports, nullptr if none (LLAMA_MOE_CPU_Q2=0 turns it off)
+q2_dot_t q2_dot_select() {
+    if (const char * e = getenv("LLAMA_MOE_CPU_Q2"); e && atoi(e) == 0) {
+        return nullptr;
+    }
+#ifdef LLAMA_MOE_Q2_X86
+    __builtin_cpu_init();
+    if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma") && __builtin_cpu_supports("f16c")) {
+        if (__builtin_cpu_supports("avxvnni")) {
+            return q2_dot_avxvnni;
+        }
+        return q2_dot_avx2;
+    }
+#endif
+    return nullptr;
+}
+
+// the multi-token i-quant kernel of a weight type (llama-moe-iq.h), nullptr if none; chosen once per type
+llama_moe_iq::dot_mt_t iq_mt(ggml_type t) {
+    static const std::vector<llama_moe_iq::dot_mt_t> tab = [] {
+        std::vector<llama_moe_iq::dot_mt_t> v(GGML_TYPE_COUNT);
+        for (int i = 0; i < GGML_TYPE_COUNT; ++i) {
+            v[i] = llama_moe_iq::select((ggml_type) i);
+        }
+        return v;
+    }();
+    return t >= 0 && t < GGML_TYPE_COUNT ? tab[t] : nullptr;
+}
+
+const char * q2_dot_name() {
+#ifdef LLAMA_MOE_Q2_X86
+    const q2_dot_t f = q2_dot_select();
+    return f == q2_dot_avxvnni ? " (Q2_0 fast path: AVX-VNNI)" : f == q2_dot_avx2 ? " (Q2_0 fast path: AVX2)" : "";
+#else
+    return "";
+#endif
+}
+} // namespace
 
 namespace {
 
-// SLRU of the experts of a group of layers, the slot of each expert is kept in the slot map of its layer
-struct moe_cache_slru {
+// LRU of the experts of a group of layers, the slot of each expert is kept in the slot map of its layer
+// lookahead prefetch: the router dot products on the host (float weights and input)
+typedef float (*la_dot_t)(const float *, const float *, int64_t);
+static float la_dot_ref(const float * a, const float * b, int64_t n) {
+    float r = 0.0f;
+    for (int64_t i = 0; i < n; ++i) r += a[i]*b[i];
+    return r;
+}
+#ifdef LLAMA_MOE_Q2_X86
+__attribute__((target("avx2,fma")))
+static float la_dot_avx2(const float * a, const float * b, int64_t n) {
+    __m256 s0 = _mm256_setzero_ps(), s1 = _mm256_setzero_ps(), s2 = _mm256_setzero_ps(), s3 = _mm256_setzero_ps();
+    int64_t i = 0;
+    for (; i + 32 <= n; i += 32) {
+        s0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i),      _mm256_loadu_ps(b + i),      s0);
+        s1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8),  _mm256_loadu_ps(b + i + 8),  s1);
+        s2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), s2);
+        s3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), s3);
+    }
+    __m256 s = _mm256_add_ps(_mm256_add_ps(s0, s1), _mm256_add_ps(s2, s3));
+    __m128 h = _mm_add_ps(_mm256_castps256_ps128(s), _mm256_extractf128_ps(s, 1));
+    h = _mm_hadd_ps(h, h);
+    h = _mm_hadd_ps(h, h);
+    float r = _mm_cvtss_f32(h);
+    for (; i < n; ++i) r += a[i]*b[i];
+    return r;
+}
+#endif
+static la_dot_t la_dot_select() {
+#ifdef LLAMA_MOE_Q2_X86
+    if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) {
+        return la_dot_avx2;
+    }
+#endif
+    return la_dot_ref;
+}
+
+struct moe_cache_lru {
     int32_t n_expert = 0;
     int32_t n_slots  = 0;
 
     std::vector<int32_t *> slot_map; // [n_layer] data of the slot maps, -1 if the expert is not cached
     std::vector<int32_t>   key_of;   // [n_slots] il*n_expert + expert, -1 if empty
 
-    // probation and protected LRU lists; hits promote slots to the protected half
+    // doubly linked list of the slots, head is the least recently used
     std::vector<int32_t> prev;
     std::vector<int32_t> next;
-    int32_t head[2] = { -1, -1 };
-    int32_t tail[2] = { -1, -1 };
-    std::vector<bool> is_protected;
-    int32_t n_protected = 0;
+    int32_t head = -1;
+    int32_t tail = -1;
 
     std::vector<uint32_t> seen; // [n_expert]
     uint32_t seen_gen = 0;
@@ -44,59 +279,27 @@ struct moe_cache_slru {
             prev[s] = s - 1;
             next[s] = s + 1 < n_slots ? s + 1 : -1;
         }
-        head[0] = 0;
-        tail[0] = n_slots - 1;
-        is_protected.assign(n_slots, false);
+        head = 0;
+        tail = n_slots - 1;
         seen.assign(n_expert, 0);
     }
 
-    // move slot s to the most recently used end of a segment
-    void move(int32_t s, bool protect) {
-        const bool from = is_protected[s];
-        if (from == protect && s == tail[from]) {
+    // move slot s to the tail (most recently used)
+    void touch(int32_t s) {
+        if (s == tail) {
             return;
         }
         if (prev[s] >= 0) {
             next[prev[s]] = next[s];
         } else {
-            head[from] = next[s];
-        }
-        if (next[s] >= 0) {
-            prev[next[s]] = prev[s];
-        } else {
-            tail[from] = prev[s];
+            head = next[s];
         }
+        prev[next[s]] = prev[s];
 
-        prev[s] = tail[protect];
+        prev[s] = tail;
         next[s] = -1;
-        if (tail[protect] >= 0) {
-            next[tail[protect]] = s;
-        } else {
-            head[protect] = s;
-        }
-        tail[protect] = s;
-        is_protected[s] = protect;
-        n_protected += int(protect) - int(from);
-    }
-
-    void touch(int32_t s) {
-        move(s, true);
-        if (n_protected > n_slots/2) {
-            move(head[1], false);
-        }
-    }
-
-    int32_t victim(int32_t il) const {
-        for (int segment = 0; segment < 2; ++segment) {
-            for (int32_t s = head[segment]; s >= 0; s = next[s]) {
-                const int32_t key = key_of[s];
-                // keep every expert selected by this request, including earlier fills
-                if (key < 0 || key / n_expert != il || seen[key % n_expert] != seen_gen) {
-                    return s;
-                }
-            }
-        }
-        GGML_ABORT("no SLRU victim");
+        next[tail] = s;
+        tail = s;
     }
 
     struct fill {
@@ -104,7 +307,7 @@ struct moe_cache_slru {
         int32_t slot;
     };
 
-    // give a slot to each expert selected by ids in layer il, evicting from probation first
+    // give a slot to each expert selected by ids in layer il, the misses evict the least recently used experts
     // returns false if the ids select more distinct experts than there are slots
     bool plan(int32_t il, const int32_t * ids, size_t n_ids, std::vector<fill> & fills, size_t & n_hit) {
         fills.clear();
@@ -128,7 +331,7 @@ struct moe_cache_slru {
 
         int32_t * slots = slot_map[il];
 
-        // promote hits before choosing victims for misses
+        // hits go to the tail first, so the head can be evicted below
         for (int32_t e : uniq) {
             if (slots[e] >= 0) {
                 touch(slots[e]);
@@ -141,17 +344,88 @@ struct moe_cache_slru {
             if (slots[e] >= 0) {
                 continue;
             }
-            const int32_t s = victim(il);
+            const int32_t s = head;
             if (key_of[s] >= 0) {
                 slot_map[key_of[s] / n_expert][key_of[s] % n_expert] = -1;
             }
             key_of[s] = il*n_expert + e;
             slots[e] = s;
-            move(s, false);
+            touch(s);
             fills.push_back({ e, s });
         }
         return true;
     }
+
+    // hybrid mode: like plan, but only n_admit of the misses get a slot (those selected by the most tokens first),
+    // the others are returned in rest and stay uncached. tail_rank > 0 (LLAMA_MOE_TAIL_SKIP, approximate): a missed expert
+    // that every token routes at rank >= tail_rank (ids are in router order, n_used per token) goes to skipped instead:
+    // neither uploaded nor computed, its contribution is zero (the other weights are not renormalized)
+    void plan_hybrid(int32_t il, const int32_t * ids, size_t n_ids, double share, std::vector<fill> & fills, std::vector<int32_t> & rest, size_t & n_hit,
+            int32_t n_used = 0, int32_t tail_rank = 0, std::vector<int32_t> * skipped = nullptr) {
+        fills.clear();
+        rest.clear();
+        if (skipped) {
+            skipped->clear();
+        }
+        n_hit = 0;
+
+        if (++seen_gen == 0) {
+            std::fill(seen.begin(), seen.end(), 0);
+            seen_gen = 1;
+        }
+        uniq.clear();
+        count.resize(n_expert);
+        if (tail_rank > 0) {
+            min_rank.resize(n_expert);
+        }
+        for (size_t i = 0; i < n_ids; ++i) {
+            GGML_ASSERT(ids[i] >= 0 && ids[i] < n_expert);
+            if (seen[ids[i]] != seen_gen) {
+                seen[ids[i]] = seen_gen;
+                count[ids[i]] = 0;
+                if (tail_rank > 0) {
+                    min_rank[ids[i]] = INT32_MAX;
+                }
+                uniq.push_back(ids[i]);
+            }
+            count[ids[i]]++;
+            if (tail_rank > 0) {
+                min_rank[ids[i]] = std::min(min_rank[ids[i]], (int32_t) (i % n_used));
+            }
+        }
+
+        int32_t * slots = slot_map[il];
+
+        std::vector<int32_t> & misses = rest;
+        for (int32_t e : uniq) {
+            if (slots[e] >= 0) {
+                touch(slots[e]);
+                n_hit++;
+            } else if (tail_rank > 0 && skipped && min_rank[e] >= tail_rank) {
+                skipped->push_back(e);
+            } else {
+                misses.push_back(e);
+            }
+        }
+        const size_t n_admit = std::min(misses.size(), (size_t) std::lround(share*misses.size()));
+        std::stable_sort(misses.begin(), misses.end(), [&](int32_t a, int32_t b) { return count[a] > count[b]; });
+        std::sort(misses.begin(), misses.begin() + n_admit);
+        for (size_t i = 0; i < n_admit; ++i) {
+            const int32_t e = misses[i];
+            const int32_t s = head;
+            if (key_of[s] >= 0) {
+                slot_map[key_of[s] / n_expert][key_of[s] % n_expert] = -1;
+            }
+            key_of[s] = il*n_expert + e;
+            slots[e] = s;
+            touch(s);
+            fills.push_back({ e, s });
+        }
+        misses.erase(misses.begin(), misses.begin() + n_admit);
+    }
+
+    std::vector<int32_t> count; // [n_expert] tokens that select each expert, valid for the experts in uniq
+    std::vector<int32_t> min_rank; // [n_expert] best router rank of each expert in uniq (tail skip)
 };
 
 // gate, up, down or gate_up, down
@@ -207,13 +481,40 @@ struct llama_moe_cache::impl {
         std::vector<ggml_tensor *> banks; // device storage of all slots, one per expert tensor
         size_t host_bytes = 0;
         int32_t n_slots = 0;
-        moe_cache_slru slru;
+        moe_cache_lru lru;
     };
 
     struct layer {
         int32_t ig = -1;                    // -1 if the layer is not cached
         ggml_tensor * slot_map = nullptr;   // I32 [1, n_expert] in host memory
         std::vector<ggml_tensor *> experts; // host expert tensors, in the order of the banks
+
+        // hybrid mode
+        ggml_tensor * up = nullptr, * gate = nullptr, * down = nullptr, * gate_up = nullptr; // host experts
+        ggml_tensor * cpu_out = nullptr;    // F32 [n_embd, max_batch] in host memory, added to the MoE output on the device
+        ggml_tensor * x_dev = nullptr;      // expert input of the current graph [n_embd, n_tokens], nullptr: not hybrid
+        ggml_tensor * w_dev = nullptr;      // router weights of the current graph [1, n_expert_used, n_tokens]
+        llama_moe_cache::act_type act = llama_moe_cache::ACT_NONE;
+        float limit = 0.0f;
+        // expert biases (gpt-oss), F32 [rows, n_expert] in host memory, nullptr: none; bias_ok false: a bias the CPU cannot read
+        const ggml_tensor * b_up = nullptr, * b_gate = nullptr, * b_down = nullptr, * b_gate_up = nullptr;
+        bool bias_ok = true;
+        std::vector<int32_t> map_dev;       // slot map as uploaded: the CPU's experts read the zero slot
+        ggml_tensor * map_pin = nullptr;    // I32 [n_expert] in host memory, the copy of map_dev that is uploaded
+        ggml_tensor * flag    = nullptr;    // I32 [4] in pinned memory, flag mode: see ggml_host_add
+        std::vector<float> x, w, out;       // host copies of the input and weights, the CPU's output
+        std::vector<int32_t> cpu_experts;   // missed experts of the current ubatch that the CPU computes
+        std::vector<int32_t> ids;           // selected experts of the current ubatch
+        bool planned = false;               // the slot map callback ran in hybrid mode for the current ubatch
+        int64_t t_start = 0;                // when the CPU job was started
+        int64_t t_job   = 0;                // how long it ran (us)
+        ggml_tensor * db = nullptr;         // doorbell mode: the control block (struct ggml_moe_db and its arrays) in pinned memory
+        // prefill share: the expert input, router weights and selected experts of the current large-batch graph (p_x nullptr: none)
+        ggml_tensor * p_x = nullptr, * p_w = nullptr;
+        const ggml_tensor * p_ids = nullptr;
+        ggml_tensor * pf_flag = nullptr;    // I32 [4] in pinned memory, the flag of the host_add of the prefill share
+        bool pf_job = false;                // the CPU job of this layer is a prefill one: its output goes to pf_out
+        ggml_tensor * la_pred = nullptr;    // lookahead: the predicted experts of the next layer [la_k, n_tokens] in the current graph
     };
 
     struct binding {
@@ -235,6 +536,213 @@ struct llama_moe_cache::impl {
     stats stats_small; // up to 8 tokens per ubatch
     stats stats_large;
     stats stats_copy;  // experts copied from the cache for large batches
+    size_t n_cpu_experts = 0; // hybrid: missed experts computed by the CPU
+    bool   flag_mode = false; // hybrid: the GPU reads the CPU's output from pinned memory after a flag (see hybrid_add)
+    size_t n_flag_waits = 0, n_flag_timeouts = 0;
+    double t_flag_wait_us = 0.0;
+    size_t n_cpu_pairs   = 0; // hybrid: (token, expert) pairs computed by the CPU
+    double t_cpu_us      = 0.0;
+    // hybrid timers (host, us): sync for the ids, uploads, between the slot map and cpu_out copies, waiting for the CPU
+    double t_sync_us = 0.0, t_upload_us = 0.0, t_gap_us = 0.0, t_wait_us = 0.0;
+    int64_t t_planned = 0;
+    size_t n_hybrid_calls = 0;
+
+    // hybrid mode
+    bool   hybrid = false;
+    double hybrid_share = 0.0; // share of the misses uploaded
+    int32_t tail_rank   = 0;   // LLAMA_MOE_TAIL_SKIP: skip missed experts that every token routes at this rank or lower (approximate)
+    size_t  n_tail_skip = 0;   // (layer, expert) misses skipped
+    std::vector<int32_t> tail_skipped;
+    // LLAMA_MOE_CPU=auto: the share follows the measured balance, so that the CPU's experts end when the GPU reaches cpu_out:
+    // every window of layer calls, the share goes up a step if the GPU mostly waited for the CPU, down if the CPU mostly idled
+    bool   hybrid_auto = false;
+    double ctl_wait_us = 0.0, ctl_idle_us = 0.0;
+    int    ctl_calls = 0;
+    int    ctl_window = 768; // layer calls (16 tokens of a 48-layer model)
+    size_t n_share_changes = 0;
+    // share rule (default in auto mode; LLAMA_MOE_SHARE_RULE=0 off, =<k> the factor): the share follows Strata's rule L/(L + k*R) (k 1.85 by default) instead of the waits:
+    // L the host-to-device link (GB/s, probed at start), R the CPU's rate on missed experts (expert bytes / the job's wall
+    // time per expert, measured). Uploads and the CPU read the same RAM, so sending more over the link slows the CPU;
+    // the wait-based controller does not see that and overshoots (4 GiB, asrok: auto 0.25 120.8 tok/s, fixed 0.20 129.7)
+    bool   share_rule = false;
+    double rule_k = 1.85, link_gbs = 0.0, expert_bytes = 0.0;
+    double rule_t_cpu = 0.0;
+    size_t rule_n_cpu = 0;
+    double rule_R_last = 0.0;
+    // auto mode A/B gate: the CPU path has a fixed cost per layer (reading the expert input back, a second split for cpu_out),
+    // so when few experts miss it is slower than uploading them all; every so often the gate runs a probe window with the CPU
+    // path off and keeps whichever was faster, by the time from the first to the last cached layer per ubatch of each size
+    bool    gate_on    = true;  // the decision: CPU path on or off
+    std::atomic<bool> path_on{true}; // the CPU path is on in the graphs being built (differs from gate_on in a probe window)
+    int     gate_phase = 0;     // 0 hold the decision, 1 probe window (the other state), 2 comparison window (the decision)
+    bool    ubatch_on  = false; // the current ubatch's graph has the CPU path
+    mutable std::atomic<bool> gate_changed{false};
+    int32_t il_first = -1, il_last = -1; // first and last cached layer
+    int64_t t_first = 0;                 // when the current ubatch reached the first cached layer
+    int64_t n_tok_cur = 0;
+    struct span_acc { double us[9] = {}; int n[9] = {}; };
+    span_acc acc_on, acc_off;
+    int     gate_count = 0;              // ubatches in the current phase
+    int     gate_hold  = 256;            // ubatches to stay before the next probe (doubles while the decision holds)
+    static constexpr int gate_probe = 48; // ubatches per probe window and per comparison window
+    size_t  n_gate_probes = 0, n_gate_off = 0, n_ubatch_on = 0, n_ubatch_off = 0;
+    // default: one less than the physical cores (hardware threads / 2, assuming SMT); LLAMA_MOE_CPU_THREADS overrides
+    int     n_threads = std::max(1, (int) std::thread::hardware_concurrency()/2 - 1);
+    int64_t n_embd = 0;
+    ggml_backend_ptr cpu_backend;
+    std::unordered_map<const ggml_tensor *, int32_t> layer_of_out; // cpu_out -> layer
+    std::vector<int32_t> pair_tok;  // CPU pairs of a layer: token, router weight index
+    std::vector<int32_t> pair_wi;
+
+    // the CPU's experts are computed by a worker thread, started before the uploads (which block the host when they read
+    // pageable memory) and joined by the copy of cpu_out; it spins between jobs to wake up quickly
+    // CPU graphs by number of pairs; all cached layers have experts of the same shape and type, so a graph is reused for any
+    // layer by pointing its mul_mat_id nodes at that layer's experts
+    struct cpu_graph {
+        ggml_context_ptr ctx;
+        ggml_backend_buffer_ptr buf;
+        ggml_cgraph * gf = nullptr;
+        ggml_tensor * x = nullptr, * sel = nullptr, * y = nullptr;
+        ggml_tensor * mm_gate_up = nullptr, * mm_up = nullptr, * mm_gate = nullptr, * mm_down = nullptr;
+        llama_moe_cache::act_type act = llama_moe_cache::ACT_NONE;
+        float limit = 0.0f;
+    };
+    std::unordered_map<int64_t, cpu_graph> cpu_graphs;
+
+    // CPU expert kernel (default; LLAMA_MOE_CPU_KERNEL=ggml uses the ggml graph above): the worker and n_threads - 1 helpers
+    // split each job by (expert, row block), with the weights of an expert read once for all of its tokens; the helpers spin
+    // between jobs and sleep after a long idle time
+    bool use_kernel = true;
+    std::vector<std::thread> helpers;
+    std::atomic<uint64_t> pool_epoch{0};
+    std::atomic<int>      pool_done{0};
+    std::atomic<int>      pool_next{0};
+    std::atomic<bool>     pool_exit{false};
+    std::function<void(int)> pool_fn;
+    // LLAMA_MOE_CPU_GUEST=1: the main thread helps with the CPU job while it would otherwise spin in the device sync for the
+    // next router (the GPU waits for this job anyway); pool_active marks a phase guests may join, guest_in counts guests in it
+    bool guest = false;
+    std::atomic<bool> pool_active{false};
+    std::atomic<int>  guest_in{0};
+    size_t n_guest_items = 0;
+    struct kexpert { int32_t e; int32_t p0, p1; };     // an expert and its pairs [p0, p1) in pair order
+    std::vector<kexpert> kexperts;
+    std::vector<int32_t> kpair_tok, kpair_wi;          // pairs sorted by expert
+    std::vector<uint8_t> kxq, kaq;                      // quantized expert input (per token) and activations (per pair)
+    std::vector<float>   kact;                          // activations [n_pairs, n_ff]
+    std::vector<float>   kdy;                           // LLAMA_MOE_CPU_DOWN_PE: down outputs [n_pairs, n_embd]
+    q2_dot_t             q2_dot = q2_dot_select();
+    q2_dot4_t            q2_dot4 = q2_dot4_select(); // LLAMA_MOE_CPU_DOT4=0: one row per call
+    std::vector<q2_act>  kxp, kap;                      // Q2_0 fast path: prepared expert input (per token), activations (per pair)
+    std::vector<int32_t> ktok;                          // tokens with CPU pairs
+    bool                 dot4_check = getenv("LLAMA_MOE_CPU_DOT4_CHECK") != nullptr;
+
+    std::thread worker;
+    std::atomic<int32_t> job_layer{-1};  // layer to compute, -1 idle, -2 exit
+
+    // doorbell mode (LLAMA_MOE_DOORBELL=1, pinned host memory, ubatches of up to 8 tokens): the slot lookup is a device op that
+    // writes the selected experts into a pinned control block and waits there; a host thread plans the layer (LRU, CPU job, uploads)
+    // and answers, and the device copies the uploads itself. No split, read back or launch per layer: one graph per ubatch
+    bool doorbell = false;
+    static constexpr int64_t db_max_tok = 8;
+    int64_t db_max_fill = 0;
+    std::vector<int32_t> db_layers;
+    std::thread db_thread;
+    std::atomic<bool> db_exit{false};
+    std::mutex mtx;                      // the LRU and the layer state: the doorbell thread and the copy callback (larger ubatches)
+    size_t n_db_calls = 0, n_db_timeouts = 0;
+
+    // lookahead prefetch (LLAMA_MOE_LOOKA=<B>, hybrid decode): at the slot map callback of layer il, a thread applies the router
+    // of layer il+1 to il's expert input (the host copy read for the CPU path), takes the experts most predicted over the
+    // ubatch's tokens (each token's top LLAMA_MOE_LOOKA_K, default 8) that are not cached, up to B, gives them slots and uploads
+    // them on a second backend of the device (its own stream) while il runs; every later callback makes the compute stream wait
+    // for those uploads first. Hash-routed layers (token id -> experts) are not predicted. Lossless: a wrong guess only costs
+    // the upload and an evicted slot
+    int     la_budget = 0, la_k = 8;
+    bool    la_pred_on = false;             // the graphs compute the next layer's predicted experts (lookahead or pre-read)
+    // LLAMA_MOE_PREREAD=1 (hybrid decode): after its job, the CPU pool reads the predicted experts of the next layer that are not
+    // cached (the ones the CPU will likely compute) into its caches, using RAM bandwidth that is idle between jobs; a new job
+    // stops it within one 64 KiB chunk. Lossless (only a read)
+    bool    preread = false;
+    std::vector<std::pair<const uint8_t *, size_t>> pr_chunks; // set by the callback before it posts the job
+    std::atomic<bool> pr_ready{false};
+    std::atomic<bool> pr_running{false}, pr_stop{false};
+    size_t  n_pr_jobs = 0, n_pr_aborted = 0, n_pr_experts = 0;
+    double  pr_bytes = 0.0, t_pr_us = 0.0;
+    volatile uint64_t pr_sink = 0;
+    ggml_backend_t la_backend = nullptr;
+    ggml_backend_event_t la_event = nullptr;
+    std::vector<ggml_tensor *> la_gi;       // [layer] router weights of the layer (on the device), nullptr: not predicted
+    std::vector<ggml_tensor *> la_bi;       // [layer] its selection bias, may be nullptr
+    std::vector<uint8_t> la_hash;           // [layer] hash-routed
+    uint32_t la_gating = 0;
+    std::thread la_thread;
+    std::atomic<int32_t> la_job{-1};        // target layer of the posted job, -1 none, -2 exit
+    std::atomic<bool>    la_busy{false};    // a job is posted or running
+    std::atomic<bool>    la_cancel{false};  // the next callback came before the job reached its slots: drop it
+    bool    la_pending = false;             // uploads issued on la_backend that the compute stream has not waited for
+    int32_t la_src = -1;                    // the layer whose callback posted the job (its slots are not evicted)
+    std::vector<int32_t> la_ids;            // the job's predicted experts [n_tok, la_k], computed in the graph
+    ggml_tensor * rb_la = nullptr;          // pinned read back of the predictions
+    std::vector<std::vector<int32_t>> la_mark; // [group][slot] key+1 of a prefetched expert not used yet, 0 none
+    // LLAMA_MOE_LOOKA_LAZY (default on): the compute stream waits for the prefetch uploads only when a layer reads or refills a
+    // slot that is still being written, so the uploads may overlap more than one layer
+    bool la_lazy = true;
+    std::vector<std::vector<uint8_t>> la_pend; // [group][slot] written by uploads the compute stream has not waited for
+    std::vector<std::pair<int32_t, int32_t>> la_pend_list;
+    size_t n_la_joins = 0;
+    size_t n_la_jobs = 0, n_la_issued = 0, n_la_useful = 0, n_la_skipped = 0, n_la_cancelled = 0;
+    double t_la_us = 0.0, t_la_wait_us = 0.0;
+
+    // prefill share (LLAMA_MOE_PREFILL_CPU=auto, or a fixed share 0..1; needs hybrid flag mode): ubatches of 33 to pf_max tokens
+    // take the large-batch path, which uploads every expert they use that is not in the cache. Here, as with Strata's
+    // STRATA_PREFILL_CPU_SHARE, the CPU computes a share of those experts, the ones fewest tokens use (at most pf_maxt), so they
+    // are not uploaded: on the device their pairs are pointed at an uploaded expert the token does not use, with router weight 0
+    // (MMQ needs distinct experts per token), and the GPU adds the CPU's output after the layer's experts (host_add).
+    // auto: share = g/(c + g), c the CPU's time per expert, g the GPU's time per uploaded expert, both measured per layer
+    bool    pf = false;
+    double  pf_fixed = -1.0;
+    int64_t pf_max = 1024;
+    int32_t pf_maxt = 8;
+    double  pf_c_us = 20.0, pf_g_us = 72.0;
+    bool    pf_measured = false;
+    ggml_tensor * pf_out = nullptr, * pf_x = nullptr, * pf_w = nullptr; // pinned: the CPU's output, read backs of x and w
+    int32_t pf_prev_il = -1;            // the previous layer of the current ubatch that had a prefill CPU job
+    int64_t pf_prev_t0 = 0, pf_prev_ncpu = 0, pf_prev_ngpu = 0;
+    size_t  n_pf_layers = 0, n_pf_cpu = 0, n_pf_stream = 0, n_pf_pairs = 0, n_pf_mismatch = 0;
+    double  t_pf_job_us = 0.0, t_pf_wait_us = 0.0, t_pf_plan_us = 0.0;
+    std::vector<int32_t> pf_cnt;
+    std::vector<uint8_t> pf_on_cpu;
+    // CPU job phases (us, summed): wake (job posted to worker start), prep (pairs, input quantization), A (gate/up), A2, B (down), post
+    double t_ph[6] = {};
+    double t_hot[2] = {}; size_t n_hot = 0;
+    // LLAMA_MOE_STATS_FILE=<path>: about once a second (at a cache callback) a JSON line of cumulative counters, for profiles
+    FILE * stats_f = nullptr;
+    int64_t stats_t = 0;
+    void stats_tick() {
+        const int64_t now = ggml_time_us();
+        if (stats_f == nullptr || now - stats_t < 1000000) {
+            return;
+        }
+        stats_t = now;
+#if defined(_MSC_VER)
+        struct timespec ts; timespec_get(&ts, TIME_UTC);
+#else
+        struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts);
+#endif
+        fprintf(stats_f, "{\"t\": %.3f, \"cpu_job_us\": %.0f, \"cpu_pairs\": %zu, \"cpu_experts\": %zu, \"up_bytes\": %zu, \"copy_bytes\": %zu, "
+            "\"hits\": %zu, \"misses\": %zu, \"flag_wait_us\": %.0f, \"la_issued\": %zu, \"pf_job_us\": %.0f, \"pf_cpu\": %zu, \"share\": %.3f, "
+            "\"calls\": %zu, \"sync_us\": %.0f, \"plan_us\": %.0f}\n",
+            ts.tv_sec + ts.tv_nsec*1e-9, t_cpu_us + t_pf_job_us, n_cpu_pairs + n_pf_pairs, n_cpu_experts + n_pf_cpu,
+            stats_small.bytes + stats_large.bytes, stats_copy.bytes, stats_small.hits, stats_small.misses, t_flag_wait_us + t_pf_wait_us,
+            n_la_issued, t_pf_job_us, n_pf_cpu, hybrid_share, n_hybrid_calls, t_sync_us, t_upload_us);
+        fflush(stats_f);
+    } // LLAMA_MOE_CPU_HOT_TEST: phases A..B cold, then hot
+    size_t n_ph = 0;
+    size_t n_ph_kexp = 0;   // (layer job, expert) groups of CPU pairs: n_cpu_pairs / n_ph_kexp = tokens per expert
+    size_t n_ph_mt   = 0;   // of those, the groups the multi-token i-quant kernels took (2+ tokens)
+    double t_db_us = 0.0, t_db_gpu_wait_us = 0.0;
+    std::atomic<bool>    job_done{true};
 
     std::vector<device> devices;
     std::vector<group> groups;
@@ -243,10 +751,11 @@ struct llama_moe_cache::impl {
     std::unordered_map<const ggml_tensor *, int32_t> layer_of; // slot map -> layer
 
     std::vector<int32_t> ids;
-    std::vector<moe_cache_slru::fill> fills;
+    std::vector<moe_cache_lru::fill> fills;
 
     // slot maps in host memory
     ggml_context_ptr ctx_host;
+    ggml_tensor * rb_ids = nullptr, * rb_x = nullptr, * rb_w = nullptr; // pinned staging of the read backs of the slot map callback
     ggml_backend_buffer_ptr buf_host;
     size_t buf_host_size = 0;
 
@@ -301,13 +810,92 @@ struct llama_moe_cache::impl {
             LLAMA_LOG_WARN("%s: no layer has all of its experts in host memory, MoE cache is disabled\n", __func__);
             return;
         }
+        for (const group & g : groups) {
+            for (int32_t il : g.layers) {
+                il_first = il_first < 0 ? il : std::min(il_first, il);
+                il_last  = std::max(il_last, il);
+            }
+        }
+
+        n_embd = model.hparams.n_embd;
+        if (const char * sf = getenv("LLAMA_MOE_STATS_FILE")) {
+            stats_f = fopen(sf, "w");
+        }
+        // the rule is the default in auto mode (DS4 16 GiB: 38.1 vs 33.4 tok/s for the wait-based controller; equal on Qwen);
+        // LLAMA_MOE_SHARE_RULE=0 goes back to the waits, =<k> sets the factor
+        share_rule = true;
+        if (const char * r = getenv("LLAMA_MOE_SHARE_RULE")) {
+            share_rule = atof(r) > 0.0;
+            rule_k = atof(r) == 1.0 || atof(r) <= 0.0 ? 1.85 : atof(r);
+        }
+        if (const char * s = getenv("LLAMA_MOE_CPU")) {
+            hybrid       = true;
+            hybrid_auto  = strcmp(s, "auto") == 0;
+            hybrid_share = hybrid_auto ? 0.25 : std::clamp(atof(s), 0.0, 1.0);
+            if (const char * t = getenv("LLAMA_MOE_TAIL_SKIP")) {
+                tail_rank = std::max(0, atoi(t));
+                if (tail_rank > 0) {
+                    LLAMA_LOG_WARN("%s: LLAMA_MOE_TAIL_SKIP=%d: missed experts that every token routes at rank %d or lower are skipped "
+                        "(approximate, not the model's output)\n", __func__, tail_rank, tail_rank);
+                }
+            }
+            if (const char * t = getenv("LLAMA_MOE_CPU_THREADS")) {
+                n_threads = std::max(1, atoi(t));
+            }
+            n_embd = model.hparams.n_embd;
+            cpu_backend.reset(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
+            if (!cpu_backend) {
+                throw std::runtime_error("MoE cache hybrid mode: failed to create a CPU backend");
+            }
+            ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(cpu_backend.get()));
+            auto set_n_threads = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
+            if (set_n_threads) {
+                set_n_threads(cpu_backend.get(), n_threads);
+            }
+            LLAMA_LOG_INFO("%s: hybrid mode: %s%.0f%% of the misses are uploaded, the CPU computes the others with %d threads\n", __func__,
+                hybrid_auto ? "auto, starting at " : "", 100.0*hybrid_share, n_threads);
+            if (const char * k = getenv("LLAMA_MOE_CPU_KERNEL")) {
+                use_kernel = strcmp(k, "ggml") != 0;
+            }
+            guest = use_kernel && getenv("LLAMA_MOE_CPU_GUEST") && atoi(getenv("LLAMA_MOE_CPU_GUEST")) != 0;
+            if (const char * p = getenv("LLAMA_MOE_PREFILL_CPU"); p && use_kernel && strcmp(p, "0") != 0) {
+                pf       = true;
+                pf_fixed = strcmp(p, "auto") == 0 || strcmp(p, "1") == 0 ? -1.0 : std::clamp(atof(p), 0.0, 1.0);
+                if (const char * m = getenv("LLAMA_MOE_PREFILL_MAX")) {
+                    pf_max = std::max(33, atoi(m));
+                }
+                if (const char * m = getenv("LLAMA_MOE_PREFILL_MAXT")) {
+                    pf_maxt = std::max(1, atoi(m));
+                }
+            }
+            pin_cpus = pin_list();
+            if (!pin_cpus.empty()) {
+                std::string l;
+                for (int i = 0; i < std::min<int>(n_threads, (int) pin_cpus.size()); ++i) l += (i ? "," : "") + std::to_string(pin_cpus[i]);
+                LLAMA_LOG_INFO("%s: hybrid mode: CPU threads pinned to %s\n", __func__, l.c_str());
+            }
+            if (use_kernel) {
+                for (int i = 1; i < n_threads; ++i) {
+                    helpers.emplace_back([this, i] { helper_loop(i); });
+                }
+            }
+            bool any_q2 = false;
+            for (const group & g : groups) {
+                for (const ggml_tensor * t : g.ref) any_q2 |= t->type == GGML_TYPE_Q2_0;
+            }
+            LLAMA_LOG_INFO("%s: hybrid mode: CPU experts by %s%s\n", __func__, use_kernel ? "the MoE cache's kernel" : "ggml graphs",
+                use_kernel && q2_dot && any_q2 ? q2_dot_name() : "");
+            worker = std::thread([this] { worker_loop(); });
+        }
 
         // one extra slot at the end, CUDA MMQ can read past the last expert
+        // hybrid mode: one more, zero filled, which the experts computed by the CPU read on the device
+        const int32_t n_extra = hybrid ? 2 : 1;
         auto alloc_size = [&](const group & g, int32_t n_slots) {
             const size_t alignment = ggml_backend_buft_get_alignment(devices[g.id].buft);
             size_t res = 0;
             for (const ggml_tensor * t : g.ref) {
-                res += GGML_PAD(t->nb[2]*(n_slots + 1), alignment);
+                res += GGML_PAD(t->nb[2]*(n_slots + n_extra), alignment);
             }
             return res;
         };
@@ -358,9 +946,9 @@ struct llama_moe_cache::impl {
                 g.n_slots = 0;
                 continue;
             }
-            g.slru.init(model.layers.size(), n_expert, g.n_slots);
+            g.lru.init(model.layers.size(), n_expert, g.n_slots);
             n_tensors[g.id] += g.ref.size()*(1 + g.layers.size());
-            n_tensors_host  += g.layers.size();
+            n_tensors_host  += g.layers.size()*(hybrid ? 6 : 2);
         }
         if (n_tensors_host == 0) {
             throw std::runtime_error("MoE cache is too small to hold the experts of one token");
@@ -383,12 +971,49 @@ struct llama_moe_cache::impl {
                 devices[id].ctx = init_ctx(n_tensors[id]);
             }
         }
-        ctx_host  = init_ctx(n_tensors_host);
+        ctx_host  = init_ctx(n_tensors_host + 6);
+        rb_ids = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_I32, max_batch*64);
+        rb_x   = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_F32, max_batch*model.hparams.n_embd);
+        rb_w   = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_F32, max_batch*64);
+        if ((getenv("LLAMA_MOE_LOOKA") && atoi(getenv("LLAMA_MOE_LOOKA")) > 0) || (getenv("LLAMA_MOE_PREREAD") && atoi(getenv("LLAMA_MOE_PREREAD")) > 0)) {
+            rb_la = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_I32, max_batch*64); // la_k <= 64
+        }
+        if (pf) {
+            pf_out = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_F32, pf_max*n_embd);
+            pf_x   = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_F32, pf_max*n_embd);
+            pf_w   = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_F32, pf_max*n_expert_used);
+        }
         ctx_views = init_ctx(2);
 
-        ggml_backend_buffer_type_t buft_host = ggml_backend_cpu_buffer_type();
+        // pinned if the device has a host buffer type: an upload from pageable memory waits for the stream (the expert
+        // uploads before it), then the host cannot launch the graph of the split while the experts are uploaded
+        ggml_backend_buffer_type_t buft_host = nullptr;
+        for (const device & d : devices) {
+            if (d.host_bytes > 0 && buft_host == nullptr && getenv("LLAMA_MOE_PAGEABLE") == nullptr) {
+                buft_host = ggml_backend_dev_host_buffer_type(ggml_backend_get_device(d.backend));
+            }
+        }
+        if (buft_host == nullptr) {
+            buft_host = ggml_backend_cpu_buffer_type();
+        }
         const size_t alignment_host = ggml_backend_buft_get_alignment(buft_host);
 
+        doorbell = getenv("LLAMA_MOE_DOORBELL") && atoi(getenv("LLAMA_MOE_DOORBELL")) != 0 && buft_host != ggml_backend_cpu_buffer_type() &&
+            !no_alloc;
+        size_t db_bytes = 0;
+        if (doorbell) {
+            size_t n_banks = 0;
+            for (const group & g : groups) {
+                n_banks = std::max(n_banks, g.ref.size());
+            }
+            db_max_fill = db_max_tok*n_expert_used*(int64_t) n_banks;
+            const int64_t n_expert_max = model.hparams.n_expert;
+            GGML_UNUSED(n_expert_max);
+            db_bytes = GGML_PAD(sizeof(ggml_moe_db), 64) + GGML_PAD(db_max_tok*n_expert_used*sizeof(int32_t), 64) +
+                GGML_PAD(db_max_tok*n_embd*sizeof(float), 64) + GGML_PAD(db_max_tok*n_expert_used*sizeof(float), 64) +
+                GGML_PAD(db_max_tok*n_expert_used*sizeof(int32_t), 64) + 8 + GGML_PAD(db_max_fill*sizeof(ggml_moe_db_fill), 64);
+        }
+
         for (size_t ig = 0; ig < groups.size(); ++ig) {
             group & g = groups[ig];
             if (g.n_slots == 0) {
@@ -396,7 +1021,7 @@ struct llama_moe_cache::impl {
             }
             ggml_context * ctx = devices[g.id].ctx.get();
             for (const ggml_tensor * t : g.ref) {
-                ggml_tensor * bank = ggml_new_tensor_3d(ctx, t->type, t->ne[0], t->ne[1], g.n_slots + 1);
+                ggml_tensor * bank = ggml_new_tensor_3d(ctx, t->type, t->ne[0], t->ne[1], g.n_slots + n_extra);
                 GGML_ASSERT(bank->nb[2] == t->nb[2]);
                 ggml_format_name(bank, "moe_cache.%zu.%s", ig, t->name);
                 g.banks.push_back(bank);
@@ -407,7 +1032,8 @@ struct llama_moe_cache::impl {
                 l.experts = llama_moe_cache_layer_experts(model.layers[il]);
                 for (size_t ip = 0; ip < l.experts.size(); ++ip) {
                     ggml_tensor * bank   = g.banks[ip];
-                    ggml_tensor * cached = ggml_view_3d(ctx, bank, bank->ne[0], bank->ne[1], g.n_slots, bank->nb[1], bank->nb[2], 0);
+                    // hybrid mode: the zero slot (index n_slots) is part of the view
+                    ggml_tensor * cached = ggml_view_3d(ctx, bank, bank->ne[0], bank->ne[1], g.n_slots + (hybrid ? 1 : 0), bank->nb[1], bank->nb[2], 0);
                     ggml_format_name(cached, "moe_cache.%s", l.experts[ip]->name);
                     bindings[l.experts[ip]] = { il, (int32_t) ip, cached };
                 }
@@ -415,30 +1041,168 @@ struct llama_moe_cache::impl {
                 ggml_format_name(l.slot_map, "moe_cache.slot_map-%d", il);
                 layer_of[l.slot_map] = il;
                 buf_host_size += GGML_PAD(ggml_nbytes(l.slot_map), alignment_host);
+                if (doorbell) {
+                    l.db = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_I32, db_bytes/sizeof(int32_t));
+                    ggml_format_name(l.db, "moe_cache.db-%d", il);
+                    db_layers.push_back(il);
+                }
+                if (hybrid) {
+                    const llama_layer & ml = model.layers[il];
+                    l.up      = ml.ffn_up_exps;
+                    l.gate    = ml.ffn_gate_exps;
+                    l.down    = ml.ffn_down_exps;
+                    l.gate_up = ml.ffn_gate_up_exps;
+                    l.b_up      = ml.ffn_up_exps_b;
+                    l.b_gate    = ml.ffn_gate_exps_b;
+                    l.b_down    = ml.ffn_down_exps_b;
+                    l.b_gate_up = ml.ffn_gate_up_exps_b;
+                    for (const ggml_tensor * b : { l.b_up, l.b_gate, l.b_down, l.b_gate_up }) {
+                        l.bias_ok = l.bias_ok && (b == nullptr || (b->type == GGML_TYPE_F32 && ggml_is_contiguous(b) &&
+                            b->buffer && ggml_backend_buffer_is_host(b->buffer)));
+                    }
+                    if (!l.bias_ok) {
+                        LLAMA_LOG_WARN("%s: hybrid mode: layer %d's expert biases are not F32 in host memory; its missed experts are all uploaded\n",
+                            __func__, il);
+                    }
+                    l.cpu_out = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_F32, n_embd, max_batch);
+                    ggml_format_name(l.cpu_out, "moe_cache.cpu_out-%d", il);
+                    layer_of_out[l.cpu_out] = il;
+                    l.map_dev.resize(g.ref[0]->ne[2]);
+                    l.map_pin = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_I32, g.ref[0]->ne[2]);
+                    ggml_format_name(l.map_pin, "moe_cache.map_pin-%d", il);
+                    l.flag = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_I32, 4);
+                    ggml_format_name(l.flag, "moe_cache.flag-%d", il);
+                    l.pf_flag = ggml_new_tensor_1d(ctx_host.get(), GGML_TYPE_I32, 4);
+                    ggml_format_name(l.pf_flag, "moe_cache.pf_flag-%d", il);
+                }
             }
             devices[g.id].buf_size += alloc_size(g, g.n_slots);
         }
 
-        // with no_alloc, dummy buffers are attached so the memory use can still be measured, see llama_context::memory_breakdown
-        for (device & d : devices) {
-            if (!d.ctx) {
-                continue;
+        if (no_alloc) {
+            // only used to measure the memory use, see llama_context::memory_breakdown
+            for (device & d : devices) {
+                if (!d.ctx) {
+                    continue;
+                }
+                d.buf.reset(ggml_backend_buft_alloc_buffer(d.buft, 0));
+                for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
+                    t->buffer = d.buf.get();
+                }
             }
-            d.buf = llama_memory_bufs::alloc_buf(d.ctx.get(), d.buft, no_alloc, "MoE cache");
-            if (!no_alloc) {
+            buf_host.reset(ggml_backend_buft_alloc_buffer(buft_host, 0));
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx_host.get()); t != nullptr; t = ggml_get_next_tensor(ctx_host.get(), t)) {
+                t->buffer = buf_host.get();
+            }
+        } else {
+            for (device & d : devices) {
+                if (!d.ctx) {
+                    continue;
+                }
+                d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
+                if (!d.buf) {
+                    throw std::runtime_error("failed to allocate the MoE cache buffers");
+                }
                 ggml_backend_buffer_clear(d.buf.get(), 0);
                 d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
             }
-        }
-        buf_host = llama_memory_bufs::alloc_buf(ctx_host.get(), buft_host, no_alloc, "MoE cache slot maps");
-        if (!no_alloc) {
+            buf_host.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_host.get(), buft_host));
+            if (!buf_host) {
+                throw std::runtime_error("failed to allocate the MoE cache buffers");
+            }
             ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1
             buf_host_size = ggml_backend_buffer_get_size(buf_host.get());
+            LLAMA_LOG_INFO("%s: slot maps in %s\n", __func__, ggml_backend_buft_name(buft_host));
+
+            // flag mode: the device reads cpu_out and the flags from pinned memory
+            flag_mode = hybrid && buft_host != ggml_backend_cpu_buffer_type() && getenv("LLAMA_MOE_CPU_COPY") == nullptr;
+            for (layer & l : layers) {
+                if (l.flag != nullptr) {
+                    memset(l.flag->data, 0, ggml_nbytes(l.flag));
+                    memset(l.pf_flag->data, 0, ggml_nbytes(l.pf_flag));
+                }
+            }
+            pf = pf && flag_mode;
+            if (pf) {
+                LLAMA_LOG_INFO("%s: hybrid prefill share: ubatches of 33 to %d tokens, the CPU computes %s of the missed experts used by at most %d tokens\n",
+                    __func__, (int) pf_max, pf_fixed < 0.0 ? "a measured share" : (std::to_string(pf_fixed).substr(0, 4) + " of the uploads").c_str(), pf_maxt);
+            }
+            doorbell = doorbell && (!hybrid || flag_mode);
+            if (doorbell) {
+                for (int32_t il : db_layers) {
+                    layer & l = layers[il];
+                    memset(l.db->data, 0, ggml_nbytes(l.db));
+                    ggml_moe_db * h = (ggml_moe_db *) l.db->data;
+                    size_t off = GGML_PAD(sizeof(ggml_moe_db), 64);
+                    h->ids_off  = off; off += GGML_PAD(db_max_tok*n_expert_used*sizeof(int32_t), 64);
+                    h->x_off    = off; off += GGML_PAD(db_max_tok*n_embd*sizeof(float), 64);
+                    h->w_off    = off; off += GGML_PAD(db_max_tok*n_expert_used*sizeof(float), 64);
+                    h->ans_off   = off;
+                    h->slots_cap = GGML_PAD(db_max_tok*n_expert_used, 2);
+                    h->max_fill  = db_max_fill;
+                    GGML_ASSERT(off + h->slots_cap*sizeof(int32_t) + 8 + db_max_fill*sizeof(ggml_moe_db_fill) <= ggml_nbytes(l.db));
+                }
+                db_thread = std::thread([this] { db_loop(); });
+                LLAMA_LOG_INFO("%s: doorbell mode: ubatches of up to %d tokens look up their experts without a split per layer\n", __func__, (int) db_max_tok);
+            }
+            if (hybrid) {
+                LLAMA_LOG_INFO("%s: hybrid: the CPU's output is %s\n", __func__, flag_mode ? "read by the GPU after a flag (flag mode)" : "copied as a host weight");
+            }
 
+                const char * a = getenv("LLAMA_MOE_LOOKA");
+                preread = getenv("LLAMA_MOE_PREREAD") && atoi(getenv("LLAMA_MOE_PREREAD")) > 0 && use_kernel;
+                if (((a && atoi(a) > 0) || preread) && hybrid && devices.size() == 1 && !doorbell && rb_la) {
+                la_budget = a ? std::max(0, atoi(a)) : 0;
+                la_k = la_budget > 0 ? 8 : 12;
+                if (const char * k = getenv("LLAMA_MOE_LOOKA_K")) {
+                    la_k = std::clamp(atoi(k), 1, 64);
+                }
+                la_pred_on = true;
+                la_gating = model.hparams.expert_gating_func;
+                la_gi.assign(layers.size(), nullptr);
+                la_bi.assign(layers.size(), nullptr);
+                size_t n_pred = 0;
+                for (size_t il = 0; il < layers.size(); ++il) {
+                    const llama_layer & ml = model.layers[il];
+                    if (layers[il].ig < 0 || ml.ffn_gate_inp == nullptr || ml.ffn_gate_tid2eid != nullptr || ml.ffn_gate_inp->ne[0] != n_embd) {
+                        continue;
+                    }
+                    la_gi[il] = ml.ffn_gate_inp;
+                    la_bi[il] = ml.ffn_exp_probs_b;
+                    n_pred++;
+                }
+                if (preread) {
+                    LLAMA_LOG_INFO("%s: pre-read: the CPU pool reads each token's top %d predicted, uncached experts of the next layer between jobs (%zu layers)\n",
+                        __func__, la_k, n_pred);
+                }
+                if (la_budget > 0) {
+                la_mark.resize(groups.size());
+                la_pend.resize(groups.size());
+                for (size_t ig = 0; ig < groups.size(); ++ig) {
+                    la_mark[ig].assign(groups[ig].n_slots, 0);
+                    la_pend[ig].assign(groups[ig].n_slots, 0);
+                }
+                la_lazy = !getenv("LLAMA_MOE_LOOKA_LAZY") || atoi(getenv("LLAMA_MOE_LOOKA_LAZY")) != 0;
+                ggml_backend_dev_t dev = ggml_backend_get_device(devices[0].backend);
+                la_backend = ggml_backend_dev_init(dev, nullptr);
+                la_event   = la_backend ? ggml_backend_event_new(dev) : nullptr;
+                if (!la_backend || !la_event) {
+                    LLAMA_LOG_WARN("%s: lookahead: no second backend or event for the device, off\n", __func__);
+                    if (la_backend) { ggml_backend_free(la_backend); la_backend = nullptr; }
+                } else {
+                    la_thread = std::thread([this] { la_loop(); });
+                    LLAMA_LOG_INFO("%s: lookahead prefetch: up to %d experts per layer from each token's top %d, %zu layers predicted\n",
+                        __func__, la_budget, la_k, n_pred);
+                }
+                }
+            }
+        if (hybrid && share_rule) {
+                probe_link();
+            }
             for (group & g : groups) {
                 for (int32_t il : g.layers) {
                     if (layers[il].slot_map != nullptr) {
-                        g.slru.slot_map[il] = (int32_t *) layers[il].slot_map->data;
+                        g.lru.slot_map[il] = (int32_t *) layers[il].slot_map->data;
                     }
                 }
             }
@@ -473,74 +1237,929 @@ struct llama_moe_cache::impl {
     }
 
     ~impl() {
+        if (stats_f) {
+            stats_t = 0;
+            stats_tick();
+            fclose(stats_f);
+        }
+        if (la_thread.joinable()) {
+            la_wait_idle();
+            la_job.store(-2);
+            la_thread.join();
+        }
+        if (la_backend) {
+            ggml_backend_synchronize(la_backend);
+        }
+        if (la_event) {
+            ggml_backend_event_free(la_event);
+        }
+        if (la_backend) {
+            ggml_backend_free(la_backend);
+        }
+        if (db_thread.joinable()) {
+            db_exit.store(true);
+            db_thread.join();
+        }
+        if (worker.joinable()) {
+            wait_cpu();
+            job_layer.store(-2);
+            worker.join();
+        }
+        pool_exit.store(true);
+        for (auto & h : helpers) {
+            h.join();
+        }
         log_stats();
     }
 
-    ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used) const {
-        if (il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0) {
-            return nullptr;
+    // LLAMA_MOE_CPU_PIN=1: the worker and the helpers are pinned one per physical core, interleaved over the L3 domains (CCDs),
+    // so that a chiplet CPU's per-CCD memory links are used evenly; =<list> pins to the given CPUs in order (e.g. 0,8,1,9)
+    std::vector<int> pin_cpus;
+    static std::vector<int> pin_list() {
+        std::vector<int> res;
+#ifdef __linux__
+        const char * e = getenv("LLAMA_MOE_CPU_PIN");
+        if (e == nullptr || strcmp(e, "0") == 0) {
+            return res;
         }
-        const layer & l = layers[il];
+        if (strchr(e, ',') != nullptr) {
+            for (const char * p = e; *p; ) {
+                res.push_back(atoi(p));
+                p = strchr(p, ',');
+                if (p == nullptr) break;
+                p++;
+            }
+            return res;
+        }
+        cpu_set_t set;
+        CPU_ZERO(&set);
+        if (sched_getaffinity(0, sizeof(set), &set) != 0) {
+            return res;
+        }
+        auto read = [](const std::string & path) { std::ifstream f(path); std::string v; std::getline(f, v); return v; };
+        auto first = [](const std::string & list) { return atoi(list.c_str()); };
+        std::map<std::string, std::vector<int>> groups; // L3 domain -> first sibling of each core
+        std::set<int> seen;
+        for (int c = 0; c < CPU_SETSIZE; ++c) {
+            if (!CPU_ISSET(c, &set)) continue;
+            const std::string base = "/sys/devices/system/cpu/cpu" + std::to_string(c);
+            const int core = first(read(base + "/topology/thread_siblings_list"));
+            if (seen.count(core)) continue;
+            seen.insert(core);
+            groups[read(base + "/cache/index3/shared_cpu_list")].push_back(core);
+        }
+        for (size_t i = 0; ; ++i) {
+            bool any = false;
+            for (auto & g : groups) {
+                if (i < g.second.size()) { res.push_back(g.second[i]); any = true; }
+            }
+            if (!any) break;
+        }
+#endif
+        return res;
+    }
+    void pin_self(int ith) {
+#ifdef __linux__
+        if (ith < (int) pin_cpus.size()) {
+            cpu_set_t set;
+            CPU_ZERO(&set);
+            CPU_SET(pin_cpus[ith], &set);
+            pthread_setaffinity_np(pthread_self(), sizeof(set), &set);
+        }
+#endif
+    }
 
-        // large batches use most experts of a layer, so they gain little from the cache and would evict the experts used in generation
-        if (n_tokens == 0 || n_tokens > max_batch || std::min(n_tokens*n_expert_used, l.slot_map->ne[1]) > groups[l.ig].n_slots) {
-            return nullptr;
+    // a job still scoring is cancelled (a late prefetch would only delay the layer it was for); one already
+    // giving out slots is waited for, that part is short
+    void la_wait_idle() {
+        if (!la_busy.load(std::memory_order_acquire)) {
+            return;
         }
-        return l.slot_map;
+        const int64_t t0 = ggml_time_us();
+        la_cancel.store(true, std::memory_order_release);
+        while (la_busy.load(std::memory_order_acquire)) {
+        }
+        la_cancel.store(false, std::memory_order_relaxed);
+        t_la_wait_us += ggml_time_us() - t0;
     }
 
-    ggml_tensor * get_experts(const ggml_tensor * w) const {
-        const auto it = bindings.find(w);
-        return it != bindings.end() ? it->second.cached : nullptr;
+    static float la_dot(const float * a, const float * b, int64_t n) {
+        static const auto fn = la_dot_select();
+        return fn(a, b, n);
     }
 
-    int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last) {
-        const auto it = bindings.find(w);
-        if (it == bindings.end()) {
-            return 0;
-        }
-        const binding & b = it->second;
-        const group & g = groups[layers[b.il].ig];
-        if (backend != devices[g.id].backend) {
-            return 0;
+    // before the compute stream touches the banks again: it waits for the prefetch uploads (called with mtx held, after la_wait_idle)
+    void la_join(ggml_backend_t backend) {
+        if (la_pending) {
+            ggml_backend_event_wait(backend, la_event);
+            la_pending = false;
+            for (const auto & p : la_pend_list) {
+                la_pend[p.first][p.second] = 0;
+            }
+            la_pend_list.clear();
+            n_la_joins++;
         }
+    }
 
-        // large batches only read the cache, so the experts used in generation stay in it
-        const int32_t * slots = g.slru.slot_map[b.il];
-        if (slots == nullptr || slots[e] < 0) {
-            return 0;
+    // lazy join: wait only if one of the slots is still being written
+    template <typename F>
+    void la_join_if(ggml_backend_t backend, int32_t ig, F slots_of) {
+        if (!la_pending) {
+            return;
         }
-        int64_t n = 1;
-        while (e + n <= last && slots[e + n] == slots[e] + n) {
-            n++;
+        if (!la_lazy) {
+            la_join(backend);
+            return;
         }
+        bool hit = false;
+        slots_of([&](int32_t s) { hit = hit || (s >= 0 && la_pend[ig][s]); });
+        if (hit) {
+            la_join(backend);
+        }
+    }
 
-        ggml_tensor * bank = g.banks[b.ip];
-        ggml_reset(ctx_views.get());
-        ggml_tensor * src_view = ggml_view_3d(ctx_views.get(), bank, bank->ne[0], bank->ne[1], n, bank->nb[1], bank->nb[2], slots[e]*bank->nb[2]);
-        ggml_tensor * dst_view = ggml_view_3d(ctx_views.get(), dst,  dst->ne[0],  dst->ne[1],  n, dst->nb[1],  dst->nb[2],  e*dst->nb[2]);
-        ggml_backend_view_init(src_view);
-        ggml_backend_view_init(dst_view);
-        ggml_backend_tensor_copy_async(backend, backend, src_view, dst_view);
-
-        stats_copy.hits  += n;
-        stats_copy.bytes += ggml_nbytes(src_view);
+    void la_loop() {
+        for (;;) {
+            int32_t tl;
+            idle_wait([&] { return (tl = la_job.load(std::memory_order_acquire)) != -1; });
+            if (tl == -2) {
+                return;
+            }
+            const int64_t t0 = ggml_time_us();
+            const layer & lt = layers[tl];
+            const int64_t n_expert = groups[lt.ig].lru.n_expert;
+            const int64_t n_tok = (int64_t) la_ids.size()/la_k;
+            // per expert: tokens that have it in their predicted top la_k, best rank
+            std::vector<int32_t> cnt(n_expert, 0), best(n_expert, 1 << 30);
+            for (int64_t t = 0; t < n_tok; ++t) {
+                for (int64_t r = 0; r < la_k; ++r) {
+                    const int32_t e = la_ids[t*la_k + r];
+                    if (e < 0 || e >= n_expert) continue;
+                    cnt[e]++;
+                    best[e] = std::min<int32_t>(best[e], (int32_t) r);
+                }
+            }
+            std::vector<int32_t> cand;
+            for (int64_t e = 0; e < n_expert; ++e) {
+                if (cnt[e] > 0) cand.push_back((int32_t) e);
+            }
+            std::sort(cand.begin(), cand.end(), [&](int32_t a, int32_t b) { return cnt[a] != cnt[b] ? cnt[a] > cnt[b] : best[a] < best[b]; });
 
-        return n;
+            std::vector<moe_cache_lru::fill> fills;
+            if (la_cancel.load(std::memory_order_acquire)) {
+                n_la_cancelled++;
+                cand.clear();
+            }
+            {
+                std::lock_guard<std::mutex> lock(mtx);
+                group & g = groups[lt.ig];
+                moe_cache_lru & lru = g.lru;
+                int32_t * slots = lru.slot_map[tl];
+                for (int32_t e : cand) {
+                    if ((int) fills.size() >= la_budget) break;
+                    if (slots[e] >= 0) continue;
+                    const int32_t s = lru.head;
+                    // the slots of the layer that is running are not evicted
+                    if (lru.key_of[s] >= 0 && lru.key_of[s] / lru.n_expert == la_src) {
+                        n_la_skipped++;
+                        break;
+                    }
+                    if (lru.key_of[s] >= 0) {
+                        lru.slot_map[lru.key_of[s] / lru.n_expert][lru.key_of[s] % lru.n_expert] = -1;
+                    }
+                    lru.key_of[s] = tl*lru.n_expert + e;
+                    slots[e] = s;
+                    lru.touch(s);
+                    la_mark[lt.ig][s] = tl*lru.n_expert + e + 1;
+                    la_pend[lt.ig][s] = 1;
+                    la_pend_list.push_back({ lt.ig, s });
+                    fills.push_back({ e, s });
+                }
+                for (const auto & f : fills) {
+                    for (size_t ip = 0; ip < lt.experts.size(); ++ip) {
+                        const ggml_tensor * w = lt.experts[ip];
+                        const size_t esz = w->nb[2];
+                        ggml_backend_tensor_set_async(la_backend, g.banks[ip], (const uint8_t *) w->data + f.expert*esz, f.slot*esz, esz);
+                    }
+                }
+                if (!fills.empty()) {
+                    ggml_backend_event_record(la_event, la_backend);
+                    la_pending = true;
+                }
+                n_la_issued += fills.size();
+                n_la_jobs++;
+                stats_small.bytes += 0; // uploads are counted in n_la_issued
+            }
+            t_la_us += ggml_time_us() - t0;
+            la_job.store(-1, std::memory_order_relaxed);
+            la_busy.store(false, std::memory_order_release);
+        }
     }
 
-    bool copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph) {
-        const auto it = layer_of.find(src);
-        if (it == layer_of.end()) {
-            return false;
+    void worker_loop() {
+        pin_self(0);
+        for (;;) {
+            int32_t il;
+            idle_wait([&] { return (il = job_layer.load(std::memory_order_acquire)) != -1; });
+            if (il == -2) {
+                return;
+            }
+            compute_cpu(layers[il]);
+            if (layers[il].pf_job) {
+                post_pf_out(layers[il]);
+            } else if (flag_mode) {
+                const int64_t tq = ggml_time_us();
+                post_cpu_out(layers[il]);
+                t_ph[5] += ggml_time_us() - tq;
+            }
+            const bool pr = preread && pr_ready.exchange(false, std::memory_order_relaxed) && !layers[il].pf_job;
+            if (pr) {
+                pr_running.store(true, std::memory_order_release); // before job_done: the callback that sees it done waits for this
+            }
+            job_layer.store(-1, std::memory_order_relaxed);
+            job_done.store(true, std::memory_order_release);
+            if (pr) {
+                run_preread();
+                pr_running.store(false, std::memory_order_release);
+            }
         }
-        const int32_t il = it->second;
-        const layer & l = layers[il];
-        group & g = groups[l.ig];
+    }
 
-        GGML_ASSERT(backend == devices[g.id].backend);
+    // reads the pre-read chunks with the pool until done or a new job is posted
+    void run_preread() {
+        const int64_t t0 = ggml_time_us();
+        const int64_t n = (int64_t) pr_chunks.size();
+        std::atomic<bool> aborted{false};
+        pool_run([&](int) {
+            uint64_t acc = 0;
+            for (int64_t i; (i = pool_next.fetch_add(1, std::memory_order_relaxed)) < n; ) {
+                if (job_layer.load(std::memory_order_relaxed) != -1 || pr_stop.load(std::memory_order_relaxed)) {
+                    aborted.store(true, std::memory_order_relaxed);
+                    break;
+                }
+                const uint64_t * p = (const uint64_t *) pr_chunks[i].first;
+                const size_t m = pr_chunks[i].second / 8;
+                for (size_t j = 0; j < m; j += 8) {
+                    acc += p[j];
+                }
+            }
+            pr_sink = pr_sink + acc;
+        });
+        n_pr_jobs++;
+        n_pr_aborted += aborted.load();
+        pr_bytes += (double) std::min<int64_t>(pool_next.load(), n) * 65536.0;
+        t_pr_us += ggml_time_us() - t0;
+    }
 
-        // the get_rows that looks up the slots of the selected experts
+    // spin until ready() (a new job), yielding after a while, sleeping briefly after ~1 s without a job (between requests)
+    template <typename F>
+    static void idle_wait(F ready) {
+        int64_t spins = 0;
+        auto t0 = std::chrono::steady_clock::now();
+        while (!ready()) {
+            if (++spins < 4096) {
+                continue;
+            }
+            if ((spins & 1023) == 0 && std::chrono::steady_clock::now() - t0 > std::chrono::seconds(1)) {
+                std::this_thread::sleep_for(std::chrono::microseconds(100));
+                spins = 4096;
+            } else {
+                std::this_thread::yield();
+            }
+        }
+    }
+
+    void helper_loop(int ith) {
+        pin_self(ith);
+        uint64_t seen = 0;
+        for (;;) {
+            uint64_t e = 0;
+            idle_wait([&] { return pool_exit.load(std::memory_order_relaxed) || (e = pool_epoch.load(std::memory_order_acquire)) != seen; });
+            if (pool_exit.load()) {
+                return;
+            }
+            seen = e;
+            pool_fn(ith);
+            pool_done.fetch_add(1, std::memory_order_acq_rel);
+        }
+    }
+
+    // runs fn(ith) on the worker (ith 0) and every helper, returns when all are done
+    void pool_run(std::function<void(int)> fn) {
+        pool_fn = std::move(fn);
+        pool_next.store(0, std::memory_order_relaxed);
+        pool_done.store(0, std::memory_order_relaxed);
+        if (guest) {
+            pool_active.store(true, std::memory_order_seq_cst);
+        }
+        pool_epoch.fetch_add(1, std::memory_order_acq_rel);
+        pool_fn(0);
+        const int n = (int) helpers.size();
+        while (pool_done.load(std::memory_order_acquire) < n) {
+        }
+        if (guest) {
+            // a guest that saw the phase active is still inside pool_fn: wait for it before the next phase reuses pool_fn
+            pool_active.store(false, std::memory_order_seq_cst);
+            while (guest_in.load(std::memory_order_seq_cst) != 0) {
+            }
+        }
+    }
+
+    // the calling thread works on the running CPU job's phases until the job is done (LLAMA_MOE_CPU_GUEST)
+    void pool_help() {
+        while (!job_done.load(std::memory_order_acquire)) {
+            if (!pool_active.load(std::memory_order_seq_cst)) {
+                continue;
+            }
+            guest_in.fetch_add(1, std::memory_order_seq_cst);
+            if (pool_active.load(std::memory_order_seq_cst)) {
+                pool_fn((int) helpers.size() + 1);
+                n_guest_items++;
+            }
+            guest_in.fetch_sub(1, std::memory_order_seq_cst);
+        }
+    }
+
+    void wait_cpu() {
+        while (!job_done.load(std::memory_order_acquire)) {
+        }
+    }
+
+    // the link speed for the share rule: uploads of 8 experts of the first cached layer into its first slots (empty at start)
+    void probe_link() {
+        for (const group & g : groups) {
+            if (g.n_slots < 8 || g.layers.empty()) {
+                continue;
+            }
+            const layer & l = layers[g.layers[0]];
+            ggml_backend_t be = devices[g.id].backend;
+            expert_bytes = 0.0;
+            for (const ggml_tensor * t : l.experts) {
+                expert_bytes += t->nb[2];
+            }
+            double best = 0.0;
+            for (int rep = 0; rep < 4; ++rep) {
+                const int64_t t0 = ggml_time_us();
+                size_t bytes = 0;
+                for (size_t ip = 0; ip < l.experts.size(); ++ip) {
+                    const ggml_tensor * w = l.experts[ip];
+                    ggml_backend_tensor_set_async(be, g.banks[ip], w->data, 0, 8*w->nb[2]);
+                    bytes += 8*w->nb[2];
+                }
+                ggml_backend_synchronize(be);
+                const double gbs = bytes/(double) std::max<int64_t>(1, ggml_time_us() - t0)/1e3;
+                best = rep > 0 ? std::max(best, gbs) : 0.0; // the first one warms up
+            }
+            link_gbs = best;
+            LLAMA_LOG_INFO("%s: hybrid share rule: link %.1f GB/s, share = L/(L + %.2f*R) from the CPU's measured rate\n", __func__, link_gbs, rule_k);
+            return;
+        }
+    }
+
+    // the layer's experts can be computed by the CPU (set_hybrid_layer's conditions apart from the graph's)
+    bool hybrid_layer_ok(int32_t il) const {
+        const layer & l = layers[il];
+        return hybrid && l.cpu_out != nullptr && l.down != nullptr && (l.gate_up || (l.up && l.gate)) && cpu_can(l, l.act);
+    }
+
+    // the CPU path can compute the layer with this activation: biases and the gpt-oss activation only in the kernel
+    bool cpu_can(const layer & l, llama_moe_cache::act_type act) const {
+        const bool biased = l.b_up || l.b_gate || l.b_down || l.b_gate_up;
+        return l.bias_ok && (use_kernel || (!biased && act != llama_moe_cache::ACT_SWIGLU_OAI));
+    }
+
+    ggml_tensor * doorbell_slots(ggml_context * ctx, int32_t il, ggml_tensor * ids, ggml_tensor * x, ggml_tensor * w, int64_t n_tokens, int64_t n_used) const {
+        if (!doorbell || il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0 || layers[il].db == nullptr) {
+            return nullptr;
+        }
+        const layer & l = layers[il];
+        if (n_tokens < 1 || n_tokens > db_max_tok || n_used > n_expert_used ||
+                std::min(n_tokens*n_used, l.slot_map->ne[1]) > groups[l.ig].n_slots) {
+            return nullptr;
+        }
+        if (x == nullptr || w == nullptr || !hybrid_layer_ok(il)) {
+            x = w = nullptr;
+        }
+        return ggml_moe_doorbell(ctx, ids, x, w, (ggml_moe_db *) l.db->data, (int) n_used);
+    }
+
+    void db_loop() {
+        int64_t spins = 0;
+        auto t_idle = std::chrono::steady_clock::now();
+        while (!db_exit.load(std::memory_order_relaxed)) {
+            bool any = false;
+            for (int32_t il : db_layers) {
+                volatile ggml_moe_db * h = (volatile ggml_moe_db *) layers[il].db->data;
+                if (h->req != h->ack) {
+                    db_service(il);
+                    any = true;
+                }
+            }
+            if (any) {
+                spins = 0;
+                continue;
+            }
+            if (++spins < 4096) {
+                continue;
+            }
+            if (spins == 4096) {
+                t_idle = std::chrono::steady_clock::now();
+            }
+            if ((spins & 1023) == 0 && std::chrono::steady_clock::now() - t_idle > std::chrono::seconds(1)) {
+                std::this_thread::sleep_for(std::chrono::microseconds(100));
+            } else {
+                std::this_thread::yield();
+            }
+        }
+    }
+
+    // a doorbell of layer il: the copy callback's work, with the ids (and x, w) read from the control block, then the answer
+    void db_service(int32_t il) {
+        std::lock_guard<std::mutex> lock(mtx);
+        const int64_t t0 = ggml_time_us();
+        std::atomic_thread_fence(std::memory_order_acquire);
+        layer & l = layers[il];
+        group & g = groups[l.ig];
+        volatile ggml_moe_db * h = (volatile ggml_moe_db *) l.db->data;
+        char * base = (char *) l.db->data;
+        const int64_t n_tok  = h->n_tok;
+        const int64_t n_used = h->n_used;
+        const bool    hyb    = h->hyb && hybrid_layer_ok(il);
+        const int32_t * h_ids = (const int32_t *) (base + h->ids_off);
+        ids.assign(h_ids, h_ids + n_tok*n_used);
+
+        if (hybrid_auto) {
+            gate_step(il, n_tok, hyb, t0);
+        }
+        if (hyb) {
+            const float * h_x = (const float *) (base + h->x_off);
+            const float * h_w = (const float *) (base + h->w_off);
+            l.x.assign(h_x, h_x + n_tok*n_embd);
+            l.w.assign(h_w, h_w + n_tok*n_used);
+            volatile uint32_t * f = (volatile uint32_t *) l.flag->data;
+            const double wait_us = f[2]/1000.0;
+            t_flag_wait_us += wait_us;
+            n_flag_waits++;
+            if (hybrid_auto) {
+                share_update(wait_us, wait_us < 2.0 ? 10.0 : 0.0);
+            }
+            f[2] = 0;
+            n_flag_timeouts = std::max<size_t>(n_flag_timeouts, f[3]);
+        }
+
+        size_t n_hit = 0;
+        if (hyb) {
+            g.lru.plan_hybrid(il, ids.data(), ids.size(), hybrid_share, fills, l.cpu_experts, n_hit, n_expert_used, tail_rank, &tail_skipped);
+            n_tail_skip += tail_skipped.size();
+        } else if (!g.lru.plan(il, ids.data(), ids.size(), fills, n_hit)) {
+            GGML_ABORT("the MoE cache is too small for the experts selected in layer %d", il);
+        }
+
+        int32_t * h_slots = (int32_t *) (base + h->ans_off);
+        const int32_t * map = (const int32_t *) l.slot_map->data;
+        const int64_t n_expert = l.slot_map->ne[1];
+        if (hyb) {
+            // the experts computed by the CPU read the zero slot on the device, so they add nothing there
+            std::copy(map, map + n_expert, l.map_dev.begin());
+            for (int32_t e : l.cpu_experts) {
+                l.map_dev[e] = g.n_slots;
+            }
+            for (int32_t e : tail_skipped) {
+                l.map_dev[e] = g.n_slots;
+            }
+            for (size_t i = 0; i < ids.size(); ++i) {
+                h_slots[i] = l.map_dev[ids[i]];
+            }
+            l.ids = ids;
+            wait_cpu();
+            l.pf_job = false;
+            l.t_start = ggml_time_us();
+            job_done.store(false, std::memory_order_relaxed);
+            job_layer.store(il, std::memory_order_release);
+            n_cpu_experts += l.cpu_experts.size();
+        } else {
+            for (size_t i = 0; i < ids.size(); ++i) {
+                h_slots[i] = map[ids[i]];
+            }
+        }
+
+        // the uploads, consecutive experts going to consecutive slots in one piece
+        uint32_t * h_nfill = (uint32_t *) (base + h->ans_off + h->slots_cap*sizeof(int32_t));
+        ggml_moe_db_fill * h_fill = (ggml_moe_db_fill *) (h_nfill + 2);
+        int64_t n_fill = 0;
+        size_t bytes = 0;
+        for (size_t ip = 0; ip < l.experts.size(); ++ip) {
+            const ggml_tensor * w = l.experts[ip];
+            ggml_tensor * bank = g.banks[ip];
+            const size_t expert_size = w->nb[2];
+            for (size_t i = 0; i < fills.size();) {
+                size_t n = 1;
+                while (i + n < fills.size() && fills[i + n].expert == fills[i].expert + (int32_t) n && fills[i + n].slot == fills[i].slot + (int32_t) n) {
+                    n++;
+                }
+                GGML_ASSERT(n_fill < db_max_fill);
+                h_fill[n_fill].src = (uint64_t) (uintptr_t) ((const uint8_t *) w->data + fills[i].expert*expert_size);
+                h_fill[n_fill].dst = (uint64_t) (uintptr_t) ((uint8_t *) bank->data + fills[i].slot*expert_size);
+                h_fill[n_fill].n   = n*expert_size;
+                n_fill++;
+                bytes += n*expert_size;
+                i += n;
+            }
+        }
+        h_nfill[0] = (uint32_t) n_fill;
+
+        stats & st = stats_small;
+        st.hits   += n_hit;
+        st.misses += fills.size() + (hyb ? l.cpu_experts.size() : 0);
+        st.bytes  += bytes;
+
+        t_db_gpu_wait_us += h->wait_ns/1000.0;
+        n_db_timeouts = std::max<size_t>(n_db_timeouts, h->timeouts);
+        std::atomic_thread_fence(std::memory_order_seq_cst);
+        h->ack = h->req;
+        std::atomic_thread_fence(std::memory_order_seq_cst);
+        n_db_calls++;
+        t_db_us += ggml_time_us() - t0;
+        if (hyb) {
+            n_hybrid_calls++;
+            t_upload_us += ggml_time_us() - t0;
+        }
+    }
+
+    // flag mode: write the CPU's output where the GPU reads it, then release the GPU's wait
+    void post_cpu_out(layer & l) {
+        memcpy(l.cpu_out->data, l.out.data(), l.out.size()*sizeof(float));
+        std::atomic_thread_fence(std::memory_order_release);
+        volatile uint32_t * f = (volatile uint32_t *) l.flag->data;
+        f[0] = f[0] + 1;
+    }
+
+    void post_pf_out(layer & l) {
+        memcpy(pf_out->data, l.out.data(), l.out.size()*sizeof(float));
+        std::atomic_thread_fence(std::memory_order_release);
+        volatile uint32_t * f = (volatile uint32_t *) l.pf_flag->data;
+        f[0] = f[0] + 1;
+    }
+
+    // lookahead: the router (and selection bias) of layer il+1, to be applied to layer il's expert input in the graph
+    ggml_tensor * lookahead_router(int32_t il, ggml_tensor ** bias, int * k) const {
+        if (!la_pred_on || il < 0 || il + 1 >= (int32_t) layers.size() || layers[il].ig < 0 || la_gi[il + 1] == nullptr ||
+                layers[il + 1].ig != layers[il].ig) {
+            return nullptr;
+        }
+        *bias = la_bi[il + 1];
+        *k    = la_k;
+        return la_gi[il + 1];
+    }
+
+    void set_lookahead_layer(int32_t il, ggml_tensor * pred) {
+        if (il >= 0 && il < (int32_t) layers.size()) {
+            layers[il].la_pred = pred;
+        }
+    }
+
+    bool prefill_enabled(int64_t n_tokens) const {
+        return pf && n_tokens > max_batch && n_tokens <= pf_max;
+    }
+
+    // a large-batch graph registers (or with x nullptr clears) the tensors of the prefill share of layer il
+    ggml_tensor * set_prefill_layer(int32_t il, ggml_tensor * x, ggml_tensor * w, const ggml_tensor * ids, llama_moe_cache::act_type act, float limit) {
+        if (il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0) {
+            return nullptr;
+        }
+        layer & l = layers[il];
+        if (!pf || x == nullptr || !hybrid_layer_ok(il) || act == llama_moe_cache::ACT_NONE || !cpu_can(l, act) || x->ne[1] > pf_max) {
+            l.p_x = nullptr;
+            return nullptr;
+        }
+        l.p_x   = x;
+        l.p_w   = w;
+        l.p_ids = ids;
+        l.act   = act;
+        l.limit = limit;
+        return pf_out;
+    }
+
+    ggml_tensor * prefill_add(ggml_context * ctx, int32_t il, ggml_tensor * moe_out) {
+        const layer & l = layers[il];
+        if (!ggml_is_contiguous(moe_out)) {
+            moe_out = ggml_cont(ctx, moe_out);
+        }
+        return ggml_host_add(ctx, moe_out, (const float *) pf_out->data, (uint32_t *) l.pf_flag->data);
+    }
+
+    // large-batch copy callback, once per ubatch and layer with the selected experts ids_data (as read, ids' strides) and the
+    // experts to upload: picks the CPU's experts, takes them out of used, points their pairs on the device elsewhere, starts the job
+    bool prefill_plan(ggml_backend_t backend, const ggml_tensor * w, const ggml_tensor * ids_t, std::vector<int32_t> & ids_data, std::vector<bool> & used) {
+        if (!pf) {
+            return false;
+        }
+        const auto it = bindings.find(w);
+        if (it == bindings.end()) {
+            return false;
+        }
+        const int32_t il = it->second.il;
+        layer & l = layers[il];
+        if (l.p_x == nullptr) {
+            return false;
+        }
+        std::lock_guard<std::mutex> lock(mtx);
+        stats_tick();
+        const int64_t t0 = ggml_time_us();
+        const int64_t n_used = ids_t->ne[0], n_tok = ids_t->ne[1];
+        const int64_t n_expert = (int64_t) used.size();
+
+        // the previous layer of this ubatch: its CPU and GPU times (the backend was synchronized before this callback)
+        if (pf_prev_il >= 0 && pf_prev_il < il && pf_fixed < 0.0) {
+            const layer & lp = layers[pf_prev_il];
+            const double wait_us = ((volatile uint32_t *) lp.pf_flag->data)[2]/1000.0;
+            const double c = (double) lp.t_job/pf_prev_ncpu;
+            const double g = std::max(0.0, (double) (t0 - pf_prev_t0) - wait_us)/std::max<int64_t>(pf_prev_ngpu, 1);
+            constexpr double a = 0.25;
+            pf_c_us = pf_measured ? pf_c_us + a*(c - pf_c_us) : c;
+            pf_g_us = pf_measured ? pf_g_us + a*(g - pf_g_us) : g;
+            pf_measured = true;
+            t_pf_wait_us += wait_us;
+        }
+        pf_prev_il = -1;
+
+        auto post_zero = [&]() {
+            // the graph adds pf_out after the layer's experts in any case
+            memset(pf_out->data, 0, n_tok*n_embd*sizeof(float));
+            std::atomic_thread_fence(std::memory_order_release);
+            volatile uint32_t * f = (volatile uint32_t *) l.pf_flag->data;
+            f[0] = f[0] + 1;
+        };
+        if (l.p_ids != ids_t || l.p_x->ne[1] != n_tok || ggml_nelements(l.p_w) != n_used*n_tok) {
+            if (n_pf_mismatch++ == 0) {
+                LLAMA_LOG_WARN("llama_moe_cache: prefill share: layer %d's registered tensors do not match the graph, no CPU share\n", il);
+            }
+            wait_cpu();
+            post_zero();
+            return true;
+        }
+        wait_cpu();
+
+        // the selected experts in token order
+        l.ids.resize(n_tok*n_used);
+        for (int64_t t = 0; t < n_tok; ++t) {
+            for (int64_t k = 0; k < n_used; ++k) {
+                l.ids[t*n_used + k] = ids_data[t*ids_t->nb[1]/sizeof(int32_t) + k*ids_t->nb[0]/sizeof(int32_t)];
+            }
+        }
+        pf_cnt.assign(n_expert, 0);
+        for (int32_t e : l.ids) {
+            pf_cnt[e]++;
+        }
+        // the candidates: experts that would be uploaded (not in the cache), used by at most pf_maxt tokens, fewest first
+        const group & g = groups[l.ig];
+        const int32_t * slots = g.lru.slot_map[il];
+        std::vector<std::pair<int32_t, int32_t>> cand;
+        int64_t n_stream = 0;
+        for (int32_t e = 0; e < n_expert; ++e) {
+            if (pf_cnt[e] == 0 || (slots != nullptr && slots[e] >= 0)) {
+                continue;
+            }
+            n_stream++;
+            if (pf_cnt[e] <= pf_maxt) {
+                cand.emplace_back(pf_cnt[e], e);
+            }
+        }
+        std::sort(cand.begin(), cand.end());
+        const double share = pf_fixed >= 0.0 ? pf_fixed : std::clamp(pf_g_us/(pf_c_us + pf_g_us), 0.05, 0.9);
+        // the CPU's pairs are pointed at experts the GPU computes and their tokens do not use: keep enough of them
+        int64_t n_used_e = 0;
+        for (int32_t e = 0; e < n_expert; ++e) {
+            n_used_e += pf_cnt[e] > 0;
+        }
+        const int64_t take = std::max<int64_t>(0, std::min<int64_t>({ (int64_t) cand.size(), std::llround(share*n_stream), n_used_e - 2*n_used }));
+        n_pf_layers++;
+        n_pf_stream += n_stream;
+        if (take == 0) {
+            post_zero();
+            t_pf_plan_us += ggml_time_us() - t0;
+            return true;
+        }
+
+        ggml_backend_tensor_get_async(backend, l.p_x, pf_x->data, 0, ggml_nbytes(l.p_x));
+        ggml_backend_tensor_get_async(backend, l.p_w, pf_w->data, 0, ggml_nbytes(l.p_w));
+
+        pf_on_cpu.assign(n_expert, 0);
+        l.cpu_experts.clear();
+        const int32_t zero = g.n_slots;
+        std::fill(l.map_dev.begin(), l.map_dev.end(), -1);
+        for (int64_t i = 0; i < take; ++i) {
+            const int32_t e = cand[i].second;
+            pf_on_cpu[e] = 1;
+            used[e] = false;
+            l.cpu_experts.push_back(e);
+            l.map_dev[e] = zero;
+        }
+        // the experts the GPU computes, most used first: the CPU's pairs are pointed at the first one their token does not use
+        std::vector<std::pair<int32_t, int32_t>> top;
+        for (int32_t e = 0; e < n_expert; ++e) {
+            if (pf_cnt[e] > 0 && !pf_on_cpu[e]) {
+                top.emplace_back(-pf_cnt[e], e);
+            }
+        }
+        std::sort(top.begin(), top.end());
+        GGML_ASSERT((int64_t) top.size() >= 2*n_used && "prefill share: too few experts left on the GPU");
+        ggml_backend_synchronize(backend);
+        l.x.assign((const float *) pf_x->data, (const float *) pf_x->data + n_tok*n_embd);
+        l.w.assign((const float *) pf_w->data, (const float *) pf_w->data + n_tok*n_used);
+        float * wd = (float *) pf_w->data;
+        size_t n_pairs = 0;
+        std::vector<int32_t> row(n_used);
+        for (int64_t t = 0; t < n_tok; ++t) {
+            const int32_t * te = l.ids.data() + t*n_used;
+            std::copy(te, te + n_used, row.begin());
+            for (int64_t k = 0; k < n_used; ++k) {
+                if (!pf_on_cpu[te[k]]) {
+                    continue;
+                }
+                // not used by the token, nor taken by one of its earlier CPU pairs
+                int32_t r = -1;
+                for (const auto & p : top) {
+                    if (std::find(row.begin(), row.end(), p.second) == row.end()) {
+                        r = p.second;
+                        break;
+                    }
+                }
+                GGML_ASSERT(r >= 0);
+                row[k] = r;
+                ids_data[t*ids_t->nb[1]/sizeof(int32_t) + k*ids_t->nb[0]/sizeof(int32_t)] = r;
+                wd[t*n_used + k] = 0.0f;
+                n_pairs++;
+            }
+        }
+        // the job: the worker reads l.ids, l.x, l.w, l.cpu_experts and l.map_dev
+        l.pf_job  = true;
+        l.t_start = ggml_time_us();
+        job_done.store(false, std::memory_order_relaxed);
+        job_layer.store(il, std::memory_order_release);
+
+        ggml_backend_tensor_set_async(backend, const_cast<ggml_tensor *>(ids_t), ids_data.data(), 0, ggml_nbytes(ids_t));
+        ggml_backend_tensor_set_async(backend, l.p_w, pf_w->data, 0, ggml_nbytes(l.p_w));
+        // the ids are pageable: that copy returned once staged, and pf_w is not written again before the next layer's sync
+        static const bool dbg = getenv("LLAMA_MOE_PREFILL_DEBUG") != nullptr;
+        if (dbg) {
+            std::vector<int32_t> chk(ids_data.size());
+            ggml_backend_tensor_get(ids_t, chk.data(), 0, ggml_nbytes(ids_t));
+            int bad = 0, dup = 0;
+            for (int64_t t = 0; t < n_tok; ++t) {
+                for (int64_t k = 0; k < n_used; ++k) {
+                    const int32_t e = chk[t*ids_t->nb[1]/4 + k*ids_t->nb[0]/4];
+                    bad += e < 0 || e >= n_expert;
+                    for (int64_t j = 0; j < k; ++j) {
+                        dup += chk[t*ids_t->nb[1]/4 + j*ids_t->nb[0]/4] == e;
+                    }
+                }
+            }
+            LLAMA_LOG_INFO("pf dbg: layer %d n_tok %d n_used %d nb %zu %zu nbytes %zu (data %zu) stream %d take %d pairs %zu bad %d dup %d same %d w %s\n",
+                il, (int) n_tok, (int) n_used, ids_t->nb[0], ids_t->nb[1], ggml_nbytes(ids_t), ids_data.size()*4, (int) n_stream, (int) take, n_pairs, bad, dup,
+                (int) (chk == ids_data), ggml_get_name(l.p_w));
+        }
+
+        pf_prev_il   = il;
+        pf_prev_t0   = l.t_start;
+        pf_prev_ncpu = take;
+        pf_prev_ngpu = n_stream - take;
+        n_pf_cpu    += take;
+        n_pf_pairs  += n_pairs;
+        t_pf_plan_us += ggml_time_us() - t0;
+        return true;
+    }
+
+    ggml_tensor * hybrid_add(ggml_context * ctx, int32_t il, ggml_tensor * moe_out) {
+        const layer & l = layers[il];
+        if (flag_mode) {
+            if (!ggml_is_contiguous(moe_out)) {
+                moe_out = ggml_cont(ctx, moe_out);
+            }
+            return ggml_host_add(ctx, moe_out, (const float *) l.cpu_out->data, (uint32_t *) l.flag->data);
+        }
+        ggml_tensor * cpu_out = ggml_view_2d(ctx, l.cpu_out, moe_out->ne[0], moe_out->ne[1], l.cpu_out->nb[1], 0);
+        return ggml_add(ctx, moe_out, cpu_out);
+    }
+
+    // auto share: the CPU was late by wait us (the GPU or host waited for it), or done early by idle us, per layer call
+    void share_update(double wait_us, double idle_us) {
+        ctl_wait_us += wait_us;
+        ctl_idle_us += idle_us;
+        if (++ctl_calls < ctl_window) {
+            return;
+        }
+        const double wait = ctl_wait_us/ctl_calls, idle = ctl_idle_us/ctl_calls;
+        const double old = hybrid_share;
+        if (share_rule && link_gbs > 0.0) {
+            const double dt = t_cpu_us - rule_t_cpu;
+            const size_t dn = n_cpu_experts - rule_n_cpu;
+            rule_t_cpu = t_cpu_us;
+            rule_n_cpu = n_cpu_experts;
+            ctl_wait_us = ctl_idle_us = 0.0;
+            ctl_calls = 0;
+            if (dn < 64 || dt <= 0.0) {
+                return;
+            }
+            const double R = expert_bytes*dn/dt/1e3; // GB/s
+            const double target = std::clamp(std::round(link_gbs/(link_gbs + rule_k*R)*20.0)/20.0, 0.10, 0.60);
+            // a step at a time toward the target
+            hybrid_share = target > hybrid_share + 0.025 ? hybrid_share + 0.05 : target < hybrid_share - 0.025 ? hybrid_share - 0.05 : hybrid_share;
+            if (hybrid_share != old) {
+                n_share_changes++;
+                LLAMA_LOG_DEBUG("llama_moe_cache: hybrid share %.2f -> %.2f (rule: link %.1f GB/s, CPU %.1f GB/s)\n", old, hybrid_share, link_gbs, R);
+            }
+            rule_R_last = R;
+            return;
+        }
+        if (wait > idle + 5.0) {
+            hybrid_share = std::min(0.95, hybrid_share + 0.05);
+        } else if (idle > wait + 5.0) {
+            hybrid_share = std::max(0.05, hybrid_share - 0.05); // keep admitting a few, so the cache follows the conversation
+        }
+        if (hybrid_share != old) {
+            n_share_changes++;
+            LLAMA_LOG_DEBUG("llama_moe_cache: hybrid share %.2f -> %.2f (CPU late %.1f us, idle %.1f us per call)\n", old, hybrid_share, wait, idle);
+        }
+        ctl_wait_us = ctl_idle_us = 0.0;
+        ctl_calls = 0;
+    }
+
+    ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used) const {
+        if (il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0) {
+            return nullptr;
+        }
+        const layer & l = layers[il];
+
+        // large batches use most experts of a layer, so they gain little from the cache and would evict the experts used in generation
+        if (n_tokens == 0 || n_tokens > max_batch || std::min(n_tokens*n_expert_used, l.slot_map->ne[1]) > groups[l.ig].n_slots) {
+            return nullptr;
+        }
+        return l.slot_map;
+    }
+
+    ggml_tensor * get_experts(const ggml_tensor * w) const {
+        const auto it = bindings.find(w);
+        return it != bindings.end() ? it->second.cached : nullptr;
+    }
+
+    int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last) {
+        if (la_backend) {
+            la_wait_idle();
+            std::lock_guard<std::mutex> lock(mtx);
+            la_join(backend);
+        }
+        const auto it = bindings.find(w);
+        if (it == bindings.end()) {
+            return 0;
+        }
+        const binding & b = it->second;
+        const group & g = groups[layers[b.il].ig];
+        if (backend != devices[g.id].backend) {
+            return 0;
+        }
+
+        // large batches only read the cache, so the experts used in generation stay in it
+        const int32_t * slots = g.lru.slot_map[b.il];
+        if (slots == nullptr || slots[e] < 0) {
+            return 0;
+        }
+        int64_t n = 1;
+        while (e + n <= last && slots[e + n] == slots[e] + n) {
+            n++;
+        }
+
+        ggml_tensor * bank = g.banks[b.ip];
+        ggml_reset(ctx_views.get());
+        ggml_tensor * src_view = ggml_view_3d(ctx_views.get(), bank, bank->ne[0], bank->ne[1], n, bank->nb[1], bank->nb[2], slots[e]*bank->nb[2]);
+        ggml_tensor * dst_view = ggml_view_3d(ctx_views.get(), dst,  dst->ne[0],  dst->ne[1],  n, dst->nb[1],  dst->nb[2],  e*dst->nb[2]);
+        ggml_backend_view_init(src_view);
+        ggml_backend_view_init(dst_view);
+        ggml_backend_tensor_copy_async(backend, backend, src_view, dst_view);
+
+        stats_copy.hits  += n;
+        stats_copy.bytes += ggml_nbytes(src_view);
+
+        return n;
+    }
+
+    bool copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph) {
+        const auto it = layer_of.find(src);
+        if (it == layer_of.end()) {
+            return false;
+        }
+        const int32_t il = it->second;
+        const layer & l = layers[il];
+        group & g = groups[l.ig];
+        if (la_backend) {
+            la_wait_idle();
+        }
+        std::lock_guard<std::mutex> lock(mtx);
+
+        GGML_ASSERT(backend == devices[g.id].backend);
+        stats_tick();
+
+        // the get_rows that looks up the slots of the selected experts
         const int n_nodes = ggml_graph_n_nodes(graph);
         const ggml_tensor * lookup = nullptr;
         for (int i = 0; i < n_nodes && lookup == nullptr; ++i) {
@@ -562,15 +2181,123 @@ struct llama_moe_cache::impl {
         }
         GGML_ASSERT(ggml_is_contiguous(sel));
 
-        ids.resize(ggml_nelements(sel));
-        ggml_backend_tensor_get_async(backend, sel, ids.data(), 0, ggml_nbytes(sel));
+        // read back into pinned memory: a read back into pageable memory is a slow staged copy
+        GGML_ASSERT(ggml_nelements(sel) <= ggml_nelements(rb_ids));
+        ggml_backend_tensor_get_async(backend, sel, rb_ids->data, 0, ggml_nbytes(sel));
+
+        // hybrid mode: also read the expert input and the router weights, the CPU computes the missed experts that are not uploaded
+        layer & lh = layers[il];
+        const bool hyb = hybrid && lh.x_dev != nullptr;
+        if (hyb) {
+            GGML_ASSERT(ggml_nelements(lh.w_dev) == ggml_nelements(sel));
+            GGML_ASSERT(ggml_nelements(lh.x_dev) <= ggml_nelements(rb_x) && ggml_nelements(lh.w_dev) <= ggml_nelements(rb_w));
+            ggml_backend_tensor_get_async(backend, lh.x_dev, rb_x->data, 0, ggml_nbytes(lh.x_dev));
+            ggml_backend_tensor_get_async(backend, lh.w_dev, rb_w->data, 0, ggml_nbytes(lh.w_dev));
+            if (la_pred_on && lh.la_pred != nullptr) {
+                GGML_ASSERT(ggml_nelements(lh.la_pred) <= ggml_nelements(rb_la));
+                ggml_backend_tensor_get_async(backend, lh.la_pred, rb_la->data, 0, ggml_nbytes(lh.la_pred));
+            }
+        }
+        const int64_t ts0 = ggml_time_us();
+        if (guest && hyb && flag_mode) {
+            pool_help();
+        }
         ggml_backend_synchronize(backend);
+        const int64_t ts1 = ggml_time_us();
+        ids.assign((const int32_t *) rb_ids->data, (const int32_t *) rb_ids->data + ggml_nelements(sel));
+        if (hyb) {
+            // the worker is idle (wait_cpu below) or computes another layer
+            lh.x.assign((const float *) rb_x->data, (const float *) rb_x->data + ggml_nelements(lh.x_dev));
+            lh.w.assign((const float *) rb_w->data, (const float *) rb_w->data + ggml_nelements(lh.w_dev));
+        }
+
+        if (hybrid_auto) {
+            gate_step(il, (int64_t) ids.size()/n_expert_used, hyb, ts1);
+        }
+
+        if (hyb && flag_mode) {
+            // the GPU's wait for the CPU in the previous ubatch of this layer (the sync above waited for it)
+            volatile uint32_t * f = (volatile uint32_t *) lh.flag->data;
+            const double wait_us = f[2]/1000.0;
+            t_flag_wait_us += wait_us;
+            n_flag_waits++;
+            if (hybrid_auto) {
+                // no wait: the CPU was done before the GPU needed it, by an unknown margin
+                share_update(wait_us, wait_us < 2.0 ? 10.0 : 0.0);
+            }
+            f[2] = 0;
+            n_flag_timeouts = std::max<size_t>(n_flag_timeouts, f[3]);
+        }
 
         size_t n_hit = 0;
-        if (!g.slru.plan(il, ids.data(), ids.size(), fills, n_hit)) {
+        if (la_backend) {
+            // the layer reads a slot that is still being written
+            const int32_t * smj = g.lru.slot_map[il];
+            la_join_if(backend, l.ig, [&](auto use) { for (int32_t e : ids) use(smj[e]); });
+            // prefetched experts used for the first time
+            const int32_t * sm = g.lru.slot_map[il];
+            for (int32_t e : ids) {
+                if (sm[e] >= 0 && la_mark[l.ig][sm[e]] == il*g.lru.n_expert + e + 1) {
+                    la_mark[l.ig][sm[e]] = 0;
+                    n_la_useful++;
+                }
+            }
+        }
+        if (hyb) {
+            g.lru.plan_hybrid(il, ids.data(), ids.size(), hybrid_share, fills, lh.cpu_experts, n_hit, n_expert_used, tail_rank, &tail_skipped);
+            n_tail_skip += tail_skipped.size();
+        } else if (!g.lru.plan(il, ids.data(), ids.size(), fills, n_hit)) {
             GGML_ABORT("the MoE cache is too small for the experts selected in layer %d", il);
         }
 
+        if (hyb) {
+            // the experts computed by the CPU read the zero slot on the device, so they add nothing there
+            const int32_t * map = (const int32_t *) src->data;
+            std::copy(map, map + lh.map_dev.size(), lh.map_dev.begin());
+            for (int32_t e : lh.cpu_experts) {
+                lh.map_dev[e] = g.n_slots;
+            }
+            for (int32_t e : tail_skipped) {
+                lh.map_dev[e] = g.n_slots;
+            }
+            lh.ids = ids;
+            wait_cpu();
+            lh.pf_job = false;
+            if (preread && lh.la_pred != nullptr && il + 1 < (int32_t) layers.size() && layers[il + 1].ig == l.ig) {
+                // the previous pre-read may still read the list
+                pr_stop.store(true, std::memory_order_release);
+                while (pr_running.load(std::memory_order_acquire)) {
+                }
+                pr_stop.store(false, std::memory_order_relaxed);
+                const layer & ln = layers[il + 1];
+                const int32_t * sm = g.lru.slot_map[il + 1];
+                const int32_t * pr = (const int32_t *) rb_la->data;
+                const int64_t np = ggml_nelements(lh.la_pred);
+                pr_chunks.clear();
+                std::vector<int32_t> seen;
+                for (int64_t i = 0; i < np; ++i) {
+                    const int32_t e = pr[i];
+                    if (e < 0 || e >= g.lru.n_expert || sm[e] >= 0 || std::find(seen.begin(), seen.end(), e) != seen.end()) continue;
+                    seen.push_back(e);
+                    for (const ggml_tensor * w : ln.experts) {
+                        const uint8_t * p = (const uint8_t *) w->data + e*w->nb[2];
+                        for (size_t o = 0; o < w->nb[2]; o += 65536) {
+                            pr_chunks.push_back({ p + o, std::min<size_t>(65536, w->nb[2] - o) });
+                        }
+                    }
+                }
+                n_pr_experts += seen.size();
+                pr_ready.store(!pr_chunks.empty(), std::memory_order_relaxed);
+            }
+            lh.t_start = ggml_time_us();
+            job_done.store(false, std::memory_order_relaxed);
+            job_layer.store(il, std::memory_order_release);
+        }
+
+        // a refilled slot that is still being written
+        if (la_backend) {
+            la_join_if(backend, l.ig, [&](auto use) { for (const auto & f : fills) use(f.slot); });
+        }
         // upload the missing experts, consecutive experts going to consecutive slots are uploaded together
         size_t bytes = 0;
         for (size_t ip = 0; ip < l.experts.size(); ++ip) {
@@ -588,17 +2315,643 @@ struct llama_moe_cache::impl {
             }
         }
 
+        if (hyb) {
+            t_sync_us   += ts1 - ts0;
+            t_upload_us += ggml_time_us() - ts1;
+            n_hybrid_calls++;
+        }
+
         stats & st = ids.size() <= (size_t) 8*n_expert_used ? stats_small : stats_large;
         st.hits   += n_hit;
-        st.misses += fills.size();
+        st.misses += fills.size() + (hyb ? lh.cpu_experts.size() : 0);
         st.bytes  += bytes;
 
-        // the next copy synchronizes the backend before it changes the slot map again
-        ggml_backend_tensor_set_async(backend, dst, src->data, 0, ggml_nbytes(src));
+        if (!hyb) {
+            // the next copy synchronizes the backend before it changes the slot map again
+            ggml_backend_tensor_set_async(backend, dst, src->data, 0, ggml_nbytes(src));
+            return true;
+        }
+
+        std::copy(lh.map_dev.begin(), lh.map_dev.end(), (int32_t *) lh.map_pin->data);
+        ggml_backend_tensor_set_async(backend, dst, lh.map_pin->data, 0, ggml_nbytes(src));
+        n_cpu_experts += lh.cpu_experts.size();
+        lh.planned = true;
+        t_planned  = ggml_time_us();
+
+        // lookahead: predict the next layer from this layer's expert input
+        const int32_t tl = il + 1;
+        if (la_backend && lh.la_pred != nullptr && tl < (int32_t) layers.size() && la_gi[tl] != nullptr && layers[tl].ig == l.ig) {
+            la_ids.assign((const int32_t *) rb_la->data, (const int32_t *) rb_la->data + ggml_nelements(lh.la_pred));
+            la_src = il;
+            la_busy.store(true, std::memory_order_relaxed);
+            la_job.store(tl, std::memory_order_release);
+        }
 
         return true;
     }
 
+    // auto mode A/B gate, at the slot map copy of each cached layer (after the sync): times each ubatch from the first to the
+    // last cached layer, and switches the CPU path for the next graphs
+    void gate_step(int32_t il, int64_t n_tok, bool hyb, int64_t t) {
+        if (n_tok < 1 || n_tok > 8) {
+            return;
+        }
+        if (il == il_first) {
+            t_first   = t;
+            n_tok_cur = n_tok;
+            ubatch_on = hyb;
+        }
+        if (il != il_last || t_first == 0 || n_tok != n_tok_cur) {
+            return;
+        }
+        span_acc & acc = ubatch_on ? acc_on : acc_off;
+        acc.us[n_tok] += (double) (t - t_first);
+        acc.n[n_tok]++;
+        (ubatch_on ? n_ubatch_on : n_ubatch_off)++;
+        t_first = 0;
+
+        auto set_path = [&](bool on) {
+            gate_changed = gate_changed || path_on != on;
+            path_on = on;
+        };
+        if (++gate_count < (gate_phase == 0 ? gate_hold : gate_probe)) {
+            return;
+        }
+        gate_count = 0;
+        if (gate_phase == 0) {
+            // probe the other state, then measure the decision right after it, so both see about the same conversation
+            acc_on = acc_off = span_acc();
+            gate_phase = 1;
+            set_path(!gate_on);
+            n_gate_probes++;
+        } else if (gate_phase == 1) {
+            gate_phase = 2;
+            set_path(gate_on);
+        } else {
+            // compare the mean span of each ubatch size seen in both windows, weighted by the smaller count
+            double t_on = 0.0, t_off = 0.0;
+            for (int b = 1; b <= 8; ++b) {
+                const int w = std::min(acc_on.n[b], acc_off.n[b]);
+                if (w >= 4) {
+                    t_on  += w*acc_on.us[b]/acc_on.n[b];
+                    t_off += w*acc_off.us[b]/acc_off.n[b];
+                }
+            }
+            // the CPU path stays on only if it wins by 2%: in a tie it costs CPU time and cache admissions for nothing
+            // (at 24 GiB of Qwen3.8 Q2_0 the two are within 2% and keeping the old decision on a tie lost 2-7%)
+            bool on = gate_on;
+            if (t_on > 0.0 && t_off > 0.0) {
+                on = t_on < 0.98*t_off;
+            }
+            gate_hold = on == gate_on ? std::min(2*gate_hold, 4096) : 256;
+            // a probe of the losing state costs its whole window (at 4 GiB, 48 ubatches that upload every miss): when one state
+            // wins by 15% or more, wait 8x longer before the next probe (LLAMA_MOE_GATE_BACKOFF=0: the old schedule)
+            static const bool backoff = !getenv("LLAMA_MOE_GATE_BACKOFF") || atoi(getenv("LLAMA_MOE_GATE_BACKOFF")) != 0;
+            if (backoff && on == gate_on && t_on > 0.0 && t_off > 0.0 && std::min(t_on, t_off) < 0.85*std::max(t_on, t_off)) {
+                gate_hold = std::min(8*gate_hold, 32768);
+            }
+            LLAMA_LOG_DEBUG("llama_moe_cache: hybrid gate: span on %.0f us, off %.0f us, hold %d\n", t_on, t_off, gate_hold);
+            if (on != gate_on) {
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid gate: CPU path %s (ubatch span with it %.0f us, without %.0f us)\n",
+                    on ? "on" : "off", t_on, t_off);
+            }
+            n_gate_off += on ? 0 : 1;
+            gate_on    = on;
+            gate_phase = 0;
+            set_path(gate_on);
+        }
+    }
+
+    // hybrid mode: the copy of cpu_out (or a view of it) to the device, after the GPU has been given the cached experts:
+    // the CPU computes the missed experts of the layer meanwhile, then uploads the weighted sum
+    bool copy_out(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) {
+        const ggml_tensor * base = src->view_src ? src->view_src : src;
+        const auto it = layer_of_out.find(base);
+        if (it == layer_of_out.end()) {
+            return false;
+        }
+        layer & l = layers[it->second];
+        GGML_ASSERT(l.planned && "MoE cache hybrid mode: cpu_out copied without a hybrid slot map copy");
+        l.planned = false;
+        // the scheduler does not wait for the expert uploads before this callback, the controller below expects it did
+        ggml_backend_synchronize(backend);
+        const int64_t tw0 = ggml_time_us();
+        wait_cpu();
+        const int64_t tw1 = ggml_time_us();
+        t_gap_us  += tw0 - t_planned;
+        t_wait_us += tw1 - tw0;
+        if (hybrid_auto) {
+            // waited: the CPU was late; idle: the CPU finished before the GPU side reached cpu_out
+            share_update(tw1 - tw0, std::max<int64_t>(0, (tw0 - l.t_start) - l.t_job));
+        }
+        GGML_ASSERT((int64_t) l.out.size() == ggml_nelements(src));
+        std::copy(l.out.begin(), l.out.end(), (float *) src->data);
+        ggml_backend_tensor_set_async(backend, dst, src->data, 0, ggml_nbytes(dst));
+        return true;
+    }
+
+    cpu_graph & get_cpu_graph(const layer & l, int64_t n_pairs) {
+        cpu_graph & cg = cpu_graphs[n_pairs];
+        if (cg.gf && cg.act == l.act && cg.limit == l.limit) {
+            return cg;
+        }
+        ggml_init_params params = {
+            /*.mem_size   =*/ 16*ggml_tensor_overhead() + ggml_graph_overhead_custom(16, false),
+            /*.mem_buffer =*/ nullptr,
+            /*.no_alloc   =*/ true,
+        };
+        cg = cpu_graph();
+        cg.ctx.reset(ggml_init(params));
+        ggml_context * c = cg.ctx.get();
+
+        cg.x   = ggml_new_tensor_3d(c, GGML_TYPE_F32, n_embd, 1, n_pairs);
+        cg.sel = ggml_new_tensor_2d(c, GGML_TYPE_I32, 1, n_pairs);
+        ggml_tensor * gate = nullptr;
+        ggml_tensor * up   = nullptr;
+        if (l.gate_up) {
+            ggml_tensor * gu = cg.mm_gate_up = ggml_mul_mat_id(c, l.gate_up, cg.x, cg.sel);
+            const int64_t n_ff = gu->ne[0]/2;
+            gate = ggml_view_3d(c, gu, n_ff, gu->ne[1], gu->ne[2], gu->nb[1], gu->nb[2], 0);
+            up   = ggml_view_3d(c, gu, n_ff, gu->ne[1], gu->ne[2], gu->nb[1], gu->nb[2], n_ff*gu->nb[0]);
+        } else {
+            up   = cg.mm_up   = ggml_mul_mat_id(c, l.up,   cg.x, cg.sel);
+            gate = cg.mm_gate = ggml_mul_mat_id(c, l.gate, cg.x, cg.sel);
+        }
+        ggml_tensor * act = l.act == llama_moe_cache::ACT_SWIGLU_CLAMP ?
+            ggml_swiglu_clamp(c, gate, up, l.limit) : ggml_swiglu_split(c, gate, up);
+        cg.y = cg.mm_down = ggml_mul_mat_id(c, l.down, act, cg.sel); // [n_embd, 1, n_pairs]
+
+        cg.gf = ggml_new_graph_custom(c, 16, false);
+        ggml_build_forward_expand(cg.gf, cg.y);
+
+        cg.buf.reset(ggml_backend_alloc_ctx_tensors(c, cpu_backend.get()));
+        GGML_ASSERT(cg.buf);
+        cg.act   = l.act;
+        cg.limit = l.limit;
+        return cg;
+    }
+
+    // the kernel: y[t] += w * down_e(act(gate_e x[t], up_e x[t])) for the CPU's pairs, with ggml's CPU dot products
+    void compute_cpu_kernel(layer & l, int64_t n_tokens, int64_t n_used) {
+        const int32_t zero = groups[l.ig].n_slots;
+        // pairs grouped by expert, in the order of the experts' first use
+        kexperts.clear();
+        kpair_tok.clear();
+        kpair_wi.clear();
+        for (int32_t e : l.cpu_experts) {
+            kexpert ke = { e, (int32_t) kpair_tok.size(), 0 };
+            for (int64_t t = 0; t < n_tokens; ++t) {
+                for (int64_t k = 0; k < n_used; ++k) {
+                    if (l.ids[t*n_used + k] == e && l.map_dev[e] == zero) {
+                        kpair_tok.push_back((int32_t) t);
+                        kpair_wi.push_back((int32_t) (t*n_used + k));
+                    }
+                }
+            }
+            ke.p1 = (int32_t) kpair_tok.size();
+            if (ke.p1 > ke.p0) {
+                kexperts.push_back(ke);
+            }
+        }
+        const int64_t n_pairs = kpair_tok.size();
+        if (n_pairs == 0) {
+            return;
+        }
+        const int64_t tp0 = ggml_time_us();
+
+        const ggml_tensor * wg = l.gate_up ? l.gate_up : l.gate;
+        const ggml_tensor * wu = l.gate_up ? l.gate_up : l.up;
+        const ggml_tensor * wd = l.down;
+        const int64_t n_ff    = l.down->ne[0];
+        const int64_t u_off   = l.gate_up ? n_ff : 0;   // row offset of up in a fused gate_up
+        const auto * tr_gu = ggml_get_type_traits_cpu(wg->type);
+        const auto * tr_d  = ggml_get_type_traits_cpu(wd->type);
+        GGML_ASSERT(tr_gu->vec_dot && tr_d->vec_dot && (l.gate_up || wu->type == wg->type));
+        const auto * tr_xq = ggml_get_type_traits_cpu(tr_gu->vec_dot_type);
+        const auto * tr_aq = ggml_get_type_traits_cpu(tr_d->vec_dot_type);
+        const size_t xq_row = ggml_row_size(tr_gu->vec_dot_type, n_embd);
+        const size_t aq_row = ggml_row_size(tr_d->vec_dot_type, n_ff);
+        const bool fast = q2_dot && wg->type == GGML_TYPE_Q2_0 && wu->type == GGML_TYPE_Q2_0 && wd->type == GGML_TYPE_Q2_0 &&
+            tr_gu->vec_dot_type == GGML_TYPE_Q8_0 && tr_d->vec_dot_type == GGML_TYPE_Q8_0 && n_embd % 64 == 0 && n_ff % 64 == 0;
+        const int64_t nbx = n_embd/64, nbf = n_ff/64;
+        // i-quant experts with 2+ tokens: one decode of each row for all of an expert's tokens (llama-moe-iq.h, ggml's results)
+        const llama_moe_iq::dot_mt_t mt_gu = fast ? nullptr : iq_mt(wg->type);
+        const llama_moe_iq::dot_mt_t mt_d  = fast ? nullptr : iq_mt(wd->type);
+
+        // the expert input of each token with CPU pairs, quantized once (and prepared for the fast path), one token per thread
+        kxq.resize(xq_row*n_tokens);
+        if (fast) {
+            kxp.resize(nbx*n_tokens);
+            kap.resize(nbf*n_pairs);
+        }
+        kact.resize(n_pairs*n_ff);
+        kaq.resize(aq_row*n_pairs);
+        ktok.clear();
+        for (int64_t t = 0; t < n_tokens; ++t) {
+            if (std::find(kpair_tok.begin(), kpair_tok.end(), (int32_t) t) != kpair_tok.end()) {
+                ktok.push_back((int32_t) t);
+            }
+        }
+        const int64_t n_qtok = ktok.size();
+        auto prep_tok = [&](int64_t t) {
+            tr_xq->from_float(l.x.data() + t*n_embd, kxq.data() + t*xq_row, n_embd);
+            if (fast) {
+                q2_prepare((const q8_block *) (kxq.data() + t*xq_row), n_embd, kxp.data() + t*nbx);
+            }
+        };
+        if (n_qtok == 1) {
+            prep_tok(ktok[0]);
+        } else {
+            pool_run([&](int) {
+                for (int64_t i; (i = pool_next.fetch_add(1, std::memory_order_relaxed)) < n_qtok; ) {
+                    prep_tok(ktok[i]);
+                }
+            });
+        }
+        const bool dot4 = fast && q2_dot4 != nullptr;
+        if (dot4 && dot4_check) {
+            // once: the 4-row kernel against the 1-row one on real data (LLAMA_MOE_CPU_DOT4_CHECK=1)
+            dot4_check = false;
+            const kexpert & ke = kexperts[0];
+            const char * bg = (const char *) wg->data + ke.e*wg->nb[2];
+            float o4[4];
+            q2_dot4(n_embd, (const q2_block *) bg, wg->nb[1], kxp.data() + kpair_tok[ke.p0]*nbx, o4);
+            for (int r = 0; r < 4; ++r) {
+                const float o1 = q2_dot(n_embd, (const q2_block *) (bg + r*wg->nb[1]), kxp.data() + kpair_tok[ke.p0]*nbx);
+                LLAMA_LOG_INFO("llama_moe_cache: dot4 check row %d: %.9g vs %.9g %s\n", r, o4[r], o1, o4[r] == o1 ? "same" : "DIFFERENT");
+            }
+        }
+        const int64_t tp1 = ggml_time_us();
+
+        // rows per gate/up work item (LLAMA_MOE_CPU_RB, a multiple of 4 up to 256; default 16): larger items read longer
+        // contiguous runs of each expert's rows (hardware prefetch); the down phase keeps 16-row output blocks
+        static const int64_t rb_a = [] {
+            const char * v = getenv("LLAMA_MOE_CPU_RB");
+            const int64_t r = v ? atoll(v) : 16;
+            return std::clamp<int64_t>(r - r % 4, 4, 256);
+        }();
+        // LLAMA_MOE_CPU_DOWN_PE=<rows> (opt-in): down work items are (expert, block of <rows> output rows), each reading one
+        // contiguous run of the expert's down rows; their outputs go to a per-pair buffer, summed afterwards in the same
+        // order as the default (expert order, then pair order), so the result is the same bits
+        static const int64_t rb_pe = [] {
+            const char * v = getenv("LLAMA_MOE_CPU_DOWN_PE");
+            const int64_t r = v ? atoll(v) : 0;
+            return r <= 0 ? (int64_t) 0 : std::clamp<int64_t>(r - r % 4, 4, 4096);
+        }();
+        constexpr int64_t rb = 16;  // rows per down output block
+        const int64_t nb_ff = (n_ff + rb_a - 1)/rb_a;
+        const int64_t nb_e  = (n_embd + rb - 1)/rb;
+        const bool clamp = l.act == llama_moe_cache::ACT_SWIGLU_CLAMP;
+        const bool oai   = l.act == llama_moe_cache::ACT_SWIGLU_OAI;
+        const float limit = l.limit;
+        // expert biases: gate and up here; down only for a prefill job (its pairs' router weights are zeroed on the device),
+        // in decode the device adds w*b_down for the CPU's pairs itself (they read the zero slot, then the bias is added)
+        const float * bgb = l.b_gate_up ? (const float *) l.b_gate_up->data : l.b_gate ? (const float *) l.b_gate->data : nullptr;
+        const float * bub = l.b_gate_up ? (const float *) l.b_gate_up->data + n_ff : l.b_up ? (const float *) l.b_up->data : nullptr;
+        const int64_t bgu_e = l.b_gate_up ? 2*n_ff : n_ff;  // bias stride per expert
+        const float * bdb = l.pf_job && l.b_down ? (const float *) l.b_down->data : nullptr;
+
+        // LLAMA_MOE_CPU_HOT_TEST=1 (measurement): the phases run twice, the second time with the experts' weights hot in the
+        // caches; the second run's output is the one kept, both times are summed (an upper bound for pre-reading the experts)
+        static const bool hot_test = getenv("LLAMA_MOE_CPU_HOT_TEST") != nullptr;
+        std::vector<float> out_save;
+        if (hot_test && !l.pf_job) {
+            out_save = l.out;
+        }
+        int64_t tp2 = 0, tp3 = 0, tp4 = 0;
+        for (int rep = 0; rep < (hot_test && !l.pf_job ? 2 : 1); ++rep) {
+        const int64_t tr0 = ggml_time_us();
+        if (rep == 1) {
+            l.out = out_save;
+        }
+        // A: gate and up rows, then the activation, per (expert, row block)
+        const int64_t n_items_a = (int64_t) kexperts.size()*nb_ff;
+        pool_run([&](int) {
+            float g[256*8], u[256*8];
+            for (int64_t it; (it = pool_next.fetch_add(1, std::memory_order_relaxed)) < n_items_a; ) {
+                const kexpert & ke = kexperts[it/nb_ff];
+                const int64_t r0 = (it % nb_ff)*rb_a, r1 = std::min(r0 + rb_a, n_ff);
+                const char * bg = (const char *) wg->data + ke.e*wg->nb[2];
+                const char * bu = (const char *) wu->data + ke.e*wu->nb[2];
+                for (int32_t p = ke.p0; p < ke.p1; p += 8) {
+                    const int32_t pn = std::min<int32_t>(8, ke.p1 - p);
+                    if (dot4 && (r1 - r0) % 4 == 0) {
+                        for (int32_t q = 0; q < pn; ++q) {
+                            const q2_act * xp = kxp.data() + kpair_tok[p + q]*nbx;
+                            for (int64_t r = r0; r < r1; r += 4) {
+                                q2_dot4(n_embd, (const q2_block *) (bg + r*wg->nb[1]), wg->nb[1], xp, &g[q*rb_a + r - r0]);
+                                q2_dot4(n_embd, (const q2_block *) (bu + (r + u_off)*wu->nb[1]), wu->nb[1], xp, &u[q*rb_a + r - r0]);
+                            }
+                        }
+                    } else if (mt_gu && pn > 1) {
+                        const void * ys[8];
+                        float og[8], ou[8];
+                        for (int32_t q = 0; q < pn; ++q) {
+                            ys[q] = kxq.data() + kpair_tok[p + q]*xq_row;
+                        }
+                        for (int64_t r = r0; r < r1; ++r) {
+                            mt_gu(n_embd, bg + r*wg->nb[1], ys, pn, og);
+                            mt_gu(n_embd, bu + (r + u_off)*wu->nb[1], ys, pn, ou);
+                            for (int32_t q = 0; q < pn; ++q) {
+                                g[q*rb_a + r - r0] = og[q];
+                                u[q*rb_a + r - r0] = ou[q];
+                            }
+                        }
+                    } else
+                    for (int64_t r = r0; r < r1; ++r) {
+                        for (int32_t q = 0; q < pn; ++q) {
+                            if (fast) {
+                                const q2_act * xp = kxp.data() + kpair_tok[p + q]*nbx;
+                                g[q*rb_a + r - r0] = q2_dot(n_embd, (const q2_block *) (bg + r*wg->nb[1]), xp);
+                                u[q*rb_a + r - r0] = q2_dot(n_embd, (const q2_block *) (bu + (r + u_off)*wu->nb[1]), xp);
+                                continue;
+                            }
+                            const void * xq = kxq.data() + kpair_tok[p + q]*xq_row;
+                            tr_gu->vec_dot((int) n_embd, &g[q*rb_a + r - r0], 0, bg + r*wg->nb[1], 0, xq, 0, 1);
+                            tr_gu->vec_dot((int) n_embd, &u[q*rb_a + r - r0], 0, bu + (r + u_off)*wu->nb[1], 0, xq, 0, 1);
+                        }
+                    }
+                    for (int32_t q = 0; q < pn; ++q) {
+                        float * a = kact.data() + (p + q)*n_ff;
+                        for (int64_t r = r0; r < r1; ++r) {
+                            float gv = g[q*rb_a + r - r0], uv = u[q*rb_a + r - r0];
+                            if (bgb) {
+                                gv += bgb[ke.e*bgu_e + r];
+                            }
+                            if (bub) {
+                                uv += bub[ke.e*bgu_e + r];
+                            }
+                            if (oai) {
+                                gv = std::min(gv, limit);
+                                uv = std::clamp(uv, -limit, limit);
+                                a[r] = gv/(1.0f + expf(1.702f*(-gv)))*(uv + 1.0f);
+                                continue;
+                            }
+                            if (clamp) {
+                                gv = std::min(gv, limit);
+                                uv = std::clamp(uv, -limit, limit);
+                            }
+                            a[r] = gv/(1.0f + expf(-gv))*uv;
+                        }
+                    }
+                }
+            }
+        });
+        tp2 = ggml_time_us();
+        // A2: the activations quantized for the down projection
+        pool_run([&](int) {
+            for (int64_t p; (p = pool_next.fetch_add(1, std::memory_order_relaxed)) < n_pairs; ) {
+                tr_aq->from_float(kact.data() + p*n_ff, kaq.data() + p*aq_row, n_ff);
+                if (fast) {
+                    q2_prepare((const q8_block *) (kaq.data() + p*aq_row), n_ff, kap.data() + p*nbf);
+                }
+            }
+        });
+        tp3 = ggml_time_us();
+        if (rb_pe > 0) {
+            // B (per expert): y[p, r] for each pair of the item's expert, then the weighted sums per output row block
+            kdy.resize(n_pairs*n_embd);
+            const int64_t nb_pe = (n_embd + rb_pe - 1)/rb_pe;
+            const int64_t n_items_b = (int64_t) kexperts.size()*nb_pe;
+            pool_run([&](int) {
+                for (int64_t it; (it = pool_next.fetch_add(1, std::memory_order_relaxed)) < n_items_b; ) {
+                    const kexpert & ke = kexperts[it/nb_pe];
+                    const int64_t r0 = (it % nb_pe)*rb_pe, r1 = std::min(r0 + rb_pe, n_embd);
+                    const char * bd = (const char *) wd->data + ke.e*wd->nb[2];
+                    if (dot4 && (r1 - r0) % 4 == 0) {
+                        for (int32_t p = ke.p0; p < ke.p1; ++p) {
+                            for (int64_t r = r0; r < r1; r += 4) {
+                                q2_dot4(n_ff, (const q2_block *) (bd + r*wd->nb[1]), wd->nb[1], kap.data() + p*nbf, kdy.data() + p*n_embd + r);
+                            }
+                        }
+                        continue;
+                    }
+                    if (mt_d && ke.p1 - ke.p0 > 1) {
+                        for (int32_t p = ke.p0; p < ke.p1; p += 8) {
+                            const int32_t pn = std::min<int32_t>(8, ke.p1 - p);
+                            const void * ys[8];
+                            float y[8];
+                            for (int32_t q = 0; q < pn; ++q) {
+                                ys[q] = kaq.data() + (p + q)*aq_row;
+                            }
+                            for (int64_t r = r0; r < r1; ++r) {
+                                mt_d(n_ff, bd + r*wd->nb[1], ys, pn, y);
+                                for (int32_t q = 0; q < pn; ++q) {
+                                    kdy[(p + q)*n_embd + r] = y[q];
+                                }
+                            }
+                        }
+                        continue;
+                    }
+                    for (int64_t r = r0; r < r1; ++r) {
+                        for (int32_t p = ke.p0; p < ke.p1; ++p) {
+                            float * y = kdy.data() + p*n_embd + r;
+                            if (fast) {
+                                *y = q2_dot(n_ff, (const q2_block *) (bd + r*wd->nb[1]), kap.data() + p*nbf);
+                            } else {
+                                tr_d->vec_dot((int) n_ff, y, 0, bd + r*wd->nb[1], 0, kaq.data() + p*aq_row, 0, 1);
+                            }
+                        }
+                    }
+                }
+            });
+            pool_run([&](int) {
+                for (int64_t it; (it = pool_next.fetch_add(1, std::memory_order_relaxed)) < nb_e; ) {
+                    const int64_t r0 = it*rb, r1 = std::min(r0 + rb, n_embd);
+                    for (const kexpert & ke : kexperts) {
+                        for (int32_t p = ke.p0; p < ke.p1; ++p) {
+                            float * o = l.out.data() + kpair_tok[p]*n_embd;
+                            const float * y = kdy.data() + p*n_embd;
+                            const float wp = l.w[kpair_wi[p]];
+                            for (int64_t r = r0; r < r1; ++r) {
+                                o[r] += wp*y[r];
+                            }
+                        }
+                    }
+                }
+            });
+        } else
+        // B: down rows per row block of the output, all pairs, so that no two threads add to the same output
+        pool_run([&](int) {
+            for (int64_t it; (it = pool_next.fetch_add(1, std::memory_order_relaxed)) < nb_e; ) {
+                const int64_t r0 = it*rb, r1 = std::min(r0 + rb, n_embd);
+                for (const kexpert & ke : kexperts) {
+                    const char * bd = (const char *) wd->data + ke.e*wd->nb[2];
+                    if (dot4 && (r1 - r0) % 4 == 0) {
+                        // per output (token, row) the pairs still add in pair order
+                        for (int32_t p = ke.p0; p < ke.p1; ++p) {
+                            float* o = l.out.data() + kpair_tok[p]*n_embd;
+                            const float wp = l.w[kpair_wi[p]];
+                            for (int64_t r = r0; r < r1; r += 4) {
+                                float y4[4];
+                                q2_dot4(n_ff, (const q2_block *) (bd + r*wd->nb[1]), wd->nb[1], kap.data() + p*nbf, y4);
+                                for (int j = 0; j < 4; ++j) {
+                                    o[r + j] += wp*y4[j];
+                                }
+                            }
+                        }
+                        continue;
+                    }
+                    if (mt_d && ke.p1 - ke.p0 > 1) {
+                        // per output (token, row) the pairs still add in pair order
+                        for (int32_t p = ke.p0; p < ke.p1; p += 8) {
+                            const int32_t pn = std::min<int32_t>(8, ke.p1 - p);
+                            const void * ys[8];
+                            float y[8];
+                            for (int32_t q = 0; q < pn; ++q) {
+                                ys[q] = kaq.data() + (p + q)*aq_row;
+                            }
+                            for (int64_t r = r0; r < r1; ++r) {
+                                mt_d(n_ff, bd + r*wd->nb[1], ys, pn, y);
+                                for (int32_t q = 0; q < pn; ++q) {
+                                    l.out[kpair_tok[p + q]*n_embd + r] += l.w[kpair_wi[p + q]]*y[q];
+                                }
+                            }
+                        }
+                        continue;
+                    }
+                    for (int64_t r = r0; r < r1; ++r) {
+                        for (int32_t p = ke.p0; p < ke.p1; ++p) {
+                            float y;
+                            if (fast) {
+                                y = q2_dot(n_ff, (const q2_block *) (bd + r*wd->nb[1]), kap.data() + p*nbf);
+                            } else {
+                                tr_d->vec_dot((int) n_ff, &y, 0, bd + r*wd->nb[1], 0, kaq.data() + p*aq_row, 0, 1);
+                            }
+                            l.out[kpair_tok[p]*n_embd + r] += l.w[kpair_wi[p]]*y;
+                        }
+                    }
+                }
+            }
+        });
+        if (bdb) {
+            // prefill job: + w*b_down per pair, per output row block (no two threads add to the same output)
+            pool_run([&](int) {
+                for (int64_t it; (it = pool_next.fetch_add(1, std::memory_order_relaxed)) < nb_e; ) {
+                    const int64_t r0 = it*rb, r1 = std::min(r0 + rb, n_embd);
+                    for (const kexpert & ke : kexperts) {
+                        const float * b = bdb + ke.e*n_embd;
+                        for (int32_t p = ke.p0; p < ke.p1; ++p) {
+                            float * o = l.out.data() + kpair_tok[p]*n_embd;
+                            const float wp = l.w[kpair_wi[p]];
+                            for (int64_t r = r0; r < r1; ++r) {
+                                o[r] += wp*b[r];
+                            }
+                        }
+                    }
+                }
+            });
+        }
+        if (hot_test && !l.pf_job) {
+            t_hot[rep] += ggml_time_us() - tr0;
+            n_hot += rep;
+        }
+        }
+        if (l.pf_job) {
+            return;
+        }
+        n_cpu_pairs += n_pairs;
+        n_ph_kexp += kexperts.size();
+        for (const kexpert & ke : kexperts) {
+            n_ph_mt += mt_gu && ke.p1 - ke.p0 > 1;
+        }
+        tp4 = ggml_time_us();
+        t_ph[0] += tp0 - l.t_start; t_ph[1] += tp1 - tp0; t_ph[2] += tp2 - tp1; t_ph[3] += tp3 - tp2; t_ph[4] += tp4 - tp3;
+        n_ph++;
+    }
+
+    // worker thread: the weighted sum of the experts of layer l that the CPU computes, into l.out [n_embd, n_tokens]
+    void compute_cpu(layer & l) {
+        const int64_t n_tokens = l.x.size()/n_embd;
+        const int64_t n_used   = l.w.size()/std::max<int64_t>(n_tokens, 1);
+
+        l.out.assign(n_embd*n_tokens, 0.0f);
+        if (l.cpu_experts.empty()) {
+            l.t_job = ggml_time_us() - l.t_start;
+            return;
+        }
+        const int64_t t0 = ggml_time_us();
+
+        if (use_kernel) {
+            compute_cpu_kernel(l, n_tokens, n_used);
+            l.t_job   = ggml_time_us() - l.t_start;
+            if (l.pf_job) {
+                t_pf_job_us += l.t_job;
+            } else {
+                t_cpu_us += ggml_time_us() - t0;
+            }
+            return;
+        }
+
+        // the (token, expert) pairs of the experts the CPU computes
+        const int32_t zero = groups[l.ig].n_slots;
+        pair_tok.clear();
+        pair_wi.clear();
+        for (int64_t t = 0; t < n_tokens; ++t) {
+            for (int64_t k = 0; k < n_used; ++k) {
+                if (l.map_dev[l.ids[t*n_used + k]] == zero) {
+                    pair_tok.push_back((int32_t) t);
+                    pair_wi.push_back((int32_t) (t*n_used + k));
+                }
+            }
+        }
+        const int64_t n_pairs = pair_tok.size();
+
+        cpu_graph & cg = get_cpu_graph(l, n_pairs);
+        auto point = [](ggml_tensor * node, ggml_tensor * w) {
+            if (node) {
+                GGML_ASSERT(w && w->type == node->src[0]->type && ggml_are_same_shape(w, node->src[0]));
+                node->src[0] = w;
+            }
+        };
+        point(cg.mm_gate_up, l.gate_up);
+        point(cg.mm_up,      l.up);
+        point(cg.mm_gate,    l.gate);
+        point(cg.mm_down,    l.down);
+        ggml_tensor * x = cg.x, * sel = cg.sel, * y = cg.y;
+        ggml_cgraph * gf = cg.gf;
+
+        float   * xd = (float   *) x->data;
+        int32_t * sd = (int32_t *) sel->data;
+        for (int64_t p = 0; p < n_pairs; ++p) {
+            memcpy(xd + p*n_embd, l.x.data() + (int64_t) pair_tok[p]*n_embd, n_embd*sizeof(float));
+            sd[p] = l.ids[pair_wi[p]];
+        }
+        ggml_backend_graph_compute(cpu_backend.get(), gf);
+
+        const float * yd = (const float *) y->data;
+        for (int64_t p = 0; p < n_pairs; ++p) {
+            const float w = l.w[pair_wi[p]];
+            float * o = l.out.data() + (int64_t) pair_tok[p]*n_embd;
+            const float * yp = yd + p*n_embd;
+            for (int64_t i = 0; i < n_embd; ++i) {
+                o[i] += w*yp[i];
+            }
+        }
+
+        n_cpu_pairs += n_pairs;
+        t_cpu_us    += ggml_time_us() - t0;
+        l.t_job      = ggml_time_us() - l.t_start;
+    }
+
+    // hybrid mode: register the current graph's tensors of a cached layer
+    ggml_tensor * set_hybrid_layer(int32_t il, ggml_tensor * cur, ggml_tensor * weights, llama_moe_cache::act_type act, float limit) {
+        if (!hybrid || il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0 || layers[il].cpu_out == nullptr) {
+            return nullptr;
+        }
+        layer & l = layers[il];
+        if (cur == nullptr || act == llama_moe_cache::ACT_NONE || l.down == nullptr || !(l.gate_up || (l.up && l.gate)) ||
+                !cpu_can(l, act)) {
+            l.x_dev = nullptr;
+            return nullptr;
+        }
+        l.x_dev = cur;
+        l.w_dev = weights;
+        l.act   = act;
+        l.limit = limit;
+        return l.cpu_out;
+    }
+
     void log_stats() const {
         auto log = [](const char * name, const stats & st) {
             const size_t n = st.hits + st.misses;
@@ -610,6 +2963,57 @@ struct llama_moe_cache::impl {
         };
         log("ubatch <= 8", stats_small);
         log("ubatch  > 8", stats_large);
+        if (hybrid) {
+            LLAMA_LOG_INFO("llama_moe_cache: hybrid: %zu missed experts (%zu token-expert pairs) computed by the CPU, %.1f us per pair\n",
+                n_cpu_experts, n_cpu_pairs, n_cpu_pairs ? t_cpu_us/n_cpu_pairs : 0.0);
+            if (tail_rank > 0) {
+                LLAMA_LOG_INFO("llama_moe_cache: tail skip (rank >= %d): %zu missed experts skipped\n", tail_rank, n_tail_skip);
+            }
+            if (hybrid_auto) {
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid auto share: %.2f at the end, %zu changes%s\n", hybrid_share, n_share_changes,
+                    share_rule ? (" (rule, CPU " + std::to_string(rule_R_last).substr(0, 5) + " GB/s, link " + std::to_string(link_gbs).substr(0, 5) + " GB/s)").c_str() : "");
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid gate: %zu probes, %zu decided off, %zu ubatches with the CPU path, %zu without, ends %s\n",
+                    n_gate_probes, n_gate_off, n_ubatch_on, n_ubatch_off, gate_on ? "on" : "off");
+            }
+            if (flag_mode) {
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid flag mode: the GPU waited %.1f us per layer call for the CPU, %zu timeouts\n",
+                    t_flag_wait_us/std::max<size_t>(n_flag_waits, 1), n_flag_timeouts);
+            }
+            if (guest) {
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid: the main thread joined %zu CPU job phases\n", n_guest_items);
+            }
+            if (n_hot > 0) {
+                LLAMA_LOG_INFO("llama_moe_cache: hot test: CPU job phases %.1f us cold, %.1f us hot (%zu jobs)\n", t_hot[0]/n_hot, t_hot[1]/n_hot, n_hot);
+            }
+            if (n_ph > 0) {
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid CPU job phases (us per job): wake %.1f, prep %.1f, gate/up %.1f, act quant %.1f, down %.1f, post %.1f (%zu jobs)\n",
+                    t_ph[0]/n_ph, t_ph[1]/n_ph, t_ph[2]/n_ph, t_ph[3]/n_ph, t_ph[4]/n_ph, t_ph[5]/n_ph, n_ph);
+                LLAMA_LOG_INFO("llama_moe_cache: hybrid CPU jobs: %.2f tokens per expert, %.1f%% of experts by the multi-token i-quant kernels\n",
+                    n_ph_kexp ? (double) n_cpu_pairs/n_ph_kexp : 0.0, n_ph_kexp ? 100.0*n_ph_mt/n_ph_kexp : 0.0);
+            }
+            if (doorbell) {
+                const double nd = std::max<size_t>(n_db_calls, 1);
+                LLAMA_LOG_INFO("llama_moe_cache: doorbell: %zu calls, host %.1f us per call, GPU waited %.1f us per call for the host, %zu timeouts\n",
+                    n_db_calls, t_db_us/nd, t_db_gpu_wait_us/nd, n_db_timeouts);
+            }
+            const double n = std::max<size_t>(n_hybrid_calls, 1);
+            LLAMA_LOG_INFO("llama_moe_cache: hybrid per layer call (us): sync %.1f, plan+uploads %.1f, until cpu_out %.1f, wait for CPU %.1f, CPU job %.1f (%zu calls)\n",
+                t_sync_us/n, t_upload_us/n, t_gap_us/n, t_wait_us/n, t_cpu_us/n, n_hybrid_calls);
+        }
+        if (preread) {
+            LLAMA_LOG_INFO("llama_moe_cache: pre-read: %zu jobs, %zu experts, %.1f GiB read, %.1f us per job, %zu stopped by the next job\n",
+                n_pr_jobs, n_pr_experts, pr_bytes/1073741824.0, t_pr_us/std::max<size_t>(n_pr_jobs, 1), n_pr_aborted);
+        }
+        if (la_backend) {
+            LLAMA_LOG_INFO("llama_moe_cache: lookahead: %zu jobs, %zu experts prefetched, %zu used (%.1f%%), %zu stopped at the running layer, %zu cancelled, "
+                "%.1f us per job, callbacks waited %.1f us in all, %zu stream waits\n", n_la_jobs, n_la_issued, n_la_useful, 100.0*n_la_useful/std::max<size_t>(n_la_issued, 1),
+                n_la_skipped, n_la_cancelled, t_la_us/std::max<size_t>(n_la_jobs, 1), t_la_wait_us, n_la_joins);
+        }
+        if (pf && n_pf_layers > 0) {
+            LLAMA_LOG_INFO("llama_moe_cache: prefill share: %zu layer calls, %zu of %zu missed experts on the CPU (%zu pairs), CPU %.0f us per expert, GPU %.0f us per upload (last), "
+                "per layer: plan %.0f us, CPU job %.0f us, GPU waited %.0f us, %zu mismatches\n",
+                n_pf_layers, n_pf_cpu, n_pf_stream, n_pf_pairs, pf_c_us, pf_g_us, t_pf_plan_us/n_pf_layers, t_pf_job_us/n_pf_layers, t_pf_wait_us/n_pf_layers, n_pf_mismatch);
+        }
         if (stats_copy.hits > 0) {
             LLAMA_LOG_INFO("llama_moe_cache: large batches: %zu experts copied from the cache, %.2f MiB\n", stats_copy.hits, stats_copy.bytes/1024.0/1024.0);
         }
@@ -635,7 +3039,33 @@ ggml_tensor * llama_moe_cache::get_experts(const ggml_tensor * w) const {
 }
 
 bool llama_moe_cache::copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph) {
-    return pimpl->copy(backend, src, dst, graph);
+    return pimpl->copy(backend, src, dst, graph) || (pimpl->hybrid && pimpl->copy_out(backend, src, dst));
+}
+
+bool llama_moe_cache::hybrid_configured() const {
+    return pimpl->hybrid;
+}
+
+bool llama_moe_cache::hybrid_enabled() const {
+    return pimpl->hybrid && (!pimpl->hybrid_auto || pimpl->path_on);
+}
+
+bool llama_moe_cache::take_graph_change() const {
+    const bool res = pimpl->gate_changed;
+    pimpl->gate_changed = false;
+    return res;
+}
+
+ggml_tensor * llama_moe_cache::set_hybrid_layer(int32_t il, ggml_tensor * cur, ggml_tensor * weights, act_type act, float limit) const {
+    return pimpl->set_hybrid_layer(il, cur, weights, act, limit);
+}
+
+ggml_tensor * llama_moe_cache::doorbell_slots(ggml_context * ctx, int32_t il, ggml_tensor * ids, ggml_tensor * x, ggml_tensor * w, int64_t n_tokens, int64_t n_used) const {
+    return pimpl->doorbell_slots(ctx, il, ids, x, w, n_tokens, n_used);
+}
+
+ggml_tensor * llama_moe_cache::hybrid_add(ggml_context * ctx, int32_t il, ggml_tensor * moe_out) const {
+    return pimpl->hybrid_add(ctx, il, moe_out);
 }
 
 int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last) {
@@ -654,3 +3084,27 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown()
     }
     return res;
 }
+
+bool llama_moe_cache::prefill_enabled(int64_t n_tokens) const {
+    return pimpl->prefill_enabled(n_tokens);
+}
+
+ggml_tensor * llama_moe_cache::set_prefill_layer(int32_t il, ggml_tensor * x, ggml_tensor * w, const ggml_tensor * ids, act_type act, float limit) const {
+    return pimpl->set_prefill_layer(il, x, w, ids, act, limit);
+}
+
+ggml_tensor * llama_moe_cache::prefill_add(ggml_context * ctx, int32_t il, ggml_tensor * moe_out) const {
+    return pimpl->prefill_add(ctx, il, moe_out);
+}
+
+bool llama_moe_cache::prefill_plan(ggml_backend_t backend, const ggml_tensor * w, const ggml_tensor * ids, std::vector<int32_t> & ids_data, std::vector<bool> & used) {
+    return pimpl->prefill_plan(backend, w, ids, ids_data, used);
+}
+
+ggml_tensor * llama_moe_cache::lookahead_router(int32_t il, ggml_tensor ** bias, int * k) const {
+    return pimpl->lookahead_router(il, bias, k);
+}
+
+void llama_moe_cache::set_lookahead_layer(int32_t il, ggml_tensor * pred) const {
+    pimpl->set_lookahead_layer(il, pred);
+}
diff --git a/src/llama-moe-cache.h b/src/llama-moe-cache.h
index 6c8e36582..59b52d1b9 100644
--- a/src/llama-moe-cache.h
+++ b/src/llama-moe-cache.h
@@ -8,8 +8,9 @@
 
 struct llama_model;
 
-// keeps recently and repeatedly used experts of host-resident MoE layers in a device buffer
+// keeps the most recently used experts of host-resident MoE layers in a device buffer
 // each layer has a slot map in host memory: when the scheduler copies it to the device, the copy callback uploads the missing experts
+// (or, in hybrid mode, part of them, the CPU computes the others)
 class llama_moe_cache {
 public:
     // backends are all the backends of the context, each GPU gets its own cache of the given size for the layers assigned to it
@@ -34,6 +35,59 @@ public:
 
     std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
 
+    // hybrid mode (LLAMA_MOE_CPU=<share of the misses uploaded, 0..1>): the CPU computes the other missed experts while the GPU
+    // computes the cached ones, and the GPU adds the CPU's output (cpu_out) to the MoE output
+    // ACT_SWIGLU_OAI: gpt-oss, min(g, limit) * sigmoid(1.702 min(g, limit)) * (clamp(u, -limit, limit) + 1)
+    enum act_type { ACT_NONE = -1, ACT_SWIGLU = 0, ACT_SWIGLU_CLAMP = 1, ACT_SWIGLU_OAI = 2 };
+
+    // hybrid mode is configured (LLAMA_MOE_CPU is set)
+    bool hybrid_configured() const;
+
+    // the graph being built should compute the missed experts on the CPU: configured, and in auto mode not switched off by
+    // the A/B gate (which compares ubatch times with the CPU path on and off)
+    bool hybrid_enabled() const;
+
+    // true once after the gate switched the CPU path on or off: the previous graph must not be reused
+    bool take_graph_change() const;
+
+    // called while building the graph of a cached layer: cur [n_embd, n_tokens] and weights [1, n_expert_used, n_tokens] are the
+    // contiguous expert input and router weights, computed before the slot lookup; returns the host tensor to add to the MoE
+    // output (a view of it for n_tokens), nullptr if the layer is not computed in hybrid mode
+    ggml_tensor * set_hybrid_layer(int32_t il, ggml_tensor * cur, ggml_tensor * weights, act_type act, float limit) const;
+
+    // moe_out [n_embd, n_tokens] plus the output of the experts the CPU computes, for a layer set_hybrid_layer returned a tensor for:
+    // with pinned host memory the GPU waits for a flag the CPU sets, so the graph is launched before the CPU is done
+    // (LLAMA_MOE_CPU_COPY=1, or pageable memory: cpu_out is copied as a host weight once the CPU is done)
+    ggml_tensor * hybrid_add(ggml_context * ctx, int32_t il, ggml_tensor * moe_out) const;
+
+    // doorbell mode (LLAMA_MOE_DOORBELL=1): the slots of the selected experts ids [n_used*n_tokens] for up to 8 tokens, from a device op
+    // that hands them (and in hybrid mode x and w, see set_hybrid_layer) to a host thread; nullptr if not used for this layer and size
+    ggml_tensor * doorbell_slots(ggml_context * ctx, int32_t il, ggml_tensor * ids, ggml_tensor * x, ggml_tensor * w, int64_t n_tokens, int64_t n_used) const;
+
+    // lookahead prefetch (LLAMA_MOE_LOOKA=<experts per layer>): the router of layer il+1 and its selection bias (may be nullptr),
+    // which the graph applies to layer il's expert input; k = predicted experts per token. nullptr: layer il does not predict
+    ggml_tensor * lookahead_router(int32_t il, ggml_tensor ** bias, int * k) const;
+
+    // registers the predicted experts [k, n_tokens] (I32, contiguous, computed before layer il's slot lookup), nullptr clears
+    void set_lookahead_layer(int32_t il, ggml_tensor * pred) const;
+
+    // prefill share (LLAMA_MOE_PREFILL_CPU, hybrid flag mode): ubatches of n_tokens on the large-batch path hand the CPU part of
+    // the missed experts
+    bool prefill_enabled(int64_t n_tokens) const;
+
+    // called while building a large-batch graph for each cached layer: x [n_embd, n_tokens] and w [1, n_expert_used, n_tokens] are
+    // the contiguous expert input and router weights (w the one the expert outputs are multiplied by), ids the selected experts
+    // the MUL_MAT_IDs read; x nullptr clears the layer. Returns non-null if the layer takes the share: then the graph must add it
+    // with prefill_add
+    ggml_tensor * set_prefill_layer(int32_t il, ggml_tensor * x, ggml_tensor * w, const ggml_tensor * ids, act_type act, float limit) const;
+
+    // moe_out [n_embd, n_tokens] plus the CPU's experts (the GPU waits for a flag, see ggml_host_add)
+    ggml_tensor * prefill_add(ggml_context * ctx, int32_t il, ggml_tensor * moe_out) const;
+
+    // large-batch copy callback, once per ubatch and layer after the selected experts were read (ids_data, with ids' strides):
+    // takes the CPU's experts out of used and rewrites ids and the router weights on the device; false if the layer has no share
+    bool prefill_plan(ggml_backend_t backend, const ggml_tensor * w, const ggml_tensor * ids, std::vector<int32_t> & ids_data, std::vector<bool> & used);
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 9ddd6a63a..bda0a71cf 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -301,7 +301,19 @@ static void llama_tensor_dequantize_impl(
 
 static bool tensor_allows_quantization(const llama_model_quantize_params * params, llm_arch arch, const ggml_tensor * tensor) {
     // trivial checks first -- no string ops needed
-    if (params->only_copy)       return false;
+    if (params->only_copy) {
+        // COPY with --tensor-type: requantize only the tensors an override names, copy everything else
+        if (params->tt_overrides == nullptr) {
+            return false;
+        }
+        const std::string name = ggml_get_name(tensor);
+        for (const auto * p = params->tt_overrides; p->pattern != nullptr; p++) {
+            if (std::regex_search(name, std::regex(p->pattern))) {
+                return true;
+            }
+        }
+        return false;
+    }
 
     // quantize only 2D+ tensors (matrices, experts, conv kernels)
     if (ggml_n_dims(tensor) < 2) return false;
@@ -757,6 +769,17 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod
     if (!tensor_allows_quantization(params, qs.model.arch, tensor)) {
         return tensor->type;
     }
+    if (params->only_copy) { // COPY with --tensor-type: the override's type, nothing else
+        const std::string tensor_name(tensor->name);
+        for (const auto & [pattern, qtype] : qs.tensor_type_patterns) {
+            if (std::regex_search(tensor_name, pattern)) {
+                LLAMA_LOG_WARN("%s: %-36s - applying manual override: %s -> %s\n",
+                               __func__, tensor_name.c_str(), ggml_type_name(tensor->type), ggml_type_name(qtype));
+                return tensor_type_fallback(qs, tensor, qtype);
+            }
+        }
+        return tensor->type;
+    }
     if (params->token_embedding_type < GGML_TYPE_COUNT && tm.category == tensor_category::TOKEN_EMBD) {
         // per_layer_token_embd follows --token-embedding-type by default, but it is a large
         // separate table, so let an explicit --tensor-type name it
diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp
index 2e15777ee..07780ef17 100644
--- a/src/models/delta-net-base.cpp
+++ b/src/models/delta-net-base.cpp
@@ -393,10 +393,12 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne
     GGML_ASSERT(g->ne[0] == 1   || g->ne[0] == S_v);
     GGML_ASSERT(                   g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);
     GGML_ASSERT(b->ne[0] == 1   && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);
-    GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v      && s->ne[3] == n_seqs);
+    GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v      && (gdn_state_ids || s->ne[3] == n_seqs));
 
     // K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs].
-    ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
+    ggml_tensor * result = gdn_state_ids || gdn_gate_dt ?
+        ggml_gated_delta_net_ext(ctx0, q, k, v, g, b, s, gdn_state_ids, gdn_gate_dt, gdn_gate_a, /*K=*/1) :
+        ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
     if (n_tokens == 1) {
         res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il});
     } else {
@@ -537,7 +539,7 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
 
     const int64_t S_v          = s->ne[0];
     const int64_t H_v          = s->ne[2];
-    const int64_t n_seqs       = s->ne[3];
+    const int64_t n_seqs       = v->ne[3];
     const int64_t n_seq_tokens = q->ne[2];
 
     const bool keep = cparams.n_rs_seq > 0;
@@ -561,7 +563,9 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
     const int64_t K = cparams.n_rs_seq + 1;
 
     // state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output.
-    ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
+    ggml_tensor * gdn_out = gdn_state_ids || gdn_gate_dt ?
+        ggml_gated_delta_net_ext(ctx0, q, k, v, g, b, s, gdn_state_ids, gdn_gate_dt, gdn_gate_a, K) :
+        ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
     if (n_seq_tokens > 1) {
         res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il});
     } else {
diff --git a/src/models/models.h b/src/models/models.h
index 6c938c458..d01253f28 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -98,6 +98,14 @@ struct llm_build_delta_net_base : public llm_graph_context {
             ggml_tensor *        b,
             ggml_tensor *        s,
             int                  il);
+
+    // when set, s is the whole state cache [S_v, S_v, H_v, n_rows] and the fused op reads row
+    // gdn_state_ids[i] for sequence i (single-sequence ubatches only, see qwen4exp)
+    ggml_tensor * gdn_state_ids = nullptr;
+    // when set, g and b are the raw alpha/beta projections and the fused op applies
+    // g = softplus(g + gdn_gate_dt) * gdn_gate_a, b = sigmoid(b)
+    ggml_tensor * gdn_gate_dt = nullptr;
+    ggml_tensor * gdn_gate_a  = nullptr;
 };
 
 struct llm_build_lfm_base : public llm_graph_context {
@@ -2768,10 +2776,15 @@ struct llama_model_qwen4exp : public llama_model_base {
         graph(const llama_model & model, const llm_graph_params & params, no_build) :
             llm_build_delta_net_base(params), model(model) {}
 
+        // LLAMA_HC_FUSE: the shared expert's output and gate logit, added by the next hc combine instead of a separate add
+        ggml_tensor * shexp_pend      = nullptr;
+        ggml_tensor * shexp_gate_pend = nullptr;
+
         // HC replaces every layer norm: residual is [n_embd, hc, n_tokens]
         struct hc_mix {
             ggml_tensor * cur;     // the mixed [n_embd, T] stream
             ggml_tensor * inject;  // [hc, T] scatter weights, null when the inject weights are null
+            float inject_scale = 1.0f; // scale the combine still applies to inject (LLAMA_HC_FUSE)
         };
 
         hc_mix build_hc_mix(
@@ -2784,10 +2797,10 @@ struct llama_model_qwen4exp : public llama_model_base {
                             int   il);
 
         ggml_tensor * build_hc_combine(
-                    ggml_tensor * residual,
-                    ggml_tensor * block_out,
-                    ggml_tensor * inject,
-                            int   il);
+                    ggml_tensor *  residual,
+                    ggml_tensor *  block_out,
+                    const hc_mix & mix,
+                            int    il);
 
         ggml_tensor * build_layer_attn(
               llm_graph_input_attn_kv * inp_attn,
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index c3e2b010d..e91fed6ea 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -363,19 +363,26 @@ llama_model_qwen4exp::graph::build_hc_mix(
     xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt);
     cb(xn, "hc_norm", il);
 
+    static const bool hc_fuse = getenv("LLAMA_HC_FUSE") != nullptr && atoi(getenv("LLAMA_HC_FUSE")) != 0;
     ggml_tensor * lo = nullptr;
     ggml_tensor * inject = nullptr;
+    float inject_scale = 1.0f;
     if (w_down_inject) {
         // one matmul for both: the hc_down rows, then the hc_inject rows
         const int64_t n_lo = w_down_inject->ne[1] - hc;
         ggml_tensor * lo_inject = build_lora_mm(w_down_inject, xn);
         // scale the combined rows before the split; both lo and inject use 1/hc
-        lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc);
+        if (!model.hc_scale_folded.count(w_down_inject)) {
+            lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc);
+        }
         lo     = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0);
         inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo));
         cb(inject, "hc_inject", il);
     } else {
-        lo = ggml_scale(ctx0, build_lora_mm(w_down, xn), 1.0f / (float) hc);
+        lo = build_lora_mm(w_down, xn);
+        if (!model.hc_scale_folded.count(w_down)) { // LLAMA_HC_FUSE: else the 1/hc scale is in the weights
+            lo = ggml_scale(ctx0, lo, 1.0f / (float) hc);
+        }
     }
     lo = ggml_silu(ctx0, lo);
     ggml_tensor * gate = build_lora_mm(w_up, lo);
@@ -407,21 +414,45 @@ llama_model_qwen4exp::graph::build_hc_mix(
     cb(mixed, "hc_mixed", il);
 
     if (!w_down_inject && w_inject) {
-        inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc);
+        inject = build_lora_mm(w_inject, xn);
         cb(inject, "hc_inject", il);
+        if (hc_fuse) {
+            // the hc combine applies the 1/hc scale itself; put the inject projection right after the mix in the node
+            // order, so the CUDA hc mix fusion can compute it with the down projection (it reads the same normed input)
+            inject_scale = 1.0f / (float) hc;
+            ggml_build_forward_expand(gf, mixed);
+            ggml_build_forward_expand(gf, inject);
+        } else {
+            inject = ggml_scale(ctx0, inject, 1.0f / (float) hc);
+        }
     }
 
-    return { mixed, inject };
+    return { mixed, inject, inject_scale };
 }
 
 ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine(
-        ggml_tensor * residual,
-        ggml_tensor * block_out,
-        ggml_tensor * inject,
-        int           il) {
+        ggml_tensor *  residual,
+        ggml_tensor *  block_out,
+        const hc_mix & mix,
+        int            il) {
     const int64_t hc = hparams.dsv4_hc_mult;
     const int64_t nt = residual->ne[2];
 
+    ggml_tensor * inject = mix.inject;
+    static const bool hc_fuse = getenv("LLAMA_HC_FUSE") != nullptr && atoi(getenv("LLAMA_HC_FUSE")) != 0;
+    if (hc_fuse && cparams.fused_dsv4_hc_post && il >= 0) {
+        // LLAMA_HC_FUSE: the op takes the injection logits and applies 2*sigmoid(x*inject_scale) itself
+        ggml_tensor * cur = ggml_dsv4_hc_post_sig(ctx0, block_out, residual, inject, nullptr, mix.inject_scale, 2.0f,
+                shexp_pend, shexp_gate_pend);
+        shexp_pend = shexp_gate_pend = nullptr;
+        res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, cur, il});
+        cb(cur, "hc_combine", il);
+        return cur;
+    }
+
+    if (mix.inject_scale != 1.0f) {
+        inject = ggml_scale(ctx0, inject, mix.inject_scale);
+    }
     // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add
     ggml_tensor * w = ggml_sigmoid(ctx0, inject);
     w = ggml_scale(ctx0, w, 2.0f);
@@ -532,7 +563,7 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
             res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);
         }
 
-        res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
+        res_hc = build_hc_combine(res_hc, cur, attn_mix, il);
 
         hc_mix ffn_mix = build_hc_mix(res_hc,
                 model.layers[il].hc_ffn_norm,
@@ -546,7 +577,7 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
         cur = build_layer_ffn(cur, il);
         cb(cur, "ffn_out", il);
 
-        res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
+        res_hc = build_hc_combine(res_hc, cur, ffn_mix, il);
 
         // "l_last" is the layer output name that build_cvec and imatrix look for
         cb(res_hc, "l_last", il);
@@ -606,15 +637,29 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
     ggml_set_input(inp->h);
     ggml_set_name(inp->h, "mtp_h_input");
 
+    // LLAMA_MTP_CHAIN: a 1-token step reads its token and h from the chain buffers (see llama_mtp_chain_*); the host
+    // inputs stay in the graph (set_input mirrors them into the buffers unless a chain is active)
+    const bool chain = model.mtp_chain_tok != nullptr && n_tokens == 1 && n_outputs == 1 && ubatch.token;
+    // any step with one output also writes the buffers, so the first draft step (the deferred catch-up plus the
+    // last token) can start a chain without a host round trip
+    const bool chain_write = model.mtp_chain_tok != nullptr && n_outputs == 1;
+
     ggml_tensor * tok_embd;
-    if (ubatch.token) {
+    if (chain) {
+        tok_embd = ggml_get_rows(ctx0, model.tok_embd, model.mtp_chain_tok);
+        ggml_build_forward_expand(gf, inp->tokens);
+        ggml_build_forward_expand(gf, inp->h);
+        inp->chain_tok    = model.mtp_chain_tok;
+        inp->chain_h      = model.mtp_chain_h;
+        inp->chain_active = &model.mtp_chain_active;
+    } else if (ubatch.token) {
         tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp->tokens);
     } else {
         tok_embd = inp->embd;
     }
     cb(tok_embd, "mtp_tok_embd", il);
 
-    ggml_tensor * h = inp->h;
+    ggml_tensor * h = chain ? ggml_reshape_2d(ctx0, model.mtp_chain_h, hparams.n_embd_out(), 1) : inp->h;
 
     res->add_input(std::move(inp));
 
@@ -647,12 +692,12 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
     hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il);
     ggml_tensor * cur    = attn_mix.cur;
     cur    = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
-    res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
+    res_hc = build_hc_combine(res_hc, cur, attn_mix, il);
 
     hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il);
     cur    = ffn_mix.cur;
     cur    = build_layer_ffn(cur, il);
-    res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
+    res_hc = build_hc_combine(res_hc, cur, ffn_mix, il);
 
     // the next draft step reads this residual as its h
     ggml_tensor * flat     = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens);
@@ -668,9 +713,24 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur, model.output_s);
+    if (model.output_draft) {
+        // score the subset only, then scatter into full-width logits that are -inf elsewhere (the drafter takes the argmax)
+        ggml_tensor * sub = ggml_mul_mat(ctx0, model.output_draft, cur);                          // [n_sub, n_out]
+        ggml_tensor * full = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, sub->ne[1], model.output->ne[1]), -INFINITY);
+        full = ggml_set_rows(ctx0, full, ggml_cont(ctx0, ggml_transpose(ctx0, sub)), model.output_draft_ids); // [n_out, n_vocab]
+        cur = ggml_cont(ctx0, ggml_transpose(ctx0, full));                                        // [n_vocab, n_out]
+    } else {
+        cur = build_lora_mm(model.output, cur, model.output_s);
+    }
     cb(cur, "result_output", -1);
     res->t_logits = cur;
+
+    if (chain_write) {
+        // the next chained step reads the greedy token and this step's h; written last, after every read
+        ggml_tensor * tok = ggml_argmax(ctx0, cur);                                                   // I32 [1]
+        ggml_build_forward_expand(gf, ggml_cpy(ctx0, tok, model.mtp_chain_tok));
+        ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_reshape_1d(ctx0, flat_out, ggml_nelements(flat_out)), model.mtp_chain_h));
+    }
 }
 
 std::pair<ggml_tensor *, ggml_tensor *> llama_model_qwen4exp::graph::build_qkvz(
@@ -1126,11 +1186,42 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
     ggml_tensor * qkv_mixed = qkvz.first;
     ggml_tensor * z         = qkvz.second;
 
+    // LLAMA_GDN_DIRECT: the fused GDN op reads the ubatch state straight from the cache by row index,
+    // which drops the per-layer gather of the whole state (single-sequence ubatches, fused op only)
+    static const bool gdn_direct = getenv("LLAMA_GDN_DIRECT") && atoi(getenv("LLAMA_GDN_DIRECT")) != 0;
+    const bool gdn_fused = cparams.n_rs_seq > 0 || (n_seq_tokens == 1 ? cparams.fused_gdn_ar : cparams.fused_gdn_ch);
+
     ggml_tensor * beta  = nullptr;
     ggml_tensor * alpha = nullptr;
-    if (model.layers[il].ssm_beta_alpha_cat) {
+    ggml_tensor * gate  = nullptr;
+    gdn_gate_dt = nullptr;
+    gdn_gate_a  = nullptr;
+    // [beta | alpha] in one weight: merged by the converter, or at load (LLAMA_GDN_GATES)
+    ggml_tensor * ba_w = model.layers[il].ssm_beta_alpha_cat ? model.layers[il].ssm_beta_alpha_cat : model.layers[il].ssm_ba_cat;
+    // up to 8 tokens (decode and verify), where float weights take the vector kernel; larger batches would go to
+    // GEMM kernels whose results depend on the row count
+    if (ba_w && gdn_fused && n_seq_tokens * n_seqs <= 8 &&
+            model.layers[il].ssm_dt->type == GGML_TYPE_F32 && model.layers[il].ssm_a->type == GGML_TYPE_F32) {
+        // LLAMA_GDN_GATES: one matvec for beta and alpha; the fused op applies the bias, softplus, A and sigmoid
+        static const int gates_mode = atoi(getenv("LLAMA_GDN_GATES"));
+        if (gates_mode == 2 && model.layers[il].ssm_beta && model.layers[il].ssm_alpha) {
+            // diagnostic: the two separate matvecs, activations in the fused op
+            beta = ggml_reshape_4d(ctx0, build_lora_mm(model.layers[il].ssm_beta, cur), 1, num_v_heads, n_seq_tokens, n_seqs);
+            gate = ggml_reshape_4d(ctx0, build_lora_mm(model.layers[il].ssm_alpha, cur), 1, num_v_heads, n_seq_tokens, n_seqs);
+        } else {
+        ggml_tensor * ba = build_lora_mm(ba_w, cur); // [2*H, n_tokens]
+        cb(ba, "beta_alpha_raw", il);
+        beta = ggml_view_4d(ctx0, ba, 1, num_v_heads, n_seq_tokens, n_seqs,
+                ggml_row_size(ba->type, 1), ba->nb[1], ba->nb[1]*n_seq_tokens, 0);
+        gate = ggml_view_4d(ctx0, ba, 1, num_v_heads, n_seq_tokens, n_seqs,
+                ggml_row_size(ba->type, 1), ba->nb[1], ba->nb[1]*n_seq_tokens, ggml_row_size(ba->type, num_v_heads));
+        }
+        gdn_gate_dt = model.layers[il].ssm_dt;
+        gdn_gate_a  = model.layers[il].ssm_a;
+    } else {
+    if (ba_w) {
         // beta and alpha read the same input: one matmul, then the beta rows and the alpha rows
-        ggml_tensor * ba = build_lora_mm(model.layers[il].ssm_beta_alpha_cat, cur);
+        ggml_tensor * ba = build_lora_mm(ba_w, cur);
         cb(ba, "beta_alpha", il);
 
         beta  = ggml_view_4d(ctx0, ba, 1, num_v_heads, n_seq_tokens, n_seqs,
@@ -1154,10 +1245,11 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
     ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);
     cb(alpha_softplus, "a_softplus", il);
 
-    ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a);  // -A_log.exp() * softplus
+    gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a);  // -A_log.exp() * softplus
     cb(gate, "gate", il);
 
     gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);
+    }
 
     ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
     ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);
@@ -1171,8 +1263,21 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
     ggml_tensor * conv_input = build_conv_state_at(inp, conv_states_all, qkv_mixed,
             conv_kernel_size - 1, conv_channels, il);
 
-    ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
-    state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
+    ggml_tensor * state = nullptr;
+    gdn_state_ids = nullptr;
+    if (gdn_direct && gdn_fused && n_seqs == 1) {
+        ggml_tensor * state_ids = nullptr;
+        auto get_state_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {
+            state     = ggml_reshape_4d(ctx, states, head_v_dim, head_v_dim, num_v_heads, states->ne[1]);
+            state_ids = ids;
+            return state;
+        };
+        build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs, get_state_rows);
+        gdn_state_ids = state_ids;
+    } else {
+        state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
+        state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
+    }
     cb(state, "state_predelta", il);
 
     ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);
@@ -1226,6 +1331,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
     cb(v_conv, "v_conv_predelta", il);
 
     ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
+    gdn_state_ids = nullptr;
+    gdn_gate_dt   = nullptr;
+    gdn_gate_a    = nullptr;
 
     ggml_tensor * z_2d = ggml_view_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,
             ggml_row_size(z->type, head_v_dim), z->nb[1], z->nb[1] * n_seq_tokens, 0);
@@ -1277,6 +1385,15 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co
         ggml_tensor * shared_gate = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur);
         cb(shared_gate, "shared_expert_gate", il);
 
+        static const bool hc_fuse = getenv("LLAMA_HC_FUSE") != nullptr && atoi(getenv("LLAMA_HC_FUSE")) != 0;
+        if (hc_fuse && cparams.fused_dsv4_hc_post && il >= 0) {
+            // LLAMA_HC_FUSE: the next hc combine adds ffn_shexp * sigmoid(shared_gate) itself
+            GGML_ASSERT(shexp_pend == nullptr);
+            shexp_pend      = ffn_shexp;
+            shexp_gate_pend = shared_gate;
+            return moe_out;
+        }
+
         shared_gate = ggml_sigmoid(ctx0, shared_gate);
         cb(shared_gate, "shared_expert_gate_sigmoid", il);
 
@@ -1421,7 +1538,14 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
     ggml_tensor * state = ggml_reshape_3d(ctx0, rows, state_cols, channels, n_seqs);
     cb(state, "conv_state_at", il);
 
-    ggml_tensor * conv_input = ggml_concat(ctx0, state, ggml_transpose(ctx0, x), 0);
+    // LLAMA_CONV_CONT=1: with more than one token the transposed input is strided and the concat takes its slow
+    // non-contiguous path (~7 us per layer at 5 tokens); a contiguous copy first is cheaper. Same values
+    static const bool conv_cont = getenv("LLAMA_CONV_CONT") && atoi(getenv("LLAMA_CONV_CONT")) != 0;
+    ggml_tensor * xt = ggml_transpose(ctx0, x);
+    if (conv_cont && x->ne[1] > 1) {
+        xt = ggml_cont(ctx0, xt);
+    }
+    ggml_tensor * conv_input = ggml_concat(ctx0, state, xt, 0);
 
     // [TAG_RECURRENT_ROLLBACK_SPLITS] keep the last state_cols columns once per rollback slot,
     // slot s ending s tokens earlier so a rollback of s tokens reads a history that never saw them
@@ -1430,6 +1554,10 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
 
     const int64_t n_slots = (int64_t) cparams.n_rs_seq + 1;
 
+    // LLAMA_CONV_TAIL=1: copy each strided tail straight into its slot (one copy kernel) instead of a contiguous copy
+    // first, which CUDA runs as a slow 2D memcpy of 12-byte rows (~6 us each, 5 slots per linear layer). Same values
+    static const bool conv_tail = getenv("LLAMA_CONV_TAIL") && atoi(getenv("LLAMA_CONV_TAIL")) != 0;
+
     for (int64_t slot = 0; slot < n_slots; ++slot) {
         const int64_t s_idx = std::max<int64_t>(0, conv_input->ne[0] - state_cols - slot);
 
@@ -1443,7 +1571,15 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
                 conv_states_all->nb[1],
                 (slot * mem_size + kv_head) * row_size);
 
-        ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst));
+        if (conv_tail) {
+            ggml_tensor * dst3 = ggml_view_3d(ctx0, conv_states_all,
+                    state_cols, channels, n_seqs,
+                    ggml_row_size(conv_states_all->type, state_cols), conv_states_all->nb[1],
+                    (slot * mem_size + kv_head) * row_size);
+            ggml_build_forward_expand(gf, ggml_cpy(ctx0, tail, dst3));
+        } else {
+            ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst));
+        }
     }
 
     return conv_input;
@@ -1462,7 +1598,14 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
     res->add_input(std::move(ple_inp));
 
     // gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
-    ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
+    ggml_tensor * emb = nullptr;
+    if (model.ple_host_mapped && ggml_backend_sched_get_n_backends(sched) > 1) {
+        // LLAMA_PLE_GPU: the GPU gathers the rows from the registered host table (see load_tensors_ple_gpu)
+        emb = ggml_get_rows_host(ctx0, model.per_layer_tok_embd, rows);
+        ggml_backend_sched_set_tensor_backend(sched, emb, ggml_backend_sched_get_backend(sched, 0));
+    } else {
+        emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
+    }
     emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
     cb(emb, "ple_embd", -1);
 
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 2bd1ab9ec..111dc04b5 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -2,6 +2,7 @@
 #include <atomic>
 #include <thread>
 #include "base64.hpp"
+#include "../../src/llama-prof.h"
 #include "server-chat.h"
 #include "server-common.h"
 #include "server-decision.h"
@@ -1013,6 +1014,191 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch
 // server_context_impl (private implementation)
 //
 
+// LLAMA_SPEC_AUTO=1 (opt-in): the draft count of each round is chosen online, up to the configured maximum. The best
+// count depends mostly on the setup (model, quant, cache size, machine); single round times swing with the text (how
+// many experts miss) and drift while a hybrid cache warms up, so counts are only compared on rounds timed close together:
+//   calibration: the counts take turns in blocks of N_BLOCK rounds (the first N_SKIP after a change are not timed: the
+//   new batch shapes rebuild and recapture graphs, ~10 ms resident); after two turns a count scoring 15% below the best
+//   leaves the rotation; when every remaining count has N_CAL timed rounds, the best one is chosen;
+//   then: every N_RECHECK rounds one block re-times a neighbour of the choice (alternately below and above), compared
+//   with the choice's own last 16 rounds; the choice moves to it for a 5% better score.
+// The score of a count n is the expected tokens per time E(n)/R(n): R(n) the trimmed mean (middle half) of the wall
+// times from one verify to the next (drafting included), E(n) = 1 + a_1 + a_1 a_2 + ... with a_k = P(k-th draft
+// accepted | the ones before were), from decayed counts. Where an extra verify token is expensive (the hybrid at a small
+// cache, slow CPU experts) fewer drafts win; resident, the maximum usually stays.
+struct server_spec_auto {
+    static constexpr int N_CAP  = 16;
+    static constexpr int N_WIN  = 64;  // timed rounds kept per count
+    static constexpr int N_CAL  = 24;  // timed rounds per count before the first choice
+    static constexpr int N_SKIP = 2;
+    static constexpr int N_BLOCK = 6;
+    static constexpr int N_RECHECK = 512;
+    bool    on = false;
+    int     cap = 0;
+    double  win[N_CAP + 1][N_WIN] = {};
+    int     nS[N_CAP + 1] = {};        // timed rounds per count
+    double  tried[N_CAP + 1] = {}, ok[N_CAP + 1] = {};
+    bool    active[N_CAP + 1] = {};    // still in the calibration rotation
+    int     turns = 0;                 // completed calibration turns
+    int64_t t_last = 0;
+    int64_t rounds = 0;
+    int64_t picked[N_CAP + 1] = {};
+    int     n_best = 0;                // the choice (0: calibrating)
+    int     blk_n = 0, blk_left = 0;   // the block in progress
+    int     blk_from = 0;              // nS[blk_n] when a re-check block started
+    double  ref_score = 0.0;           // the choice's score just before a re-check block
+    int64_t last_check = 0;
+    int     run_n = 0, run_len = 0;
+    int64_t n_switch = 0;
+
+    void init(int n_max) {
+        const char * e = getenv("LLAMA_SPEC_AUTO");
+        on  = e && atoi(e) != 0 && n_max > 1;
+        cap = std::min(n_max, N_CAP);
+        for (int k = 1; k <= cap; ++k) {
+            active[k] = true;
+        }
+    }
+    // trimmed mean of the last m timed rounds of count n (all kept ones for m = 0)
+    double R(int n, int m = 0) const {
+        const int have = std::min(nS[n], N_WIN);
+        m = m > 0 ? std::min(m, have) : have;
+        if (m == 0) {
+            return 0.0;
+        }
+        double v[N_WIN];
+        for (int i = 0; i < m; ++i) {
+            v[i] = win[n][(nS[n] - 1 - i) % N_WIN];
+        }
+        std::sort(v, v + m);
+        const int i0 = m/4, i1 = std::max(i0 + 1, m - m/4);
+        double sum = 0.0;
+        for (int i = i0; i < i1; ++i) {
+            sum += v[i];
+        }
+        return sum/(i1 - i0);
+    }
+    double expect(int n) const {
+        double e = 1.0, p = 1.0;
+        for (int k = 1; k <= n; ++k) {
+            p *= (ok[k] + 1.0)/(tried[k] + 2.0);
+            e += p;
+        }
+        return e;
+    }
+    double score(int n, int m = 0) const { const double r = R(n, m); return r > 0.0 ? expect(n)/r : 0.0; }
+    int best_active() const {
+        int kb = 0;
+        for (int k = 1; k <= cap; ++k) {
+            if (active[k] && (kb == 0 || score(k) > score(kb))) { kb = k; }
+        }
+        return kb;
+    }
+    // the draft count for the next round
+    int pick() {
+        if (!on) {
+            return cap;
+        }
+        if (blk_left == 0) {
+            if (n_best == 0) {
+                // the next count of the rotation, from the maximum down
+                int k = blk_n;
+                do {
+                    k = k <= 1 ? cap : k - 1;
+                    if (k == cap && blk_n != 0) {
+                        turns++;
+                        const int kb = best_active();
+                        bool done = true;
+                        for (int j = 1; j <= cap; ++j) {
+                            if (!active[j]) { continue; }
+                            if (turns >= 2 && j != kb && score(j) < 0.85*score(kb)) { active[j] = false; continue; }
+                            done = done && nS[j] >= N_CAL;
+                        }
+                        if (done) {
+                            n_best = best_active();
+                            break;
+                        }
+                    }
+                } while (!active[k]);
+                if (n_best == 0) {
+                    blk_n = k;
+                    blk_left = N_BLOCK;
+                }
+            } else {
+                // a finished re-check block: move to the neighbour if it scored 5% better than the choice did just before
+                if (blk_n != n_best && blk_n != 0) {
+                    const int m = nS[blk_n] - blk_from;
+                    if (m > 0 && score(blk_n, m) > 1.05*ref_score) {
+                        n_best = blk_n;
+                        n_switch++;
+                    }
+                    blk_n = 0;
+                }
+                if (rounds - last_check >= N_RECHECK) {
+                    last_check = rounds;
+                    const int d = (rounds/N_RECHECK) % 2 ? 1 : -1;
+                    int k = std::clamp(n_best + d, 1, cap);
+                    if (k == n_best) {
+                        k = std::clamp(n_best - d, 1, cap);
+                    }
+                    if (k != n_best) {
+                        ref_score = score(n_best, 16);
+                        blk_n = k;
+                        blk_from = nS[k];
+                        blk_left = N_BLOCK + 2;
+                    }
+                }
+            }
+        }
+        int n = n_best;
+        if (blk_left > 0) {
+            blk_left--;
+            n = blk_n;
+        }
+        picked[n]++;
+        return n;
+    }
+    // after the verify of a round with n_draft drafts, n_acc of them accepted
+    void observe(int n_draft, int n_acc) {
+        if (!on) {
+            return;
+        }
+        const int64_t now = ggml_time_us();
+        const int64_t dt  = t_last > 0 ? now - t_last : -1;
+        t_last = now;
+        rounds++;
+        for (int k = 1; k <= cap; ++k) {
+            tried[k] *= 0.999; ok[k] *= 0.999;
+        }
+        for (int k = 1; k <= std::min(n_draft, cap); ++k) {
+            if (n_acc >= k - 1) { tried[k] += 1.0; }
+            if (n_acc >= k)     { ok[k]    += 1.0; }
+        }
+        // a round of another request, or one with a prompt in between, is not a decode round
+        if (n_draft < 1 || n_draft > cap || dt <= 0 || dt > 500000) {
+            return;
+        }
+        run_len = n_draft == run_n ? run_len + 1 : 1;
+        run_n   = n_draft;
+        if (run_len > N_SKIP) {
+            win[n_draft][nS[n_draft]++ % N_WIN] = (double) dt;
+        }
+    }
+    // a new request starts: its first round has no previous one to time against
+    void reset_round() { t_last = 0; }
+    ~server_spec_auto() {
+        if (!on || rounds == 0) {
+            return;
+        }
+        fprintf(stderr, "lc-spec-auto: %lld rounds, choice n=%d after %d turns, %lld switches; per draft count n: picked, R(n) ms, E(n) tokens, E/R tok/ms\n",
+            (long long) rounds, n_best, turns, (long long) n_switch);
+        for (int k = 1; k <= cap; ++k) {
+            fprintf(stderr, "lc-spec-auto:   n=%d picked %6lld  R %6.2f  E %5.2f  E/R %.3f\n", k, (long long) picked[k], R(k)/1e3, expect(k),
+                score(k)*1e3);
+        }
+    }
+};
+
 struct server_context_impl {
     friend struct server_context;
 
@@ -1091,6 +1277,7 @@ private:
     common_context_seq_rm_type ctx_dft_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
 
     common_speculative_ptr spec;
+    server_spec_auto       spec_auto;
 
     bool add_bos_token = true;
 
@@ -1491,6 +1678,12 @@ private:
         if (ctx_tgt_seq_rm_type != COMMON_CONTEXT_SEQ_RM_TYPE_NO) {
             try {
                 spec.reset(common_speculative_init(params_base.speculative, params_base.n_parallel));
+                if (spec) {
+                    spec_auto.init(common_speculative_n_max(spec.get()));
+                    if (spec_auto.on) {
+                        SRV_INF("LLAMA_SPEC_AUTO: the draft count is chosen per round, up to %d\n", spec_auto.cap);
+                    }
+                }
             } catch (const std::exception & e) {
                 SRV_ERR("failed to initialize speculative decoding context: %s\n", e.what());
                 if (params_base.speculative.has_synth()) {
@@ -2131,6 +2324,7 @@ private:
         n_empty_consecutive = 0;
 
         SLT_INF(slot, "processing task, is_child = %d\n", slot.task->is_child());
+        spec_auto.reset_round();
         return true;
     }
 
@@ -2735,6 +2929,7 @@ private:
                 SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
                         it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024);
 
+                common_prompt_checkpoint_recycle(*it);
                 it = slot.prompt.checkpoints.erase(it);
                 continue;
             }
@@ -2750,15 +2945,32 @@ private:
             SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
                     cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024);
 
+            common_prompt_checkpoint_recycle(slot.prompt.checkpoints.front());
             slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin());
         }
 
+        // LLAMA_SRV_CKPT_KEEP (default on, =0 off): a checkpoint at the same n_tokens and positions is kept, not copied again.
+        // Checkpoints past a divergence are erased before the prompt is processed, so one that remains holds this prefix's
+        // state; after a restore the server would otherwise copy the state it just restored (112 MiB, ~55 ms here)
+        static const bool keep_same = !getenv("LLAMA_SRV_CKPT_KEEP") || atoi(getenv("LLAMA_SRV_CKPT_KEEP")) != 0;
+        if (keep_same) {
+            const int64_t n_tokens_new = slot.prompt.n_tokens() - n_tokens_cur;
+            for (auto & ck : slot.prompt.checkpoints) {
+                if (ck.n_tokens == n_tokens_new && ck.pos_min == pos_min && ck.pos_max == pos_max && !ck.data_tgt.empty()) {
+                    ck.id_task = id_task;
+                    SLT_TRC(slot, "keeping context checkpoint at n_tokens = %" PRId64 " (same state)\n", ck.n_tokens);
+                    return;
+                }
+            }
+        }
+
         // replace an existing checkpoint at the same n_tokens instead of appending a duplicate
         {
             const int64_t n_tokens_new = slot.prompt.n_tokens() - n_tokens_cur;
             for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) {
                 if (it->n_tokens == n_tokens_new) {
                     SLT_TRC(slot, "superseding context checkpoint at n_tokens = %" PRId64 "\n", it->n_tokens);
+                    common_prompt_checkpoint_recycle(*it);
                     it = slot.prompt.checkpoints.erase(it);
                 } else {
                     ++it;
@@ -3538,6 +3750,8 @@ private:
     }
 
     void update_slots() {
+        llama_prof_lap("lap.loop_to_update_slots");
+        LC_PROF("srv.update_slots");
 #ifdef DEBUG_TIMINGS
         static int64_t t_prev = 0;
         int64_t t_start = ggml_time_us();
@@ -3581,6 +3795,7 @@ private:
 
         try {
             scoped_timer t(t_pre_decode, n_pre_decode);
+            LC_PROF("srv.pre_decode");
             pre_decode();
         } catch (const std::exception & e) {
             SRV_ERR("pre_decode() failed: %s\n", e.what());
@@ -3636,7 +3851,12 @@ private:
                 // TODO @ngxson : maybe handle n_batch == 1 here instead of inside decode()
 
                 batch.render(off, n_tokens);
-                bool ok = decode(n_batch, off);
+                bool ok;
+                {
+                    LC_PROF("srv.decode_tgt");
+                    ok = decode(n_batch, off);
+                }
+                llama_prof_lap("lap.decode_tgt_rest");
 #ifdef DEBUG_TIMINGS
                 llama_synchronize(ctx_tgt);
 #endif
@@ -3659,7 +3879,9 @@ private:
 
             try {
                 scoped_timer t(t_post_decode, n_post_decode);
+                llama_prof_lap("lap.to_post_decode");
                 post_decode(n_tokens, off);
+                llama_prof_lap("lap.post_decode");
             } catch (const std::exception & e) {
                 SRV_ERR("post_decode() failed: %s\n", e.what());
                 abort_all_slots("post_decode() failed: " + std::string(e.what()));
@@ -3836,6 +4058,21 @@ private:
         std::vector<server_slot *> generating;
         std::vector<server_slot *> drafting;
 
+        // lc-: with the MoE cache, a ubatch of more than 8 tokens leaves its decode path (cache lookups, CPU hybrid) for the
+        // large-batch one, which streams every expert: with several requests drafting, cap the drafts so that the step's
+        // tokens stay within 8 (2 requests: 3 drafts each, 4: 1). LLAMA_SRV_MOE_BATCH_CAP=<tokens>, 0 = off
+        static const int moe_batch_cap = getenv("LLAMA_SRV_MOE_BATCH_CAP") ? atoi(getenv("LLAMA_SRV_MOE_BATCH_CAP")) : 8;
+        int n_draft_cap = 1 << 30;
+        if (params_base.moe_cache_size > 0 && moe_batch_cap > 0) {
+            int n_gen = 0;
+            for (const auto & sl : slots) {
+                n_gen += sl.state == SLOT_STATE_GENERATING;
+            }
+            if (n_gen > 1) {
+                n_draft_cap = std::max(0, moe_batch_cap/n_gen - 1);
+            }
+        }
+
         // determine which slots are generating and drafting
         iterate(slots, [&](server_slot & slot) {
             if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
@@ -3861,7 +4098,10 @@ private:
                 const bool use_ckpt_tgt = ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
                 const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
 
-                const int n_draft_max = slot.get_n_draft_max();
+                int n_draft_max = std::min(slot.get_n_draft_max(), n_draft_cap);
+                if (spec_auto.on && n_draft_max > 0 && slot.spec_draft.empty()) {
+                    n_draft_max = std::min(n_draft_max, spec_auto.pick());
+                }
 
                 if (n_draft_max > 0) {
                     GGML_ASSERT(slot.can_speculate());
@@ -3910,12 +4150,15 @@ private:
 
         // generate the actual drafts (if any)
         if (!drafting.empty()) {
+            llama_prof_lap("lap.to_spec_draft");
+            LC_PROF("srv.spec_draft");
             queue_tasks.yield_to_queue([&]() {
                 common_speculative_draft(spec.get());
             });
         }
 
         // make checkpoints if needed
+        llama_prof_scope lc_prof_ck("srv.ckpt");
         iterate(drafting, [&](server_slot & slot) {
             auto & draft = slot.spec_draft;
             auto & ckpt  = slot.spec_ckpt;
@@ -3963,10 +4206,12 @@ private:
             }
         });
 
+        lc_prof_ck.~llama_prof_scope(); new (&lc_prof_ck) llama_prof_scope("srv.batch_gen");
         // update the batch with the sampled/drafted tokens
         iterate(generating, [&](server_slot & slot) {
             slot.handle_last_sampled_token(batch);
         });
+        lc_prof_ck.~llama_prof_scope(); new (&lc_prof_ck) llama_prof_scope("srv.prompts");
 
         // process in chunks of params.n_batch
         int32_t n_batch  = llama_n_batch(ctx_tgt);
@@ -4304,6 +4549,7 @@ private:
                                     const auto & cur = *it;
                                     if (cur.pos_max > pos_next || cur.n_tokens > n_prefix) {
                                         SLT_TRC(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024);
+                                        common_prompt_checkpoint_recycle(*it);
                                         it = slot.prompt.checkpoints.erase(it);
                                     } else {
                                         ++it;
@@ -4500,7 +4746,10 @@ private:
                         slot.prompt.tokens.push_back(cur_tok);
 
                         // break at the last user message, or at user messages at least min step past the last checkpoint
-                        if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
+                        // LLAMA_SRV_USER_BREAK=0: not at user messages (the checkpoints before the prompt's end remain). With host
+                        // experts each prompt batch above 32 tokens uploads most experts, so a turn's two batches cost two uploads
+                        static const bool user_break = !getenv("LLAMA_SRV_USER_BREAK") || atoi(getenv("LLAMA_SRV_USER_BREAK")) != 0;
+                        if (do_checkpoint && user_break && spans.is_user_start(slot.prompt.n_tokens())) {
                             const auto pos = slot.prompt.n_tokens();
                             const auto & checkpoints = slot.prompt.checkpoints;
 
@@ -4754,9 +5003,12 @@ private:
         //       ref: https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4400925384
         if (spec) {
             bool ok = true;
+            llama_prof_lap("lap.to_spec_process");
+            LC_PROF("srv.spec_process");
             queue_tasks.yield_to_queue([&]() {
                 ok = common_speculative_process(spec.get(), batch.view);
             });
+            llama_prof_lap("lap.spec_process");
 
             if (!ok) {
                 SRV_ERR("%s", "failed to process speculative batch\n");
@@ -4795,6 +5047,7 @@ private:
     }
 
     void post_decode(int32_t n_batch_tokens, int32_t off) {
+        LC_PROF("srv.post_decode");
         // for checking if a given batch index is inside the current sub-batch
         auto is_inside_view = [&](int32_t idx) {
             return idx >= off && idx < off + n_batch_tokens;
@@ -4931,6 +5184,7 @@ private:
 
             // verify and try to accept the draft
             {
+                LC_PROF("srv.verify_accept");
                 common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
 
                 GGML_ASSERT(spec_i_batch.size() == n_draft + 1);
@@ -5014,6 +5268,7 @@ private:
             // update how many tokens out of those tested were accepted
             slot.stats.n_draft_accepted += n_accepted;
             slot.stats.n_draft_verif_steps += 1;
+            spec_auto.observe((int) n_draft, (int) n_accepted);
 
             auto & n_accepted_per_pos = slot.n_accepted_per_pos;
             if (n_accepted_per_pos.empty()) {
@@ -5030,8 +5285,12 @@ private:
             slot.sampled = ids.back(); // last accepted token
             SLT_DBG(slot, "add accepted tokens: sampled=%d, ids.size=%zu, n_draft=%zu\n", slot.sampled, ids.size(), n_draft);
 
-            slot.mem.seq_rm(slot.id, slot.prompt.tokens.pos_next(), -1);
+            {
+                LC_PROF("srv.seq_rm");
+                slot.mem.seq_rm(slot.id, slot.prompt.tokens.pos_next(), -1);
+            }
 
+            llama_prof_scope lc_prof_pt("srv.process_tokens");
             for (size_t i = 0; i < ids.size(); ++i) {
                 completion_token_output result;