feat(ds4): add concurrent serving on Strix Halo and R9700 + Strix

.github/workflows/ci.yml

@@ -84,6 +84,7 @@ jobs:
             test_dflash test_generate test_flash_attn_sparse test_server_unit \
             test_deepseek4_unit test_feature_gate test_seq_slot_manager \
             test_seq_engine_contract test_seq_batch_plan test_client_send_buffer \
+            test_paged_kv_pool test_rope_tail \
             test_model_smoke test_batched_gdn test_concat_transpose \
             test_kernel_qualification_core test_kernel_qualification_gdn \
             test_kernel_qualification_moe_ds4 -j$(nproc)
@@ -92,7 +93,7 @@ jobs:
         run: |
           cd server/build
           ctest --output-on-failure \
-            -R "server_unit|deepseek4_unit|feature_gate|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer|batched_gdn_cpu|test_kernel_qualification_core" \
+            -R "server_unit|deepseek4_unit|feature_gate|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer|paged_kv_pool|rope_tail|batched_gdn_cpu|test_kernel_qualification_core" \
             --no-tests=error
 
       - name: Populate venv with cu128 torch + setuptools

server/CMakeLists.txt

@@ -469,6 +469,8 @@ add_library(dflash_common STATIC
     src/deepseek4/deepseek4_loader.cpp
     src/deepseek4/deepseek4_graph.cpp
     src/deepseek4/deepseek4_roctx.cpp
+    src/deepseek4/deepseek4_paged_cache.cpp
+    src/deepseek4/deepseek4_seq_engine.cpp
     src/deepseek4/deepseek4_backend.cpp
     src/deepseek4/deepseek4_daemon.cpp
     src/deepseek4/deepseek4_layer_split_adapter.cpp
@@ -505,7 +507,7 @@ add_library(dflash_common STATIC
     src/common/draft_swa.cpp
     src/common/dflash_spec_decode.cpp
     src/common/concurrency/paged_kv_pool.cpp
-    src/qwen35/concurrency/qwen35_slot_manager.cpp
+    src/common/concurrency/seq_slot_manager.cpp
     src/common/layer_split_backend.cpp
     src/common/layer_split_runtime.cpp
     src/qwen35/graph_builders.cpp
@@ -985,8 +987,23 @@ if(DFLASH27B_TESTS)
             hip::host)
         list(APPEND _raw_unit_test_targets test_rocmfpx_mmq)
 
+        add_executable(test_gfx1151_width4_dispatch test/test_gfx1151_width4_dispatch.cpp)
+        set_source_files_properties(test/test_gfx1151_width4_dispatch.cpp PROPERTIES LANGUAGE HIP)
+        set_target_properties(test_gfx1151_width4_dispatch PROPERTIES HIP_ARCHITECTURES "${_dflash_archs}")
+        target_include_directories(test_gfx1151_width4_dispatch PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include)
+        target_link_libraries(test_gfx1151_width4_dispatch PRIVATE
+            ggml
+            ggml-base
+            ${DFLASH27B_GGML_BACKEND_TARGET}
+            hip::host)
+        list(APPEND _raw_unit_test_targets test_gfx1151_width4_dispatch)
 
-
+        add_executable(test_softmax_sink_col test/test_softmax_sink_col.cpp)
+        set_source_files_properties(test/test_softmax_sink_col.cpp PROPERTIES LANGUAGE HIP)
+        set_target_properties(test_softmax_sink_col PROPERTIES HIP_ARCHITECTURES "${_dflash_archs}")
+        target_link_libraries(test_softmax_sink_col PRIVATE ggml ggml-base ggml-cpu ${DFLASH27B_GGML_BACKEND_TARGET} hip::host)
+        list(APPEND _raw_unit_test_targets test_softmax_sink_col)
 
         # Benchmark, not a test: built with the tests but deliberately NOT added to
         # _raw_unit_test_targets, so ctest stays deterministic. Measures the launch-count
@@ -1003,6 +1020,10 @@ if(DFLASH27B_TESTS)
     target_include_directories(test_qwen35_tensor_parallel PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
     target_link_libraries(test_qwen35_tensor_parallel PRIVATE dflash_common)
 
+    add_executable(test_rope_tail test/test_rope_tail.cpp)
+    target_compile_definitions(test_rope_tail PRIVATE DFLASH_ROPE_TEST_GPU=1)
+    target_link_libraries(test_rope_tail PRIVATE ggml ggml-base ggml-cpu ${DFLASH27B_GGML_BACKEND_TARGET})
+
     if(DFLASH27B_GPU_BACKEND STREQUAL "cuda" AND _dflash_cuda_min_sm GREATER_EQUAL 80 AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_flashprefill_kernels.cpp")
         add_executable(test_flashprefill_kernels test/test_flashprefill_kernels.cpp)
         set_target_properties(test_flashprefill_kernels PROPERTIES CUDA_ARCHITECTURES "${_dflash_archs}")
@@ -1481,7 +1502,11 @@ if(DFLASH27B_TESTS)
         # Pure host-side allocator test: no model, ggml, or GPU is required.
         add_executable(test_paged_kv_pool
             test/test_paged_kv_pool.cpp
+            test/test_deepseek4_page_layout.cpp
+            test/test_deepseek4_paged_cache.cpp
+            src/deepseek4/deepseek4_paged_cache.cpp
             src/common/concurrency/paged_kv_pool.cpp)
+        target_compile_definitions(test_paged_kv_pool PRIVATE DFLASH_DS4_PLAN_ONLY=1)
         target_include_directories(test_paged_kv_pool PRIVATE
             ${CMAKE_CURRENT_SOURCE_DIR}/src)
         list(APPEND _raw_unit_test_targets test_paged_kv_pool)
@@ -1490,7 +1515,7 @@ if(DFLASH27B_TESTS)
         # Host-side slot bookkeeping test (concurrent serving): no GPU.
         add_executable(test_seq_slot_manager
             test/test_seq_slot_manager.cpp
-            src/qwen35/concurrency/qwen35_slot_manager.cpp
+            src/common/concurrency/seq_slot_manager.cpp
             src/common/concurrency/paged_kv_pool.cpp)
         target_include_directories(test_seq_slot_manager PRIVATE
             ${CMAKE_CURRENT_SOURCE_DIR}/src)
@@ -1917,6 +1942,8 @@ if(DFLASH27B_TESTS)
         test_cuda_comm_api
         test_mmq_streamk_iq4_xs
         test_rms_norm_hip
+        test_rope_tail
+        test_paged_kv_pool
         test_rocmfp3_mix_registry
         test_rocmfp_mix_slice_matvec
         test_rocmfp_mix_gateup_glu

server/README.md

@@ -299,10 +299,10 @@ See the [current six-expert Strix Halo profile](https://www.lucebox.com/blog/dee
 | `--prefix-cache-slots <N>` | `32` | In-memory prefix-cache slots; `0` disables. |
 | `--agent-turn-cache` | off | Extend prefix caching through generated tool calls. |
 | `--prefill-cache-slots <N>` | `0` | Full-prompt cache slots. |
-| `--paged-attention` | off | Enable 16-token paged KV blocks for supported Qwen targets. |
-| `--max-concurrency <N>` | `1` | Maximum concurrent decode sequences; values above 1 enable paged attention. |
+| `--paged-attention` | off | Enable paged KV for supported Qwen targets (16-token blocks) or DeepSeek4 on Strix Halo and R9700 plus Strix Halo (128-token pages). DeepSeek4 paged serving is AR-only. |
+| `--max-concurrency <N>` | `1` | Maximum concurrent decode sequences. Qwen supports up to 64; DeepSeek4 supports up to 6. Values above 1 enable paged attention. |
 | `--admission-coalesce-ms <N>` | `20` | Idle-to-busy batching window from 0 through 1000 ms. |
-| `--kv-pool-tokens <N>` | auto | Shared physical K/V capacity for concurrent serving. |
+| `--kv-pool-tokens <N>` | auto | Shared physical K/V capacity for concurrent serving, rounded to the backend page size. |
 | `--kv-cache-dir <path>` | none | Enable persistent disk KV cache in this directory. |
 | `--kv-cache-budget <MB>` | `4096` | Disk KV-cache size cap. |
 | `--kv-cache-min-tokens <N>` | `512` | Minimum prefix length to persist. |

server/deps/llama.cpp/ggml/include/ggml.h

@@ -252,6 +252,10 @@
 #define GGML_ROPE_TYPE_MROPE  8
 #define GGML_ROPE_TYPE_VISION 24
 #define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000
+// Lucebox: rotate the LAST n_dims of each row (NORMAL pairing) and pass the
+// leading ne0 - n_dims through unchanged, so a tail-rotary head (DeepSeek4)
+// is one launch instead of split, rotate, and concat. Combine with NORMAL only.
+#define GGML_ROPE_TYPE_TAIL   64
 
 #define GGML_MROPE_SECTIONS   4
 
@@ -1807,6 +1811,16 @@ extern "C" {
             float                 scale,
             float                 max_bias);
 
+    // Lucebox: softmax over [a | sink] per row with the sink as a virtual last
+    // column, scale folded in (a*scale), no mask. sinks holds one value per row
+    // of a (ne[0] == ggml_nrows(a)). Bit-identical to scale -> concat(sink) ->
+    // soft_max -> view, in one launch with a contiguous result of a's shape.
+    GGML_API struct ggml_tensor * ggml_soft_max_ext_sink_col(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * a,
+            struct ggml_tensor  * sinks,
+            float                 scale);
+
     GGML_API void ggml_soft_max_add_sinks(
             struct ggml_tensor * a,
             struct ggml_tensor * sinks);

server/deps/llama.cpp/ggml/src/ggml-backend.cpp

@@ -2322,7 +2322,10 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
     std::vector<int32_t> ids;
     std::vector<ggml_bitset_t> used_ids;
     size_t batch_staging_cursors[GGML_SCHED_MAX_BACKENDS] = {};
-    const bool copy_destinations_ready = sched->backends_synchronized;
+    const bool staging_arena_reusable = sched->backends_synchronized;
+    bool copy_destinations_ready[GGML_SCHED_MAX_BACKENDS] = {};
+    std::fill_n(copy_destinations_ready, sched->n_backends,
+                sched->backends_synchronized);
     // From this point onward an early return must remain conservative: a
     // backend may have accepted work even if a later split fails.
     sched->backends_synchronized = false;
@@ -2344,7 +2347,8 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
 
         ggml_backend_sched_split_copy_state copy_state{
             sched, split_backend_id, split_backend,
-            copy_destinations_ready, copy_destinations_ready,
+            staging_arena_reusable,
+            copy_destinations_ready[split_backend_id],
             batch_staging_cursors};
 
         // copy the input tensors to the split backend
@@ -2537,6 +2541,8 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
                 j0 = j1;
             }
         }
+        copy_destinations_ready[split_backend_id] =
+            sched->callback_eval != nullptr;
 
         // Publish producer completion before a later consumer-backend graph
         // reaches its in-graph event wait. Recording is asynchronous and does

server/deps/llama.cpp/ggml/src/ggml-cann/aclnn_ops.cpp

@@ -2669,6 +2669,7 @@ void ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst) {
     // const int n_past     = ((int32_t *) dst->op_params)[0];
     const int n_dims     = ((int32_t *) dst->op_params)[1];
     const int mode       = ((int32_t *) dst->op_params)[2];
+    GGML_ASSERT((mode & GGML_ROPE_TYPE_TAIL) == 0);
     // const int n_ctx      = ((int32_t *) dst->op_params)[3];
     const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
 
@@ -3028,6 +3029,7 @@ void ggml_cann_rope_cache_preload(ggml_backend_cann_context & ctx, ggml_tensor *
     int       sections[4];
     const int n_dims     = ((int32_t *) dst->op_params)[1];
     const int mode       = ((int32_t *) dst->op_params)[2];
+    GGML_ASSERT((mode & GGML_ROPE_TYPE_TAIL) == 0);
     const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
 
     GGML_TENSOR_UNARY_OP_LOCALS

server/deps/llama.cpp/ggml/src/ggml-cann/ggml-cann.cpp

@@ -2364,6 +2364,11 @@ static enum ggml_status ggml_backend_cann_graph_compute(ggml_backend_t backend,
  *              otherwise false.
  */
 static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+    if ((op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK) &&
+        (op->op_params[2] & GGML_ROPE_TYPE_TAIL)) {
+        return false;
+    }
+
     switch (op->op) {
         case GGML_OP_UNARY:
             switch (ggml_get_unary_op(op)) {

server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c

@@ -3018,7 +3018,7 @@ struct ggml_cplan ggml_graph_plan(
                             // conversion between F32 and I32
                             (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) ||
                             (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) {
-                            cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
+                            cur = ggml_type_size(GGML_TYPE_F32) * (node->ne[0] + 1) * n_tasks; // +1: sink-column mode
                         }
                     } break;
                 case GGML_OP_ADD:

server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp

@@ -5356,6 +5356,12 @@ static void ggml_compute_forward_soft_max_f32(
 
     // sinks
     const float * sk = src2 ? (float *)((char *) src2->data) : nullptr;
+    // Lucebox sink-as-virtual-column mode (see ggml_soft_max_ext_sink_col)
+    int32_t sink_col = 0;
+    memcpy(&sink_col, (int32_t *) dst->op_params + 2, sizeof(int32_t));
+    if (sink_col) {
+        GGML_ASSERT(sk && !src1);
+    }
 
     for (int64_t i03 = 0; i03 < ne03; i03++) {
         for (int64_t i02 = 0; i02 < ne02; i02++) {
@@ -5396,6 +5402,20 @@ static void ggml_compute_forward_soft_max_f32(
                 }
 #endif // NDEBUG
 
+                if (sink_col) {
+                    // Append the row's sink as one more column and run the
+                    // plain softmax over ne00 + 1 values, exactly as the
+                    // concat form would; only the first ne00 outputs are kept.
+                    const int64_t row = i01 + ne01*(i02 + ne02*i03);
+                    wp[ne00] = sk[row];
+                    float max = -INFINITY;
+                    ggml_vec_max_f32(ne00 + 1, &max, wp);
+                    ggml_float sum = ggml_vec_soft_max_f32(ne00 + 1, wp, wp, max);
+                    assert(sum > 0.0);
+                    sum = 1.0/sum;
+                    ggml_vec_cpy_f32(ne00, dp, wp);
+                    ggml_vec_scale_f32(ne00, dp, sum);
+                } else {
                 float max = -INFINITY;
                 ggml_vec_max_f32(ne00, &max, wp);
 
@@ -5413,6 +5433,7 @@ static void ggml_compute_forward_soft_max_f32(
 
                 sum = 1.0/sum;
                 ggml_vec_scale_f32(ne00, dp, sum);
+                }
 
 #ifndef NDEBUG
                 for (int i = 0; i < ne00; ++i) {
@@ -5871,9 +5892,13 @@ static void ggml_compute_forward_rope_flt(
     float corr_dims[2];
     ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
 
-    const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope
-    const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE;  // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope
-    const bool is_vision = mode == GGML_ROPE_TYPE_VISION;
+    const bool is_tail = mode & GGML_ROPE_TYPE_TAIL;      // rotate the last n_dims, pass the head through
+    const int  mode_base = mode & ~GGML_ROPE_TYPE_TAIL;
+    GGML_ASSERT(!is_tail || mode_base == GGML_ROPE_TYPE_NORMAL);
+    const int64_t rot_off = is_tail ? ne0 - n_dims : 0;
+    const bool is_imrope = mode_base == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope
+    const bool mrope_used = mode_base & GGML_ROPE_TYPE_MROPE;  // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope
+    const bool is_vision = mode_base == GGML_ROPE_TYPE_VISION;
 
     if (mrope_used) {
         GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0);
@@ -5909,7 +5934,7 @@ static void ggml_compute_forward_rope_flt(
                 if (last_i2 != i2) {
                     if (!mrope_used) {
                         const int64_t p = pos[i2];
-                        ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
+                        ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, is_tail ? n_dims : ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
                     }
                     else {
                         const int64_t p_t = pos[i2];
@@ -5927,9 +5952,12 @@ static void ggml_compute_forward_rope_flt(
                 T * src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
                 T * dst_data  = (T *)((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1);
 
-                switch (mode) {
+                switch (mode_base) {
                     case GGML_ROPE_TYPE_NORMAL:
-                        rotate_pairs<T>(n_dims, 1, cache, src, dst_data, 1);
+                        // For the tail mode the cache entries [0, n_dims) hold
+                        // the angles of the rotated span, as they would for an
+                        // extracted n_dims-wide tail tensor.
+                        rotate_pairs<T>(n_dims, 1, cache, src + rot_off, dst_data + rot_off, 1);
                         break;
                     case GGML_ROPE_TYPE_NEOX:
                     case GGML_ROPE_TYPE_MROPE:
@@ -5945,7 +5973,11 @@ static void ggml_compute_forward_rope_flt(
 
                 if (!is_vision) {
                     // fill the remain channels with data from src tensor
-                    for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {
+                    // (the head [0, rot_off) for the tail mode, the tail
+                    // [n_dims, ne0) otherwise)
+                    const int64_t copy_begin = is_tail ? 0 : n_dims;
+                    const int64_t copy_end   = is_tail ? rot_off : ne0;
+                    for (int64_t i0 = copy_begin; i0 < copy_end; i0 += 2) {
                         const T * const src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
                         T * dst_data  = (T *)((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1  + i0*nb0);