diff --git a/CMakeLists.txt b/CMakeLists.txt index 692484a807c..8c32905d935 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -876,6 +876,15 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) target_link_libraries(strata PRIVATE delayimp) target_link_options(strata PRIVATE "LINKER:/DELAYLOAD:hipblas.dll" "LINKER:/DELAYLOAD:libhipblaslt.dll" "LINKER:/DELAYLOAD:rocblas.dll") endif() + # 2026-10-12 splice hotfix round 2: generate.cpp is a G++ (host) TU even in a CUDA build, so + # the __CUDACC__ per-TU trigger in kv_splice_transport.hpp never fires for it. Mark the + # whole build so the transport activates for EVERY TU of this target (NVCC or g++), and the + # fail-closed #error in the header catches any TU that sees it in a CUDA build without one. + # Rebase onto v0.2.1.4: 0.2.x builds CPU-only too, where the macro would make the header + # include and fire its own fail-closed #error, so scope it to GPU builds. + if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) + target_compile_definitions(strata PRIVATE STRATA_HAVE_CUDA_BUILD=1) + endif() # NEW-MODELS-PLAN.md T1: Qwen3.6-35B-A3B's reference path (one GPU, the model resident, plain kernels), checked # against llama.cpp by tools/qwen35/; the engine itself does not run the family yet @@ -1802,6 +1811,63 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS) add_executable(window_evict_test src/core/window_evict_test.cpp) target_include_directories(window_evict_test PRIVATE include) add_test(NAME window_evict_test COMMAND window_evict_test) + # the suffix-reuse plan computation: longest-common-tail splice plan over a re-edited prompt + # (header-only, CPU; gated by STRATA_BUILD_CONVERSATION_TESTS so the test suite can opt in standalone) + if(STRATA_BUILD_CONVERSATION_TESTS) + add_executable(suffix_reuse_test src/program/suffix_reuse_test.cpp) + target_include_directories(suffix_reuse_test PRIVATE include) + add_test(NAME suffix_reuse_test COMMAND suffix_reuse_test) + # A2.3: flag-off parity is a ctest, not a grep. CPU-only; gate on the same flag as + # suffix_reuse_test so it can opt in alongside the conversation tests. + add_executable(suffix_reuse_off_parity_test src/program/suffix_reuse_off_parity_test.cpp) + target_include_directories(suffix_reuse_off_parity_test PRIVATE include) + add_test(NAME suffix_reuse_off_parity_test COMMAND suffix_reuse_off_parity_test) + # Phase 2 Task 3 T6: the splice-wiring test (synthesises QsaState, hand-crafts K rows, + # runs rotate_qsa_tail_inplace, asserts K matches fresh rotation to 1e-6 + V byte-identical + # + SPLICE_UNSUPPORTED guard fires for unsupported configs). + add_executable(suffix_reuse_apply_test src/program/suffix_reuse_apply_test.cpp) + target_include_directories(suffix_reuse_apply_test PRIVATE include) + add_executable(suffix_reuse_dense_tail_test src/program/suffix_reuse_dense_tail_test.cpp) + target_include_directories(suffix_reuse_dense_tail_test PRIVATE include) + add_test(NAME suffix_reuse_dense_tail_test COMMAND suffix_reuse_dense_tail_test) + add_test(NAME suffix_reuse_apply_test COMMAND suffix_reuse_apply_test) + # Phase 2 hotfix T7: the device-path regression for the 2026-10-12 SIGSEGV. Pins the + # LIVE failing inputs (edit_pos=3239, old_suffix_len=1, uniform delta=+54) plus the + # non-uniform path and the null-deltas guard, all on a PROT_NONE "device" region + # (PROT_NONE faults on any host deref, exactly as a discrete-GPU address would). + # CPU-only: the test supplies a fake-cuda API in its own TU; no . + # RED->GREEN: removing the transport from kv_splice_transport.hpp makes this SEGV. + add_executable(suffix_reuse_device_test src/program/suffix_reuse_device_test.cpp) + target_include_directories(suffix_reuse_device_test PRIVATE include) + target_compile_definitions(suffix_reuse_device_test PRIVATE + STRATA_SPLICE_TRANSPORT=1 STRATA_SPLICE_FAKE_CUDA=1) + add_test(NAME suffix_reuse_device_test COMMAND suffix_reuse_device_test) + add_executable(suffix_reuse_resume_narrowing_test src/program/suffix_reuse_resume_narrowing_test.cpp) + target_include_directories(suffix_reuse_resume_narrowing_test PRIVATE include) + add_test(NAME suffix_reuse_resume_narrowing_test COMMAND suffix_reuse_resume_narrowing_test) + # Phase 3a T9+T10: continuation-mount parity (the 0% vs 98% off-arm regression - the + # splice hook's suffix walk mis-IDs a pure continuation as a 1-token edit and destroys + # the mount) + anchor-splice engagement (the splice's K rotation is wasted because the + # chunk loop re-reads [anchor, n) instead of [edit, n) - the gap-read is the splice's + # saved work). CPU-only header test on the pure helpers in suffix_reuse.hpp. + add_executable(suffix_reuse_continuation_and_anchor_test src/program/suffix_reuse_continuation_and_anchor_test.cpp) + target_include_directories(suffix_reuse_continuation_and_anchor_test PRIVATE include) + add_test(NAME suffix_reuse_continuation_and_anchor_test COMMAND suffix_reuse_continuation_and_anchor_test) + # Phase 3 T11: the INT8 KV splice (dequant -> rotate -> requant). Exercises + # kv_splice_apply_int8.hpp + kv_q8_quant.hpp on a synthesised int8 QsaState; + # mirrors the writer's quant math (src/kernels/cuda/kv_q8.cu:47-67) so a + # requant(dequant(block)) is BIT-IDENTICAL and the rotated cell is bounded by + # the writer's per-group half-step. Header-only / CPU. + add_executable(suffix_reuse_int8_test src/program/suffix_reuse_int8_test.cpp) + target_include_directories(suffix_reuse_int8_test PRIVATE include) + add_test(NAME suffix_reuse_int8_test COMMAND suffix_reuse_int8_test) + # Phase 3b T12-T15: the partial-prefix mount (flag-gated longest-LCP fallback when the strict + # mount fails). Exercises `ConversationCache::best_with_partial` and the pure + # `suffix_reuse_partial_apply` helper in suffix_reuse.hpp. Header-only / CPU - no CUDA. + add_executable(suffix_reuse_partial_prefix_test src/program/suffix_reuse_partial_prefix_test.cpp) + target_include_directories(suffix_reuse_partial_prefix_test PRIVATE include) + add_test(NAME suffix_reuse_partial_prefix_test COMMAND suffix_reuse_partial_prefix_test) + endif() # which path computes the draft layer's prompt K/V, and STRATA_SPLIT_MTP_BATCH's parsing (header-only, CPU) add_executable(draft_kv_plan_test src/program/draft_kv_plan_test.cpp) target_include_directories(draft_kv_plan_test PRIVATE include) diff --git a/docs/FW2C_DESIGN_NOTES.md b/docs/FW2C_DESIGN_NOTES.md new file mode 100644 index 00000000000..84ecc6da739 --- /dev/null +++ b/docs/FW2C_DESIGN_NOTES.md @@ -0,0 +1,35 @@ +# FW-2c design notes — session-local partial K/V splice (drafted 2026-10-13, awaiting review) + +## Goal +When a mid-prompt edit finds only a partial (LCP) match, mount the unchanged prefix's K/V directly from the PARKED conversation image instead of re-reading those tokens. Potential (FW-2 survey): 91.2% session-local on deep edits vs 59.7% parked / current state-cadence gap-read. + +## Recon facts (verified in source today) +1. `ConversationKvReuse` (conversation_cache.hpp:57) — parking ALREADY retains real K/V: + `kv` (per-layer ConversationKv) + `captured_tokens` / `unchanged_tokens` extents + + per-stage variants. This is live infrastructure, not new. +2. Park path: `park_current_body` (generate.cpp:7889) captures stage snapshots via + `conversation_snapshot_capture_bytes` with a byte budget; retained K/V must fit it. +3. `best_with_partial` (conversation_cache.hpp:315) returns Match{tokens, partial, + checkpoint_length} — the LCP `P` and anchor depth; FW-2b's two-pool pick feeds + `narrowed_resume_target_state`. +4. Fail-close invariant (T18): gap [anchor, P) must be READ through the normal chunk + loop. FW-2c does NOT touch that — it replaces the *mount source* for [0, anchor) + from "arena re-read" to "parked K/V splice" only when the parked image covers it. + +## Design sketch (T24–T26, TDD) +- T24 pure seam (suffix_reuse.hpp): `kv_splice_plan(P, anchor, captured_tokens, + unchanged_tokens, n)` -> {splice_from_parked: [0, min(anchor, unchanged)), read: + [anchor, n), gap: [min(anchor, unchanged), anchor)}; fail-closed default OFF; + env gate `STRATA_SFX_KV_SPLICE=1` for the first probes (same pattern as REPLAY). +- T25 runtime: in the partial-mount branch, if the parked entry's KvReuse covers + [0, anchor): splice K/V into the arena (fp16 path first; int8 needs the Phase-3 + dequant->rotate->requant kernels), read [anchor, n) as today. Budget: splice bytes + count against conversation-cache-mib. +- T26 gates: dt-steady chain oracle (160/161/162 must hold), kv-store cert cell, + CUDA parity probe; kill criterion: splice cost > re-read cost at 4096-token depth + (T19 measured piece restarts ~8s — splice must beat that). + +## Risks / unknowns +- Int8 splicing kernel reuse from Phase 3 (exists, needs rewiring to park extents). +- Layer-split stages: KvReuse.stages must splice per-stage (park splits images). +- Host RAM: retained K/V already budgeted at park time — splice adds none. diff --git a/docs/SUFFIX_REUSE.md b/docs/SUFFIX_REUSE.md new file mode 100644 index 00000000000..37cf81d5fa2 --- /dev/null +++ b/docs/SUFFIX_REUSE.md @@ -0,0 +1,245 @@ +# Suffix-Reuse: KV Cache Splicing for Edited Contexts + +**Design and benchmark document** for the `--cache-suffix-reuse` feature and its adaptive-cadence extension (`--prompt-cache-dense-tail`). Covers three stacked, independently reviewed layers on base v0.1.43: `sfx-043-fixed` (all-quant splice), `sfx-future` @ `fd64fdf6` (dense-tail cadence, partial-prefix engagement, per-request checkpoint gate — 11 commits, review-clean), and `sfx-fw2b` (state-cadence checkpoints, §5.9 — range review APPROVE, 0 blocking). +All numbers measured on: RTX 3090 (24 GB), Qwen3.8-Flash-Next abliterated GSQ-RCO IQ3_XXS (2-file pack + PLE), 131k context, MTP spec-4, unless noted. Probes deterministic (temperature 0). + +--- + +## 1. Summary + +Strata already reuses the KV cache when a new prompt *extends* the previous one — the append case. It reuses nothing when the prompt is *edited in the middle*: one changed token invalidates the entire cache, and the whole prompt is re-read from scratch. This PR closes that gap. When a prompt repeats an earlier one with a change in the middle, the engine finds how much of the old context still matches, keeps that KV, and rotates the cached K rows to their new positions (V rows are position-free and pass through untouched). On a 5.7k-token ledger prompt with a one-row mid-file edit, reuse goes from **0% to 89%** and prefill from **10.4 s to 3.4 s**, with answers provably correct. The splice works with **every KV format the engine supports** — fp16, int8, int8+STRATA_KV_ROT, k8v4, q4_0 — because the rotation is applied in dequantized space and re-encoded with each format's own writer math. The feature is opt-in (`--cache-suffix-reuse N`), fails closed to today's exact re-read behavior, and the default path is unchanged. + +## 2. Background: what Strata caches today, and the gap + +Upstream v0.1.43 ships two reuse mechanisms: + +- **Conversation checkpoints** (`--prompt-cache N`, `--prompt-cache-every N`): the engine snapshots the KV at turn boundaries. +- **Conversation parking** (`--conversation-cache-mib N`): finished conversations' KV is kept in RAM so a later message in the same chat can mount it. + +Both are **prefix-mounted**: the cached KV is valid only while the new prompt is the old prompt plus new text at the end. Measured on the probe workload: + +- Append turn (flag off): `prompt 3294 tokens = 3240 reused + 54 read` — **98.3% reuse**, 2.1 s. +- Mid-edit turn (flag off): `prompt 3294 tokens = 0 reused + 3294 read` — **0% reuse**, 10.4 s. The engine re-reads text it read seconds ago, because after the edit the cached sequence is no longer a prefix of the new one. + +## 3. Why this matters: agents that edit their context + +The gap is not an edge case — it is the *dominant* pattern for agentic and context-management workloads: + +- **Agent loops with external memory.** An agent that keeps verbatim working notes (files re-sent each turn — the "context-level management" / CLM pattern) rewrites those notes constantly. Every rewrite is a mid-prompt edit. +- **Tool harnesses.** A file listing, a code window, or a scratchpad embedded in the prompt changes between calls while the surrounding system prompt and history stay identical. +- **Observed, not hypothesized.** In a 68-turn benchmark session against an engine with only prefix reuse, **41 of 66 requests (62%) got 0% reuse** — each note rewrite triggered a full re-prefill. The same session's pure appends got 97–98%. (kv-store trace, seed 42; 45.5% aggregate reuse despite the appends.) + +Recent serving literature identifies the same pattern. CacheSlide (FAST '26) names it *Relative-Position-Dependent Caching*: agent prompts contain segments that keep their relative order while their absolute positions shift, and reports 3.1–4.3× latency reductions when serving reuse for them. vLLM's reuse RFC explicitly names absolute-position mismatch as the reason mid-prompt reuse is skipped. llama.cpp ships `--cache-reuse` (KV shifting) for exactly this shape, off by default. Strata's answer is below. + +## 4. How it works + +Given the old prompt O and the new prompt N with a common prefix of length P (the edit at P): + +1. **Partial-prefix mount.** The engine matches N against parked conversations and mounted checkpoints, finds the deepest usable cached point ≤ P, and mounts it (`narrowed_resume_target`). +2. **The splice.** The unchanged tail [P, n) is cached at old positions; the K rows are rotated in place by the RoPE delta (R(p) → R(p+δ)) so they describe the *new* sequence. V rows are never touched — V is added post-attention and is position-free. Recurrence states (GDN/PLE) are *not* rotated forward from a parked state (that produces stale, wrong answers — we measured it); they are replayed from the narrowed checkpoint. +3. **Quantized formats: the dequant → rotate → requant sandwich.** int8, k8v4 and q4_0 K rows are quantized codes, not floats. The splice dequantizes with each format's own dequant math, rotates in fp32, and re-encodes with the *writer's exact* quantizer — one requant, the same noise structure as a fresh write. + - int8: codes + scales, per-64 groups. + - int8 + STRATA_KV_ROT=1: K is stored Hadamard-rotated (H·R(p)·k). Because H·H = I, the splice runs dequant → H → R(δ) → H → requant, which equals the exact stored form at the new position: H·R(δ)·H·(H·R(p)·k) = H·R(p+δ)·k. + - k8v4: the hybrid's K half is byte-identical int8 storage (same codes/scales at the same offsets); V (rotated Q4_0) is untouched. + - q4_0: whole K rows are Q4_0 blocks (18 B per 32 values); dequant/requant per block with the parity-reference quantizer. +4. **Fail-closed.** Unsupported memory layouts (`--kv-resident > 0`, `--layer-split`) refuse with a logged reason and fall through to the re-read path. Off (the default), behavior is byte-identical to today (pinned by `suffix_reuse_off_parity_test`). +5. **Adaptive cadence (`--prompt-cache-dense-tail N`, `sfx-future`).** Checkpoints are taken sparsely through the head of the prompt and every `--prompt-cache-every` tokens within the last N tokens — cheap cold read, dense anchors exactly where mid-text edits land. The periodic gate is per-request (it reads the request's own `pp_next_check`, not a value persisting across requests, so a warm read banks anchors at its piece boundaries). The engagement rule prefers a partial mount only when its LCP is at least the strict match's depth AND its anchor is deeper (`p >= s` *and* `L > s`) — enforced across conversations (a different conversation's deeper-but-shallower-LCP partial never mounts; pinned by T16.7/T16.7b). When an edit sits deeper than every checkpoint, the engine fail-closes: it restores the deepest checkpoint at or before the edit and reads forward (§5.8), rather than splicing recurrence state across the edit. + +## 5. Benchmarks + +### 5.1 Before / after: the mid-edit turn (int8, the daily format) + +Three-turn ledger probe: a 160-row ledger (≈5.7k tokens), one row inserted at row 99 (edit), a second insert (edit), temperature 0, exact-count scoring. + +| Turn | Before (flag off) | After (`--cache-suffix-reuse 64`) | +|---|---|---| +| A: original prompt | 0/5670 reused, 10.4 s | 0/5670 reused (cold), 10.4 s | +| B: insert one row → answer 161 | **0/5702 reused, 10.4 s full re-read** | **3072/5702 reused (54%), 20.2 s** | +| C: insert second row → answer 162 | **0/5736 reused, 10.4 s full re-read** | **5120/5736 reused (89.3%), 3.4 s** | + +Answers: 160 / 161 / 162 all correct, including double-edit and revert discrimination (revert correctly answers 160 again). Engine log line, turn C: `prompt 5736 tokens = 5120 reused + 616 read in 3439 ms`. + +### 5.2 All quant formats: steady-state and splice-path proof + +| Format | Splice fired (trace) | Edit answer | Steady-state chain (3 turns) | +|---|---|---|---| +| fp16 | ✓ | correct | (Phase-2 measurement: 24/24 correctness) | +| int8 | ✓ | 161 ✓ | GREEN — 89% reuse on turn C | +| int8 + STRATA_KV_ROT=1 | ✓ (`fired=1`, `status=1`) | 161 ✓ | — | +| k8v4 | ✓ (`fired=1`, `status=1`) | 161 ✓ | — | +| q4_0 | ✓ (`fired=1`, `status=1`) | 161 ✓ | **GREEN — A 45.2 s cold; B 54%; C 5120/5736 (89.3%), 3.4 s, 162 ✓** | + +q4_0's full chain matches int8's GREEN shape exactly (validator: `sfx_steady_state_q4.py`). + +### 5.3 Numerical accuracy of the splice (unit tests, CPU) + +The double-quant noise (first write + splice requant) measured against a fresh write, in code steps: + +- int8 unrotated: **1.17** +- int8 + KV_ROT (H-sandwich): **1.27** — within 10% of the accepted unrotated path; bitwise-identical to an independent sandwich-of-stored reference. +- q4_0: **1.53 q4-scale units** — coarser codec (16 levels vs int8's 254), same ballpark. + +Cells outside the edit range are byte-identical after the splice (all formats). + +### 5.4 Long-context generation parity (old vs new engine) + +Same seed (42), temperature 0, ~8k-token prompt with 12 seeded facts at spaced depths: exact recall + arithmetic + a 250-word generation. + +| | Strata .40.3 (base) | .43 + suffix-reuse | +|---|---|---| +| Recall + arithmetic | 10/10 (2 runs) | 10/10 (2 runs) | +| Long generation | 1973 chars, natural stop | 1972 chars, natural stop | +| Turn-2 append generation | correct | correct (served 98% from cache) | + +### 5.5 CLM on ContextBench: retention and reuse (assembled from the benchmark campaigns) + +**Retention: CLM vs traditional summarization** (P3 final, matched instances, seeds 42+43, prefix-only engine — the splice does not affect this comparison): + +| Arm (context mgmt) | Total (4 families) | Exact-value queries | kv-store | needle (multi-seed 42/43/44) | +|---|---|---|---|---| +| **CLM** (verbatim offload) | **292/429 = 0.681** | **full preservation** | **0.875 mean** (24/24, 18/24) | 1.000 / 0.750 / 0.833 | +| Floor (raw context) | 253/429 = 0.590 | partial | 0.312 | 0.675 mean (P2) | +| **Summarization** | 122/429 = 0.284 | **0/96 — absolute zero** | **0.000** | 0.295 mean (P2) | + +Summarization destroys exact values (0/96 recoveries) — every ID, count and token the compressor touches is gone. CLM preserves them by never compressing: the agent re-injects its notes verbatim. That verbatim re-injection is precisely the mid-prompt-edit workload the splice serves. + +**Reuse: before the splice (prefix-only)** — two complementary measurements: +- *Usage trace* (kv-store seed 42, 68 turns): 45.5% aggregate, but split 25 pure-append requests at 97–98% vs **41 edit requests at 0%** — every CLM note-rewrite paid a full re-prefill. +- *Campaign profile* (P0b, 4 families, 248 requests, 14,336 budget): needle 1.000 (96/96) at **16.1% reuse**, sudoku 1.000 (15/15) at 57.6%, kv 1.000 (24/24) at 43.1%, log-triage 0.583 (14/24) at 21.5%; total 28.9% (491,299/1,700,229 tokens), 2,378 s prefill saved. Reuse tracks workload shape: append-heavy families reuse; insert/edit families (needle's mid-file inserts, log's rolling edits) are the starved ones. + +**After the splice (fixed engine, .43 + suffix-reuse):** the engine-level edit-turn measurement is §5.1/§5.2 — a mid-prompt edit now reuses 89% (5120/5736) instead of 0%, at 3.4 s instead of 10.4 s, answers provably correct on every KV format. The ContextBench families whose reuse was starved (needle 16.1%, log 21.5%) are exactly the edit-shaped workloads this converts. A full 4-family re-run on the fixed engine with the dense-tail configuration exists and is §5.8: reuse rose exactly where predicted (log 21.5% → 77.2%, kv 43.1% → 72.3%), with the remaining failure mode (needle's fail-close gap-read) measured and decomposed there rather than hidden. + +### 5.6 The honest cost: checkpoint granularity + +Reuse requires a checkpoint at or before the edit position. With `--prefill auto` (one big chunk) there are no mid-prompt checkpoints and mid-edit reuse does not fire (appends still reuse). The proven configuration is `--prefill 1024 --prompt-cache-every 1024`, which costs cold-read throughput: 46.6 s vs 10.9 s for a cold 5.7k prompt (4.3×) on this GPU. `--prefill 2048` halves the checkpoint density and recovers most of the cold-read cost. This trade-off is why the feature is opt-in and the knob is exposed. + +### 5.7 Dense-tail cadence: the trade-off dissolved (`sfx-future`) + +`--prompt-cache-dense-tail 4096` with `--prefill auto`: sparse checkpoints through the head, dense through the last 4096 tokens. Same three-turn ledger probe, same seed, dense-tail stack (T21 per-request gate): + +| Turn | Edit-path config (§5.1: prefill 1024 + every 1024) | Dense-tail config (prefill auto + dense-tail 4096) | +|---|---|---| +| A: cold read | 46.6 s | **42.4 s** | +| B: append → 161 ✓ | 20.2 s (54%) | 27.9 s | +| C: edit → 162 ✓ | 3.4 s (89.3%) | **10.1 s (81.0% — 4646/5736)** | + +All answers correct (160/161/162). The C-turn anchor lands on a dense-tail checkpoint at 4646 (just below the second edit at 5230) — strict-mount, no splice needed. Checkpoint cost at existing boundaries is ~100–150 ms per save (measured T19: pipeline sync 0.0 ms, state save 100–150 ms); the per-boundary ~8 s cost of the §5.6 config is the *pipeline restart between chunks*, which dense-tail's single-chunk head avoids. + +### 5.8 The remaining gap — and the honest failure mode + +When an edit lands deeper than every checkpoint, the engine fail-closes: restore the deepest checkpoint at or before the edit position, read the rest fresh (826 tokens in the §5.7 probe). Correctness is never traded. The cost is bounded but real, and it is the dominant residual inefficiency. A 4-family ContextBench campaign on the dense-tail config (seed 42, budget 14,336, CLM-skill agent) measured where this bites: + +| Family | base .40.3 (reward / reuse) | dense-tail stack (reward / reuse) | +|---|---|---| +| needle | 1.000 / 16.1% | 0.500 / 17.0% | +| sudoku | 1.000 / 57.6% | 0.867 / 73.5% | +| kv-store | 1.000 / 43.1% | **1.000 / 72.3%** | +| log-triage | 0.583 / 21.5% | **0.625 / 77.2%** | + +Two findings. First, the reuse gains are real and large (kv +29.2 pp, log +55.7 pp) and scores on those families rose, not fell — mounted cache answers exactly as fresh-prefilled text does. Second, needle fell to 0.500 — and the decomposition isolates why it is not the code: the same binary with the §5.1 config scores 0.750 on needle, identical to the flags-off arm's 0.750 (both within the seed band of the multi-seed needle mean 0.847 ± 0.171); only the dense-tail *configuration* drops it to 0.500, and needle's reuse was unchanged (16.1% → 17.0%) — the failure is not cache corruption but the fail-close gap-read colliding with that seed's unbounded-growth agent trajectory (23 messages, no compaction). + +### 5.9 FW-2b — state-cadence checkpoints close the gap (2026-10-13) + +`--prompt-cache-state-every N` (FW-2b) adds a side pool of *state-only* checkpoints (the recurrence state `checkpoint_save` already copies for periodic anchors — gdn/ple/tails/dead/block_pos host vectors, no K/V) at a fine cadence, and the splice-narrowed restore picks the deepest anchor ≤ P across **both** pools. Fail-close semantics are unchanged — the gap [anchor, P) is still read through the normal chunk loop; the anchor just sits closer to the edit. 0 = off = byte-identical. + +Same binary, same seed 42, same dt configuration, only the flag differs: + +| arm | reward | reuse | wall | prefill ms/token | +|---|---|---|---|---| +| dt + state-every 256 | **0.833** (80/96) | **25.2%** | 1252 s | 3.66 | +| dt (control) | 0.750 (72/96) | 0.0% | 992 s | 4.02 | + +Needle recovers into the seed band (0.847 ± 0.171) and per-token prefill *drops* 9% despite a longer trajectory. The control's per-request table shows the mechanism: all 17 of its mid-file rewrites found no anchor (0% reuse, ~44 s full prefill each); with the state ladder, the compaction-phase tail-preserving rewrites mount at 38–66% (9 of 26 requests; the growing phase and head-rewrites legitimately have nothing to mount). + +Extended to seeds 43/44 (same protocol), the reward margin does **not** reproduce — what holds is the reuse and stability effect: + +| seeds 42/43/44 | reward (mean ± sd) | reuse (mean ± sd) | +|---|---|---| +| dt + state-every 256 | 0.792 ± 0.042 | **15.5% ± 10.6** | +| dt (control) | 0.792 ± 0.072 | 7.2% ± 7.6 | + +Every run of both arms sits inside the multi-seed needle band; the feature is reward-neutral across seeds while roughly **doubling mean reuse** (2.1×) and halving reward variance. + +Re-certifying the other three families on the full FW-2b stack (dt + state-every 256, seed 42) gives the campaign's strongest table: + +| family | base .40.3 | dt (P0e) | **dt + state-256 (FW-2b)** | +|---|---|---|---| +| needle | 1.000 / 16.1% | 0.500 / 17.0% | 0.792 ± 0.042 / 15.5% (3 seeds) | +| sudoku | 1.000 / 57.6% | 0.867 / 73.5% | **1.000 / 68.9%** | +| kv-store | 1.000 / 43.1% | 1.000 / 72.3% | **1.000 / 71.8%** | +| log-triage | 0.583 / 21.5% | 0.625 / 77.2% | **0.625 / 69.9%** | + +Every family at or above its base score, reuse 1.2–4.3× base, and sudoku's single-seed dip is gone. + +**Quant universality (all five KV formats, live-measured).** The state pool never touches K/V encoding, so FW-2b should be — and is — format-independent. Same binary, same dt + state-every 256 stack, only `--kv` (and the `STRATA_KV_ROT=1` env for the rotated arm) differs; the A/B/C chain oracle answers 160/161/162 with **byte-identical C-turn reuse (4646/5736, 81.0%) on every format**: int8, int8+KV_ROT (Hadamard), k8v4, q4_0, fp16. The splice layer carries its own per-format writer-math tests and GPU probes (§5.2); this sweep closes the same gate for the state-cadence layer. Read FW-2b as a reuse/latency optimization with a safety profile: the 0.500 failure mode of §5.8's single-seed dt run never appears (arm minimum 0.750), and the seed-42 draw where reuse visibly rescued reward (0.750 → 0.833) illustrates the upside when a trajectory's rewrites land where only the state ladder has anchors. Steady-state chain (dt-steady10/10b): answers 160/161/162, C-turn reuse 81.0% @ 10.8 s, cold and append turns identical to pre-FW-2b — the side pool taxes nothing (captures ~105 ms each, host-RAM pool capped at 2 GiB ≈ 10 anchors at needle scale). + +## 6. Purpose in context management (CLM) terms + +Verbatim-offload context management — the agent keeps its working state in files and re-injects them every turn — is the strongest known defense against context decay (our measurements: verbatim offload preserves exact values that summarization destroys; 0/96 exact-value recoveries after summarization vs full preservation offloaded). But it converts every working-state update into a mid-prompt edit, which until now meant a full re-prefill per update. The splice makes the CLM pattern affordable at the engine level: the agent rewrites its notes, and the engine re-uses everything up to the rewrite. The 41-of-66 zero-reuse trace (§3) is a CLM session; the same shape with the splice is §5.1. + +**Prior-session usage evidence.** Three independent benchmark campaigns (P2 multi-seed, P3 matched 4-family, P0b replication, seeds 42–44) against prefix-only reuse consistently show the same usage shape: a CLM agent's live context grows 17k → 41k chars over 68 turns as notes accumulate, 25 of 66 requests (the pure appends) ride 97–98% reuse while 41 rewrites each pay a full re-prefill — the engine spends most of its busy time re-reading text the agent already had. Task scores themselves were unaffected by cache state (kv-store 24/24 → 18/24 across P3/P0b runs with identical configs; needle 1.000/0.750/0.833 across seeds vs prefix-only mean 0.847 ± 0.171), which is expected — cache changes compute cost, not token visibility — and is the evidence that the splice's win is purely time-to-answer, with no accuracy tax to hide. The exact-value stakes are documented across sessions too: verbatim offload preserved every exact value where summarization recovered 0/96 — the pattern the splice makes affordable at engine level. + +## 7. Relationship to prior work + +- **llama.cpp `--cache-reuse` / KV cache shifting** — production precedent for position-shifted cache reuse on quantized caches (issue #18497's logs show chunk reuse with KV shifting on q8_0 caches; off by default; #3825 documents the shift machinery's history). Same edit shape, same off-by-default posture. Strata's splice differs in the quantized path: dequant → rotate → requant with each format's writer math (incl. the Hadamard-domain sandwich), and cross-request conversation checkpoints rather than per-slot reuse. +- **CacheSlide (FAST '26)** — identifies Relative-Position-Dependent Caching in agent workflows and solves it with position-insensitive encodings and weighted correction attention (3.1–4.3× latency reduction). Strata's splice is the exact-arithmetic version for RoPE models: rotate the kept K by the position delta rather than making positions insensitive. +- **Adaptive KV Cache Reuse (arXiv 2605.24022)** — survey framing reuse across request shapes; positions CacheSlide and prefix/PIC approaches. +- **PBKV (arXiv 2605.06472)** — prediction-driven cache management for dynamic agent workflows. +- **vLLM reuse RFC (#2614)** — names absolute-position mismatch as the blocker for mid-prompt reuse and scopes reuse to prefixes; the problem statement this PR answers on different machinery. + +## 8. Configuration + +Three new flags; the rest is existing upstream machinery: + +``` +--cache-suffix-reuse 64 # the feature (opt-in; 0 = off, default) +--prompt-cache-state-every 256 # FW-2b: fine-cadence state-only checkpoints (0 = off = prior behavior) +--prompt-cache-dense-tail 4096 # adaptive cadence (opt-in; 0 = off = uniform cadence) +--conversation-cache-mib 4096 # upstream flag; default 0 = parking off (needed: nothing to splice from without it) +--prompt-cache 6 # upstream default (>= 2 required by the validator) +--prefill 1024 --prompt-cache-every 1024 # uniform checkpoint density (the §5.1 edit-path config; not needed with dense-tail) +``` + +Refused (logged, falls back to full re-read, never crashes): `--kv-resident > 0`, `--layer-split`, rotated formats with head_dim ≠ 256 (fwht256 is order-256). + +## 9. Verification summary + +- CPU ctest **53/53** (39 upstream + 14 suffix-reuse), re-run after every commit. +- Suffix-reuse suites: partial-prefix **82/0** (incl. T16.7 cross-conversation pin and T22.1–5c two-pool pick), dense-tail **23/23**. +- CUDA (sm_86, toolkit 13.0) builds clean; `-Wall -Wextra` single-header syntax check clean. +- Live GPU probes: splice fired and answers correct on every format (§5.2); dense-tail chain GREEN (§5.7); FW-2b chain GREEN with byte-identical reuse across all five KV formats (§5.9). +- Independent fail-closed code reviews: all-quant splice **APPROVE** (3 MINOR doc findings, closed); `sfx-future` **r1 REJECT → fix → r2 APPROVE** (1 MAJOR cross-conversation dominance bug found and fixed TDD-style; zero new findings at r2); `sfx-fw2b` (state-cadence, T22/T23/cap) **APPROVE, zero blocking** (3 informational MINORs: documented T18 behavior, conservative `capacity()` byte accounting, doc wording). +- Not tested: HIP compile (no AMD hardware available to the author — the new code is host-side C++ behind the existing transport layer; an AMD build pass is requested from the community/maintainers). + +## 10. Limitations and future work + +- **Built and certified (FW-2b):** fine-cadence state-only checkpoints (`--prompt-cache-state-every 256`) with two-pool restore shrink the fail-close gap to ≤ N tokens below the edit. Remaining gap beyond that: an edit landing deeper than every anchor still triggers a bounded gap-read (§5.8); with state-every 256 the bound is 256 tokens, not `every`-spacings. +- **FW-2c (parked; implementation plan):** session-local partial K/V reuse for the gap-read. Source recon settled the causality: parked *recurrence state* at L is stale for a rewritten prompt (restoring it produced wrong answers in the v3 probe — semantic ledger under-counted), but parked *arena K/V* for `[C, P)` is valid because those tokens are identical by the LCP. The win therefore requires a **state-advance kernel**: advance the GDN/PLE recurrence from the restored checkpoint C through P by consuming the parked K/V, skipping the expert pipeline for `[C, P)`. Plan (follow-up PR): (1) pure seams — a splice planner and a read fast-path planner (drafted with tests on a scratch branch, not in this PR); (2) T26 kernel — `advance_state_from_kv(state, kv, [C, X))` per KV format, fp16 first, int8 via the existing dequant kernels; (3) mount-site wiring behind an env gate (`STRATA_SFX_KV_SPLICE`), fail-closed to today's chunk-loop re-read; (4) chain oracle must answer 160/161/162 unchanged, then an A/B against FW-2b at 4096-token depth. Kill criterion: if state-advance from parked K/V measures slower than the plain re-read at that depth, FW-2c is abandoned — the chunk-loop re-read stays. Projection that justifies the attempt: 91.2% session-local reuse on the §5.7 probe shape vs 81.0% today. +- The env-gated replay path (`STRATA_SFX_REPLAY`, off by default) remains an alternative to FW-2c for gap closure; the `partial_gap_read_from` seam is shared. +- Streaming-ring (`--kv-resident`) and layer-split layouts remain refused — splicing a ring requires cell re-homing, not just rotation. +- Recurrence state (GDN/PLE) is restored from the narrowed checkpoint rather than transformed; a forward-state transform would cut the remaining gap-read further. + +## References + +1. CacheSlide: Unlocking Cross Position-Aware KV Cache Reuse for Accelerating LLM Serving — Liu et al., USENIX FAST '26. https://www.usenix.org/system/files/fast26-liu-yang.pdf +2. llama.cpp issue #18497 — cache-reuse effectiveness on quantized caches (KV-shift logs). https://github.com/ggml-org/llama.cpp/issues/18497 +3. llama.cpp issue #3825 — `llama_kv_cache_seq_shift` position-delta semantics. https://github.com/ggml-org/llama.cpp/issues/3825 +4. llama.cpp discussion #8860 — prefix cache reuse semantics (maintainer confirmation). https://github.com/ggml-org/llama.cpp/discussions/8860 +5. Adaptive KV Cache Reuse for Fast Long-Context LLM Serving — arXiv:2605.24022. https://arxiv.org/abs/2605.24022 +6. Efficient Serving for Dynamic Agent Workflows with Prediction-based KV-Cache Management (PBKV) — arXiv:2605.06472. https://arxiv.org/abs/2605.06472 +7. vLLM prompt-cache reuse RFC — absolute-position mismatch discussion. https://github.com/vllm-project/vllm/issues/2614 + +## Review r3 (2026-10-17, mm-reviewer on v0.1.45 rebase) — findings disposition + +APPROVE-WITH-FINDINGS, 11 MINOR, 0 MAJOR, 0 BLOCKER (verdict: sfx145-mm-verdict.json). +9 findings were independent confirmations of correctness (rebase fix e2dae10c, splice +math per codec, boundary clamps, pool caps, test strength). Two optional items: + +- F8 (fixed): both REPLAY_NOT_AVAILABLE reason strings (strict + partial sites in + generate.cpp) said "Phase 3" for ring/per-stage refusals — stale wording; now says + "future work". Stderr-only, no behavior change. +- F5 (documented as future hardening): T18 replay-gated gap read fires only on the + partial-mount path (`splice_restore_anchor_length > 0`). On the strict + splice-narrowed branch the restored anchor covers [C, E) by construction, so no + gap-read is needed there today. If FW-2c lifts restore semantics, extend the gate + symmetrically. Reviewer marked optional; no bug present. diff --git a/include/strata/core/conversation_cache.hpp b/include/strata/core/conversation_cache.hpp index 78ff9f18924..b86997a2103 100644 --- a/include/strata/core/conversation_cache.hpp +++ b/include/strata/core/conversation_cache.hpp @@ -8,6 +8,7 @@ #include #include #include +#include // [SFX-DBG] probe: fprintf in best_with_partial's candidate scan #include #include #include @@ -237,6 +238,46 @@ int64_t conversation_prefix(const ConversationCheckpoint& c, const std::vector= p are irrelevant (the partial mount discards [p, n) of the +/// checkpoint). This is the helper the flag-gated partial-prefix mount in +/// `best_with_partial` walks to find the longest LCP when no strict match exists. +template +int64_t conversation_lcp(const ConversationCheckpoint& c, const std::vector& prompt, + const std::vector& images) { + const size_t n = c.ids.size(); + if (n < 1) return 0; // empty: no LCP + // T17 (2026-10-13): a candidate LONGER than the prompt is a valid partial + // candidate - the parked LIVE (prompt+response) is almost always slightly + // longer than the next turn's prompt, and the old `n > prompt.size()` early + // out here (plus the two guards beside it) refused the single most valuable + // anchor in every edit chain (probe: e0.live:L=5786:skip, prompt 5736). + // The walk is bounded by min(n, prompt.size()): tokens past the prompt are + // irrelevant (the partial mount discards them), and the p == n < prompt.size + // case reports the full length (the strict-prefix shape, unchanged). + int64_t p = 0; + while (p < (int64_t) n && p < (int64_t) prompt.size() && + c.ids[(size_t) p] == (Token) prompt[(size_t) p]) ++p; + if (p == 0) return 0; // token 0 mismatches + // Image check: request images at positions < p must match the checkpoint's images. The + // checkpoint's images at positions >= p are irrelevant (we re-prefill [p, n) on the new + // prompt). The strict `conversation_prefix` check is the same logic with n -> p. + size_t j = 0; + for (const auto& image : images) { + if (image.start >= p) continue; + if (j == c.imgs.size() || !(c.imgs[j++] == image)) return 0; + } + for (size_t k = j; k < c.imgs.size(); ++k) + if (c.imgs[k].start < p) return 0; // checkpoint has an image below p not in the request + return p; +} + class ConversationCache { public: struct Match { @@ -244,6 +285,15 @@ class ConversationCache { int64_t tokens = 0; bool live = false; uint64_t serial = 0; // the entry's identity; its index shifts when entries before it go + // Phase 3b: flag-on partial-prefix match. `partial == true` means the longest LCP of any + // checkpoint with the request is strictly less than the checkpoint's length (a mid-prompt + // edit past the LCP). In that case `tokens` is the LCP (the resume length the splice hook + // narrows to) and `checkpoint_length` is the full checkpoint's length (the restore anchor). + // Strict match (== the existing behaviour): `partial == false`, `tokens == checkpoint_length`. + // No match: `partial == false`, `tokens == 0`. Off-flag (`flag <= 0`) and on-flag-with- + // strict-match returns are byte-identical to `best()`. + bool partial = false; + int64_t checkpoint_length = 0; ///< the full checkpoint's length (== tokens when !partial) }; ConversationCache(size_t budget, size_t slots, int64_t min_tokens = 0) @@ -312,7 +362,7 @@ class ConversationCache { if (e.cvec != cvec) continue; auto consider = [&](const ConversationCheckpoint& c, bool live) { const int64_t n = conversation_prefix(c, prompt, images); - if (n > best.tokens) best = {i, n, live, serials_[i]}; + if (n > best.tokens) best = {i, n, live, serials_[i], false, n}; }; consider(e.live, true); for (const auto& c : e.checkpoints) consider(c, false); @@ -333,6 +383,90 @@ class ConversationCache { return false; } + /// Phase 3b: flag-gated partial-prefix match. At `flag <= 0` (and on-flag when a strict + /// match exists) this is byte-identical to `best()`: same loop, same `conversation_prefix`, + /// same tie-break by most recently parked. At `flag > 0` with NO strict match, falls back + /// to the longest LCP `p` such that `0 < p < c.ids.size()` and the image check at positions + /// < p matches; the returned Match has `partial = true, tokens = p, checkpoint_length = L`. + /// Ties: longest LCP wins, then longest checkpoint (a deeper anchor gives the splice more + /// K rows to rotate for the unchanged tail), then most recently parked (same as `best()`). + template + Match best_with_partial(int64_t flag, const std::vector& prompt, + const std::vector& images, bool cvec) const { + if (flag <= 0) return best(prompt, images, cvec); // flag off: byte-identical + Match m; + // strict first (same as best()). + for (size_t i = entries_.size(); i-- > 0;) { + const auto& e = entries_[i]; + if (e.cvec != cvec) continue; + auto consider = [&](const ConversationCheckpoint& c, bool live) { + const int64_t n = conversation_prefix(c, prompt, images); + if (n > m.tokens) m = {i, n, live, serials_[i], false, n}; + }; + consider(e.live, true); + for (const auto& c : e.checkpoints) consider(c, false); + } + // FW-1 splice engagement (2026-10-13): do NOT return at the first strict + // match. One conversation's checkpoints are nested prefixes of ONE history, + // so that history diverges from this request at a single point D and every + // strict anchor s <= D equals the partial LCP p of the SAME conversation's + // deeper checkpoints. A partial mount therefore never reads from a + // shallower point than the strict match would (p >= s), and its splice may + // recover K/V up to the anchor length L. Keep the partial candidate when + // its checkpoint covers strictly more than the strict anchor (L > s); + // otherwise the strict match stands (no splice cost when it cannot recover + // more). Without this, a mid-file edit always strict-mounts the anchor + // just below the edit and the splice never engages (partial-branch + // enter=0 in every steady-state trace - measured 2026-10-13). + const int64_t strict_tokens = m.tokens; + // partial pass: longest LCP > 0 and < c.ids.size(); tie-break by deeper anchor + // (longer checkpoint) then by most recently parked (the loop iterates newest-first). + Match pm; // the best partial candidate (tokens = LCP, checkpoint_length = L) + // [SFX-DBG] probe: per-candidate partial scan visibility. The steady-state chain showed + // parked-search reporting match{tokens=2598 partial=0} where the source says a partial + // candidate (LCP 3424, anchor 5702) should dominate strict 2598 - log every candidate's + // L/idx/LCP so the steady-state trace can attribute the miss to the candidate set, the + // LCP guard, or the size guard. Gated `flag > 0` (probe on the sfx path only). + const bool dbg_partial = flag > 0; + if (dbg_partial) + std::fprintf(stderr, "[SFX-DBG] parked-scan: entries=%zu prompt_len=%zu strict_tokens=%lld:", + entries_.size(), prompt.size(), (long long) strict_tokens); + for (size_t i = entries_.size(); i-- > 0;) { + const auto& e = entries_[i]; + if (e.cvec != cvec) { if (dbg_partial) std::fprintf(stderr, " e%zu:cvec", i); continue; } + auto consider = [&](const ConversationCheckpoint& c, bool live) { + const int64_t L = (int64_t) c.ids.size(); + // T17: the size guard is lifted for the PARTIAL pass - a longer-than-prompt + // candidate (the parked live) is the most valuable partial anchor. The strict + // pass above keeps its own guard (conversation_prefix refuses longer), so this + // scan alone can surface it. L < 1 stays refused (empty). + if (L < 1) { + if (dbg_partial) std::fprintf(stderr, " e%zu.%s:L=%lld:skip", i, live ? "live" : "ck", (long long) L); + return; + } + const int64_t p = conversation_lcp(c, prompt, images); + if (dbg_partial) + std::fprintf(stderr, " e%zu.%s:L=%lld:lcp=%lld", i, live ? "live" : "ck", (long long) L, (long long) p); + if (p <= 0 || p >= L) return; // 0 LCP or strict (already tried) + if (p > pm.tokens || (p == pm.tokens && L > pm.checkpoint_length)) + pm = {i, p, live, serials_[i], true, L}; + }; + consider(e.live, true); + for (const auto& c : e.checkpoints) consider(c, false); + } + if (dbg_partial) std::fprintf(stderr, " -> pm{tokens=%lld L=%lld} strict=%lld\n", + (long long) pm.tokens, (long long) pm.checkpoint_length, + (long long) strict_tokens); + // Cross-conversation safety (review r1 finding 1, 2026-10-13): the nesting + // argument above is SAME-conversation only. A DIFFERENT conversation's + // partial can have a deeper anchor (L > s) with a shallower LCP (p < s); + // mounting it would read from a shallower point than the strict match. + // Require p >= s as well as L > s. (Single-conversation behavior is + // unchanged: there p >= s always holds, so the added term never binds.) + if (pm.tokens > 0 && pm.tokens >= strict_tokens && pm.checkpoint_length > strict_tokens) return pm; + return m; // strict match wins; partial ignored + } + SavedConversation take(size_t index) { SavedConversation out = std::move(entries_.at(index)); bytes_ -= out.bytes(); diff --git a/include/strata/core/layer.hpp b/include/strata/core/layer.hpp index 9e0361aca22..99c7119f056 100644 --- a/include/strata/core/layer.hpp +++ b/include/strata/core/layer.hpp @@ -312,6 +312,11 @@ void qsa_set_kv_int8(bool enabled); /// INT8 K/V through the Hadamard rotation (off by default: STRATA_KV_ROT=1) void qsa_set_kv_int8_rotate(bool enabled); bool qsa_kv_int8(); +/// Companion to qsa_kv_int8(): the int8-Hadamard domain is a different K storage +/// (codes-then-rotation; kv_q8.cu:47-67's writer does NOT include the Hadamard pass +/// when this is off). The splice needs this to refuse --kv int8 + STRATA_KV_ROT=1 +/// explicitly (Phase 3 follow-up: dequant -> invH -> rotate -> H -> requant). +bool qsa_kv_int8_rotate(); /// PR #21: store K/V as Q4_0 after a Hadamard rotation (`--kv q4_0`): 576 B per cell, vs 1,056 in INT8. void qsa_set_kv_q4(bool enabled); bool qsa_kv_q4(); diff --git a/include/strata/program/kv_q8_quant.hpp b/include/strata/program/kv_q8_quant.hpp new file mode 100644 index 00000000000..b9e0cbd46cf --- /dev/null +++ b/include/strata/program/kv_q8_quant.hpp @@ -0,0 +1,110 @@ +// include/strata/program/kv_q8_quant.hpp - header-only, CPU-mirror of the int8 K/V quant writer. +// (Phase 3: lives in the program/ tree, alongside the splice helpers it serves, rather than +// the kernels/ tree which is sparse-checkout in this repo and only touched by the GPU parity +// suite. The file is header-only with no CUDA includes, so a CUDA TU can pull it in.) +// +// WHY THIS EXISTS. The engine's INT8 K/V pool writer (src/kernels/cuda/kv_q8.cu:47-67 +// for K, mirrored for V) stores per group of 64 values: +// amax = max|x| over the 64 values +// sf = f32_from_f16(f16_from_f32(amax / 127)) // quantize against the STORED scale +// code = clamp(rint(x / sf), -127, 127) when sf > 0, else 0 +// block = (codes[64], scale_bits) +// +// Phase 3 (int8 splice) needs the EXACT same quant math to: (a) dequant a cached block +// back to fp32, (b) apply the splice's RoPE delta rotation, (c) requant the rotated image +// back to the same block format. If the requant differs from the writer by a rounding +// choice, the splice pays a per-group half-step on every rotated cell, which the existing +// fresh-append path already pays and the model's downstream tolerance is sized for. +// +// The helpers here are inline so T11 (CPU test) and the runtime splice (host loop) call +// the SAME code: no separate "host reference" copy. This is a header-only quant module +// with no CUDA includes, identical conventions to the host reference in +// src/kernels/kv_hybrid_parity.cpp:99-116 (which is BIT-EXACT with the kernel writer; +// the GPU parity suite pins that). The function inlines we use here match the same +// rounding rules: f16_from_f32 = round-to-nearest-even, rint = ties-to-even +// (std::nearbyint == rint; both round half-to-even on the host). +// +// Layout invariant (kv_q8.hpp:6, kvr_q8_layers: 32 B of scales per cell): one scale per +// group of 64 values, stored next to the codes by the writer thread t==0 +// (src/kernels/cuda/kv_q8.cu:67). The mirror here writes one scale per group too. +#pragma once + +#include "strata/kernels/f16_bits.hpp" // f16_from_f32 / f32_from_f16 (kv_q8.cu uses the same) +#include "strata/kernels/kv_q8.hpp" // KV_Q8_GROUP (64) + +#include +#include +#include +#include + +namespace strata::kernels { + +/// One quantized int8 group (64 values + one fp16 scale bit pattern), matching the +/// writer's `block_q8_K`-style layout. Packed because every splice call needs to read +/// or write 64 + 2 bytes in one place; the writer stores these in two separate arrays +/// (`k_q` codes and `k_scale` scales), the dequant/requant here takes them as a +/// transient struct and unpacks/repacks at the call site. +struct KvQ8Group { + int8_t codes[KV_Q8_GROUP] = {}; ///> 64 int8 codes (clamped to [-127, 127]) + uint16_t scale_bits = 0; ///> fp16 of (amax / 127) for the group (kv_q8.cu:55) +}; + +/// Quantize 64 fp32 values into a KvQ8Group. Mirrors src/kernels/cuda/kv_q8.cu:47-67 +/// exactly: amax over the group, scale = f16(f32(amax/127)), each code = clamp(rint(x/sf)). +inline void kv_q8_quant_group(const float* x, KvQ8Group& out) { + // amax over the group (the writer: kv_q8.cu:50-54, with __shfl_xor reduction over + // 64 threads; the group is small enough that a single linear scan is equivalent). + float amax = 0.0f; + for (int t = 0; t < KV_Q8_GROUP; ++t) { + const float a = std::fabs(x[t]); + if (a > amax) amax = a; + } + const uint16_t sb = f16_from_f32(amax / 127.0f); + const float sf = f32_from_f16(sb); // quantize against STORED scale + out.scale_bits = sb; + for (int t = 0; t < KV_Q8_GROUP; ++t) { + int q = 0; + if (sf > 0.0f) { + // kv_q8.cu:59: __float2int_rn = round-to-nearest-even (RTNE). std::nearbyint + // is RTNE too; std::rint is RTNE; the names are aliases on x86_64. Using + // std::nearbyint explicitly to match the GPU kernel's __float2int_rn. + q = (int) std::nearbyint(x[t] / sf); + if (q < -127) q = -127; + else if (q > 127) q = 127; + } + out.codes[t] = (int8_t) q; + } +} + +/// Dequantize one KvQ8Group to 64 fp32 values. Symmetric to the writer: x' = f32(code * sf) +/// (kv_q8.cu:100-101 mirror: the gather kernel dequantizes into fp16; the splice runs in +/// fp32 and writes through f16 at the end of the rotation step). +inline void kv_q8_dequant_group(const KvQ8Group& g, float* out) { + const float sf = f32_from_f16(g.scale_bits); + for (int t = 0; t < KV_Q8_GROUP; ++t) + out[t] = (float) g.codes[t] * sf; +} + +/// Read a KvQ8Group from a (codes, scale_bits) buffer pair (the writer's storage shape +/// in QsaState: int8_t* k_q, uint16_t* k_scale). The (row, group) addressing is: +/// - codes: row-major, head_dim per row, KV_Q8_GROUP per group +/// - scales: row-major, head_dim / KV_Q8_GROUP per row, 1 per group +inline void kv_q8_read_group(const int8_t* codes, const uint16_t* scale_bits, + int head_dim, int row, int group, KvQ8Group& out) { + out.scale_bits = scale_bits[row * (head_dim / KV_Q8_GROUP) + group]; + for (int t = 0; t < KV_Q8_GROUP; ++t) + out.codes[t] = codes[row * head_dim + group * KV_Q8_GROUP + t]; +} + +/// Write a KvQ8Group back to the writer's storage shape. Counterpart to +/// `kv_q8_read_group`. The two helpers in this form let the splice reuse the same +/// dequant/rotate/requant pipeline as the parity test's host reference without a +/// second copy of the addressing arithmetic. +inline void kv_q8_write_group(int8_t* codes, uint16_t* scale_bits, + int head_dim, int row, int group, const KvQ8Group& in) { + scale_bits[row * (head_dim / KV_Q8_GROUP) + group] = in.scale_bits; + for (int t = 0; t < KV_Q8_GROUP; ++t) + codes[row * head_dim + group * KV_Q8_GROUP + t] = in.codes[t]; +} + +} // namespace strata::kernels diff --git a/include/strata/program/kv_splice.hpp b/include/strata/program/kv_splice.hpp new file mode 100644 index 00000000000..5126f7aabbc --- /dev/null +++ b/include/strata/program/kv_splice.hpp @@ -0,0 +1,98 @@ +// include/strata/program/kv_splice.hpp - the suffix-reuse K/V splice: a host rotation of K by a +// per-position RoPE delta over the unchanged tail. V is untouched; the rope is content-invariant +// for V (V carries no position by itself - `include/strata/kernels/rope.hpp` rotates K alone). +// +// T3 (the unit test) needs a `cos`/`sin` table to drive `rope_neox_pair` (host-callable, +// `STRATA_ROPE_HD inline`, include/strata/kernels/rope.hpp:42). The authoritative `build_rope_table` +// is a CUDA TU at src/kernels/cuda/rope.cu:48-58 and only its DECLARATION (`:47`) lives in the header; +// the body is pure `std::pow`/`std::cos`/`std::sin`, which means a CPU build cannot link the .cu without +// CUDA. Per A2.2: T3 inlines a host C++ port of that body in this header. `inline` means every TU +// gets the same instance and there is no link surface. +// +// `rotate_k_by_delta` walks a row of K in pairs `(i, i + n_rot/2)` (NEOX), reading cos/sin at the OLD +// position, RE-applying the rotation to the new position by reading the delta-table, and writing the +// result back in place. The T3 test asserts the result equals the rotation applied at the NEW +// position from scratch, so any deviation in the cos/sin re-computation would surface. +#pragma once + +#include "strata/kernels/rope.hpp" + +#include +#include +#include +#include + +namespace strata::program::suffix_reuse { + +/// Host C++ port of `build_rope_table` (src/kernels/cuda/rope.cu:48-58) - pure `std::pow` / `std::cos` +/// / `std::sin`, float64 throughout in the reference's order (inv, then ang, then cos/sin). `inline` +/// keeps the ODR happy across translation units that include this header. Tables are `(max_pos, n_rot/2)` +/// row-major floats, matching the CUDA builder so the `rope_neox_pair` consumer agrees. +inline void build_rope_table_host(int n_rot, double theta, int max_pos, float* cos_tab, float* sin_tab) { + const int half = n_rot / 2; + for (int p = 0; p < max_pos; ++p) { + for (int i = 0; i < half; ++i) { + const double inv = std::pow(theta, -2.0 * (double) i / (double) n_rot); + const double ang = (double) p * inv; + cos_tab[(size_t) p * (size_t) half + (size_t) i] = (float) std::cos(ang); + sin_tab[(size_t) p * (size_t) half + (size_t) i] = (float) std::sin(ang); + } + } +} + +/// In-place NEOX partial rotation of one row of K. `k` is one (head_dim,) row; `n_rot` is the +/// partial-rotation width (only the first `n_rot` of `head_dim` are touched, the tail passes through). +/// The host port of `rope_neox_pair` applies `(c, s)` at the chosen position; this wrapper chooses +/// (c, s) from the table using a per-row position offset. +/// +/// `cos_tab` / `sin_tab` are `(max_pos, n_rot/2)` matching the CUDA builder. `k` is a CACHED K row, +/// i.e. POST-RoPE at `old_pos` - the engine applies RoPE before the cell enters the cache. We +/// must re-rotate by the DELTA angle `(new_pos - old_pos) * inv(i)` so `R(diag)·R(old)·x = R(new)·x`. +/// Reading the table at `new_pos` (the bug the F2 review caught) would apply `R(new)` on top of the +/// already-rotated cached row, giving `R(new)·R(old)·x = R(old+new)·x` - wrong. +/// +/// `max_pos` is the table extent. When `new_pos` overflows it (long suffix at a very deep position), +/// both `new_pos` and `delta` use a CLAMPED position so the table lookup stays in range - the cached +/// row's rotation at the clamped position approximates the extrapolated angle, which is the +/// pre-existing A2.2 convention the splice layer assumes. +inline void rotate_row_by_delta(float* k, int head_dim, int n_rot, int old_pos, int delta, + const float* cos_tab, const float* sin_tab, int max_pos) { + const int half = n_rot / 2; + const int new_pos = old_pos + delta; + if (new_pos < 0) return; // the rotation domain is nonnegative + // F2 fix: rotation by the DELTA ANGLE, not by the full new-position angle. The effective delta + // is the offset between the clamped new position (where the table can land us) and old_pos: + // effective_delta = clamp(new_pos, 0, max_pos-1) - old_pos + // For pair i the additional rotation angle is `effective_delta * inv(i)`. When `new_pos` is in + // range this is just `delta`; when it overflows the table the additional rotation is reduced + // (capped at `max_pos-1 - old_pos`). For delta>0 we read the table at the absolute effective + // delta; for delta<0 we use `cos(-x)=cos(x), sin(-x)=-sin(x)` and flip the sign. + const int new_pos_clamped = new_pos < max_pos ? new_pos : max_pos - 1; + const int effective_delta = new_pos_clamped - old_pos; + const int delta_abs = effective_delta < 0 ? -effective_delta : effective_delta; + const int p_delta = delta_abs < max_pos ? delta_abs : max_pos - 1; + const float s_sign = effective_delta < 0 ? -1.0f : 1.0f; + for (int i = 0; i < half; ++i) { + const float c = cos_tab[(size_t) p_delta * (size_t) half + (size_t) i]; + const float s = s_sign * sin_tab[(size_t) p_delta * (size_t) half + (size_t) i]; + float a = k[i], b = k[i + half]; + float oa, ob; + strata::kernels::rope_neox_pair(a, b, c, s, oa, ob); + k[i] = oa; + k[i + half] = ob; + } + // the tail [n_rot, head_dim) passes through - `include/strata/kernels/rope.hpp:4` + (void) head_dim; +} + +/// Apply the splice rotation over a (rows, head_dim) K slab in place (the unchanged tail of the prompt). +/// `old_pos0` is the OLD prompt's position for row 0; `delta` is the per-token position offset +/// (uniform over a uniform edit; otherwise the caller iterates per-row). `n_rot` matches the model. +inline void rotate_k_by_delta(float* k, int rows, int head_dim, int n_rot, int old_pos0, int delta, + const float* cos_tab, const float* sin_tab, int max_pos) { + for (int r = 0; r < rows; ++r) + rotate_row_by_delta(k + (size_t) r * (size_t) head_dim, head_dim, n_rot, old_pos0 + r, delta, + cos_tab, sin_tab, max_pos); +} + +} // namespace strata::program::suffix_reuse diff --git a/include/strata/program/kv_splice_apply.hpp b/include/strata/program/kv_splice_apply.hpp new file mode 100644 index 00000000000..b4702eae693 --- /dev/null +++ b/include/strata/program/kv_splice_apply.hpp @@ -0,0 +1,253 @@ +// include/strata/program/kv_splice_apply.hpp - Phase 2 Task 2: the runtime `apply_splice` that walks +// the QSA K pool and rotates the unchanged tail by the splice plan's per-position delta. V is +// untouched. This is a SIBLING header to `suffix_reuse.hpp` (which holds the PURE plan computation +// and the runtime hook); the split keeps `suffix_reuse.hpp` lightweight for tests and call sites +// that only need the plan, while `apply_splice` brings in `QsaState`, `ModelGeometry`, the rope +// helpers, and the splice functions. Nothing here allocates - the session owns `k_pool`, +// `cos_tab`, `sin_tab`, and `max_cells`. +// +// v1 SCOPE (review-r2 NEW-2 / r1 Blockers 1+2): the splice SUPPORTS ONLY fp16 K, no kv-resident +// ring, no layer-split. Anything else - int8 codes, q4_0 rotated codes, hybrid k8v4, a streamed +// slot-ring, a multi-stage layer split - logs `SPLICE_UNSUPPORTED: ` +// to stderr and returns WITHOUT touching K. The runtime hook at generate.cpp falls through to +// today's re-read path, which is byte-identical to before. The startup guard (Task 4) refuses +// the same combination at flag-parse time, so this runtime guard is the second layer of the +// A1.3 fail-closed pattern. Dequant-rotate-requant (int8/q4_0) and streaming-ring splice = Phase 3. +// +// `void* stream` is a future CUDA seam (review-r2 NEW-3): today's path is a HOST LOOP, no kernel +// launch (T3 already established the host port of `rotate_row_by_delta` and pinned it to 1e-6). +// The parameter is reserved for a future GPU kernel that operates on the same per-row layout; +// leaving it in the signature now keeps the call site from churning when that lands. The +// SUPPORTS guard checks happen BEFORE the stream is consulted. +// +// FP16 STORAGE NOTES: `QsaState.k_pool` is documented `uint16_t* fp16 (null in INT8 mode)` at +// include/strata/core/layer.hpp:210 - one fp16 per cell of head_dim. The math runs in fp32 +// (T3's tolerance is 1e-6, fp32 ULP), so each row is converted fp16->fp32 on entry, rotated, and +// written back to fp16 on exit. The fp16 rounding is the SAME on the splice path and the fresh- +// compute path (both produce the same fp16-stored cell), so the round-trip preserves the cache's +// numerical ground truth. No re-quantization; the fp16 stored K is rotated and stored. +// +// POOLED-LAYOUT NOTES: K is laid out `[page][kv_head][page_size][head_dim]` (qsa.cu:130) - a +// single position `p` has its K split across `n_head_kv` different rows, one per kv-head. Phase 2 +// v1 supports the guard-checked mode (fp16, kv_resident==0, no layer-split) where `st.kv_mode==0` +// is identity: position `p` lives in page `p/page_size`, offset `p%page_size`, and row index +// `page_idx * n_head_kv * page_size + h * page_size + offset_within_page`. The splice walks +// every kv_head's row at each position in the tail. V is untouched (separate `st.v_pool`). +// +// TWO-TIER API: `rotate_qsa_tail_inplace(st, plan, head_dim)` is the lower-level helper that +// operates on a single QsaState (T6's unit test target). `apply_splice(outcome, ss, g, stream)` +// is the SessionState-level wrapper that walks the layer range and calls the helper per owned +// QSA layer. +#pragma once + +#include "strata/kernels/kv_stream.hpp" // for kKvF16 / kKvInt8 / kKvQ4 / kKvHybrid enum (kv_stream.hpp:48) +#include "strata/core/layer.hpp" // QsaState, qsa_kv_int8/q4/hybrid/resident (header-only, no CUDA) +#include "strata/core/layout.hpp" // ModelGeometry + is_qsa_layer +#include "strata/program/kv_splice.hpp" // rotate_k_by_delta + build_rope_table_host +#include "strata/program/kv_splice_transport.hpp" // host/device staging (identity on CPU builds) +#include "strata/program/suffix_reuse.hpp" // SuffixHookOutcome + +#include +#include +#include +#include + +namespace strata::program::suffix_reuse { + +/// Splice outcome tag for the runtime caller - what happened. `Rotated` is the success path; +/// `Unsupported*` are the SPLICE_UNSUPPORTED log lines (caller falls through to today's re-read); +/// `NoOp` is the contract-false path (fired=false or old_suffix_len<=0, no log). +enum class SpliceStatus { + NoOp = 0, ///< contract not satisfied - fired=false or empty tail. No log. + Rotated, ///< K rotated in place. V untouched. + UnsupportedFormat, ///< kv != fp16 (int8 / q4_0 / hybrid). Logged + no-op. + UnsupportedResident, ///< kv_resident > 0 (streaming ring). Logged + no-op. + UnsupportedSplit, ///< layer split active. Logged + no-op. +}; + +/// Pure predicate for the runtime scope guard (review-r2 NEW-2 / r1 Blockers 1+2). When this +/// returns Rotated/`NoOp` we proceed; otherwise we log SPLICE_UNSUPPORTED and return without +/// touching K. Factored out of `apply_splice` so T6 (and the test suite) can pin the guard +/// against pure inputs without linking to layer.cpp (where `qsa_kv_int8()`/etc. live as globals). +inline SpliceStatus splice_scope_guard(bool kv_int8, bool kv_q4, bool kv_hybrid, + int64_t kv_resident, bool layer_split_active) { + if (kv_int8 || kv_q4 || kv_hybrid) return SpliceStatus::UnsupportedFormat; + if (kv_resident > 0) return SpliceStatus::UnsupportedResident; + if (layer_split_active) return SpliceStatus::UnsupportedSplit; + return SpliceStatus::NoOp; // contract-false (not enough signal to say Rotated yet) - the + // caller also checks fired and old_suffix_len +} + +namespace detail { +/// fp16 <-> fp32 round-trip (IEEE 754 binary16, no NaN preservation - K never holds NaN, and +/// `__half` is not portable here so we hand-roll the conversions; they round to nearest even). +inline uint16_t fp32_to_fp16_bits(float f) { + uint32_t x = 0u; std::memcpy(&x, &f, 4); + uint32_t sign = (x >> 16) & 0x8000u; + int32_t exp = (int32_t) ((x >> 23) & 0xff) - 127 + 15; + uint32_t mant = x & 0x7fffffu; + if (exp <= 0) { + if (exp < -10) return (uint16_t) sign; + mant = (mant | 0x800000u) >> (uint32_t) (1 - exp); + if (mant & 0x1000u) mant += 0x2000u; + return (uint16_t) (sign | (mant >> 13)); + } + if (exp >= 31) return (uint16_t) (sign | 0x7bf0u | (((x & 0x7fffffu) != 0u) ? 0x1u : 0u)); + if (mant & 0x1000u) { mant += 0x2000u; if (mant & 0x800000u) { mant = 0u; ++exp; if (exp >= 31) return (uint16_t) (sign | 0x7bf0u); } } + return (uint16_t) (sign | ((uint32_t) exp << 10) | (mant >> 13)); +} +inline float fp16_bits_to_fp32(uint16_t h) { + uint32_t sign = ((uint32_t) h & 0x8000u) << 16; + uint32_t exp = ((uint32_t) h >> 10) & 0x1fu; + uint32_t mant = (uint32_t) h & 0x3ffu; + uint32_t bits; + if (exp == 0) { + if (mant == 0) bits = sign; + else { + uint32_t e = 0; + while ((mant & 0x400u) == 0) { mant <<= 1; ++e; } + mant &= 0x3ffu; + bits = sign | ((uint32_t) (113 - e) << 23) | (mant << 13); + } + } else if (exp == 31) { + bits = sign | 0x7f800000u | (mant << 13); + } else { + bits = sign | ((exp + 112u) << 23) | (mant << 13); + } + float f = 0.0f; std::memcpy(&f, &bits, 4); return f; +} +} // namespace detail + +/// T6's test target: rotate the k_pool rows of ONE QsaState that correspond to positions +/// `[edit_pos, R)` in the OLD prompt, where `R = edit_pos + old_suffix_len`. This is the +/// pure-CPU loop the SessionState wrapper invokes per owned Qsa layer. +/// +/// MEMORY TRANSPORT (the 2026-10-12 SIGSEGV hotfix). In a CUDA build `st.k_pool`, +/// `st.cos_tab` and `st.sin_tab` are DEVICE addresses (the session arena is cudaMalloc'd, +/// and qsa_state_init uploads the rope table INTO st.cos_tab). This loop therefore never +/// dereferences those pointers directly: `stage_for_readwrite` brings a contiguous range +/// into a host staging buffer (identity on a CPU-only build, cudaMemcpy otherwise) and +/// `commit` writes it back. Three ranges are staged per call: +/// 1. the K rows of `[edit_pos, R)` - read, rotated, written back; +/// 2. the rope-table SLICE the rotation reads - `[0, min(max_pos, p_delta_max + 1))` +/// rows of cos/sin, which for a mid-prompt edit is a small prefix of the 64 MiB table; +/// 3. nothing else - V is untouched. +/// +/// Parameters: +/// - `st`: the QsaState whose `k_pool` is rotated (must be fp16, kv_mode==0). +/// - `head_dim`: 256 for the artifact. +/// - `n_rot`: 64 for the artifact. +/// - `n_head_kv`: 2 for the artifact. +/// - `max_pos`: the rope-table extent (== the state's max_cells). +/// - `page_size`: the kv_stream page_size (qsa_real_shapes().page_size; 4 in the +/// artifact - the SAME source qsa_state_init sizes the pool with). +/// - `pool_rows`: the k_pool's ROW count (st.n_slots * n_head_kv * page_size). The +/// row index of every touched position is checked against it; an out-of-range index +/// (a plan the pool cannot hold) refuses the rotation instead of writing past the +/// pool. 0 disables the check (T6 synthesises states without n_slots set). +/// - `edit_pos`: the splice's plan_edit_pos. +/// - `old_suffix_len`: the splice's plan.old_suffix_len. +/// - `uniform`: true for single delta, false for per-token deltas. +/// - `delta`: the uniform position offset when `uniform` is true. +/// - `deltas`: per-row deltas when `uniform` is false (size == old_suffix_len; may be +/// null ONLY when `uniform` is true - a non-uniform plan without a table is refused +/// rather than dereferenced, the empty-deltas trap the live plan can hit when the +/// suffix is empty). +/// - `skip_rotation`: T6's RED hook - skips the `rotate_k_by_delta` call so the test proves +/// a regression that drops the rotation surfaces as `diff_count > 0`. Production callers +/// pass `false`. +/// +/// Returns true on success (K rotated). No allocation on the CPU path; the CUDA path +/// stages through the shared transport buffer (kv_splice_transport.hpp). +inline bool rotate_qsa_tail_inplace(strata::core::QsaState& st, + int head_dim, int n_rot, int n_head_kv, int max_pos, + int64_t page_size, + int64_t edit_pos, int64_t old_suffix_len, + bool uniform, int delta, const int64_t* deltas, + bool skip_rotation = false, + int64_t pool_rows = 0) { + if (st.k_pool == nullptr) return false; + if (st.cos_tab == nullptr || st.sin_tab == nullptr) return false; + if (st.kv_mode != 0) return false; // v1: identity-page mode only + if (edit_pos < 0 || old_suffix_len <= 0) return false; + if (!uniform && deltas == nullptr) return false; // no table to walk: refuse, don't fault + if (page_size <= 0) page_size = 1; + // Bounds: every position the loop touches must land inside the pool's rows. Checked + // BEFORE the first staging copy so a bad plan never reads or writes anything. + const int64_t last_pos = edit_pos + old_suffix_len - 1; + if (pool_rows > 0) { + const int64_t last_page = last_pos / page_size; + const int64_t last_row = last_page * n_head_kv * page_size + (int64_t) (n_head_kv - 1) * page_size + + last_pos % page_size; + if (last_row >= pool_rows) return false; // plan exceeds the pool: no-op, caller re-reads + } + // The rope-table slice: the rotation reads rows [0, p_delta_max] where p_delta_max is + // the largest |effective delta| (rotate_row_by_delta clamps to max_pos-1 itself, so + // staging min(max_pos, max|delta|+1) rows covers every read). + int64_t max_abs_delta = 0; + for (int64_t k = 0; k < old_suffix_len; ++k) + max_abs_delta = std::max(max_abs_delta, + uniform ? (delta < 0 ? -(int64_t) delta : (int64_t) delta) + : (deltas[k] < 0 ? -deltas[k] : deltas[k])); + const int half = n_rot / 2; + const int64_t tab_rows = std::min(max_pos, max_abs_delta + 1); + if (tab_rows <= 0 || half <= 0) return false; + // Stage the rope-table slice and the K rows (transport = identity on a CPU build). + const size_t tab_floats = (size_t) tab_rows * (size_t) half; + transport::Staged cos_s = transport::stage_for_readwrite(st.cos_tab, tab_floats * 4); + transport::Staged sin_s = transport::stage_for_readwrite(st.sin_tab, tab_floats * 4); + if (cos_s.host == nullptr || sin_s.host == nullptr) return false; // staging failed: refuse the layer + const size_t first_row = (size_t) (edit_pos / page_size) * (size_t) n_head_kv * (size_t) page_size; + const size_t row_count = (size_t) ((last_pos / page_size) - (edit_pos / page_size) + 1) + * (size_t) n_head_kv * (size_t) page_size; + const size_t k_elems = row_count * (size_t) head_dim; + transport::Staged k_s = transport::stage_for_readwrite( + st.k_pool + first_row * (size_t) head_dim, k_elems * 2); + if (k_s.host == nullptr) return false; + const float* cos_tab = (const float*) cos_s.host; + const float* sin_tab = (const float*) sin_s.host; + // The staged table covers rows [0, tab_rows); rotate_row_by_delta clamps its p_delta + // into [0, max_pos-1] - with the slice staged, every in-slice row it picks is present. + // Production reads fp16 -> fp32, rotates in fp32, writes back to fp16. The fp16 round-trips + // add 2 quantizations (cached image at OLD_pos stored fp16, splice-path output stored fp16). + // The fresh-comparison path does 2 quantizations of its own (raw -> fp16, rope -> fp16), so + // both paths have the same structural fp16 footprint. T6 compares the two paths using the + // SAME fp16 round-trip step, so the test tolerates ~fp16 ULP errors between them (T3 pinned + // the underlying rotation math to 1e-6 absolute tolerance; this function is fp16-correct). + float row_scratch[ /*head_dim*/ 4096]; + if (head_dim > 4096) return false; // stack scratch bound; artifact is 256 + for (int64_t k = 0; k < old_suffix_len; ++k) { + const int64_t p = edit_pos + k; // position in OLD prompt + const int64_t page = p / page_size; + const int64_t off = p % page_size; + const int delta_row = uniform ? delta : (int) deltas[(size_t) k]; + for (int h = 0; h < n_head_kv; ++h) { + const int64_t row_idx = page * n_head_kv * page_size + (int64_t) h * page_size + off; + uint16_t* row = (uint16_t*) k_s.host + + (size_t)(row_idx - (int64_t) first_row) * (size_t) head_dim; + for (int i = 0; i < head_dim; ++i) row_scratch[i] = detail::fp16_bits_to_fp32(row[i]); + // Production rotation. The `skip_rotation=true` arg lets T6 (and any future + // RED-armed regression test) skip the rotation step to prove the assertion fails + // without it. The brief's RED proof was exercised during build: masking this call + // makes T6.y's `ulp == strict+ulp+bad` and the regression-guardrail checks fail. + if (!skip_rotation) { + strata::program::suffix_reuse::rotate_k_by_delta( + row_scratch, 1, head_dim, n_rot, (int) p, delta_row, + cos_tab, sin_tab, max_pos); + } + for (int i = 0; i < head_dim; ++i) row[i] = detail::fp32_to_fp16_bits(row_scratch[i]); + } + } + transport::commit(k_s, k_elems * 2); + return true; +} + +/// Walk the QSA K pools of the session and rotate, in place, the rows corresponding to positions +/// `[plan_edit_pos, plan_edit_pos + old_suffix_len)` of every owned QSA layer by the splice plan's +/// per-position delta. V is untouched. +/// +/// Defined in `kv_splice_apply_session.hpp` (separate header that pulls `SessionState`/CUDA-runtime +/// include); the kernel-only helper above + the runtime's `apply_splice` are the two layers, with +/// T6 covering the helper directly without paying the CUDA include cost. + +} // namespace strata::program::suffix_reuse diff --git a/include/strata/program/kv_splice_apply_int8.hpp b/include/strata/program/kv_splice_apply_int8.hpp new file mode 100644 index 00000000000..f4c967ecc99 --- /dev/null +++ b/include/strata/program/kv_splice_apply_int8.hpp @@ -0,0 +1,368 @@ +// include/strata/program/kv_splice_apply_int8.hpp - Phase 3: the INT8 K/V splice. +// +// WHAT THIS IS. The fp16 splice (kv_splice_apply.hpp) rotates the cached K rows of +// [edit_pos, edit_pos + old_suffix_len) by a per-position RoPE delta, leaving V +// untouched. INT8 K cannot be rotated in place: each cell is 64 int8 codes + 1 fp16 +// scale, and rotating the codes would break the [code * scale] contract the dequant +// kernel relies on. The fix: dequant the 64 codes to fp32, apply the rotation in fp32 +// (the splice's RoPE math already runs in fp32: kv_splice_apply.hpp:228-238), requant +// the rotated image with the EXACT writer math (src/kernels/cuda/kv_q8.cu:47-67, mirrored +// in include/strata/kernels/kv_q8_quant.hpp:kv_q8_quant_group), write the codes + scale +// back. V is untouched. +// +// MATH-MIRROR PROPERTY. For the UNCHANGED range (cells outside the splice), the helper +// is a no-op: cells OUTSIDE [edit_pos, R) are not visited, and their codes+scales are +// BIT-IDENTICAL. For the CHANGED range, requant(dequant(dequant->rotate->requant)) +// equals dequant->rotate->requant on the original stored image (the dequant/requant +// pair round-trips BIT-EXACTLY because the writer's `rint(x/sf) * sf = x` is exact for +// the 254 quantable values: the only difference is the second quant's rounding of the +// ROTATED fp32, which pays the same per-group half-step the fresh-append path pays when +// it writes the rotated image to the cache). T11.c pins this bit-for-bit; T11.b bounds +// the rotation-requant error to the same half-step the engine's other write paths pay. +// +// TRANSPORT. The cached K is staged through the existing splice transport +// (kv_splice_transport.hpp): in a CUDA build the k_q and k_scale buffers are device +// pointers and the host loop must go through `stage_for_readwrite` / `commit`. The +// transport's identity path (CPU build) aliases the buffers directly. V is never +// touched; this header only stages K codes + K scales. +// +// FAIL-CLOSED. Outside the supported config the helpers return +// SpliceStatus::Unsupported* WITHOUT side effects, matching the A1.3 fail-closed pattern. +// Supported: fp16, int8, int8 + STRATA_KV_ROT=1 (the H -> R(delta) -> H sandwich below), +// and k8v4 (the hybrid K half is byte-identical int8 storage; V is never touched). +// --kv q4_0 splices through rotate_qsa_tail_inplace_q4 (the Q4 codec helper here). +// Still refused everywhere: kv_resident > 0 and layer_split (memory layouts, not codecs); +// head_dim != 256 in a rotated domain (fwht256 is order-256; fail-closed inside the +// helpers). The startup guard in suffix_reuse.hpp is the first layer; these are second. +#pragma once + +#include "strata/core/layer.hpp" // QsaState (kv_int8 / k_q / k_scale / kv_mode) +#include "strata/core/layout.hpp" // ModelGeometry +#include "strata/kernels/f16_bits.hpp" // f16_from_f32 / f32_from_f16 +#include "strata/kernels/kv_q8.hpp" // KV_Q8_GROUP +#include "strata/kernels/kv_q4.hpp" // block_q4_0, QK4_0 (the q4_0 splice codec) +#include "strata/program/kv_q8_quant.hpp" // KvQ8Group + kv_q8_{quant,dequant,read,write}_group +#include "strata/program/kv_splice.hpp" // rotate_k_by_delta + build_rope_table_host +#include "strata/program/kv_splice_apply.hpp" // SpliceStatus, fp32/fp16 helpers +#include "strata/program/kv_splice_transport.hpp" // Staged + stage_for_readwrite + commit +#include "strata/program/suffix_reuse.hpp" // SuffixHookOutcome + +#include +#include +#include + +namespace strata::program::suffix_reuse { + +/// Host Q4_0 dequant of one 32-value block (the writer's exact math, kv_q4.cu's writer / +/// quantize_block_q4_0_host in kv_q4_parity.cpp:64-87). x must hold 32 floats. +inline void kv_q4_dequant_block_host(const strata::kernels::block_q4_0& blk, float* x) { + const float d = strata::kernels::f32_from_f16(blk.d); + for (int i = 0; i < 16; ++i) { + const int q0 = blk.qs[i] & 0xF; + const int q1 = blk.qs[i] >> 4; + x[i] = (q0 - 8.0f) * d; + x[i + 16] = (q1 - 8.0f) * d; + } +} + +/// Host Q4_0 quant of one 32-value block - byte-identical to quantize_block_q4_0_host +/// (the parity reference the CUDA writer is pinned to). x must hold 32 floats. +inline void kv_q4_quant_block_host(const float* x, strata::kernels::block_q4_0& blk) { + float amax = 0.0f, max_val = 0.0f; + for (int i = 0; i < 32; ++i) { + const float v = x[i]; + if (amax < std::fabs(v)) { amax = std::fabs(v); max_val = v; } + } + const float d = max_val / -8.0f; + const float id = d != 0.0f ? 1.0f / d : 0.0f; + blk.d = strata::kernels::f16_from_f32(d); + for (int i = 0; i < 16; ++i) { + const float x0 = x[i] * id; + const float x1 = x[i + 16] * id; + const int q0 = (int) (x0 + 8.5f); + const int q1 = (int) (x1 + 8.5f); + const uint8_t xi0 = (uint8_t) std::min(15, std::max(0, q0)); + const uint8_t xi1 = (uint8_t) std::min(15, std::max(0, q1)); + blk.qs[i] = (uint8_t) (xi0 | (xi1 << 4)); + } +} + +/// Host FWHT-256 (Sylvester construction, 1/sqrt(256) normalized) - bit-exact same math +/// as the engine's kv_q4.hpp fwht256_cuda (kv_q4_parity.cpp:46-62 is the reference the +/// parity test pins it to). H is its own inverse (H*H = I), which is what the kv_rot +/// splice sandwich relies on. The splice is a host-loop helper by design (a tail of a +/// few thousand rows), so the row is already staged on the host when this runs. +inline void fwht256_row_host(float* x) { + constexpr int n = 256; + for (int s = 1; s < n; s *= 2) + for (int i = 0; i < n; i += 2 * s) + for (int j = 0; j < s; ++j) { + const float u = x[i + j]; + const float v = x[i + j + s]; + x[i + j] = u + v; + x[i + j + s] = u - v; + } + const float inv = 1.0f / 16.0f; // 1/sqrt(256) + for (int i = 0; i < n; ++i) x[i] *= inv; +} + +/// Runtime scope guard for the int8 splice. Returns NoOp when the config is +/// supported; otherwise one of the `Unsupported*` codes WITHOUT side effects. +/// Accepted: plain --kv int8, int8 + STRATA_KV_ROT=1 (the H-sandwich in +/// rotate_qsa_tail_inplace_int8), and k8v4 hybrid (the K half is byte-identical +/// int8 storage; V is never touched). --kv q4_0 is dispatched to the q4 helper +/// (rotate_qsa_tail_inplace_q4) before this guard runs, but still refuses here if +/// it reaches the int8 domain. Still refused: kv_resident > 0, layer_split +/// (memory layouts, not codecs); head_dim != 256 in a rotated domain refuses +/// fail-closed inside the rotation helpers (fwht256 is order-256 only). +inline SpliceStatus splice_scope_guard_int8(bool kv_int8, bool kv_q4, bool kv_hybrid, + int64_t kv_resident, bool layer_split_active, + bool kv_rot) { + (void) kv_rot; // accepted since the H-sandwich landed; head_dim!=256 refuses inside the helper + (void) kv_hybrid; // accepted (k8v4): the hybrid K half is int8; V is untouched + if (!kv_int8) { + // fp16 path: never comes here (the dispatch is the caller's); the q4/hybrid/resident + // /split refusals are still enforced for safety. + if (kv_q4 || kv_hybrid) return SpliceStatus::UnsupportedFormat; + if (kv_resident > 0) return SpliceStatus::UnsupportedResident; + if (layer_split_active) return SpliceStatus::UnsupportedSplit; + return SpliceStatus::NoOp; // fp16 K is the OTHER helper's domain + } + // int8 path. kv_rot on == STRATA_KV_ROT=1 and k8v4 hybrid are accepted (see above and + // the H-sandwich). q4_0 is accepted too: the q4 helper below does the same sandwich + // with Q4_0 dequant/requant blocks (K rows are stored H*R(p)*k in Q4_0; the fwht and + // rope math are identical, only the storage codec differs). Refusals that remain: + // kv_resident > 0 and layer split (memory layouts, not quant formats). + if (kv_resident > 0) return SpliceStatus::UnsupportedResident; + if (layer_split_active) return SpliceStatus::UnsupportedSplit; + return SpliceStatus::NoOp; // contract gate; the caller also checks fired + old_suffix_len +} + +/// Phase 3 T11.c's test target: rotate the K blocks of ONE QsaState that correspond to +/// positions `[edit_pos, R)` in the OLD prompt, where `R = edit_pos + old_suffix_len`. +/// Mirrors `rotate_qsa_tail_inplace` (kv_splice_apply.hpp:162-243) in the int8 layout: +/// +/// 1. Bounds-check the splice range against the pool's row count (the same out-of-pool +/// refusal the fp16 path has). +/// 2. Compute the rope-table slice the rotation reads (p_delta_max based on deltas). +/// 3. Stage cos_tab + sin_tab (transport: identity on CPU, cudaMemcpy on device). +/// 4. Stage the K codes range [first_row, last_row) AND the K scales range covering +/// the same rows. The two are separate transport views (same Staged pool), so the +/// pair is read-write consistent: the host rotates codes+scales in lockstep. +/// 5. For each (position, kv_head) the splice touches: +/// a. dequant every 64-value group to fp32 +/// b. apply the splice's RoPE delta rotation in fp32 (rotate_k_by_delta, fp16-input +/// row staged via the local helper so the rope math stays the same as the fp16 +/// splice path; the difference is the read/dequant before and the requant write +/// after). +/// c. requant every 64-value group with the writer's math (kv_q8_quant_group). +/// 6. commit the two staged views (no-op on CPU, cudaMemcpy on device). +/// +/// Parameters mirror `rotate_qsa_tail_inplace` (kv_splice_apply.hpp:162-168) for symmetry; +/// the only added constraint is `head_dim % KV_Q8_GROUP == 0` (the engine's qsa_state_init +/// already enforces this, kv_q8.cu:22-26 validate()). V is untouched. Returns true on +/// success; false on a guard failure (null pool, no table, page out of pool, etc.) with +/// NO side effects on the staged buffers (the staging slots are only mutated after a +/// successful loop). +inline bool rotate_qsa_tail_inplace_int8(strata::core::QsaState& st, + int head_dim, int n_rot, int n_head_kv, int max_pos, + int64_t page_size, + int64_t edit_pos, int64_t old_suffix_len, + bool uniform, int delta, const int64_t* deltas, + int64_t pool_rows = 0) { + // ---- Guard: identical to the fp16 path's fail-closed pattern ---- + if (st.k_q == nullptr || st.k_scale == nullptr) return false; + if (st.cos_tab == nullptr || st.sin_tab == nullptr) return false; + if (st.kv_mode != 0) return false; // v1: identity-page mode only + if (edit_pos < 0 || old_suffix_len <= 0) return false; + if (!uniform && deltas == nullptr) return false; + if (page_size <= 0) page_size = 1; + if (head_dim % strata::kernels::KV_Q8_GROUP != 0) return false; // writer's invariant (kv_q8.cu:22-26) + // STRATA_KV_ROT=1: K rows are stored Hadamard-rotated (fwht256, kv_q4.hpp). The splice + // must undo H, apply the RoPE delta, re-apply H (H*H = I). fwht256 is order-256 only: + // a head_dim other than 256 in the rotated domain is refused fail-closed (the splice + // helper would mix wrong). The engine's rotated configs are all head_dim 256 today. + if (st.kv_rot && head_dim != 256) return false; + if ((int64_t) n_rot > head_dim) return false; + const int groups = head_dim / strata::kernels::KV_Q8_GROUP; + // Bounds: every position the loop touches must land inside the pool's rows. + const int64_t last_pos = edit_pos + old_suffix_len - 1; + if (pool_rows > 0) { + const int64_t last_page = last_pos / page_size; + const int64_t last_row = last_page * n_head_kv * page_size + (int64_t) (n_head_kv - 1) * page_size + + last_pos % page_size; + if (last_row >= pool_rows) return false; + } + // ---- Rope-table slice (the rotation reads cos/sin at |p_delta|) ---- + int64_t max_abs_delta = 0; + for (int64_t k = 0; k < old_suffix_len; ++k) + max_abs_delta = std::max(max_abs_delta, + uniform ? (delta < 0 ? -(int64_t) delta : (int64_t) delta) + : (deltas[k] < 0 ? -deltas[k] : deltas[k])); + const int half = n_rot / 2; + const int64_t tab_rows = std::min(max_pos, max_abs_delta + 1); + if (tab_rows <= 0 || half <= 0) return false; + const size_t tab_floats = (size_t) tab_rows * (size_t) half; + transport::Staged cos_s = transport::stage_for_readwrite(st.cos_tab, tab_floats * 4); + transport::Staged sin_s = transport::stage_for_readwrite(st.sin_tab, tab_floats * 4); + if (cos_s.host == nullptr || sin_s.host == nullptr) return false; + // ---- Stage the K codes + K scales ranges covering the splice rows ---- + // The first/last touched ROWS in the pool (row-major across pages and kv_heads). + const size_t first_row = (size_t) (edit_pos / page_size) * (size_t) n_head_kv * (size_t) page_size; + const size_t row_count = (size_t) ((last_pos / page_size) - (edit_pos / page_size) + 1) + * (size_t) n_head_kv * (size_t) page_size; + const size_t k_codes_bytes = row_count * (size_t) head_dim * sizeof(int8_t); + const size_t k_scales_elems = row_count * (size_t) groups; + const size_t k_scales_bytes = k_scales_elems * sizeof(uint16_t); + transport::Staged codes_s = transport::stage_for_readwrite( + st.k_q + first_row * (size_t) head_dim, k_codes_bytes); + transport::Staged scales_s = transport::stage_for_readwrite( + st.k_scale + first_row * (size_t) groups, k_scales_bytes); + if (codes_s.host == nullptr || scales_s.host == nullptr) return false; + int8_t* codes = (int8_t*) codes_s.host; + uint16_t* scales = (uint16_t*) scales_s.host; + const float* cos_tab = (const float*) cos_s.host; + const float* sin_tab = (const float*) sin_s.host; + // ---- Rotate: dequant per group, apply RoPE delta, requant per group ---- + // Scratch: one fp32 row of head_dim. The fp16 splice uses an fp16 stack scratch and + // does the convert in-line; the int8 splice does TWO conversions (dequant + requant) + // per group, so the scratch is fp32. head_dim == 256 in the artifact, < 4096 stack + // scratch limit (same as the fp16 helper, kv_splice_apply.hpp:218). + float row_scratch[4096]; + if (head_dim > 4096) return false; + for (int64_t k = 0; k < old_suffix_len; ++k) { + const int64_t p = edit_pos + k; // position in OLD prompt + const int64_t page = p / page_size; + const int64_t off = p % page_size; + const int delta_row = uniform ? delta : (int) deltas[(size_t) k]; + for (int h = 0; h < n_head_kv; ++h) { + const int64_t row_idx = page * n_head_kv * page_size + (int64_t) h * page_size + off; + const size_t local_row = (size_t)(row_idx - (int64_t) first_row); + // 1. Dequant: every group -> fp32. The dequant math is `x' = code * sf` + // (mirror of src/kernels/cuda/kv_q8.cu:100-101). + for (int g = 0; g < groups; ++g) { + const float sf = strata::kernels::f32_from_f16(scales[local_row * groups + g]); + for (int t = 0; t < strata::kernels::KV_Q8_GROUP; ++t) + row_scratch[g * strata::kernels::KV_Q8_GROUP + t] = + (float) codes[local_row * head_dim + g * strata::kernels::KV_Q8_GROUP + t] * sf; + } + // 2. Splice rotation (fp32, same as the fp16 helper's math but no fp16 round-trip + // around the rotation - the dequant is fp32, the rotation is fp32, the requant + // is fp32, the dequant in the gather is fp16. The fp16 path does 2 fp16 + // round-trips; the int8 path does 1 int8 round-trip on each end. Bounded by T11.b). + // STRATA_KV_ROT=1: the stored row is H*R(p)*k (queries are H'd too, H^T*H = I), + // so dequant gives H*R(p)*k; undo H, rotate by delta, re-apply H to get + // H*R(delta)*H * (H*R(p)*k) = H*R(p+delta)*k - the exact stored form at the + // new position. SAME single requant as the unrotated path (no extra noise). + if (st.kv_rot) fwht256_row_host(row_scratch); + strata::program::suffix_reuse::rotate_k_by_delta( + row_scratch, 1, head_dim, n_rot, (int) p, delta_row, + cos_tab, sin_tab, max_pos); + if (st.kv_rot) fwht256_row_host(row_scratch); + // 3. Requant: writer's math. Per group, amax over the 64 rotated fp32 values + // (fabs), scale = f16(f32(amax/127)) (the round-to-nearest-even bit pattern + // the writer stores), code = clamp(rint(x / sf), -127, 127). Mirrors + // src/kernels/cuda/kv_q8.cu:47-67 EXACTLY. + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group qg{}; + strata::kernels::kv_q8_quant_group(&row_scratch[g * strata::kernels::KV_Q8_GROUP], qg); + scales[local_row * groups + g] = qg.scale_bits; + for (int t = 0; t < strata::kernels::KV_Q8_GROUP; ++t) + codes[local_row * head_dim + g * strata::kernels::KV_Q8_GROUP + t] = qg.codes[t]; + } + } + } + // ---- Commit: copy the host-staged codes + scales back to the device on a CUDA build ---- + transport::commit(codes_s, k_codes_bytes); + transport::commit(scales_s, k_scales_bytes); + return true; +} + +/// Q4_0 K splice: the same H -> R(delta) -> H sandwich, with Q4_0 as the storage codec. +/// `--kv q4_0` stores the WHOLE K row as Q4_0 blocks after the Hadamard rotation +/// (kv_q4.hpp), so dequant is per-32-value block (18 B each: f16 d + 16 offset nibbles), +/// and the requant re-encodes with the writer's exact math (kv_q4_quant_block_host, the +/// parity-reference quantizer). Same structure and guards as the int8 helper; returns +/// true on success, false fail-closed with no side effects. Measured noise (T11.f): +/// double-quant code-steps ~1.9x the int8 path's - Q4_0's 16-level codes are coarser than +/// int8's 254, so the requant after rotation is lossier; parity vs a fresh write is +/// asserted in code-steps, and the engine's q4 path itself accepts that codec's noise. +inline bool rotate_qsa_tail_inplace_q4(strata::core::QsaState& st, + int head_dim, int n_rot, int n_head_kv, int max_pos, + int64_t page_size, + int64_t edit_pos, int64_t old_suffix_len, + bool uniform, int delta, const int64_t* deltas, + int64_t pool_rows = 0) { + if (st.k_q4 == nullptr) return false; + if (st.cos_tab == nullptr || st.sin_tab == nullptr) return false; + if (st.kv_mode != 0) return false; + if (edit_pos < 0 || old_suffix_len <= 0) return false; + if (!uniform && deltas == nullptr) return false; + if (page_size <= 0) page_size = 1; + if (head_dim % strata::kernels::QK4_0 != 0) return false; // writer's invariant (whole-row blocks) + if (st.kv_q4 && head_dim != 256) return false; // fwht256 is order-256 only (fail-closed) + if ((int64_t) n_rot > head_dim) return false; + const int blocks_per_head = head_dim / strata::kernels::QK4_0; + const int64_t last_pos = edit_pos + old_suffix_len - 1; + if (pool_rows > 0) { + const int64_t last_page = last_pos / page_size; + const int64_t last_row = last_page * n_head_kv * page_size + (int64_t) (n_head_kv - 1) * page_size + + last_pos % page_size; + if (last_row >= pool_rows) return false; + } + int64_t max_abs_delta = 0; + for (int64_t k = 0; k < old_suffix_len; ++k) + max_abs_delta = std::max(max_abs_delta, + uniform ? (delta < 0 ? -(int64_t) delta : (int64_t) delta) + : (deltas[k] < 0 ? -deltas[k] : deltas[k])); + const int half = n_rot / 2; + const int64_t tab_rows = std::min(max_pos, max_abs_delta + 1); + if (tab_rows <= 0 || half <= 0) return false; + const size_t tab_floats = (size_t) tab_rows * (size_t) half; + transport::Staged cos_s = transport::stage_for_readwrite(st.cos_tab, tab_floats * 4); + transport::Staged sin_s = transport::stage_for_readwrite(st.sin_tab, tab_floats * 4); + if (cos_s.host == nullptr || sin_s.host == nullptr) return false; + const size_t first_row = (size_t) (edit_pos / page_size) * (size_t) n_head_kv * (size_t) page_size; + const size_t row_count = (size_t) ((last_pos / page_size) - (edit_pos / page_size) + 1) + * (size_t) n_head_kv * (size_t) page_size; + // K Q4 blocks: the writer's layout (kv_q4.cu:94,114) is one packed row of + // head_dim/QK4_0 blocks per (page, head, off) - the row id encodes the head, so the + // per-row byte stride is blocks_per_head * 18 B with NO per-head sub-array. + const size_t q4_bytes_per_row = (size_t) blocks_per_head * sizeof(strata::kernels::block_q4_0); + const size_t k_bytes = row_count * q4_bytes_per_row; + transport::Staged kq4_s = transport::stage_for_readwrite( + (uint8_t*) st.k_q4 + first_row * q4_bytes_per_row, k_bytes); + if (kq4_s.host == nullptr) return false; + strata::kernels::block_q4_0* blocks = (strata::kernels::block_q4_0*) kq4_s.host; + const float* cos_tab = (const float*) cos_s.host; + const float* sin_tab = (const float*) sin_s.host; + float row_scratch[4096]; + if (head_dim > 4096) return false; + for (int64_t k = 0; k < old_suffix_len; ++k) { + const int64_t p = edit_pos + k; + const int64_t page = p / page_size; + const int64_t off = p % page_size; + const int delta_row = uniform ? delta : (int) deltas[(size_t) k]; + for (int h = 0; h < n_head_kv; ++h) { + const int64_t row_idx = page * n_head_kv * page_size + (int64_t) h * page_size + off; + const size_t local_row = (size_t)(row_idx - (int64_t) first_row); + // one packed block row per (page, head, off) - the writer's kv_q4.cu layout + strata::kernels::block_q4_0* row_blocks = blocks + local_row * (size_t) blocks_per_head; + // 1. dequant the whole row (per 32-value block) + for (int b = 0; b < blocks_per_head; ++b) + kv_q4_dequant_block_host(row_blocks[b], &row_scratch[b * strata::kernels::QK4_0]); + // 2. the exact sandwich: stored = Q4(H*R(p)*k), so dequant ~ H*R(p)*k + fwht256_row_host(row_scratch); + strata::program::suffix_reuse::rotate_k_by_delta( + row_scratch, 1, head_dim, n_rot, (int) p, delta_row, cos_tab, sin_tab, max_pos); + fwht256_row_host(row_scratch); + // 3. requant with the writer's exact math + for (int b = 0; b < blocks_per_head; ++b) + kv_q4_quant_block_host(&row_scratch[b * strata::kernels::QK4_0], row_blocks[b]); + } + } + transport::commit(kq4_s, k_bytes); + return true; +} + +} // namespace strata::program::suffix_reuse diff --git a/include/strata/program/kv_splice_apply_session.hpp b/include/strata/program/kv_splice_apply_session.hpp new file mode 100644 index 00000000000..b691b189fb0 --- /dev/null +++ b/include/strata/program/kv_splice_apply_session.hpp @@ -0,0 +1,192 @@ +// include/strata/program/kv_splice_apply_session.hpp - Phase 2 Task 2: SessionState wrapper for +// `apply_splice`. The kernel-only helper (`rotate_qsa_tail_inplace`) lives in `kv_splice_apply.hpp` +// (header-only, no CUDA include) so T6 can exercise it directly. This header pulls in +// `SessionState`/`cuda_runtime.h` and provides the production runtime wrapper that walks every +// owned QSA layer of the session and invokes the helper. +#pragma once + +#include "strata/core/session.hpp" // SessionState (CUDA-runtime include, runtime-only) +#include "strata/core/layout.hpp" // ModelGeometry + is_qsa_layer +#include "strata/kernels/qsa.hpp" // qsa_real_shapes().page_size (the pool's page granule) +#include "strata/program/kv_splice_apply.hpp" // SpliceStatus + rotate_qsa_tail_inplace +#include "strata/program/kv_splice_apply_int8.hpp" // Phase 3: int8 splice (dequant->rotate->requant) + +#include + +namespace strata::program::suffix_reuse { + +/// Walk the QSA K pools of the session and rotate, in place, the rows corresponding to positions +/// `[plan_edit_pos, plan_edit_pos + old_suffix_len)` of every owned QSA layer by the splice plan's +/// per-position delta. V is untouched. +/// +/// **GUARDS BEFORE TOUCHING K (each returns SpliceStatus::Unsupported* WITHOUT side effects and +/// logs `SPLICE_UNSUPPORTED: ` to stderr):** +/// 1. KV format != fp16 (`qsa_kv_int8() / qsa_kv_q4() / qsa_kv_hybrid()` returns true) +/// 2. `qsa_kv_resident() > 0` (slot-ring / streamed mode) +/// 3. layer split active (`g.layer_lo != 0 || g.layer_hi != g.n_layers`) +/// +/// The runtime guard complements Task 4's startup guard (which refuses the same combination at +/// flag parse), forming the two-layer A1.3 fail-closed pattern. When the guard fires the caller +/// falls through to today's re-read path byte-identically. +inline SpliceStatus apply_splice(const SuffixHookOutcome& out, + strata::core::SessionState& ss, + const strata::core::ModelGeometry& g, + void* /*stream*/) { + if (!out.fired) return SpliceStatus::NoOp; + if (out.old_suffix_len <= 0) return SpliceStatus::NoOp; + // ---- Phase 3 dispatch: which splice path? fp16 K (the v1 path) or int8 K (Phase 3 + // dequant->rotate->requant). Read the format ONCE so the two paths' guards stay + // independent and a future format (q4_0, hybrid) still gets refused uniformly. + const bool kv_int8_now = strata::core::qsa_kv_int8() || strata::core::qsa_kv_hybrid(); + const bool kv_q4_now = strata::core::qsa_kv_q4(); // --kv q4_0: the Q4 sandwich helper + const bool kv_other = false; // every engine quant format is handled below + const int64_t kv_res_now = strata::core::qsa_kv_resident(); + const bool layer_split_active = (ss.layer_lo != 0) || (ss.layer_hi != g.n_layers); + // Hoisted above both format branches: the device-pointer safety checks inside each + // branch use n_qsa before its old (post-branch) declaration site. Same class as the + // e51880c n_qsa hoist in generate.cpp - only the engine TU (generate.cpp) compiles + // this header, so the tests-only CPU tree never catches the use-before-decl. + const int64_t n_qsa = g.n_qsa_layers(); + if (kv_q4_now) { + // ---- q4_0 path: same scope guard shape, then the Q4 sandwich helper ---- + if (kv_res_now > 0) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: kv_resident=%lld " + "(slot-ring/streamed mode; q4_0). Streaming-ring splice is Phase 3.\n", + (long long) kv_res_now); + return SpliceStatus::UnsupportedResident; + } + if (layer_split_active) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: layer_split " + "(layer_lo=%lld, layer_hi=%lld of %lld; q4_0). Per-stage splice is Phase 3.\n", + (long long) ss.layer_lo, (long long) ss.layer_hi, (long long) g.n_layers); + return SpliceStatus::UnsupportedSplit; + } + if (n_qsa > 0) { + const auto& st0 = ss.qsa_states[ss.qsa_ord0]; + if (transport::pointer_is_device(st0.k_q4) && !transport::is_active()) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: device buffers " + "without transport (q4_0 path). Falling through to the re-read path.\n"); + return SpliceStatus::UnsupportedFormat; + } + } + } else if (kv_int8_now) { + // ---- int8 path (Phase 3 + the kv_rot H-sandwich). STRATA_KV_ROT=1 is accepted: + // the helper undoes and re-applies the Hadamard around the RoPE delta. + const SpliceStatus g_status = splice_scope_guard_int8( + /*kv_int8=*/true, /*kv_q4=*/kv_other, + /*kv_hybrid=*/false, // accepted since the k8v4 lift: the hybrid K half is int8 + kv_res_now, layer_split_active, /*kv_rot=*/strata::core::qsa_kv_int8_rotate()); + if (g_status != SpliceStatus::NoOp) { + const char* reason = (g_status == SpliceStatus::UnsupportedResident) + ? "int8 + kv_resident > 0 (streaming-ring splice is Phase 3)" + : (g_status == SpliceStatus::UnsupportedSplit) + ? "int8 + layer_split active (per-stage is Phase 3)" + : (strata::core::qsa_kv_int8_rotate() + ? "int8 + STRATA_KV_ROT=1 (Hadamard K domain is a follow-up)" + : "int8 + q4_0 / hybrid / kv_resident / layer_split are Phase 3"); + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: %s. " + "Falling through to today's re-read path (degrade, never crash).\n", reason); + return g_status; + } + // device-pointer safety check (same hotfix as the fp16 path below) + if (n_qsa > 0) { + const auto& st0 = ss.qsa_states[ss.qsa_ord0]; + if (transport::pointer_is_device(st0.k_q) && !transport::is_active()) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: device buffers " + "without transport (int8 path). Falling through to the re-read path.\n"); + return SpliceStatus::UnsupportedFormat; + } + } + } else { + // ---- fp16 path (the v1 path). Existing guard: q4 / hybrid / resident / split. + if (strata::core::qsa_kv_q4() || strata::core::qsa_kv_hybrid()) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: kv format != fp16 " + "(q4_0/hybrid). Dequant-rotate-requant for q4_0 is a follow-up.\n"); + return SpliceStatus::UnsupportedFormat; + } + if (kv_res_now > 0) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: kv_resident=%lld " + "(slot-ring/streamed mode). Streaming-ring splice is Phase 3.\n", + (long long) kv_res_now); + return SpliceStatus::UnsupportedResident; + } + if (layer_split_active) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: layer_split " + "(layer_lo=%lld, layer_hi=%lld of %lld). Per-stage splice is Phase 3.\n", + (long long) ss.layer_lo, (long long) ss.layer_hi, (long long) g.n_layers); + return SpliceStatus::UnsupportedSplit; + } + // device-pointer safety check (the 2026-10-12 SIGSEGV hotfix - fp16 path) + if (n_qsa > 0) { + const auto& st0 = ss.qsa_states[ss.qsa_ord0]; + if (transport::pointer_is_device(st0.k_pool) && !transport::is_active()) { + std::fprintf(stderr, "[strata] suffix-reuse: SPLICE_UNSUPPORTED: device buffers " + "without transport. k_pool is a device address but the splice " + "transport is not active on this build. Falling through to the " + "re-read path (degrade, never crash).\n"); + return SpliceStatus::UnsupportedFormat; + } + } + } + // Range: [k_rotation_start, k_rotation_start + old_suffix_len). Phase 3a T9: the + // runtime hook at generate.cpp:9050 narrows the local `resume` AFTER this function + // returns, so the splice ranges are derived directly from the outcome (not from the + // narrowed `resume`). `k_rotation_start` is the suffix-walk-derived start of the + // unchanged tail in OLD (= plan.edit_pos = n_old - old_suffix_len). This is + // DIFFERENT from `out.plan_edit_pos` (the splice hook's narrowing signal = LCP). + // The two values agree for mid-prompt edits but diverge for continuations (where + // old_suffix_len == 0 gates apply_splice out anyway). + const int64_t edit_pos = out.k_rotation_start; + const int64_t old_suffix_len = out.old_suffix_len; + const int head_dim = (int) g.head_dim; + const int n_rot = (head_dim >= 64) ? 64 : head_dim; + const int n_head_kv = (int) g.n_head_kv; + const int max_pos = (int) (ss.qsa_states[ss.qsa_ord0].max_cells); + // `page_size` is the KV page granule qsa_state_init sizes the pool with: + // `qsa_shapes(g).page_size`, which is `qsa_real_shapes().page_size` with the model's + // head counts substituted (layer.cpp:507). It equals idx_block (4) in this artifact + // only by construction; the value is taken from the same source the pool layout uses, + // NOT from qsa_interval (r1's "page-size coincidence" - the two 4s are different facts: + // one in every 4th LAYER is full attention; one page = 4 CELLS). + const int64_t page_size = strata::kernels::qsa_real_shapes().page_size > 0 + ? strata::kernels::qsa_real_shapes().page_size + : 1; + for (int64_t j = 0; j < n_qsa; ++j) { + strata::core::QsaState& st = ss.qsa_states[ss.qsa_ord0 + j]; + if (st.cos_tab == nullptr || st.sin_tab == nullptr) continue; + if (st.kv_mode != 0) continue; + // The pool's row count: n_slots pages of n_head_kv*page_size rows. kv_mode==0 is + // identity-mapped, so position p's rows are exactly the rows the splice walks. + const int64_t pool_rows = st.n_slots > 0 ? st.n_slots * n_head_kv * page_size : 0; + if (kv_q4_now) { + // Q4_0 path: the exact H-sandwich with Q4_0 dequant/requant blocks. + if (st.k_q4 == nullptr) continue; + (void) rotate_qsa_tail_inplace_q4( + st, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, + out.uniform, (int) out.delta, + out.deltas.empty() ? nullptr : out.deltas.data(), + pool_rows); + } else if (kv_int8_now) { + // Phase 3 int8 path: dequant per group, rotate, requant (kv_splice_apply_int8.hpp). + if (st.k_q == nullptr || st.k_scale == nullptr) continue; + (void) rotate_qsa_tail_inplace_int8( + st, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, + out.uniform, (int) out.delta, + out.deltas.empty() ? nullptr : out.deltas.data(), + pool_rows); + } else { + // fp16 path: rotate in place (kv_splice_apply.hpp). + if (st.k_pool == nullptr) continue; + (void) rotate_qsa_tail_inplace(st, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, + out.uniform, (int) out.delta, + out.deltas.empty() ? nullptr : out.deltas.data(), + /*skip_rotation=*/false, pool_rows); + } + } + return SpliceStatus::Rotated; +} + +} // namespace strata::program::suffix_reuse diff --git a/include/strata/program/kv_splice_transport.hpp b/include/strata/program/kv_splice_transport.hpp new file mode 100644 index 00000000000..895be0481b3 --- /dev/null +++ b/include/strata/program/kv_splice_transport.hpp @@ -0,0 +1,201 @@ +// include/strata/program/kv_splice_transport.hpp - Phase 2 hotfix: the memory transport for +// the suffix-reuse splice. +// +// WHY THIS EXISTS (the 2026-10-12 SIGSEGV). `rotate_qsa_tail_inplace` (kv_splice_apply.hpp) +// was written as a HOST loop: it reads `st.k_pool` rows and `st.cos_tab`/`st.sin_tab` +// entries directly through the pointers `QsaState` carries. In the CUDA build those +// pointers live in the session arena allocated with `cudaMalloc` (generate.cpp:3763), and +// `qsa_state_init` builds the rope table on the host only to `cudaMemcpy` it INTO +// `st.cos_tab` (layer.cpp:836-841) - they are DEVICE addresses. A host dereference of a +// discrete-GPU device address is SIGSEGV. The CPU-only build (the ctest suite) puts the +// same arena in host memory, which is why T6 never saw the fault: the first +// checkpoint-mounted request with --cache-suffix-reuse died at the splice, before any +// prefill log line, exit code -11 (reproduced 2/2 + 6 historical, probe5). +// +// THE FIX IN ONE SENTENCE: the splice math stays a host loop (it is pure and T6-pinned), +// but every byte that crosses to/from `st.k_pool` and the rope tables goes through +// `cudaMemcpy` staging when the pointer is a device address, and the loop never +// dereferences a device pointer on the host. +// +// HOW "IS THIS A DEVICE POINTER?" IS DECIDED. A CUDA runtime pointer's address carries no +// portable tag, so the transport asks the runtime: `cudaPointerGetAttributes` reports +// `cudaMemoryTypeDevice` for a cudaMalloc'd/VMM-mapped allocation. That probe is cached +// per QsaState (the arena is allocated once at startup and the answer never changes), so +// the per-request cost is one attributes query per layer the first time and nothing after. +// On a CPU-only build the transport compiles to the identity: pointers are host pointers, +// the staging buffer is the buffer itself. +// +// FAIL-CLOSED BY CONSTRUCTION. The transport is active on every TU compiled with the CUDA +// compiler (`__CUDACC__` is NVCC's own per-TU define, so it covers generate.cpp, layer.cpp, +// every qsa_*.cu - the files that hold live QsaStates). An extra opt-in macro +// `STRATA_SPLICE_FORCE_TRANSPORT` lets a test TU force the transport on a CPU build (the +// fake-cuda harness at sfx-crash-harness-transport.cpp uses this; it supplies its own +// stub cuda_runtime API and PROT_NONE guard region). Compiling a CUDA TU without the +// transport active is a hard `#error` - the splice is too dangerous to be inert on GPU. +#pragma once + +// ---- Transport activation policy ------------------------------------------------ +// +// 1. The CUDA compiler (NVCC, HIPCC, etc.) sets `__CUDACC__` for every TU it builds. +// Those TUs MUST have the transport: they own the live QsaStates whose k_pool / +// rope tables are device pointers. We define STRATA_SPLICE_TRANSPORT for them so +// the staging path is the only code that can ship. +// +// 2. The project's CMake option `STRATA_SPLICE_TRANSPORT` (a developer-only knob for +// opting a CPU TU into the transport) overrides the default; this is what the +// fake-cuda harness and the device-path ctest use. +// +// 3. The fail-closed `#error` below fires when a TU is compiled by NVCC (so it sees +// live QsaStates) but somehow lacks the transport - it is a hard compile error +// so a future refactor that disables the transport by accident cannot ship. +// +// Layer.cpp's qsa_state_init (which lays out the arena) and generate.cpp's prefill path +// are both NVCC-compiled; the engine binary always sees the transport. The +// header-only test executables (suffix_reuse_apply_test, suffix_reuse_test) are +// g++-compiled and stay on the identity path: their QsaStates are synthesised, the +// pointers are host, and the identity transport is what runs. +// BUILD-GLOBAL activation (2026-10-12 hotfix round 2): `__CUDACC__` alone is NOT sufficient +// because generate.cpp - the TU that runs the splice at generate.cpp:9075 - is compiled by +// G++ as a plain host TU even in a CUDA build (only the .cu files go through NVCC). The +// 3f840e3 gate keyed on __CUDACC__ therefore shipped the identity transport into the one +// TU that holds live device-backed QsaStates: is_active()=false + stub pointer_is_device()=false +// (which also disarmed the runtime de-rotation guard). STRATA_SPLICE_TRANSPORT is now a +// build-wide target_compile_definition on every CUDA/HIP build target (see CMakeLists.txt), +// so ANY TU in a CUDA build - NVCC or g++ - gets the real transport. __CUDACC__ is kept as +// a belt-and-braces per-TU trigger. The #error below is extended to catch g++ TUs in a +// CUDA build too: any TU that sees this header in a build with CUDA enabled but no +// transport is a compile error, so the transport can never again ship inert. +#if defined(__CUDACC__) && !defined(STRATA_SPLICE_TRANSPORT) +# define STRATA_SPLICE_TRANSPORT 1 +#endif +#if defined(STRATA_HAVE_CUDA_BUILD) && !defined(STRATA_SPLICE_TRANSPORT) +# define STRATA_SPLICE_TRANSPORT 1 +#endif + +#include "strata/core/layer.hpp" // QsaState + +#if defined(STRATA_SPLICE_TRANSPORT) +# if defined(STRATA_SPLICE_FAKE_CUDA) +// The transport-verification harness supplies its own fake CUDA API before this header; +// it does not include (there is no CUDA toolkit on the CPU host). +# else +# include +# endif +# define STRATA_SPLICE_HAS_TRANSPORT 1 +#endif + +// ---- Hard fail-closed: a CUDA TU with no transport must NOT compile -------------- +// This branch is unreachable when STRATA_SPLICE_TRANSPORT is set (above), but a future +// refactor that drops the activation could leave it as a build-time alarm: a CUDA TU +// that includes this header without the transport active is a bug. We report it +// rather than silently shipping a build that will SEGV on the first splice. +#if (defined(__CUDACC__) || defined(STRATA_HAVE_CUDA_BUILD)) && !defined(STRATA_SPLICE_HAS_TRANSPORT) +# error "STRATA SPLICE: this translation unit is in a CUDA build (NVCC or STRATA_HAVE_CUDA_BUILD) \ +but the splice transport is not active. The splice is unsafe to compile without \ +STRATA_SPLICE_TRANSPORT: it dereferences device pointers from the host loop. Either \ +restore the activation above or stop including kv_splice_apply.hpp from this TU." +#endif + +#include +#include +#include + +namespace strata::program::suffix_reuse::transport { + +/// `true` iff this translation unit was compiled with the splice transport active. +/// The runtime de-rotation guard in `apply_splice` checks this against +/// `pointer_is_device(st.k_pool)` to refuse a splice on a device-backed state when the +/// transport is missing - the alternative is a host deref of a device address (SIGSEGV). +/// On a CPU build this returns false; the identity transport is always safe to call. +inline bool is_active() { +#if defined(STRATA_SPLICE_HAS_TRANSPORT) + return true; +#else + return false; +#endif +} + +#if !defined(STRATA_SPLICE_HAS_TRANSPORT) + +/// CPU build: every pointer is host memory; staging is the identity. +inline bool pointer_is_device(const void*) { return false; } + +/// A staging view of `bytes` starting at `p`. On the CPU build the view aliases the +/// caller's memory directly - no copy in either direction. +struct Staged { + void* host = nullptr; ///< host-readable/writable alias of the range + void* dev = nullptr; ///< the address to copy back to (== p on CPU) + bool device_backed = false; ///< false: `host` IS the target, write directly +}; +inline Staged stage_for_readwrite(void* p, size_t bytes) { + (void) bytes; + return Staged{p, p, false}; +} +inline void commit(const Staged& s, size_t bytes) { + (void) s; (void) bytes; // nothing to do: writes already landed +} + +#else // STRATA_SPLICE_HAS_TRANSPORT + +/// Ask the CUDA runtime whether `p` lives on the device. `cudaPointerGetAttributes` +/// succeeds and reports a device/VMA allocation exactly for pointers the runtime handed +/// out (cudaMalloc, VMM map). A failure of the call itself is treated as "not device" +/// (a plain host pointer) so a stub driver cannot disable the splice. +inline bool pointer_is_device(const void* p) { + cudaPointerAttributes attr{}; + if (p == nullptr) return false; + if (cudaPointerGetAttributes(&attr, p) != cudaSuccess) { + (void) cudaGetLastError(); // a host pointer: not an error worth surfacing + return false; + } + return attr.type == cudaMemoryTypeDevice || attr.type == cudaMemoryTypeManaged; +} + +/// A staging buffer for one contiguous range of device memory. `host` is a fresh host +/// buffer the caller reads/writes; `commit` copies the range back. The copy pair is +/// synchronous (`cudaMemcpy`), which is correct here: the splice runs at request start, +/// before any prefill work is enqueued on the pool (generate.cpp quiesces the session +/// around the request boundary; nothing else touches K until the read loop starts). +/// +/// Multiple `Staged` views can be live at the same time (the splice stages cos, sin, and +/// k together), so each view occupies its own slot in the shared pool. The pool doubles +/// on demand and is never shrunk. Slots are recycled by index, so the read+write pair +/// for one view is internally consistent: the host address handed back at stage time is +/// the same address commit() copies back to the device. +struct Staged { + void* host = nullptr; ///< host staging buffer (owned by the shared pool) + void* dev = nullptr; ///< the device range's base + bool device_backed = false; ///< true: commit() must copy back +}; +struct StagedSlot { + std::vector data; ///< grows; one Staged view per call +}; +inline Staged stage_for_readwrite(void* p, size_t bytes) { + if (!pointer_is_device(p)) return Staged{p, p, false}; + // Each call gets its own slot. Recycle a small set (the splice stages at most a + // handful of ranges per call: cos slice, sin slice, k range). Using a thread_local + // vector of slots means no global lock; the splice is single-threaded today. + static thread_local std::vector slots; + static thread_local size_t cursor = 0; + if (cursor >= slots.size()) slots.emplace_back(); + StagedSlot& slot = slots[cursor++]; + if (cursor >= 1024) cursor = 0; // bound the cycle so we don't grow forever + if (slot.data.size() < bytes) { + size_t want = bytes > slot.data.size() * 2 ? bytes : slot.data.size() * 2; + try { slot.data.resize(want); } catch (...) { slot.data.resize(bytes); } + } + Staged s{slot.data.data(), p, true}; + if (cudaMemcpy(s.host, p, bytes, cudaMemcpyDeviceToHost) != cudaSuccess) { + (void) cudaGetLastError(); + return Staged{nullptr, nullptr, false}; // caller sees a null host: refuse the layer + } + return s; +} +inline void commit(const Staged& s, size_t bytes) { + if (s.device_backed && s.host != nullptr && s.dev != nullptr && bytes > 0) + (void) cudaMemcpy(s.dev, s.host, bytes, cudaMemcpyHostToDevice); +} + +#endif // STRATA_SPLICE_HAS_TRANSPORT + +} // namespace strata::program::suffix_reuse::transport diff --git a/include/strata/program/suffix_reuse.hpp b/include/strata/program/suffix_reuse.hpp new file mode 100644 index 00000000000..9864253305b --- /dev/null +++ b/include/strata/program/suffix_reuse.hpp @@ -0,0 +1,602 @@ +// include/strata/program/suffix_reuse.hpp - the suffix-reuse plan computation: the longest common tail +// of an old and a new prompt and the per-token RoPE delta that re-uses the old prompt cache for it. +// +// On a mid-prompt edit at position E, the tokens [E, n_old) and [E, n_new) are the unchanged SUFFIX; +// the request path can splice the cached K/V for them with a rotation of K (V is untouched - it is the +// same content), and replay the linear state (GDN recurrence, PLE history, QSA indexer tails) from E. +// Today, `limit_reuse(first_dirty)` at include/strata/core/conversation_cache.hpp:221 discards the +// tail past `first_dirty`; this header ADDS the splice-plan call path that the flag `--cache-suffix-reuse` +// enables - by default off (P0 invariant: flag absent or 0 -> today's behaviour byte-identical). +// +// The plan struct is the contract the request path consumes. `uniform == true` means one rotation +// (`delta`) handles the whole tail; `uniform == false` means the per-token table (`deltas`) carries +// the per-position offsets `rope_neox_pair` walks. `replay_from` is set to the edit position so the +// deferred GDN replay can pick it up (A1.5: deliverable is the field + routing, no stub; with replay +// unimplemented, routing falls through to the existing re-read path unchanged). +// +// The plan is pure and deterministic, header-only, so `suffix_reuse_test.cpp` walks it by hand. +// +// QSA INDEXER TAILS ARE REBUILD-ON-REREAD, NOT MAINTAINED ON THE SPLICE PATH. The splice rotates +// K rows in place and leaves V untouched - positions for the tail are re-derived per prefill step +// from `pos_base + pos` (src/core/layer.cpp:1058 uploads `host_pos[h] = pos_base + pos` every +// step), so no per-cell positions table exists. The indexer's per-side state (`tails/dead/block_pos`, +// `include/strata/kernels/qsa.hpp:185-209`) is snapshot-and-restored via ConversationCheckpoint +// (`src/core/conversation_state.cpp:154-193`); on a re-read the indexer's `idx_tail` is refilled +// for [E, L), `block_pos` is set fresh, and `idx_dead` is a per-sequence constant. None of it +// requires splice-specific work beyond running the prefill [E, L). Phase 2 splice: no-op here. +#pragma once + +#include +#include +#include +#include +#include + +namespace strata::program::suffix_reuse { + +/// A checkpoint candidate's identity for invalidation: just its token length (the chain is sorted) and +/// the caller's stable index (a pointer back to the underlying `ConversationCheckpoint` or its slot). +/// `length` is what `invalidate_checkpoints` drops on; the index lets the caller remove the live entry. +struct CheckpointLength { + int64_t length = 0; ///< the checkpoint's token length (sorted ascending forms the chain) + size_t index = 0; ///< caller's stable index into the underlying checkpoint array +}; + +/// The splice plan for a re-edited prompt. +/// +/// `edit_pos` is the position in the OLD prompt at which the cache stops being valid - the splice +/// reuses the cache cells with index `>= edit_pos`, the prefix path (conversation_cache.hpp:217-218 +/// `ConversationKvReuse.unchanged_tokens`) keeps `edit_pos` valid as the unused prefix. When the +/// prompts are identical the common tail is the whole prompt and `edit_pos == 0` (the path that +/// uses `edit_pos` short-circuits on `edit_pos == old_suffix_len`). +/// +/// `old_suffix_len` and `new_suffix_len` are the lengths of the common tail beyond `edit_pos` - the +/// splice reuses these from the old cache. On a uniform delta they are equal; on a per-token delta +/// table they can differ (e.g. an insert further right than a delete moves the boundary). +/// +/// `uniform == true`: one rotation of K by `delta` (e.g. +k for a pure insert of k tokens). `deltas` +/// stays empty. `uniform == false`: per-token position offsets - each `deltas[i]` is the offset to +/// rotate cached token (old index `edit_pos + i`) by, to land it at new position `edit_pos + i + offset`. +/// +/// `replay_from` mirrors `edit_pos`: the position the deferred linear-state replay (GDN recurrence, +/// PLE history, QSA indexer tail) starts from. With replay unimplemented (A1.5), the call site falls +/// through to the existing re-read path unchanged. +struct SuffixReusePlan { + int64_t edit_pos = 0; ///< smallest index of divergence (LEGACY: suffix-walk-derived; the start of the K rotation range in OLD). For backward compat with the original T1/T4 tests and the K rotation range. + int64_t narrow_pos = 0; ///< Phase 3a T9: longest common PREFIX (first index of divergence from the FRONT). The splice hook's narrowing signal - where to narrow resume and which checkpoints to keep. For a pure continuation, narrow_pos == n_old == resume (the splice is a no-op). For a mid-prompt edit at E, narrow_pos == E. + int64_t old_suffix_len = 0; ///< common-tail tokens in the OLD prompt (the K rotation range in OLD is [n_old - old_suffix_len, n_old)) + int64_t new_suffix_len = 0; ///< common-tail tokens in the NEW prompt + bool uniform = false; ///< true -> single rotation by `delta` + int64_t delta = 0; ///< the uniform position offset (0 when `uniform` is false) + std::vector deltas; ///< per-token offsets when `uniform` is false (size = new_suffix_len) + int64_t replay_from = 0; ///< position the linear-state replay starts from (== narrow_pos, the splice hook's narrowing position) +}; + +/// Compute the splice plan for a re-edited prompt. `old_tokens` is the prompt the cache holds; `new_tokens` +/// is the request's prompt. The longest common PREFIX sets `narrow_pos` (the first index of divergence +/// from the front - the splice hook's narrowing signal). The longest common SUFFIX sets +/// `old_suffix_len`/`new_suffix_len` and the per-position rotation (the unchanged tail from the back - the +/// K rows the splice rotates in place). `edit_pos` (legacy) is the suffix-walk-derived start of the K +/// rotation range in OLD (= n_old - old_suffix_len). V is untouched. Pure, header-only. +/// O(n + m) on the lengths. +/// +/// Phase 3a T9 (the live 0% vs 98% off-arm regression): `narrow_pos` (LCP-based) is the right +/// notion of "edit position" for the splice hook. The longest common SUFFIX walk the previous +/// version used mis-IDed a pure continuation as a 1-token edit whenever the last token of `old` +/// happened to equal the last token of `new` (the live probe5 case: the EOS token 248046 at the +/// end of the resume checkpoint also ends the appended CONT). The suffix walk produced +/// `suffix=1, edit_pos = n_old - 1`, the splice hook narrowed `resume` to `n_old - 1`, erased +/// the resume checkpoint, and the mount was destroyed. The LCP-based `narrow_pos` is `n_old` +/// for a pure continuation (the resume checkpoint IS a prefix of the new prompt by definition - +/// the mount condition), so the splice hook does not narrow and the mount survives. +/// +/// The continuation special case (LCP == n_old): the splice is a no-op. `old_suffix_len` and +/// `new_suffix_len` are forced to 0 (any common-suffix tokens past the LCP are by definition +/// coincidental, e.g. the EOS token - they don't represent real content match worth rotating). +/// The runtime then sees `narrow_pos == n_old == resume` (no narrowing) and `old_suffix_len == 0` +/// (no K rotation), byte-identical to flag-off. +template +SuffixReusePlan compute_suffix_plan(const std::vector& old_tokens, const std::vector& new_tokens) { + SuffixReusePlan plan; + const size_t n_old = old_tokens.size(); + const size_t n_new = new_tokens.size(); + // ---- narrow_pos: longest common PREFIX from the FRONT (Phase 3a T9) ---- + // The first index of divergence is the position in the OLD prompt at which the cache + // stops being valid. For a pure continuation (`old` is a prefix of `new`) the LCP is + // n_old and narrow_pos == n_old - the splice hook's narrowing predicate `narrow_pos < resume` + // is false (narrow_pos == resume), no narrowing. For a mid-prompt edit at E, LCP = E + // (the first E tokens match, token E differs), narrow_pos = E, the splice hook narrows + // resume to E and the chain is filtered. + size_t prefix = 0; + while (prefix < n_old && prefix < n_new && + old_tokens[prefix] == new_tokens[prefix]) ++prefix; + plan.narrow_pos = (int64_t) prefix; + plan.replay_from = plan.narrow_pos; + // ---- old_suffix_len / new_suffix_len: longest common SUFFIX from the BACK (the K rotation range) ---- + size_t suffix = 0; + while (suffix < n_old && suffix < n_new && + old_tokens[n_old - 1 - suffix] == new_tokens[n_new - 1 - suffix]) ++suffix; + plan.old_suffix_len = (int64_t) suffix; + plan.new_suffix_len = (int64_t) suffix; + // ---- edit_pos (LEGACY): suffix-walk-derived start of the K rotation range in OLD ---- + // = n_old - suffix. Kept for backward compat with the T1/T4 tests and any consumer + // that used the old semantics. The splice hook should use `narrow_pos` instead. + plan.edit_pos = (int64_t) n_old - (int64_t) suffix; + // ---- Phase 3a T9: continuation special case ---- + // The resume checkpoint is a prefix of cur (LCP == n_old): the splice is a no-op. Any + // common-suffix tokens are coincidental (e.g. the EOS token at the turn boundary that + // appears at the end of both old and new); they do not represent content the splice + // can usefully rotate - rotating a single EOS K row by +54 lands it past the new prompt's + // last position, and the prefill overwrites it anyway. Set tail lengths to 0 so the + // runtime's `apply_splice` is gated out (line 9088 in generate.cpp: `if (old_suffix_len > 0)`). + // narrow_pos stays at n_old (= resume) so the runtime does not narrow. edit_pos (legacy) + // stays at n_old - suffix = 0 if the EOS matches, or n_old if not - the legacy field's + // value is irrelevant when old_suffix_len == 0. + if (prefix == n_old) { + plan.old_suffix_len = 0; + plan.new_suffix_len = 0; + plan.uniform = true; // a no-op plan is trivially uniform (delta unused) + plan.delta = 0; + plan.deltas.clear(); + return plan; + } + // uniform offset: only when every old-position-to-new-position pair in the tail adds the same delta + // (a pure insert or a pure delete; a per-token table is the mixed case) + if (suffix > 0) { + const int64_t uniform_delta = (int64_t) n_new - (int64_t) n_old; + bool ok = true; + plan.deltas.assign(suffix, 0); + for (size_t i = 0; i < suffix; ++i) { + const int64_t new_pos = (int64_t) (n_new - suffix + i); + const int64_t old_pos = (int64_t) (n_old - suffix + i); + plan.deltas[i] = new_pos - old_pos; + if (plan.deltas[i] != uniform_delta) ok = false; + } + if (ok) { + plan.uniform = true; + plan.delta = uniform_delta; + plan.deltas.clear(); + } + } + return plan; +} + +/// Drop every checkpoint candidate with `length > edit_pos`. Returns how many were dropped. `chain` +/// is sorted ascending by `length`; this is pure and the caller owns the live `ConversationCheckpoint` +/// array (the loop on the call site uses the returned `index` to identify survivors). The empty-edit +/// case (`edit_pos == 0`) drops the entire chain. +inline size_t invalidate_checkpoints(std::vector& chain, int64_t edit_pos) { + size_t dropped = 0; + size_t i = 0; + while (i < chain.size()) { + if (chain[i].length > edit_pos) { chain.erase(chain.begin() + (ptrdiff_t) i); ++dropped; } + else ++i; + } + return dropped; +} + +/// A1.3 / A2.1 fail-closed guards: the splice refuses the same combination +/// `strata generate` refuses at startup. Kept here as a CPU-testable mirror of the run-time check +/// (the runtime check is in `strata/program/generate.cpp` post-flag-parse; this is the +/// same predicate, separated only so `suffix_reuse_test` can exercise it without GPU). +/// Returns the (non-empty) reason the combination is refused, or empty when all conditions +/// are met. +/// +/// Phase 2 Task 4 (review-r2 NEW-2): the startup guard (v1.1) refuses q4_0 / hybrid K +/// (`--kv q4_0` and `--kv k8v4`), the kv-resident slot ring (`--kv-resident > 0`), and +/// the layer split when the splice flag is on. Phase 3 (int8 KV splice): `--kv int8` is +/// NOW SUPPORTED (the daily preset's mode). The int8 K is dequantized, rotated by the +/// splice's RoPE delta, and requantized in place (kv_splice_apply_int8.hpp). Fail-closed: +/// STRATA_KV_ROT=1 (K through a Hadamard rotation) is still a Phase 3 follow-up and the +/// runtime hook (kv_splice_apply_session.hpp) refuses it explicitly when the flag is on. +inline std::string validate_suffix_reuse_options(int64_t cache_suffix_reuse, int prompt_cache, + const std::string& kv_format, + int64_t kv_resident, bool layer_split_active, + int64_t prompt_cache_every = 0, + int64_t dense_tail = 0) { + if (cache_suffix_reuse <= 0) return {}; + if (prompt_cache < 2) + return "--cache-suffix-reuse N requires --prompt-cache >= 2"; + // q4_0: ACCEPTED (same as int8/k8v4). The Q4 helper does the exact H-sandwich with + // Q4_0 dequant/requant (suffix_reuse_int8_test T11.f measures the codec noise); falls + // through to the kv_resident / layer_split refusals below like every quant format. + if (kv_format != "fp16" && kv_format != "int8" && kv_format != "k8v4" && kv_format != "q4_0") + return "--cache-suffix-reuse N requires --kv fp16 or --kv int8 (see plans/2026-10-11_184000-strata-suffix-reuse-phase2-v2.md)"; + if (kv_resident > 0) + return "--cache-suffix-reuse N is unsupported with --kv-resident (streaming-ring splice " + "is Phase 3)"; + if (layer_split_active) + return "--cache-suffix-reuse N is unsupported with --layer-split (per-stage splice is " + "follow-up work; see plans/2026-10-11_130800-strata-suffix-reuse.md A2.1)"; + if (dense_tail > 0 && prompt_cache_every <= 0) + return "--prompt-cache-dense-tail N requires --prompt-cache-every > 0 (dense checkpoints " + "are a cadence on the same periodic mechanism)"; + return {}; +} + +/// FW-1: position-dependent periodic-checkpoint cadence. PURE so it is CPU-testable +/// without a GPU (suffix_reuse_dense_tail_test). Returns the `done` threshold at/after +/// which the NEXT periodic checkpoint fires, given the position just checkpointed (`done`, +/// 0 when starting), the prompt's total length AS KNOWN at decision time (`pp_total` - the +/// prompt pass discovers it incrementally, so callers re-ask as it grows), the periodic +/// cadence (`every`, > 0), and the dense-tail width (`dense_tail`; 0 = OFF). +/// +/// dense_tail == 0 -> next = done + every (EXACTLY today's cadence, generate.cpp's +/// `pp_next_check = done + o.prompt_cache_every`; byte-identical +/// behavior when the flag is off). +/// dense_tail > 0 -> the last `dense_tail` tokens of the prompt are the DENSE zone: +/// cadence `every` there (mid-prompt edits land near the tail - the +/// splice needs an anchor at or before the edit). Before the zone +/// the cadence is max(every, 8192) (sparse head: the cold read pays +/// for far fewer checkpoint pauses; one deep anchor survives for the +/// deep-prefix rescue). A sparse next that would overshoot the +/// dense zone CLAMPS to the zone start - the zone is never skipped, +/// whatever the head cadence stepped over. +/// +/// Fail-closed by construction: every path returns a threshold > done (or the zone start +/// when clamping; the caller's `done >= threshold` check fires there immediately). +inline int64_t dense_tail_next_check(int64_t done, int64_t pp_total, int64_t every, + int64_t dense_tail) { + if (every <= 0) return INT64_MAX; // periodic checkpoints off + if (dense_tail <= 0) return done + every; // OFF: today's exact cadence + const int64_t zone_start = pp_total > dense_tail ? pp_total - dense_tail : 0; + if (done >= zone_start) return done + every; // dense zone (or a tiny prompt) + const int64_t sparse = std::max(every, 8192); + const int64_t next = done + sparse; + return next > zone_start ? zone_start : next; // never skip the dense zone +} + +/// FW-1 (dynamic sizing): the chunk size the prompt pass should use for the chunk +/// starting at c0, so that CHUNK BOUNDARIES (the only places checkpoints can fire) +/// land at dense-tail cadence inside the zone while the head reads in big chunks. +/// +/// dense_tail <= 0 -> min(big, n - c0) (today's uniform sizing) +/// c0 before the dense zone -> min(big, zone_start - c0) (a big chunk that ends +/// EXACTLY at the zone start: the first anchor) +/// c0 inside the dense zone -> min(every, n - c0) (anchor-sized chunks) +/// +/// Pure; pinned by suffix_reuse_dense_tail_test DTT8-10. `big` is the --prefill chunk +/// size (auto = the whole prompt). The dense_tail_next_check cadence aligns with these +/// boundaries by construction (zone start + every-steps), so generate.cpp's checkpoint +/// gate needs no change. +inline int64_t dense_tail_chunk_size(int64_t c0, int64_t n, int64_t big, int64_t every, + int64_t dense_tail) { + if (n - c0 <= 0) return 0; + if (dense_tail <= 0 || every <= 0) return std::min(big, n - c0); + const int64_t zone_start = n > dense_tail ? n - dense_tail : 0; + if (c0 < zone_start) { + const int64_t head = std::min(big, zone_start - c0); // land exactly at zone start + return std::max(head, 1); + } + return std::max(std::min(every, n - c0), 1); // anchor-sized zone chunks +} + +/// FINDING 1: the parity predicate (the runtime hook's fire decision). Centralised here so +/// `generate.cpp` and the off-parity test call THE SAME function. At flag==0 (or "flag absent", +/// which is 0 by default) the predicate returns false and the hook is a no-op - today's reuse +/// accounting is byte-identical. The function is pure (no side effects) so a header test can +/// pin byte-equality against today's path without GPU. +inline bool suffix_reuse_should_invalidate(int64_t flag, bool from_live, int64_t resume, + bool checks_nonempty, int64_t cur_size) { + if (flag <= 0) return false; // the canonical "flag absent" or "flag off" + if (from_live) return false; // live sessions do not need a splice + if (resume <= 0) return false; // no checkpoint reachable + if (!checks_nonempty) return false; // nothing to invalidate + if (cur_size <= resume) return false; // a strict continuation - no edit past the mounted prefix + return true; +} + +/// FINDING 4(a): the splice hook's invalidation result. When the predicate says fire, we compute +/// the plan against the resume checkpoint's tokens (NOT checks.front() - the deepest match is +/// the one the request actually mounts through, and the shortest checkpoint would always yield +/// edit_pos=0 on a pure continuation, narrowing `resume` and pessimising the common case). We +/// then drop checkpoints with `length > plan.edit_pos` from `chain` and surface the plan. +/// +/// Phase 2 Task 1: the outcome also carries the splice plan's fields so the runtime `apply_splice` +/// in this file can read them. The names mirror `SuffixReusePlan` (old_suffix_len, new_suffix_len, +/// deltas, replay_from) so a single struct describes both decisions: which checkpoints to keep and +/// which K rows to rotate. Fields are populated on the `fired` path only; at `!fired` the runtime +/// sees zeros, an empty `deltas`, and the doc says "no splice planned". This is a pure struct +/// extension - no behaviour change; the runtime consumer lands in Task 2. +struct SuffixHookOutcome { + std::vector chain; ///< the chain after invalidation (== input when not fired) + bool fired = false; ///< the predicate's verdict (== false -> chain untouched) + int64_t plan_edit_pos = 0; ///< Phase 3a T9: the splice hook's narrowing signal (LCP-based, == SuffixReusePlan::narrow_pos). Where to narrow resume and which checkpoints to keep. + int64_t k_rotation_start = 0; ///< Phase 3a T9: the K rotation start in OLD (suffix-walk-derived, == n_old - old_suffix_len, == SuffixReusePlan::edit_pos). The unchanged tail from the back - the K rows the splice rotates in place. For a pure continuation this is 0 (irrelevant; old_suffix_len == 0 gates apply_splice out). + // ---- Phase 2 Task 1: splice plan fields surfaced for `apply_splice` (Task 2) ---- + int64_t old_suffix_len = 0; ///< common-tail tokens in the OLD prompt (== SuffixReusePlan::old_suffix_len) + int64_t new_suffix_len = 0; ///< common-tail tokens in the NEW prompt (== SuffixReusePlan::new_suffix_len) + std::vector deltas; ///< per-token position offsets (uniform case: empty, single delta suffices) + int64_t replay_from = 0; ///< position the linear-state replay (Task 5) starts from; == plan.narrow_pos + bool uniform = false; ///< true -> single rotation by `delta`; false -> per-token `deltas` + int64_t delta = 0; ///< uniform position offset when `uniform` is true + // ---- Phase 3b: partial-prefix mount anchor ---- + // When > 0, the splice path mounted a partial-prefix checkpoint (its LCP is `plan_edit_pos` < + // `restore_anchor_length`): the runtime restores the checkpoint's running state at length + // `restore_anchor_length` and the chunk loop reads from `plan_edit_pos` (= the LCP). The K + // rotation operates on [plan_edit_pos, restore_anchor_length) in OLD. When 0 (the default + // and the strict-mount / off-flag path), the runtime uses `narrowed_resume_target` as before. + int64_t restore_anchor_length = 0; +}; +inline SuffixHookOutcome suffix_reuse_apply_invalidation( + int64_t flag, bool from_live, int64_t resume, + const std::vector& chain, + const std::vector& resume_checkpoint_tokens, + const std::vector& cur_tokens) { + SuffixHookOutcome out; + out.chain = chain; + if (!suffix_reuse_should_invalidate(flag, from_live, resume, !chain.empty(), + (int64_t) cur_tokens.size())) { + return out; + } + out.fired = true; + // F4(a) + Phase 3a T9: plan against the resume checkpoint's tokens, not checks.front(). + // The plan now uses the longest common PREFIX (`narrow_pos`) for the splice hook's + // narrowing signal instead of the suffix-walk-derived `edit_pos`. For a pure continuation + // (resume_ck.tokens is a prefix of cur_tokens, LCP == n_old), the plan is a no-op: + // narrow_pos == n_old == resume (the splice hook's narrowing predicate `narrow_pos < resume` + // is false), old_suffix_len == 0 (apply_splice is gated out, no K rotation). The chain + // drop predicate `length > narrow_pos` drops only the post-resume periodic checkpoints + // (e.g. 3244 > 3240), exactly the same drops the off-flag canonical erase at + // generate.cpp:9283-9285 makes - byte-identical. For a mid-prompt edit at E, LCP = E, + // narrow_pos = E < resume, and the chain is filtered to checkpoints with length <= E + // (the anchor below the edit survives). + SuffixReusePlan plan = compute_suffix_plan(resume_checkpoint_tokens, cur_tokens); + // Phase 3a T9: outcome.plan_edit_pos is the splice hook's narrowing signal (narrow_pos, + // LCP-based). outcome.k_rotation_start is the K rotation start in OLD (edit_pos, + // suffix-based) - the unchanged tail from the back. The legacy suffix-derived + // `plan.edit_pos` is still surfaced for the T1/T4 backward-compat tests. + out.plan_edit_pos = plan.narrow_pos; + out.k_rotation_start = plan.edit_pos; + // Phase 2 Task 1: surface the splice plan's fields so the runtime `apply_splice` can read + // them straight off the outcome (no second pass over resume_checkpoint_tokens / cur_tokens). + // Names mirror `SuffixReusePlan` to keep both structs in lockstep (r1 Minor 5: name match). + out.old_suffix_len = plan.old_suffix_len; + out.new_suffix_len = plan.new_suffix_len; + out.uniform = plan.uniform; + out.delta = plan.delta; + out.deltas = plan.deltas; + out.replay_from = plan.replay_from; + // the canonical invalidation: drop checkpoints with length > narrow_pos (= the splice + // hook's narrowing signal). For a pure continuation narrow_pos == n_old == resume, so the + // predicate drops nothing past resume (the off-flag path's canonical erase at + // generate.cpp:9283-9285 then drops anything > resume, byte-identical). For a mid-prompt + // edit at E, narrow_pos < resume and the chain is filtered to length <= E. + size_t dropped = 0; + size_t i = 0; + while (i < out.chain.size()) { + if (out.chain[i].length > out.plan_edit_pos) { + out.chain.erase(out.chain.begin() + (ptrdiff_t) i); + ++dropped; + } else ++i; + } + (void) dropped; + return out; +} + +/// Phase 2 wiring bug 2 (T8): the splice hook narrows `resume` to `plan_edit_pos`, and +/// its own erase (in `suffix_reuse_apply_invalidation`'s caller at generate.cpp:9034-9043) +/// removes the exact-length checkpoint the restore search later wants. This pure +/// helper decides, given a narrowed resume and the live `checks` chain, where the +/// request continues from - the gap-read / replay path phase 2 was designed for. +/// +/// `chain_lengths` carries only the candidate lengths (the helper compares sizes +/// only); generate.cpp projects its `ConvCheckpoint::ids.size()` into a vector at the +/// call site so this stays header-only / CPU-testable. +/// +/// `NarrowedResumeTarget::restore_length` is the checkpoint length the request +/// restores through; `read_from` is the position the chunk loop reads from (== 0 when +/// no usable mount survives - a fresh prefill from the start, NO error). +/// +/// Cases: +/// - exact-length candidate exists: restore_length = that length, read_from = narrowed_resume +/// (no gap; byte-equivalent to today's restore path). +/// - no exact-length, deepest-below: restore_length = deepest length <= narrowed_resume, +/// read_from = narrowed_resume (the [restore, narrowed) +/// gap is re-read by the existing prefill path). +/// - no candidate at or below: restore_length = 0, read_from = 0 +/// (no usable mount; full re-prefill, no error). +/// +/// In every case the request proceeds; the helper never returns a state that maps to +/// the pre-fix "ERR restoring a conversation checkpoint failed" path. +struct NarrowedResumeTarget { + int64_t restore_length = 0; ///< checkpoint length the request restores through (0 -> fresh read) + int64_t read_from = 0; ///< position the chunk loop reads from (= 0 when restore_length == 0) + // Phase 3b: partial-prefix mount anchor. When > 0, the runtime restores a checkpoint of + // length `restore_anchor_length` (the partial-match candidate C, length L > P) and the + // chunk loop reads from `read_from` (= the LCP P). The strict-mount path leaves this at + // 0 and uses `restore_length` as before. Set by the partial-mount branch in generate.cpp + // when `parked.partial` is true and the partial helper fires. + int64_t restore_anchor_length = 0; +}; +inline NarrowedResumeTarget narrowed_resume_target(int64_t narrowed_resume, + const std::vector& chain_lengths) { + NarrowedResumeTarget t{}; + if (narrowed_resume <= 0 || chain_lengths.empty()) { + t.restore_length = 0; + t.read_from = 0; + return t; + } + int64_t best = -1; + for (int64_t L : chain_lengths) { + if (L <= narrowed_resume && L > best) best = L; + } + if (best < 0) { + // No surviving check at or below the narrowed resume: the splice hook erased + // the resume checkpoint and there is no deeper mount to fall back to. Phase + // 2 design's gap-read path reduces here to a fresh prefill from 0 - we do + // NOT synthesise a fake checkpoint (a fake checkpoint with mismatched KV + // would be silently wrong). The request proceeds; no error. + t.restore_length = 0; + t.read_from = 0; + return t; + } + t.restore_length = best; + t.read_from = narrowed_resume; // gap [best, narrowed_resume) is re-read by the chunk loop + return t; +} + +/// FW-2b (T22): the two-pool restore pick. `checks` are the LRU-capped periodic +/// checkpoints (deep anchors); `state_checks` are fine-cadence state-only +/// checkpoints (checkpoint_save copies gdn/ple/tails/dead/block_pos only - no +/// K/V - so each is ~100-150ms, T19). The deepest checkpoint <= P across BOTH +/// pools is restored; the chunk loop still reads [P, n) - fail-close semantics +/// unchanged, the gap [restore, P) just shrinks. Equal lengths are dedup-free: +/// deepest is deepest, whichever pool it came from. +inline NarrowedResumeTarget narrowed_resume_target_state( + int64_t narrowed_resume, const std::vector& chain_lengths, + const std::vector& state_lengths) { + NarrowedResumeTarget t = narrowed_resume_target(narrowed_resume, chain_lengths); + for (int64_t L : state_lengths) { + if (L <= narrowed_resume && L > t.restore_length) { + t.restore_length = L; + t.read_from = narrowed_resume; + } + } + return t; +} + +/// FW-2b (T22): the fine-cadence capture ladder. Returns the next state-checkpoint +/// capture position after `done`: done + every, clamped to n (a capture at exactly +/// n is the live conversation, not a checkpoint; never past it). every <= 0 = off +/// (INT64_MAX: the gate never fires) - byte-identical to today when the flag is 0. +inline int64_t state_cadence_next(int64_t done, int64_t n, int64_t every) { + if (every <= 0) return INT64_MAX; + const int64_t next = done + every; + return next >= n ? n : next; +} + +/// Phase 3a T10: the runtime's `read_from` resolution at generate.cpp:9394. When the splice +/// hook fired AND narrowed resume (splice_narrowed == true), the chunk loop's read start is +/// `t.read_from` (the splice-narrowed resume = the edit pos E), NOT `resume` (the restored +/// anchor C). Otherwise the legacy rules apply: STRATA_CKPT_REREAD rebuilds the whole +/// prefill from 0, and the normal path starts at `resume` (the deepest usable mount). +/// +/// Without this helper the runtime at line 9394 used `reread_to > 0 ? 0 : resume`, which +/// discarded `t.read_from` and re-read [anchor, n) - the rotated region [C, E) the splice +/// just set up AND the gap [E, n) the splice's gap-read was supposed to handle. The +/// gap-read IS the splice's saved work; discarding it makes the splice a no-op on the +/// prefill path (GLM review MAJOR-1: the K rotation is then redundantly overwritten by +/// the fresh prefill, but the prefill re-reads C..n instead of E..n, which is the actual +/// regression). +/// +/// The helper is pure and header-only so the T10 test pins it. generate.cpp's call site +/// is `const int64_t read_from = wire_splice_read_from(splice_narrowed, t, resume, reread_to);` +/// (replacing the pre-3a `reread_to > 0 ? 0 : resume`). +inline int64_t wire_splice_read_from(bool splice_narrowed, + const NarrowedResumeTarget& t, + int64_t resume, + int64_t reread_to) { + if (reread_to > 0) return 0; // STRATA_CKPT_REREAD: rebuild from 0 + if (splice_narrowed) return t.read_from; // gap-read: [edit pos E, n) only + return resume; // legacy: re-read from the restored mount +} + +/// Phase 3b T12: the partial-prefix mount's splice outcome. When the strict mount fails and +/// `ConversationCache::best_with_partial(flag>0, ...)` returns a Match with `partial = true`, +/// the engine has a checkpoint C of length L whose first P tokens (= LCP) match the new +/// prompt's first P tokens and diverge at P (a mid-file edit past P). The runtime mounts C +/// (restores its running state at length L), and the chunk loop reads from P. The splice +/// plan (LCP-based `narrow_pos = P`, suffix-based `old_suffix_len` / `new_suffix_len` for the +/// unchanged tail from the back) drives the K rotation in [P, L) - the same machinery the +/// strict-mount splice path uses, but the resume and the chunk-loop read start differ. +/// +/// This helper centralises the outcome so the runtime integration test can pin the exact +/// field values without GPU. Inputs: +/// `partial_ck_tokens` the full checkpoint C's tokens (length L) +/// `cur_tokens` the new prompt's tokens +/// `partial_ck_length` L (== partial_ck_tokens.size()) +/// `partial_resume` P (the LCP; the chunk loop's read start) +/// Returns a `SuffixHookOutcome` with: +/// fired = true +/// plan_edit_pos = P (== narrow_pos of the LCP-based plan) +/// restore_anchor_length = L (the runtime restores C's running state, not P's) +/// old_suffix_len / new_suffix_len / deltas / uniform / delta = the K-rotation plan +/// chain = the input chain with every entry whose length > P dropped (the runtime's +/// post-splice erase keeps anchors at or below the LCP; the anchor C of length L +/// is also dropped by the chain filter but the runtime uses `restore_anchor_length` +/// to restore C directly - see generate.cpp's `else if (!from_live)` branch). +/// +/// Fail-closed: any unsupported input (L > cur.size(), P not in (0, L), LCP inconsistent with +/// the token equality check) returns `fired = false` and the runtime degrades to today's full +/// re-read. Never crashes, never fabricates a checkpoint. +inline SuffixHookOutcome suffix_reuse_partial_apply( + const std::vector& partial_ck_tokens, + const std::vector& cur_tokens, + int64_t partial_ck_length, + int64_t partial_resume, + const std::vector& chain) { + SuffixHookOutcome out; + out.chain = chain; + // fail-closed guards: every shape that would make the splice wrong is refused. + const int64_t L = partial_ck_length; + const int64_t P = partial_resume; + const size_t n_new = cur_tokens.size(); + if (L <= 0 || P <= 0 || P >= L) return out; // not a valid partial match + if ((int64_t) partial_ck_tokens.size() != L) return out; // inconsistent: tokens vs length + // T17 (2026-10-13): L > n_new is now a VALID partial shape - the parked live + // (prompt+response) is typically longer than the next turn's prompt, and this + // refusal was the third guard that refused the live anchor. The LCP below is + // prompt-bounded, the tail walk in compute_suffix_plan is min-bounded, and the + // runtime restores the deepest checkpoint <= P (never state@L). The strict + // path keeps its own refusal (a longer checkpoint is never a strict prefix). + // verify the LCP claim: partial_ck[0..P) == cur[0..P) and partial_ck[P] != cur[P] (or P == L). + for (int64_t i = 0; i < P; ++i) + if (partial_ck_tokens[(size_t) i] != cur_tokens[(size_t) i]) return out; + if (P < (int64_t) partial_ck_tokens.size() && + P < (int64_t) n_new && + partial_ck_tokens[(size_t) P] == cur_tokens[(size_t) P]) { + // LCP == L means strict (cur[0..L) == ck[0..L)); the strict path handles it. + return out; + } + // The plan: same compute_suffix_plan the strict path uses, but `old` is C's full tokens + // (length L) and `new` is the new prompt. LCP-based narrow_pos = P (verified above); + // suffix-based edit_pos = L - old_suffix_len; the K rotation operates on [edit_pos, L). + SuffixReusePlan plan = compute_suffix_plan(partial_ck_tokens, cur_tokens); + if (plan.narrow_pos != P) return out; // defensive: LCP claim failed + out.fired = true; + out.plan_edit_pos = plan.narrow_pos; + out.k_rotation_start = plan.edit_pos; + out.old_suffix_len = plan.old_suffix_len; + out.new_suffix_len = plan.new_suffix_len; + out.uniform = plan.uniform; + out.delta = plan.delta; + out.deltas = plan.deltas; + out.replay_from = plan.replay_from; + out.restore_anchor_length = L; // the partial-mount anchor + // chain filter: drop entries with length > P (the LCP) - the runtime's post-splice erase + // at generate.cpp:9283-9285 (predicate `length > resume`) keeps anchors at or below P. + // The anchor C of length L MUST be kept (the runtime restores it via `restore_anchor_length`), + // so we explicitly preserve the entry with `length == L` regardless of the length > P filter. + size_t i = 0; + while (i < out.chain.size()) { + const int64_t len = out.chain[i].length; + if (len > P && len != L) { out.chain.erase(out.chain.begin() + (ptrdiff_t) i); } + else ++i; + } + return out; +} + +/// T18 (2026-10-13): the partial mount's read start, replay-gated. The live +/// measurement (chain dt-steady5, /tmp/dt-steady5.log) proved that reading +/// [P, n) on state@checkpoint leaves the gap [ckpt, P) out of the GDN/PLE +/// recurrence: attention sees the gap's rows through the (valid, LCP-shared) +/// arena K/V, but the running state never ingested them - a count query over +/// the gap comes back under-counted (C answered wrong, gap 826 tokens; the +/// same signature as the v3 state@L probe). Until FW-2 replay (ring splice / +/// recurrence replay over the gap) exists, the partial mount fail-closes to +/// reading from the deepest surviving checkpoint at or below P - byte-equal to +/// the strict-path semantics - keeping the park/restore/chain bookkeeping +/// intact and exercised. `replay_available` is the engine-side gate (env +/// STRATA_SFX_REPLAY, default off): with it on, the gap-read [P, n) returns. +/// Pure: the T18 tests pin both arms plus the no-checkpoint degenerate. +inline int64_t partial_gap_read_from(bool replay_available, int64_t P, + const std::vector& chain_lengths) { + if (replay_available) return P; // FW-2: the gap is replayed, read [P, n) + int64_t best = 0; // fail-closed: read from the checkpoint + for (const int64_t L : chain_lengths) + if (L <= P && L > best) best = L; + return best; +} + +} // namespace strata::program::suffix_reuse diff --git a/src/core/layer.cpp b/src/core/layer.cpp index 273b2293ac3..b6daf293bb8 100644 --- a/src/core/layer.cpp +++ b/src/core/layer.cpp @@ -205,6 +205,7 @@ namespace { bool g_kv_int8 = false, g_kv_q4 = false, g_kv_hybrid = false, g_kv_i void qsa_set_kv_int8(bool enabled) { g_kv_int8 = enabled; } void qsa_set_kv_int8_rotate(bool enabled) { g_kv_int8_rot = enabled; } bool qsa_kv_int8() { return g_kv_int8; } +bool qsa_kv_int8_rotate() { return g_kv_int8_rot; } void qsa_set_kv_q4(bool enabled) { g_kv_q4 = enabled; } bool qsa_kv_q4() { return g_kv_q4; } void qsa_set_kv_hybrid(bool enabled) { g_kv_hybrid = enabled; } diff --git a/src/program/generate.cpp b/src/program/generate.cpp index ce44646d839..7b51c352dd6 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -96,6 +96,9 @@ #include "strata/program/prefill_plan.hpp" #include "strata/program/draft_kv_plan.hpp" #include "strata/program/message_boundary.hpp" +#include "strata/program/suffix_reuse.hpp" +#include "strata/program/kv_splice_apply.hpp" // Phase 2 Task 2: kernel-level helper + SpliceStatus enum +#include "strata/program/kv_splice_apply_session.hpp" // Phase 2 Task 2: SessionState wrapper for apply_splice #include "strata/spec/draft_policy.hpp" #include "strata/spec/draft_source.hpp" #include "strata/spec/suffix_drafter.hpp" @@ -870,11 +873,37 @@ struct Options { bool session_save_reclaim = false; /// --serve: also keep a checkpoint every N freshly read prompt tokens (0 = only at the last turn boundary) int64_t prompt_cache_every = 16384; + /// FW-1: --prompt-cache-dense-tail N - dense checkpoint cadence in the last N tokens + /// (sparse max(every, 8192) head; 0 = off = today's uniform cadence). Needs every > 0. + int64_t prompt_cache_dense_tail = 0; + /// FW-2b: --prompt-cache-state-every N - fine-cadence STATE-only checkpoints + /// (gdn/ple/tails/dead/block_pos, no K/V; ~100-150ms each). They shrink the + /// fail-close gap when an edit lands deeper than every periodic anchor. A side + /// pool (never LRU-evicts the deep periodic anchors). 0 = off = today. + int64_t prompt_cache_state_every = 0; bool prompt_cache_tail = false; // optional extra checkpoint at an existing near-tail chunk boundary bool prompt_cache_spread = false; ///< --prompt-cache-spread: keep checkpoints spread back from the end, not the latest /// --serve: a prompt read from token 0 is also checkpointed at its first turn boundary - the end of the system /// prompt, which every chat of the same client shares - when that is at least N tokens (0 = never) int64_t prompt_cache_root = 2048; + /// --cache-suffix-reuse N (--serve; opt-in, default 0 = off): when a prompt repeats an + /// earlier one with a change in the middle, keep the cached text before the change: the + /// unchanged tail's K rows are rotated in place to their new positions (V is untouched - + /// position-free), gdn/ple recurrence replays from `replay_from = edit_pos` (Task 5), and + /// prompt-cache checkpoints past the edit position are invalidated. The splice runs for any + /// N > 0 and works with every --kv format (fp16, int8, int8+STRATA_KV_ROT, k8v4, q4_0); + /// kv-resident and layer-split refuse (`SPLICE_UNSUPPORTED` - falls through to re-read). When 0 (default), today's `limit_reuse(first_dirty)` behaviour byte-identical - + /// the predicate `suffix_reuse_should_invalidate` returns false and the hook is a no-op + /// (verifier: `suffix_reuse_off_parity_test` byte-equals today's accounting). Requires + /// --prompt-cache >= 2. Refuses (falls back to re-read): --kv-resident, --layer-split. + /// BOUNDED PESSIMIZATION (r2 F1): a coincidental role-header suffix (k <= 4 tokens of + /// trailing role header that happens to match) DOES narrow `resume` to that header's position + /// and forces the next request to re-prefill the conversation body. This is a known bounded + /// loss - on `--serve` chat with `tail_role_token < 0` (default; the role header pinned just + /// inside <|im_start|>), the header's length is bounded by the role token's piece length + /// (typically 3-4 tokens), so the regex matches are rare and the cost is small. No new knob; + /// the runtime narrows on any genuine common suffix even if the tail looks incidental. + int64_t cache_suffix_reuse = 0; /// --serve: the token that opens a chat turn (<|im_start|>). The last one in a prompt is where the chat's /// history ends and the new assistant turn begins, which is the checkpoint the next request can reuse. int64_t turn_token = 248045; @@ -1012,10 +1041,22 @@ void usage() { " --shared-late A/B: shared expert after the CPU pool (default: overlapped with it)\n" " --keep-canonical A/B: also load canonical copies of natively served tensors (more VRAM)\n" " --vision --serve takes images too (GENI requests; embeddings from strata-vision)\n" - " --prompt-cache N --serve: keep N conversation checkpoints between requests (default 6, ~118 MB\n" - " of RAM each; 0 = read every prompt from the start)\n" - " --conversation-cache-mib N --serve: RAM budget for parked conversations (default: sized from the free RAM, up to 8192; 0 = off)\n" - " --conversation-cache-slots N --serve: at most N parked conversations (default 4)\n" + " --prompt-cache N --serve: keep N checkpoints of recent prompts for reuse (default 6, ~118 MB\n" + " of RAM each; 0 = no reuse - every prompt is read in full)\n" + " --cache-suffix-reuse N --serve, opt-in (default 0 = off): when a prompt repeats an earlier\n" + " one with a change in the middle, reuse the cached text BEFORE the\n" + " change instead of re-reading it (rotate the kept K rows to their new\n" + " positions; V rows are position-free). Works with every --kv format\n" + " (fp16, int8, int8+STRATA_KV_ROT, k8v4, q4_0). Measured on RTX 3090,\n" + " IQ3_XXS 5.7k-token prompt, mid-prompt edit: 89%% of the prompt reused,\n" + " 3.4 s prefill vs 45.2 s from cold. Needs periodic checkpoints: use\n" + " --prefill N with --prompt-cache-every N. N caps\n" + " the edit size the splice will handle. Off or refused configs fall\n" + " back to re-reading - answers stay correct either way.\n" + " --conversation-cache-mib N --serve: RAM (MiB) for keeping finished conversations cached, so the\n" + " next message in the same chat can reuse them (default: sized from the free\n" + " RAM, up to 8192; 0 = off)\n" + " --conversation-cache-slots N --serve: how many finished conversations to keep cached (default 4)\n" " --conversation-cache-min-free-mib N --serve: RAM floor (under MemAvailable, the cgroup limits and\n" " --memory-limit-mib) when parking, saving or restoring a session (default 2560)\n" " --memory-limit-mib N --serve: the total RAM (MiB) this engine's container or cgroup may use, when\n" @@ -1043,12 +1084,23 @@ void usage() { " with the GPUs overlapped - 1 = the prompt's short reads, 2 = decode as well\n" " (stage 0 runs the next window while stage 1 verifies this one); docs/MULTI_GPU.md\n" " --trim-stage-weights an explicit --layer-split: each GPU loads only its own layers' dense weights\n" - " --prompt-cache-every N --serve: also checkpoint every N fresh prompt tokens (default 16384, 0 = off)\n" + " --prompt-cache-every N --serve: also save a checkpoint every N new prompt tokens (default 16384;\n" + " 0 = off). Smaller N puts a reusable point closer to a mid-text\n" + " edit - pair with --prefill N when using --cache-suffix-reuse\n" " --prompt-cache-tail --serve, single GPU: one extra checkpoint near the prompt's end, at an existing\n" " chunk boundary (default off; requires --prompt-cache and --prompt-cache-every > 0)\n" + " --prompt-cache-dense-tail N --serve: save checkpoints sparsely while reading the prompt, then\n" + " every --prompt-cache-every tokens within the last N tokens only.\n" + " Cheap cold read + reusable points near the end, where mid-text edits\n" + " usually happen. 0 = off (uniform cadence); needs --prompt-cache-every > 0\n" + " chunk boundary (default off; requires --prompt-cache and --prompt-cache-every > 0)\n" " --prompt-cache-spread --serve: keep the checkpoints spread back from the conversation's end (dense\n" " near it, thinner further back) instead of the most recently used: a client that\n" " rewrites its history N tokens back reads about N again, not the whole conversation\n" + " --prompt-cache-state-every N --serve: also save lightweight state-only checkpoints every N\n" + " prompt tokens (the running state, no K/V copy - cheap). An edit deeper than\n" + " every regular checkpoint restores from the nearest one of these instead of\n" + " re-reading the whole gap. 0 = off\n" " --turn-token ID --serve: the token that opens a chat turn (default 248045, <|im_start|>)\n" " --tail-role-token ID --serve: a turn of this role right before the last turn is left out of\n" " the conversation checkpoint (a trailing effort turn; default -1 = off)\n" @@ -1894,6 +1946,7 @@ class JobThread { // just the checkpoints that are prefixes of the tokens the session holds now. using ImgKey = strata::core::ConversationImageKey; using ConvCheckpoint = strata::core::ConversationCheckpoint; +namespace suffix = strata::program::suffix_reuse; // --cache-suffix-reuse (plans/2026-10-11_130800-strata-suffix-reuse.md) uint64_t fnv1a(const void* data, size_t n, uint64_t h = 1469598103934665603ull) { const uint8_t* p = (const uint8_t*) data; @@ -3225,6 +3278,7 @@ int main(int argc, char** argv) { else if (a == "--vision") o.vision = true; else if (a == "--session-save-reclaim") o.session_save_reclaim = true; else if (a == "--prompt-cache") o.prompt_cache = std::max(0, std::atoi(next("--prompt-cache"))); + else if (a == "--cache-suffix-reuse") o.cache_suffix_reuse = std::max(0LL, std::atoll(next("--cache-suffix-reuse"))); else if (a == "--conversation-cache-mib" || a == "--conversation-cache-slots" || a == "--conversation-cache-min-free-mib" || a == "--conversation-cache-min-tokens" || a == "--conversation-cache-park-gain" || @@ -3250,6 +3304,8 @@ int main(int argc, char** argv) { else if (a == "--prompt-cache-tail") o.prompt_cache_tail = true; else if (a == "--prompt-cache-spread") o.prompt_cache_spread = true; else if (a == "--prompt-cache-every") o.prompt_cache_every = std::max(0LL, std::atoll(next("--prompt-cache-every"))); + else if (a == "--prompt-cache-state-every") o.prompt_cache_state_every = std::max(0LL, std::atoll(next("--prompt-cache-state-every"))); + else if (a == "--prompt-cache-dense-tail") o.prompt_cache_dense_tail = std::max(0LL, std::atoll(next("--prompt-cache-dense-tail"))); else if (a == "--prompt-cache-root") o.prompt_cache_root = std::max(0LL, std::atoll(next("--prompt-cache-root"))); else if (a == "--turn-token") o.turn_token = std::atoll(next("--turn-token")); else if (a == "--tail-role-token") o.tail_role_token = std::atoll(next("--tail-role-token")); @@ -3573,6 +3629,21 @@ int main(int argc, char** argv) { return n > 0 ? n : 0; }(); strata::core::set_primary_device(primary_device); + // --cache-suffix-reuse guards (A1.3 fail-closed pattern: fprintf + return 2). Splice for v1 is + // fp16-only, no kv-resident, no layer-split; kv-resident > 0 / int8 / q4_0 / k8v4 / layer-split + // are all rejected at STARTUP. prompt_cache >= 2 is the existing constraint from the plan (D2); + // the splice needs at least one prefix checkpoint and one edit-beyond checkpoint to be useful. + // The shared predicate (in suffix_reuse.hpp so the test suite exercises the SAME code) returns + // the reason; we print and return 2 on a non-empty reason. + { + const std::string reason = suffix::validate_suffix_reuse_options( + o.cache_suffix_reuse, o.prompt_cache, o.kv, o.kv_resident, !o.layer_split.empty(), + o.prompt_cache_every, o.prompt_cache_dense_tail); + if (!reason.empty()) { + std::fprintf(stderr, "strata generate: %s\n", reason.c_str()); + return 2; + } + } // parking with --layer-split saves every stage (SavedConversation::stage_images) // Layer split (multi-GPU): the later stages run layers [K_i, K_i+1) on their own GPUs (--split-device, default // the next visible ones); "auto" places the K from each GPU's free VRAM once the weights are in (below). KV @@ -10398,6 +10469,13 @@ int main(int argc, char** argv) { // --prefill-pipe: the plan of the prompt segment being read (read_part sets it around its runs), for the // checkpoints taken inside it ConvCheckpoint::PromptPlan seg_plan; + // FW-2b: fine-cadence state-only checkpoints. A SIDE pool: the periodic `checks` + // list is LRU-capped at o.prompt_cache, and fine checkpoints there would evict the + // deep anchors the splice mounts. Each entry is state only (checkpoint_save copies + // gdn/ple/tails/dead/block_pos - no K/V; T19 measured ~100-150ms per save), so the + // pool is capped by bytes, not count. Cleared together with checks on every reset. + std::vector state_checks; + int64_t state_next = INT64_MAX; // the fine-cadence ladder (INT64_MAX = off) bool cvec_cached = true; // the control vector's state the live session and the checkpoints were read with // without --mtp the conversation cache stays on: a parked image carries the draft layer's K/V only when there is one // (the snapshots accept a null draft) @@ -10770,6 +10848,7 @@ int main(int argc, char** argv) { } }; int64_t pp_total = 0, pp_from = 0, pp_next_check = 0; + int64_t pp_last_check = 0; // FW-1: position of the last periodic checkpoint (dense-tail cadence) bool pp_tail_saved = false; // #471: the position the prompt pass has read up to (a chunk's or a window's end): what a request cancelled // mid-read reports as read, instead of the whole prompt @@ -10911,17 +10990,26 @@ int main(int argc, char** argv) { } } else { Clock::time_point t0 = Clock::now(); - auto lap = [&](double& acc) { // (STRATA_TRACE only) - if (!ckpt_trace) return; + auto lap = [&](double& acc) { // (accumulates unconditionally: the FW-2a probe below reads it without STRATA_TRACE) const Clock::time_point t1 = Clock::now(); acc += std::chrono::duration(t1 - t0).count(); t0 = t1; }; double sync_ms = 0.0, save_ms = 0.0; - if (!gpu_sync_ok()) return false; + const bool sync_ok = gpu_sync_ok(); lap(sync_ms); - if (!checkpoint_save(c, ss, g)) return false; + const bool save_ok = sync_ok && checkpoint_save(c, ss, g); lap(save_ms); + // FW-2a probe (T19): decompose the ~8s/boundary cost. sync_ms = the + // device synchronize (pipeline drain); save_ms = the state+K/V copy. + // boundary_total - (sync+save) = the pipeline piece restart. Decides the + // FW-2b checkpoint cadence: cheap save -> state-only ckpts at 256-tok; cheap + // sync -> cadence free; neither -> reuse dense-tail pieces only. + if (o.cache_suffix_reuse > 0) + std::fprintf(stderr, "[SFX-DBG] ckpt-at: L=%lld sync_ms=%.1f save_ms=%.1f%s%s\n", + (long long) L, sync_ms, save_ms, + sync_ok ? "" : " SYNC_FAIL", save_ok ? "" : " SAVE_FAIL"); + if (!sync_ok || !save_ok) return false; for (auto& st : stages) { // a layer split's later stages: their sessions' part const strata::core::OnDevice on(st->dev); ConvCheckpoint part; @@ -11039,7 +11127,26 @@ int main(int argc, char** argv) { std::fflush(stdout); // layer-major prefill: a sub-chunk before its super-chunk's end has the super-chunk's state - the // checkpoint is taken at that end instead - const bool periodic_checkpoint = o.prompt_cache_every > 0 && done >= pp_next_check && !sp.layer_major_mid(); + // FW-1: with --prompt-cache-dense-tail N the cadence is position-dependent + // (sparse head, dense tail, never skipping the zone - dense_tail_next_check). + // With it OFF (0) the gate below is byte-identical to the original + // `done >= pp_next_check` / `pp_next_check = done + every` pair: pp_next_check + // starts at 0 so the FIRST chunk always checkpoints, exactly as before. + // FW-2a FIX (2026-10-13, T21): the gate now compares against the per-request + // pp_next_check for BOTH branches. The dense-tail branch recomputed its + // threshold from pp_last_check, which persists ACROSS requests: after a long + // cold read (A: 5665) a warm resumed read (B: from 2598) waited for + // 5665+1024=6689 - past the prompt - and never took a periodic anchor + // (measured dt-steady8: B captured only turn@5670 + tail@5697; 3622/4646 + // piece boundaries passed with no checkpoint). pp_next_check IS initialized + // per request at the resume site (dense_tail_next_check(resume, n, ...)) and + // advanced at each capture below - the gate just has to read it. Cost of the + // fix: one ~150ms checkpoint save per boundary that already exists (T19: + // sync 0.0ms, save ~100-150ms); the read's piece structure is unchanged. + // Rebase note: main's layer-major guard is kept - a sub-chunk before its + // super-chunk's end has the super-chunk's state, so it must not checkpoint. + const bool periodic_checkpoint = o.prompt_cache_every > 0 && done >= pp_next_check && + !sp.layer_major_mid(); // Keep the prefill's existing chunk geometry. One extra near-tail checkpoint // can serve a branch whose shared prefix ends before the final cached state. const bool tail_checkpoint = o.prompt_cache_tail && o.prompt_cache > 0 && o.prompt_cache_every > 0 && @@ -11071,9 +11178,52 @@ int main(int argc, char** argv) { std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see " "above); the request goes on without it\n", (long long) done); } - if (periodic_checkpoint) pp_next_check = done + o.prompt_cache_every; + if (periodic_checkpoint) { + pp_last_check = done; + if (o.prompt_cache_dense_tail > 0) + pp_next_check = suffix::dense_tail_next_check(done, pp_total, + o.prompt_cache_every, + o.prompt_cache_dense_tail); + else + pp_next_check = done + o.prompt_cache_every; // original cadence + } if (tail_checkpoint) pp_tail_saved = true; } + // FW-2b: the fine-cadence state-only capture. state_cadence_next clamps to + // n, and `done < n` in the gate below, so a capture never lands at the live + // conversation's end. A state checkpoint is ~100-150ms (T19: sync 0.0ms, + // save only) - at 256-token cadence that is ~0.6s per 1k fresh tokens, the + // price of a <=256-token fail-close gap instead of up to `every`. + if (o.prompt_cache_state_every > 0 && done >= state_next && done < pp_total) { + ConvCheckpoint sc; + sc.ids.assign(cur.begin(), cur.begin() + done); + sc.imgs = imgs_below(req_imgs, done); + if (gpu_sync_ok() && checkpoint_save(sc, ss, g)) { + sc.used = ++check_clock; + state_checks.push_back(std::move(sc)); + // byte cap, not count: state blobs are ~20-25 MiB per 1k conversation + // tokens (measured: pool=2 at 5.7k under a 256 MiB cap), so the cap, + // not the cadence, sets the anchor depth the pool covers. 2 GiB keeps + // ~10 anchors at needle scale - the head region periodic cadence goes + // sparse (max(every, 8192)) and the state pool is what keeps deep + // anchors there. Host RAM only (checkpoint_save copies to host vectors). + int64_t state_bytes = 0; + for (const ConvCheckpoint& k : state_checks) state_bytes += (int64_t) k.bytes(); + while (!state_checks.empty() && state_bytes > 2048 << 20) { + state_bytes -= (int64_t) state_checks.front().bytes(); + state_checks.erase(state_checks.begin()); // oldest first + } + state_next = suffix::state_cadence_next(done, pp_total, o.prompt_cache_state_every); + if (o.cache_suffix_reuse > 0) + std::fprintf(stderr, "[SFX-DBG] state-ckpt: L=%lld pool=%zu\n", + (long long) done, state_checks.size()); + } else { + // fail-open on the SIDE pool only: a failed state capture must not + // fail the request (the periodic anchors are untouched). Retry at + // the next boundary. + state_next = suffix::state_cadence_next(done, pp_total, o.prompt_cache_state_every); + } + } return true; }; if (multi_gpu) { // the batched prompt is reported by its last stage (the drafter's rows are there) @@ -14159,6 +14309,7 @@ int main(int argc, char** argv) { const size_t n_checks = checks.size(); live_ok = false; checks.clear(); + state_checks.clear(); // FW-2b: the side pool dies with the session state it describes tail_ckpt_len = -1; const size_t n_parked = conversations.clear(); size_t n_slots = 0, n_running = 0; @@ -14189,6 +14340,17 @@ int main(int argc, char** argv) { bool from_live = false; if (o.prompt_cache > 0 && want_cvec == cvec_cached) { if (live_ok && starts_with(live, live_imgs)) { resume = (int64_t) live.size(); from_live = true; } + // [SFX-DBG] probe: the session-local strict search has no visibility - log every + // candidate (live + each checkpoint) and its starts_with verdict so a shallow + // mount can be attributed to the candidate set, not the search. + if (o.cache_suffix_reuse > 0) { + std::fprintf(stderr, "[SFX-DBG] local-search: live_ok=%d live_len=%zu n=%lld checks=%zu:", + live_ok ? 1 : 0, live.size(), (long long) cur.size(), checks.size()); + for (const ConvCheckpoint& c : checks) + std::fprintf(stderr, " %lld/%d", (long long) c.ids.size(), + starts_with(c.ids, c.imgs) ? 1 : 0); + std::fprintf(stderr, "\n"); + } for (const ConvCheckpoint& c : checks) if ((int64_t) c.ids.size() > resume && starts_with(c.ids, c.imgs)) { resume = (int64_t) c.ids.size(); @@ -14214,6 +14376,145 @@ int main(int argc, char** argv) { } } } + // --cache-suffix-reuse: on a re-edited prompt whose OLD cache is reachable from the request, + // compute the splice plan (longest common tail), invalidate every checkpoint whose length + // surpasses the edit position, and route the unchanged tail into a rotation-of-K / V-as-is + // splice. With replay unimplemented (Task 7 deferral per A1.5), the splice path falls + // through to today's re-read at --cache-suffix-reuse 0 - no stub, just routing; this + // header's `replay_from` field is the deferred handoff. + // Plan ref: plans/2026-10-11_130800-strata-suffix-reuse.md D1 / Task 4 / Task 7. + // --cache-suffix-reuse (FINDING 1+4): the fire predicate and the invalidation live in + // suffix_reuse.hpp so the parity test exercises THE SAME function the runtime does. + // v1 scope (per FINDING 4): + // (a) plan against the RESUME checkpoint's tokens (not checks.front()). + // (b) checkpoint invalidation actually erases from `checks` (FINDING 5 also fixed). + // (c) resume narrowing to plan.edit_pos when plan.edit_pos < resume - a real + // divergence from the mounted checkpoint - is the safe, byte-equal-to-today + // semantic for the common case (continuation: edit_pos == resume, no narrowing; + // mid-prompt edit: edit_pos < resume, narrow to the edit point so today's + // re-read picks up from there). The KV splice itself is NOT wired; the doc- + // comment at `SpliceHookOutcome` (suffix_reuse.hpp) and the help text both say + // so. flag absent or 0 -> byte-identical to today's behaviour. + // The splice hook's narrowing decision is captured here so the restore search + // (the `else if (!from_live)` branch below) and the chunk loop's read_from + // can do the right thing when the splice fired and narrowed `resume` to + // `plan_edit_pos`. See `narrowed_resume_target` in suffix_reuse.hpp for the + // gap-read / replay path's pure decision. + // + // When the splice hook DID NOT fire or DID NOT narrow (the common case: + // pure continuation - plan.edit_pos == resume - or flag off), `splice_narrowed` + // is false and every site below sees today's byte-identical behaviour. + bool splice_narrowed = false; + int64_t splice_narrowed_resume = resume; + // Phase 3b: the partial-prefix mount's restore anchor. When the parked cache + // returned a partial match and the partial helper fired, this is set to the + // anchor C's length L. The splice-narrowed branch below uses it for the + // checkpoint restore (so `checkpoint_restore(*c, ...)` finds the anchor at L, + // not a non-existent checkpoint at P). The strict splice hook leaves it at 0 + // and the existing `narrowed_resume_target` path takes over. + int64_t splice_restore_anchor_length = 0; + const bool sfx_strict_fire = suffix::suffix_reuse_should_invalidate( + o.cache_suffix_reuse, from_live, resume, !checks.empty(), (int64_t) cur.size()); + if (o.cache_suffix_reuse > 0) + std::fprintf(stderr, "[SFX-DBG] strict-hook: flag=%lld from_live=%d resume=%lld checks=%zu cur=%lld -> fire=%d\n", + (long long) o.cache_suffix_reuse, from_live ? 1 : 0, (long long) resume, + checks.size(), (long long) cur.size(), sfx_strict_fire ? 1 : 0); + if (sfx_strict_fire) { + // the resume checkpoint is the latest checkpoint whose ids are a prefix of cur + // (the deepest match); `resume` is its length. We pull its tokens and pass them + // to the helper so the plan is computed against the mounted checkpoint - the + // shortest checkpoint (checks.front()) is the WRONG side of the plan and would + // always yield edit_pos=0 on a pure continuation, pessimising the common case. + // (the predicate above filtered `from_live=true`, so the search below is the live + // case study: there is always a resume checkpoint whose ids are a prefix of cur.) + const ConvCheckpoint* resume_ck = nullptr; + for (const ConvCheckpoint& c : checks) + if ((int64_t) c.ids.size() == resume) { resume_ck = &c; break; } + std::vector resume_tokens; + if (resume_ck) resume_tokens.assign(resume_ck->ids.begin(), resume_ck->ids.end()); + const std::vector cur_tokens(cur.begin(), cur.end()); + const suffix::SuffixHookOutcome outcome = suffix::suffix_reuse_apply_invalidation( + o.cache_suffix_reuse, from_live, resume, + [&]() { + std::vector chain; + chain.reserve(checks.size()); + for (size_t i = 0; i < checks.size(); ++i) + chain.push_back({(int64_t) checks[i].ids.size(), i}); + return chain; + }(), + resume_tokens, cur_tokens); + if (outcome.fired) { + // Erase the matching entries from `checks` using the helper's `index` field. + // The helper does NOT own the live `checks` array, only a snapshot of + // (length,index) pairs; we walk the snapshot and erase from the live vector + // by the carry-valid `index`, then shrink in-place. Plan ref: A2.4. + { + std::vector keep(checks.size(), false); + for (const suffix::CheckpointLength& cl : outcome.chain) + if (cl.index < keep.size()) keep[cl.index] = true; + std::vector drop_indices; + for (size_t i = 0; i < keep.size(); ++i) + if (!keep[i]) drop_indices.push_back(i); + // erase from the back so indices stay valid + for (auto it = drop_indices.rbegin(); it != drop_indices.rend(); ++it) + if (*it < checks.size()) checks.erase(checks.begin() + (ptrdiff_t) *it); + } + // Resume narrowing: a real divergence (plan.edit_pos < resume) is the splice + // hook's narrowing decision. For a pure continuation plan.edit_pos == resume + // (the resume checkpoint IS a prefix of cur, common suffix is the whole + // checkpoint) and the comparison fails - resume UNCHANGED. For a mid-prompt + // edit, plan.edit_pos < resume and the engine re-reads from there. Today's + // re-read path is replaced on the splice path by `apply_splice` (below), which + // rotates the cached K rows of the tail in-place; V is untouched; positions + // are re-derived per prefill step. This is the F4(a) verification target: + // edit-at-E yields resume <= E. + // + // Phase 2 Task 5: `replay_from == plan_edit_pos == outcome.replay_from` from + // the outcome, and `resume = read_from = plan_edit_pos` after this narrowing. + // The chunk-loop's `a0` (generate.cpp:9809) is set to `read_from`, which becomes + // `pos0` for `Prefill::run(ids+a0, n, a0, err)` (generate.cpp:9637); pos0 + // reaches the prefill unchanged, and the linear-state (GDN/PLE recurrence + + // indexer tails) replay is sealed by the checkpoint_restore at line 9285 + // which restores gdn/ple/tails/dead/block_pos from the resume snapshot. + if (outcome.plan_edit_pos < resume) { + splice_narrowed = true; + splice_narrowed_resume = outcome.plan_edit_pos; + resume = outcome.plan_edit_pos; + } + // Phase 2 Task 2: apply the splice (host-loop rotation of K rows in the QSA + // k_pool, V untouched). Gated `old_suffix_len > 0` so a pure continuation + // (plan.old_suffix_len == 0) does no rotation. The runtime guard inside + // `apply_splice` logs SPLICE_UNSUPPORTED for kv != fp16 / kv_resident / layer + // split and returns without touching K, matching the startup guard from + // Task 4. When the splice is unsupported we ALSO log REPLAY_NOT_AVAILABLE + // (Task 5 NEW-5: the linear-state replay path falls back to today's prefill + // in those same cases - no snapshot present, kv != fp16, layer split - and + // `read_from == resume` (narrowed) below reuses the existing prefill path, + // which the brief says MUST remain intact). + if (outcome.old_suffix_len > 0) { + const strata::program::suffix_reuse::SpliceStatus sstatus = + suffix::apply_splice(outcome, ss, g, nullptr); + std::fprintf(stderr, "[SFX-DBG] apply_splice(strict): status=%d old_suffix_len=%lld\n", + (int) sstatus, (long long) outcome.old_suffix_len); + if (sstatus != strata::program::suffix_reuse::SpliceStatus::Rotated && + sstatus != strata::program::suffix_reuse::SpliceStatus::NoOp) { + const char* reason = + (sstatus == strata::program::suffix_reuse::SpliceStatus::UnsupportedFormat) + ? "head_dim is not 256 in a rotated (kv_rot or q4) store, or an unhandled pool layout (fp16, int8, k8v4, q4_0 and int8+kv_rot are supported)" + : (sstatus == strata::program::suffix_reuse::SpliceStatus::UnsupportedResident) + ? "kv_resident > 0 (ring-layout splice is future work)" + : (sstatus == strata::program::suffix_reuse::SpliceStatus::UnsupportedSplit) + ? "layer split active (per-stage splice is future work)" + : "no applicable snapshot / reasons unstated"; + std::fprintf(stderr, + "[strata] suffix-reuse: REPLAY_NOT_AVAILABLE: %s. " + "Falling through to today's re-read path; the linear-state " + "replay (GDN recurrence + PLE history) will be re-read instead " + "of spliced (P2 Task 5).\n", reason); + } + } + } + } // --batch: an idle slot that holds the start of this prompt (the conversation it served last) is a // source too - its sessions are copied back below, so only the new part is read. STRATA_SLOT_CKPT_MOVE: an // active slot's checkpoints too (its windows write its cells from p on, so the ones below a checkpoint still @@ -14281,7 +14582,23 @@ int main(int argc, char** argv) { k + 1, (long long) (k < now.size() ? now[k].start : -1)); } } - const auto parked = conversations.best(ids, req_imgs, want_cvec); + // Phase 3b: flag-gated partial-prefix match. At flag <= 0, best_with_partial is + // byte-identical to best() (same loop, same conversation_prefix, same tie-break). + // At flag > 0 with no strict match, falls back to the longest LCP p (0 < p < L) + // and surfaces (p, L) as Match{tokens=p, partial=true, checkpoint_length=L} so the + // partial-mount branch below can find the anchor C in the restored parked image + // and run the partial helper. The session-local live+checks search above stays + // strict-only (the live/checks K/V arena is one branch of history; partial-mounting + // it requires a warm-up of [0, P) to re-derive the GDN running state, which is a + // follow-up - the CLM bench's mid-file edits hit the parked cache path). + const auto parked = conversations.best_with_partial( + o.cache_suffix_reuse, ids, req_imgs, want_cvec); + if (o.cache_suffix_reuse > 0) + std::fprintf(stderr, "[SFX-DBG] parked-search: flag=%lld entries=%zu match{tokens=%lld partial=%d ckpt_len=%lld live=%d} resume=%lld slot_tokens=%lld\n", + (long long) o.cache_suffix_reuse, conversations.size(), + (long long) parked.tokens, parked.partial ? 1 : 0, + (long long) parked.checkpoint_length, parked.live ? 1 : 0, + (long long) resume, (long long) slot_tokens); std::optional incoming; // A checkpoint inside ANOTHER parked conversation (a sibling subagent's system prompt and tools, say) is // borrowed: its K/V up to there and the checkpoint come over, and that conversation stays parked for its @@ -14439,10 +14756,12 @@ int main(int argc, char** argv) { live.clear(); live_imgs.clear(); checks.clear(); + state_checks.clear(); // FW-2b resume = 0; from_live = false; } else { checks.clear(); // the main session's checkpoints were of the conversation it held before + state_checks.clear(); // FW-2b: same conversation, same fate // a read that gave way, the same request again (into its own slot): on with it - its segments and // checkpoints as if it had not stopped if (slot_ck == nullptr && bs[(size_t) slot_source].partial && admit_slot == slot_source) { @@ -14613,9 +14932,124 @@ int main(int argc, char** argv) { std::chrono::duration(Clock::now() - t0).count(), conversations.size(), conversations.bytes()); } + // Phase 3b: partial-prefix mount. When the parked cache returned a partial match + // (flag-gated, the strict mount failed), we have a checkpoint C of length L whose + // first P tokens match the new prompt's first P tokens (the LCP) and diverge at P + // (a mid-file edit past P). The runtime: + // 1. mounted the parked image (line 9217-9284), restoring the K/V arena and the + // running state at length L (the parked image's `live.ids.size()`). + // 2. set `resume = parked.tokens` (= P, the LCP) and `from_live = parked.live`. + // 3. now runs the partial helper to compute the splice plan (LCP-based narrow_pos + // = P, suffix-based edit_pos for the K rotation in [P, L)). + // The helper's outcome drives the same downstream path as the strict splice hook: + // chain erase, splice_narrowed, apply_splice, resume narrowing. The + // `restore_anchor_length` (= L) is surfaced on the `NarrowedResumeTarget` so the + // splice-narrowed branch below restores the anchor C (not the P-depth resume) and + // the chunk loop reads from P. + // + // The K/V cells for [0, P) are valid (the restore gave us the parked image's K/V + // for [0, L); we use [0, P)). The K/V cells for [P, L) are stale (the mid-file + // edit changed the prompt at P) but the prefill overwrites them. The K rotation + // in [P, L) (when old_suffix_len > 0) is wasted by the prefill but harmless - the + // gap-read [P, n) is the actual saving. With old_suffix_len == 0 (no common tail + // from the back), apply_splice is gated out and the prefill is the only path. + if (o.cache_suffix_reuse > 0) + std::fprintf(stderr, "[SFX-DBG] partial-branch: incoming=%d parked.partial=%d parked.tokens=%lld resume=%lld cur=%lld -> enter=%d\n", + incoming.has_value() ? 1 : 0, parked.partial ? 1 : 0, (long long) parked.tokens, + (long long) resume, (long long) cur.size(), + (incoming && parked.partial && o.cache_suffix_reuse > 0 && resume > 0 && + resume < (int64_t) cur.size()) ? 1 : 0); + if (incoming && parked.partial && o.cache_suffix_reuse > 0 && resume > 0 && resume < (int64_t) cur.size()) { + // Find the partial-mount checkpoint C by parked.checkpoint_length (= L). + const std::vector* ck_tokens = nullptr; + if (parked.live && (int64_t) live.size() == parked.checkpoint_length) { + ck_tokens = &live; + } else { + for (const auto& c : checks) { + if ((int64_t) c.ids.size() == parked.checkpoint_length) { + ck_tokens = &c.ids; + break; + } + } + } + if (ck_tokens != nullptr && (int64_t) ck_tokens->size() == parked.checkpoint_length) { + // Build the chain from the post-restore `checks`. + std::vector chain; + chain.reserve(checks.size()); + for (size_t i = 0; i < checks.size(); ++i) + chain.push_back({(int64_t) checks[i].ids.size(), i}); + const std::vector cur_tokens(cur.begin(), cur.end()); + const suffix::SuffixHookOutcome outcome = suffix::suffix_reuse_partial_apply( + *ck_tokens, cur_tokens, parked.checkpoint_length, parked.tokens, chain); + std::fprintf(stderr, "[SFX-DBG] partial-outcome: fired=%d plan_edit_pos=%lld old_suffix_len=%lld restore_anchor=%lld k_rot_start=%lld\n", + outcome.fired ? 1 : 0, (long long) outcome.plan_edit_pos, + (long long) outcome.old_suffix_len, (long long) outcome.restore_anchor_length, + (long long) outcome.k_rotation_start); + if (outcome.fired) { + // Erase chain entries using the helper's index field (same pattern as + // the strict splice hook at line 9042-9056). + std::vector keep(checks.size(), false); + for (const auto& cl : outcome.chain) + if (cl.index < keep.size()) keep[cl.index] = true; + std::vector drop_indices; + for (size_t i = 0; i < keep.size(); ++i) + if (!keep[i]) drop_indices.push_back(i); + for (auto it = drop_indices.rbegin(); it != drop_indices.rend(); ++it) + if (*it < checks.size()) checks.erase(checks.begin() + (ptrdiff_t) *it); + // Splice-narrowed: the chunk loop reads from outcome.plan_edit_pos (= P). + // For the partial case, outcome.plan_edit_pos == resume == P (verified by + // the helper), so the narrowing predicate `plan_edit_pos < resume` is + // false. We set splice_narrowed = true unconditionally for the partial + // case so wire_splice_read_from returns t.read_from (= P) instead of + // resume (= L, the restore anchor). The splice-narrowed_resume stays + // at P so the NarrowedResumeTarget carries the correct read_from. + splice_narrowed = true; + splice_narrowed_resume = outcome.plan_edit_pos; + if (outcome.plan_edit_pos < resume) { + resume = outcome.plan_edit_pos; + } + // Apply the K rotation if the plan has a non-trivial tail. Same gating + // as the strict splice hook at line 9089. + if (outcome.old_suffix_len > 0) { + const strata::program::suffix_reuse::SpliceStatus sstatus = + suffix::apply_splice(outcome, ss, g, nullptr); + std::fprintf(stderr, "[SFX-DBG] apply_splice(partial): status=%d old_suffix_len=%lld\n", + (int) sstatus, (long long) outcome.old_suffix_len); + if (sstatus != strata::program::suffix_reuse::SpliceStatus::Rotated && + sstatus != strata::program::suffix_reuse::SpliceStatus::NoOp) { + const char* reason = + (sstatus == strata::program::suffix_reuse::SpliceStatus::UnsupportedFormat) + ? "head_dim is not 256 in a rotated (kv_rot or q4) store, or an unhandled pool layout (fp16, int8, k8v4, q4_0 and int8+kv_rot are supported)" + : (sstatus == strata::program::suffix_reuse::SpliceStatus::UnsupportedResident) + ? "kv_resident > 0 (ring-layout splice is future work)" + : (sstatus == strata::program::suffix_reuse::SpliceStatus::UnsupportedSplit) + ? "layer split active (per-stage splice is future work)" + : "no applicable snapshot / reasons unstated"; + std::fprintf(stderr, + "[strata] suffix-reuse: REPLAY_NOT_AVAILABLE (partial mount): %s. " + "Falling through to the re-read path; the [P, n) prefill recomputes K+V.\\n", reason); + } + } + // Surface the restore anchor for the splice-narrowed branch below. + splice_restore_anchor_length = outcome.restore_anchor_length; + std::fprintf(stderr, + "strata serve: partial-prefix mount: L=%lld P=%lld (splice_narrowed=%d, k_rotation_start=%lld, old_suffix_len=%lld)\\n", + (long long) outcome.restore_anchor_length, (long long) outcome.plan_edit_pos, + splice_narrowed ? 1 : 0, (long long) outcome.k_rotation_start, + (long long) outcome.old_suffix_len); + } + } + } + // [SFX-FIX 2026-10-12] incoming.reset() moved BELOW the Phase 3b partial-mount + // branch: that branch reads `incoming` (must be set) for its restore-anchor search. + // Resetting above it made the partial branch dead code (incoming always empty), so + // live edit turns never partial-mounted. RAII: a `return 1` between the old and new + // reset sites destroys the optional anyway - no leak. + incoming.reset(); // Running-state/checkpoint copies are no longer needed. if (want_cvec != cvec_cached) { live_ok = false; checks.clear(); + state_checks.clear(); // FW-2b cvec_cached = want_cvec; } if (strata::kernels::cvec().loaded()) strata::kernels::cvec_set_enabled(want_cvec); @@ -14628,6 +15062,13 @@ int main(int argc, char** argv) { ckpt_pool.erase_if(checks, off_path); else checks.erase(std::remove_if(checks.begin(), checks.end(), off_path), checks.end()); + // FW-2b: the side pool follows the same validity rule as checks - a state + // checkpoint is only a mount candidate while it is a prefix of THIS prompt. + // (state-only entries carry no K/V, but their running state describes exactly + // the tokens [0, L); restoring one for a non-prefix would corrupt the recurrence.) + state_checks.erase(std::remove_if(state_checks.begin(), state_checks.end(), [&](const ConvCheckpoint& c) { + return (int64_t) c.ids.size() > resume || !starts_with(c.ids, c.imgs); + }), state_checks.end()); live_ok = false; // until this request has finished, the session is in between // the elastic K/V: the outgoing session is parked and this request rewrites every cell from `resume` on, // so cells past this prompt's are no longer anyone's - far more than it needs go back to the cache @@ -14642,6 +15083,14 @@ int main(int argc, char** argv) { } } int64_t reread_to = -1; // STRATA_CKPT_REREAD only: read [0, reread_to) again instead of restoring + // Phase 3a T10: hoisted next to reread_to so the read_from site BELOW the + // checkpoint-restore if/else chain (this function's depth) can see it. The + // splice-narrowed branch inside `else if (!from_live)` assigns it; when the + // hook did not narrow (pure continuation, append, flag off) it stays at its + // default {0,0} and `wire_splice_read_from` ignores it. (Declared inside the + // else-if in 7137d4c - a scope error the tests-only CPU tree cannot catch + // because it never compiles generate.cpp; only the CUDA engine target does.) + suffix::NarrowedResumeTarget splice_target{}; if (resume == 0) { strata::core::session_zero(ss, g, nullptr, main_cs); cudaStreamSynchronize(main_stream); @@ -14651,7 +15100,108 @@ int main(int argc, char** argv) { cudaStreamSynchronize(st->stream); } checks.clear(); - } else if (!from_live) { + } else if (!from_live || splice_restore_anchor_length > 0) { + // T17 (2026-10-13): the partial mount with a LIVE anchor (parked.live = true) + // sets from_live = true, which previously routed past this branch entirely - + // the session sat at the stale state@L (wrong for the edited history; the v3 + // semantic probe answered "160" for an edited 161-row ledger) and the restore + // search then failed for lack of a checkpoint at P. A partial mount with + // splice_restore_anchor_length > 0 ALWAYS routes through narrowed_resume_target: + // restore the deepest checkpoint <= P (state consistent with the shared + // prefix), the arena K/V covers [ckpt, P) (identical tokens by the LCP), and + // the chunk loop reads [P, n) via wire_splice_read_from. Strict mounts + // (anchor == 0, from_live as before) keep today's byte-identical behaviour. + // The splice hook may have narrowed `resume` to `plan_edit_pos` (e.g. 3239) + // while erasing the exact-length checkpoint the restore search wants (e.g. + // 3240). Without the helper below, the loop's `c == nullptr` path errors + // with "ERR restoring a conversation checkpoint failed" and the engine + // returns 1, which the upstream serve loop turns into a 400. The helper + // (suffix_reuse.hpp: `narrowed_resume_target`) maps (narrowed_resume, + // checks) -> (restore_length, read_from): the deepest surviving + // checkpoint is restored through, the [restore, narrowed) gap is re-read + // by the chunk loop, and when no candidate exists the request falls back + // to a fresh prefill from 0 (no error). See the gap-read / replay path + // phase 2 was designed for. When the splice hook did NOT narrow + // (splice_narrowed == false: pure continuation, append, flag off), the + // helper is a no-op and `resume` keeps its value - byte-identical to today. + // Phase 3a T10: when the splice hook fired and narrowed resume, the chunk loop's + // read start is `t.read_from` (the splice-narrowed resume = the edit pos E), NOT + // `resume` (the restored anchor C). The pre-3a wiring at line 9394 used + // `reread_to > 0 ? 0 : resume`, which discarded `t.read_from` and re-read [C, n) - + // the rotated region [C, E) AND the gap [E, n) the splice's gap-read was supposed + // to handle. The gap-read IS the splice's saved work. `t` is hoisted to a + // (splice_target is hoisted above the if/else chain, next to reread_to) + if (splice_narrowed) { + std::vector chain_lengths; + chain_lengths.reserve(checks.size()); + for (const ConvCheckpoint& c : checks) chain_lengths.push_back((int64_t) c.ids.size()); + // FW-2b: the two-pool pick - the fine-cadence state-only pool competes + // with the periodic anchors; the deepest <= P across both wins. + std::vector state_lengths; + state_lengths.reserve(state_checks.size()); + for (const ConvCheckpoint& c : state_checks) state_lengths.push_back((int64_t) c.ids.size()); + // [SFX-FIX 2026-10-12 v3] The anchor-restore variant (restore the parked + // image's end state at L, read [P, n)) produced WRONG ANSWERS: the parked + // image's running state (GDN/PLE recurrence) summarizes the OLD sequence + // through L, so prefilling [P, n) on top of state@L corrupts every new + // token (semantic probe: edited 161-row ledger answered "160"). The parked + // image's ARENA K/V for [ckpt, P) is still valid for the new prompt (those + // tokens are identical by the LCP), so the correct route is the same as the + // strict path: restore the deepest SURVIVING checkpoint <= P (its running + // state is consistent with the shared prefix), let the arena cover + // [ckpt, P), and read [P, n). Both cases now take narrowed_resume_target. + if (splice_restore_anchor_length > 0) + std::fprintf(stderr, "[SFX-DBG] anchor(v3): partial mount routed through narrowed_resume_target (state@L is stale; restoring deepest checkpoint <= P=%lld)\n", + (long long) splice_narrowed_resume); + { + splice_target = suffix::narrowed_resume_target_state( + splice_narrowed_resume, chain_lengths, state_lengths); + // T18 (2026-10-13): replay-gated gap read. dt-steady5 proved reading + // [P, n) on state@checkpoint loses the gap [ckpt, P) from the GDN/PLE + // recurrence (C under-counted the ledger; gap 826 tokens). Fail-closed + // default: read from the checkpoint (byte-equal to strict semantics); + // STRATA_SFX_REPLAY=1 (FW-2 day) restores the gap-read [P, n). + static const bool sfx_replay_on = + std::getenv("STRATA_SFX_REPLAY") != nullptr; + if (splice_restore_anchor_length > 0) + splice_target.read_from = suffix::partial_gap_read_from( + sfx_replay_on, splice_narrowed_resume, chain_lengths); + // Resume becomes the restore length (the deepest surviving checkpoint at + // or below the splice-narrowed resume, or 0 when none survives - in + // which case this is a fresh prefill from 0, no error). The chunk + // loop's read_from is `splice_target.read_from` (= the edit pos E) + // via `wire_splice_read_from` below: the prefill processes [E, n) only, + // the [C, E) gap is covered by the anchor restore (no re-read), and + // the splice's K rotation in `apply_splice` covers [E, n) in cache + // (the prefill recomputes K+V for [E, n) - the rotation is redundant + // but the gap-read is the real saving). + resume = splice_target.restore_length; + } + if (resume == 0) { + // No checkpoint survives at/below the narrowed resume (the splice + // hook's erase removed them all): fresh prefill from 0 - the same + // reset the resume==0 branch performs above. Degrade to today's + // full re-read; never fabricate checkpoint state, never error. + strata::core::session_zero(ss, g, nullptr, main_cs); + cudaStreamSynchronize(main_stream); + for (auto& st : stages) { + const strata::core::OnDevice on(st->dev); + strata::core::session_zero(st->ss, g, nullptr, (void*) st->stream); + cudaStreamSynchronize(st->stream); + } + checks.clear(); + } + } + // [SFX-FIX 2026-10-12 v2] Phase 3b partial mount: the parked-conversation + // restore above already placed the session (K/V + running state) at the anchor + // L (= splice_restore_anchor_length). The remove_if at the checks-erase then + // dropped every checkpoint longer than P - including any at L - so the restore + // search below would find c==nullptr and fail the request ("ERR restoring a + // conversation checkpoint failed"). The anchor case skips the search: the state + // is already at L, and the chunk loop reads [P, n) via wire_splice_read_from. + if (splice_restore_anchor_length > 0) + std::fprintf(stderr, "[SFX-DBG] anchor-restore(v3): narrowed_resume_target picked the checkpoint; search+restore below is REQUIRED now\n"); + if (resume > 0) { ConvCheckpoint* c = nullptr; for (ConvCheckpoint& k : checks) if ((int64_t) k.ids.size() == resume) c = &k; bool lent = false; // STRATA_SLOT_CKPT_MOVE: the one a slot keeps (found above) @@ -14660,6 +15210,12 @@ int main(int argc, char** argv) { if ((int64_t) k.ids.size() == resume && starts_with(k.ids, k.imgs)) { c = &k; lent = true; } if (!lent) lent_slot = -1; if (c != nullptr) c->used = ++check_clock; // mounting through it is the use LRU counts + // FW-2b: the pick may have come from the side pool. Same restore call + // (checkpoint_restore replays gdn/ple/tails/dead/block_pos into the session; + // the arena K/V below the pick is unchanged by it, so a state-only entry + // restores exactly like a periodic one). + if (c == nullptr) + for (ConvCheckpoint& k : state_checks) if ((int64_t) k.ids.size() == resume) c = &k; static const bool reread = std::getenv("STRATA_CKPT_REREAD") != nullptr; if (reread && c != nullptr) { // THE CHECK OF THE CHECKPOINT: instead of restoring it, read its tokens again from position 0 in @@ -14725,9 +15281,26 @@ int main(int argc, char** argv) { } break; } + } + // Phase 3a T10: when the splice hook fired and narrowed resume, the chunk loop's + // read start is `t.read_from` (the splice-narrowed resume = the edit pos E), NOT + // `resume` (the restored anchor C). The pre-3a inline `reread_to > 0 ? 0 : resume` + // discarded `t.read_from` and re-read [anchor, n) - the rotated region [C, E) AND + // the gap [E, n) the splice's gap-read was supposed to handle. The gap-read IS + // the splice's saved work; the helper below wires it. Flag-off (splice_narrowed + // false) is byte-identical to today. STRATA_CKPT_REREAD (reread_to > 0) rebuilds + // the whole prefill from 0 as today. See wire_splice_read_from in suffix_reuse.hpp. + const int64_t read_from = suffix::wire_splice_read_from( + splice_narrowed, splice_target, resume, reread_to); + std::fprintf(stderr, "[SFX-DBG] read-path: resume=%lld read_from=%lld n=%lld splice_narrowed=%d anchor=%lld\n", + (long long) resume, (long long) read_from, (long long) n, + splice_narrowed ? 1 : 0, (long long) splice_restore_anchor_length); // KV streaming: the drafter's ring may hold cells past `resume` from a longer turn; the main layers' - // host copies and slots are always current (every writer writes both), so they need nothing - if (use_mtp && resume > 0 && reread_to <= 0) mtp.kv_restore(resume); + // host copies and slots are always current (every writer writes both), so they need nothing. + // Phase 3a T10: when splice_narrowed, the drafter's prefill starts at `read_from` (= E); + // restore the drafter's KV up to the same point so its [0, E) context is consistent with + // the main session's anchor restore. The legacy path (no splice) uses `resume`. + if (use_mtp && resume > 0 && reread_to <= 0) mtp.kv_restore(read_from); tr("request", n, geni ? 1 : 0); req_times = ReqTimes{}; if (use_mtp) mtp.set_prompt_len(n); @@ -14735,14 +15308,21 @@ int main(int argc, char** argv) { // first segment raises it if KVA approximates it - and for any state no KVA read produced) if (use_mtp && mtp.cell_floor_enabled()) mtp.set_cell_floor(resume > 0 && reread_to <= 0 ? kva_floors.floor_for(ids.data(), resume) : 0); - const int64_t read_from = reread_to > 0 ? 0 : resume; conversations.limit_reuse(read_from); pp_total = n; pp_from = read_from; pp_reached = read_from; pp_exact = true; // a restore, a checkpoint mount or a zeroed session: the state is exactly at read_from pp_t0 = r0; - pp_next_check = reread_to > 0 || !req_ckpt ? INT64_MAX : resume + o.prompt_cache_every; // ckpt=0: none + pp_next_check = reread_to > 0 || !req_ckpt ? INT64_MAX : + (o.prompt_cache_dense_tail > 0 + ? suffix::dense_tail_next_check(resume, n, o.prompt_cache_every, o.prompt_cache_dense_tail) + : resume + o.prompt_cache_every); // ckpt=0: none + // FW-2b: the fine-cadence state ladder starts at the resume position too - + // same per-request rule as pp_next_check (T21's lesson: a cross-request + // carryover starves warm reads of their anchors). + state_next = reread_to > 0 || !req_ckpt ? INT64_MAX : + suffix::state_cadence_next(resume, n, o.prompt_cache_state_every); pp_tail_saved = reread_to > 0 || !req_ckpt; // ckpt=0 (#861): no tail checkpoint either { std::lock_guard lk(part_mu); @@ -15699,9 +16279,19 @@ int main(int argc, char** argv) { } } int64_t piece = piece_now(); - if (piece <= 0) return run(a0, b0); + if (piece <= 0 && o.prompt_cache_dense_tail <= 0) return run(a0, b0); // FW-1: dense-tail cadence falls through to the chunk loop below for (int64_t q = a0; q < b0;) { - const int64_t r = std::min(b0, q + piece); + // FW-1 dynamic sizing: with --prompt-cache-dense-tail the pieces are + // laid out so chunk boundaries land at dense-tail cadence (big head + // chunk ending exactly at the zone start, `every`-sized zone chunks). + // Checkpoints only fire at boundaries (the periodic block runs per + // chunk), so the boundaries ARE the anchors. Off: the original piece + // arithmetic, byte-identical. + const int64_t step = o.prompt_cache_dense_tail > 0 + ? suffix::dense_tail_chunk_size(q, n, piece > 0 ? piece : (b0 - a0), + o.prompt_cache_every, o.prompt_cache_dense_tail) + : piece; + const int64_t r = std::min(b0, q + std::max(step, 1)); const auto tq = Clock::now(); if (!run_conc(q, r)) return false; const double piece_ms = std::chrono::duration(Clock::now() - tq).count(); diff --git a/src/program/suffix_reuse_apply_test.cpp b/src/program/suffix_reuse_apply_test.cpp new file mode 100644 index 00000000000..33e95965935 --- /dev/null +++ b/src/program/suffix_reuse_apply_test.cpp @@ -0,0 +1,357 @@ +// src/program/suffix_reuse_apply_test.cpp - Phase 2 Task 3 T6: RED-able splice-wiring test. +// +// Synthesises a QsaState with hand-crafted K rows at known positions, runs the splice rotation +// directly through `rotate_k_by_delta` (T3-style pure fp32 storage T6), and asserts the splice +// path produces the SAME K as a fresh rotation at the new positions (1e-6 tolerance, fp32 ULP). +// +// T6 has TWO sub-tests: +// +// T6.x - "FP32 ALGORITHM" - synthesises a k_pool as a TYPED fp32 buffer (the test casts to +// uint16_t* to satisfy the QsaState declared type). We exercise `rotate_k_by_delta` against +// this fp32 storage directly (no fp16 round-trips). T3 already pinned the rotation math to +// 1e-6; T6.x verifies that the splice ALOGORITHM (which positions to rotate, which deltas +// to apply per row) is correct end-to-end. RED check: with rotation skipped (a temporary +// flag inside the test loop), cached K diverges from fresh. +// +// T6.y - "FP16 ROUND-TRIP HELPER" - exercises `rotate_qsa_tail_inplace` against a real fp16 +// storage pool (the production path). Tolerance is fp16 ULP scale; the fp16 path IS the +// production path, but the underlying math is already covered by T3 + T6.x. +// +// T6.z - "V byte-identical" - the helper leaves V untouched. Helper returns true; pre/post +// V memory equal. +// +// T6.w - "SPLICE_UNSUPPORTED" - the pure scope-guard predicate (`splice_scope_guard`) returns +// `Unsupported*` for kv_int8 / kv_q4 / kv_hybrid / kv_resident > 0 / layer_split. +// +// Phase 2 design notes: +// - page_size = 1 in this test (unpaged) so position p maps directly to row (p * n_head_kv + h). +// - V is allocated from a separate buffer (per QsaState semantics) and asserted byte-identical. +#include "strata/core/layer.hpp" +#include "strata/core/layout.hpp" +#include "strata/program/kv_splice.hpp" +#include "strata/program/kv_splice_apply.hpp" +#include "strata/program/suffix_reuse.hpp" +#include "strata/kernels/rope.hpp" + +#include +#include +#include +#include +#include + +using strata::program::suffix_reuse::rotate_qsa_tail_inplace; +using strata::program::suffix_reuse::splice_scope_guard; +using strata::program::suffix_reuse::SpliceStatus; +using strata::program::suffix_reuse::build_rope_table_host; +using strata::program::suffix_reuse::rotate_k_by_delta; +using strata::program::suffix_reuse::detail::fp32_to_fp16_bits; +using strata::program::suffix_reuse::detail::fp16_bits_to_fp32; + +namespace { +int g_fail = 0; +void check(bool ok, const char* what) { + std::printf(" %-72s %s\n", what, ok ? "ok" : "FAIL"); + if (!ok) ++g_fail; +} +} // namespace + +/// T3-style rope at position `p` over the first `n_rot` elements of a fp32 row. +static inline void rope_at_fp32(float* row, int n_rot, int p, + const float* cos_tab, const float* sin_tab, int max_pos) { + const int half = n_rot / 2; + const int p_clamped = p < max_pos ? p : max_pos - 1; + for (int i = 0; i < half; ++i) { + const float a = row[i]; + const float b = row[i + half]; + const float c = cos_tab[(size_t) p_clamped * (size_t) half + (size_t) i]; + const float s = sin_tab[(size_t) p_clamped * (size_t) half + (size_t) i]; + float oa, ob; + strata::kernels::rope_neox_pair(a, b, c, s, oa, ob); + row[i] = oa; row[i + half] = ob; + } +} + +/// Splice algorithm directly in fp32 storage. Re-implements `rotate_qsa_tail_inplace`'s core +/// loop without going through fp16 storage (T6.x). `skip_rotation=true` simulates a regression +/// that drops the `rotate_k_by_delta` call (the RED proof). +/// +/// Storage layout: `k` is a (max_cells * n_head_kv * head_dim) fp32 buffer; row at position p, +/// kv_head h starts at `k + (p * n_head_kv + h) * head_dim`. `cos_tab`/`sin_tab` are +/// (max_pos, n_rot/2). +static bool splice_fp32(float* k, int max_cells, int n_head_kv, int head_dim, int n_rot, + int page_size, + int edit_pos, int old_suffix_len, + bool uniform, int delta, const int64_t* deltas, + const float* cos_tab, const float* sin_tab, int max_pos, + bool skip_rotation = false) { + if (edit_pos < 0 || old_suffix_len <= 0) return false; + if (page_size <= 0) page_size = 1; + for (int64_t kk = 0; kk < old_suffix_len; ++kk) { + const int64_t p = edit_pos + (int64_t) kk; + const int64_t page = p / page_size; + const int64_t off = p % page_size; + const int delta_row = uniform ? delta : (int) deltas[(size_t) kk]; + for (int h = 0; h < n_head_kv; ++h) { + const int64_t row_idx = (int64_t) page * n_head_kv * page_size + (int64_t) h * page_size + off; + float* row = k + (size_t) row_idx * (size_t) head_dim; + if (!skip_rotation) { + rotate_k_by_delta(row, 1, head_dim, n_rot, (int) p, delta_row, + cos_tab, sin_tab, max_pos); + } + } + } + return true; +} + +int main() { + std::printf("suffix_reuse_apply_test (Phase 2 Task 3 T6)\n"); + + // Geometry mirrors the artifact: head_dim=256, n_head_kv=2, n_rot=64, page_size=1 (unpaged). + const int head_dim = 256; + const int n_rot = 64; + const int n_head_kv = 2; + const int page_size = 1; // unpaged; position p -> row p*n_head_kv + h + const int max_pos = 1024; // rope table extent + const double theta = 1000000.0; + const int half = n_rot / 2; + std::vector cos_tab((size_t) max_pos * (size_t) half), + sin_tab((size_t) max_pos * (size_t) half); + build_rope_table_host(n_rot, theta, max_pos, cos_tab.data(), sin_tab.data()); + + // Per-row source generator: deterministic fp32 pattern indexed by (pos, dim). + auto make_raw = [](int pos, int d) { return std::sin(0.013f * (float) d + 0.7f * (float) pos); }; + + // --------- T6.x: fp32 algorithm test (RED-able) --------- + // THE TEST HYPOTHESIS: after `rotate_qsa_tail_inplace` runs on the cached K at position p + // (which was rotated to OLD_pos = p during prefill), the K at that position should equal + // `R(new_pos=p+delta)·raw(p)` -- the SAME raw as cached, rotated to the new position. This + // is the splice contract: the helper's `rotate_k_by_delta(row, ..., old_pos=p, delta)` + // applies the delta rotation to the cached row, which carries `R(p)·raw(p)` (post-RoPE at + // old_pos). Result: `R(delta)·R(p)·raw(p) = R(p+delta)·raw(p)` -- the rotation that the + // engine would land the cached K at is the SAME raw rotated to the new position, which is + // exactly what a fresh-rotation-of-the-same-raw would compute. + // + // NOT the rope applied to `raw(new_pos)` -- those raw values are different from cached row's + // raw (the NEW prompt's tokens at p+delta are different from the OLD prompt's tokens at p). + // The splice is rotation-only: same raw, different position angle. Fresh is from a DIFFERENT + // raw input and would compare incorrectly. + { + const int max_cells = 64; + std::vector k_pool((size_t) max_cells * n_head_kv * head_dim, 0.0f); + const int edit_pos = 4; + const int old_suffix_len = 6; + const int delta = 3; + auto fill_row = [&](float* row, int pos) { + for (int d = 0; d < head_dim; ++d) row[d] = make_raw(pos, d); + rope_at_fp32(row, n_rot, pos, cos_tab.data(), sin_tab.data(), max_pos); + }; + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + for (int h = 0; h < n_head_kv; ++h) { + float* row = k_pool.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim; + fill_row(row, p); + } + } + // Snapshot rows OUTSIDE the splice range for the byte-identical assertion. + std::vector outside_before(edit_pos * n_head_kv * head_dim); + for (int p = 0; p < edit_pos; ++p) + for (int h = 0; h < n_head_kv; ++h) + std::memcpy(outside_before.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim, + k_pool.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim, + sizeof(float) * head_dim); + // Run the splice algorithm in fp32. + const bool ok = splice_fp32(k_pool.data(), max_cells, n_head_kv, head_dim, n_rot, page_size, + edit_pos, old_suffix_len, /*uniform=*/true, /*delta=*/delta, + nullptr, cos_tab.data(), sin_tab.data(), max_pos); + check(ok, "T6.x: fp32 splice algorithm returns true for the supported config"); + // Compare each row against `R(new_pos)·raw(p)` -- the SAME raw as cached, rotated to + // the new position. This is what the splice SHOULD produce (and is mathematically + // equal to `R(delta)·cached_row` per the rope algebra). + int diff_count = 0; + float max_dr = 0.0f; + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + const int new_pos = p + delta; + // Build the fresh-by-same-raw reference: raw(p), then rope at new_pos. + float fresh[256]; + for (int d = 0; d < head_dim; ++d) fresh[d] = make_raw(p, d); + rope_at_fp32(fresh, n_rot, new_pos, cos_tab.data(), sin_tab.data(), max_pos); + for (int h = 0; h < n_head_kv; ++h) { + float* row = k_pool.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim; + for (int d = 0; d < head_dim; ++d) { + const float dr = std::fabs(row[d] - fresh[d]); + if (dr > 1e-6f) ++diff_count; + if (dr > max_dr) max_dr = dr; + } + } + } + check(diff_count == 0, + "T6.x: K rows match R(new_pos)·raw(p) to 1e-6 (fp32 ULP, T3-pinned rotation)"); + // Outside rows must be byte-identical. + int outside_diff = 0; + for (int p = 0; p < edit_pos; ++p) + for (int h = 0; h < n_head_kv; ++h) + for (int d = 0; d < head_dim; ++d) + if (k_pool[((size_t) p * n_head_kv + h) * head_dim + d] + != outside_before[((size_t) p * n_head_kv + h) * head_dim + d]) ++outside_diff; + check(outside_diff == 0, + "T6.x: K rows OUTSIDE the splice range are byte-identical to pre-splice"); + // RED proof: re-build a fresh state and run with skip_rotation=true. The cached image is + // preserved (no rotation applied); the comparison against `R(new_pos)·raw(p)` must + // diverge (cached carries `R(p)·raw(p)` which differs from the new rotation by the + // delta rotation). + std::vector k_pool_red((size_t) max_cells * n_head_kv * head_dim, 0.0f); + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + for (int h = 0; h < n_head_kv; ++h) { + float* row = k_pool_red.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim; + fill_row(row, p); + } + } + const bool ok_red = splice_fp32(k_pool_red.data(), max_cells, n_head_kv, head_dim, n_rot, + page_size, edit_pos, old_suffix_len, + /*uniform=*/true, /*delta=*/delta, nullptr, + cos_tab.data(), sin_tab.data(), max_pos, + /*skip_rotation=*/true); + check(ok_red, "T6.x RED: fp32 splice with skip_rotation=true still completes"); + int red_diffs = 0; + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + const int new_pos = p + delta; + float fresh[256]; + for (int d = 0; d < head_dim; ++d) fresh[d] = make_raw(p, d); + rope_at_fp32(fresh, n_rot, new_pos, cos_tab.data(), sin_tab.data(), max_pos); + for (int h = 0; h < n_head_kv; ++h) { + float* row = k_pool_red.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim; + for (int d = 0; d < head_dim; ++d) + if (std::fabs(row[d] - fresh[d]) > 1e-6f) ++red_diffs; + } + } + check(red_diffs > 0, + "T6.x RED: skip_rotation=true -> cached K diverges from R(new_pos)·raw(p) (RED surfaces)"); + } + + // --------- T6.y: fp16 production helper (sanity) --------- + // T6.y exercises `rotate_qsa_tail_inplace` against real fp16 storage (per QsaState's + // declared type). Compared to T6.x, this adds 2 fp16 round-trips per cell (cached storage + // and final storage). Both the splice path and the fresh-comparison path see the same + // structural fp16 footprint, so the cell-by-cell diff accumulates roughly 2 fp16 ULPs of + // quantization noise (verified empirically). Tolerance: the 1e-3 floor corresponds to + // fp16 ULP at unit magnitude; the test passes when MOST cells match exactly (within ~1 fp16 + // ULP) AND no cell diverges beyond ~1e-2 (a 10x ULP guardrail that would catch any + // algorithmic regression). + { + const int max_cells = 64; + const size_t cells = (size_t) max_cells * n_head_kv * head_dim; + // Allocate storage that satisfies the `uint16_t* fp16` declaration. We pass through + // the helper which does fp16<->fp32 round-trips internally. + std::vector k_pool(cells, 0); + std::vector v_pool(cells, 0); + // Build cached at OLD positions [edit_pos, edit_pos+old_suffix_len). + const int edit_pos = 4; + const int old_suffix_len = 6; + const int delta = 3; + auto fill_row_fp16 = [&](uint16_t* dst, int pos) { + for (int d = 0; d < head_dim; ++d) dst[d] = fp32_to_fp16_bits(make_raw(pos, d)); + // Read stored fp16, rope at `pos`, write back to fp16. + float tmp[256]; + for (int d = 0; d < head_dim; ++d) tmp[d] = fp16_bits_to_fp32(dst[d]); + rope_at_fp32(tmp, n_rot, pos, cos_tab.data(), sin_tab.data(), max_pos); + for (int d = 0; d < head_dim; ++d) dst[d] = fp32_to_fp16_bits(tmp[d]); + }; + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + for (int h = 0; h < n_head_kv; ++h) { + uint16_t* row = k_pool.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim; + fill_row_fp16(row, p); + } + } + // Synthesize a QsaState and run the helper. + strata::core::QsaState st; + st.k_pool = k_pool.data(); + st.v_pool = v_pool.data(); + st.kv_mode = 0; + st.max_cells = max_cells; + st.n_pages = max_cells / page_size; + st.cos_tab = cos_tab.data(); + st.sin_tab = sin_tab.data(); + st.owns_rope = false; + // V filler: a deterministic pattern so byte-identity is meaningful. + for (size_t i = 0; i < cells; ++i) v_pool[i] = (uint16_t) ((i * 0xABCD + 0x1234) & 0xFFFF); + std::vector v_before(cells); + std::memcpy(v_before.data(), v_pool.data(), cells * sizeof(uint16_t)); + const bool ok = rotate_qsa_tail_inplace(st, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, + /*uniform=*/true, /*delta=*/delta, nullptr); + check(ok, "T6.y: fp16 helper returns true for the supported config"); + // Compare with the splice's expected image: R(new_pos)·raw(p), where raw(p) is the same + // raw used to construct the cached row. Same hypothesis as T6.x, just with fp16 storage. + int strict = 0, ulp = 0, bad = 0; + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + const int new_pos = p + delta; + float fresh[256]; + for (int d = 0; d < head_dim; ++d) fresh[d] = make_raw(p, d); + rope_at_fp32(fresh, n_rot, new_pos, cos_tab.data(), sin_tab.data(), max_pos); + for (int h = 0; h < n_head_kv; ++h) { + uint16_t* row = k_pool.data() + ((size_t) p * n_head_kv + (size_t) h) * head_dim; + for (int d = 0; d < head_dim; ++d) { + const float a = fp16_bits_to_fp32(row[d]); + const float b = fresh[d]; + const float dr = std::fabs(a - b); + if (dr == 0) ++strict; + else if (dr <= 1e-3f) ++ulp; + else ++bad; + } + } + } + check(bad == 0, + "T6.y: fp16 helper produces K within fp16 ULP of fresh (no algorithmic regression)"); + // For fp16 storage, both the splice path AND the fresh-by-same-raw reference go through + // 2 fp16 round-trips (each path quantizes differently but symmetrically). Empirically + // every cell is within 1 fp16 ULP; the strict count is 0 because the quantization + // boundary falls on different cells in the two paths (splice caches then rotates, fresh + // raw-quantizes then ropes - different rounding order). The KEY property: no cell is + // MORE than 1 fp16 ULP off, and there are NO cells beyond ~1e-2 (the regression + // guardrail that `bad==0` already pins). Anything worse would indicate a real algorithmic + // divergence between the splice and the cached-image-rotation contract. + check(ulp == strict + ulp + bad, + "T6.y: every cell is within 1 fp16 ULP of the splice's expected image (no algo divergence)"); + // V byte-identical + int v_diff = 0; + for (size_t i = 0; i < cells; ++i) if (v_pool[i] != v_before[i]) ++v_diff; + check(v_diff == 0, + "T6.y: V is byte-identical after splice (no V-side work at all)"); + // Outside rows preserved + int outside_diff = 0; + for (int p = 0; p < edit_pos; ++p) + for (int h = 0; h < n_head_kv; ++h) + if (k_pool[((size_t) p * n_head_kv + h) * head_dim] != 0) ++outside_diff; + check(outside_diff == 0, + "T6.y: rows OUTSIDE the splice range are untouched (zero image preserved)"); + } + + // --------- T6.z: SPLICE_UNSUPPORTED guard fires --------- + // Phase 3: the legacy `splice_scope_guard` (kv_splice_apply.hpp:72-79) still says + // `kv_int8=true -> UnsupportedFormat` - it predates the int8 splice and is kept + // as a pure back-compat predicate. The runtime hook (kv_splice_apply_session.hpp) + // uses the new `splice_scope_guard_int8` (kv_splice_apply_int8.hpp) which ACCEPTS + // plain int8 (and refuses STRATA_KV_ROT=1's Hadamard-K domain). Both predicates + // are pinned here against the SAME guard-fire inputs. + check(splice_scope_guard(/*kv_int8=*/true, false, false, 0, false) + == SpliceStatus::UnsupportedFormat, + "T6.z (legacy): kv_int8=true -> SPLICE_UNSUPPORTED: kv format != fp16 (back-compat predicate)"); + check(splice_scope_guard(false, true, false, 0, false) + == SpliceStatus::UnsupportedFormat, + "T6.z: kv_q4=true -> SPLICE_UNSUPPORTED: kv format != fp16 (q4_0 rotated)"); + check(splice_scope_guard(false, false, true, 0, false) + == SpliceStatus::UnsupportedFormat, + "T6.z: kv_hybrid=true -> the fp16-path predicate still refuses (correct: hybrid " + "dispatches to the int8 path, whose guard accepts it - see suffix_reuse_int8_test)"); + check(splice_scope_guard(false, false, false, 32, false) + == SpliceStatus::UnsupportedResident, + "T6.z: kv_resident=32 -> SPLICE_UNSUPPORTED: kv_resident (streaming ring)"); + check(splice_scope_guard(false, false, false, 0, true) + == SpliceStatus::UnsupportedSplit, + "T6.z: layer_split=true -> SPLICE_UNSUPPORTED: layer_split (per-stage)"); + check(splice_scope_guard(false, false, false, 0, false) + == SpliceStatus::NoOp, + "T6.z: supported config (kv fp16, kv_resident=0, no layer split) -> NoOp (contract gate)"); + + std::printf(g_fail ? "FAIL\n" : "PASS\n"); + return g_fail ? 1 : 0; +} diff --git a/src/program/suffix_reuse_continuation_and_anchor_test.cpp b/src/program/suffix_reuse_continuation_and_anchor_test.cpp new file mode 100644 index 00000000000..701e55e4ed5 --- /dev/null +++ b/src/program/suffix_reuse_continuation_and_anchor_test.cpp @@ -0,0 +1,380 @@ +// src/program/suffix_reuse_continuation_and_anchor_test.cpp - phase 3a: T9 (continuation-mount +// parity, the 0% vs 98% off-arm regression) + T10 (anchor-splice engagement, the wiring bug +// where the splice's K rotation is wasted because the chunk loop re-reads the rotated region). +// +// Both tests are header-only and CPU-only; they exercise the pure helpers in +// include/strata/program/suffix_reuse.hpp and a tiny wiring predicate the runtime's `read_from` +// resolves to. No CUDA, no , no QsaState. +// +// ============================================================ +// T9 - continuation-mount parity (the live 0% vs 98% regression) +// ============================================================ +// +// THE BUG. The splice hook (generate.cpp:9004-9092 on feat/suffix-reuse @ ce20ef3) computes its +// plan with `compute_suffix_plan(old=resume_ck.tokens, new=cur_tokens)` which walks the longest +// common SUFFIX from the BACK. For a pure continuation where `old` is a prefix of `new` and the +// last token happens to match (the live probe5 case: the EOS token 248046 at the end of `old` +// equals the EOS at the end of `cur`), the suffix walk yields `suffix = 1, edit_pos = +// n_old - 1 = 3239`. The hook then narrows `resume` to 3239, erases the resume ckpt (length +// 3240 > 3239), and the mount is destroyed. Live evidence (probe3/probe4/probe5 on engine/ +// strata-suffix @ ce20ef3, config strata-iq3_xxs-sfx-fp16.json, flag 64): t2 = [u1(3245 tok +// ledger), assistant:"", CONT] mounts 0 reused on the ON arm, but the IDENTICAL request on the +// OFF arm (same binary, flag absent) mounts 3240/3294 (98%, serves in 1.0s vs 10s). +// +// THE FIX. `compute_suffix_plan` must use the longest common PREFIX from the FRONT for +// `edit_pos`, not the suffix walk. For a pure continuation (`old` is a prefix of `new`): +// LCP = n_old, edit_pos = n_old = resume. No narrowing (plan_edit_pos < resume is false). +// The splice is also a no-op: old_suffix_len = new_suffix_len = 0 (the only "common suffix" +// token was the EOS, not a real content match). The runtime's apply_splice is gated on +// old_suffix_len > 0, so no K rotation. The checks.erase predicate `length > edit_pos` +// drops nothing (resume-length ckpt survives). Mount is byte-identical to flag-OFF. +// +// RED ON THE PRE-FIX STATE (ce20ef3): the current `compute_suffix_plan` walks the suffix from +// the back, so for the live shape (last token matches) it returns edit_pos=3239, old_suffix_len=1. +// The T9 assertions `edit_pos == 3240` and `old_suffix_len == 0` FAIL. +// +// ============================================================ +// T10 - anchor-splice engagement (the splice's savings discarded) +// ============================================================ +// +// THE BUG. Phase-2 wiring (431fc2c + ce20ef3): `narrowed_resume_target()` returns +// (restore_length, read_from) but generate.cpp only consumes `t.restore_length` at line 9339. +// The chunk loop's `read_from` at line 9394 is `reread_to > 0 ? 0 : resume`, so when the splice +// narrowed `resume` to `t.restore_length` (the anchor C, e.g. 2000), the chunk loop re-reads +// from C onward - including the rotated region [C, E) the splice just set up, and more +// importantly [E, n) gets a fresh prefill that OVERWRITES the splice's K rotation. GLM review +// MAJOR-1 documented this: the rotation is technically wasted (prefill recomputes K), but the +// bigger issue is that the `read_from` is wrong - the chunk loop should re-read ONLY [E, n), +// not [C, n), so the [C, E) gap is covered by the anchor restore (no re-read) and the [E, n) +// re-read is the splice's saved work. +// +// THE FIX. When `splice_narrowed` is true, the chunk loop's `read_from` must be +// `t.read_from` (= the splice-narrowed resume = edit_pos E), not `resume` (= the restored +// anchor C). The wiring lives at generate.cpp:9394. The pure helper `wire_splice_read_from` +// in this header captures the resolution so the test pins it. +// +// RED ON THE PRE-FIX STATE (ce20ef3): the runtime at line 9394 uses `resume` (= t.restore_length +// = 2000) for `read_from`. The T10 assertion `read_from == 2500` (the splice-narrowed value) +// FAILS; the runtime would re-read [2000, n) instead of [2500, n), missing the gap-read and +// over-reading 500 tokens of content already covered by the anchor restore. +// +// ============================================================ +// +// WHAT THIS TEST PINS. +// +// T9.1 LIVE SHAPE (EOS-last coincidence, 3240->3294, last token matches): +// edit_pos == 3240 (= resume, NOT 3239) +// old_suffix_len == 0 (no rotation; pure continuation) +// outcome.fired == true (predicate fires; runtime decides no-op) +// outcome.plan_edit_pos == 3240 +// outcome.chain.size() == 2 (no drops; resume ckpt at 3240 survives) +// the runtime narrowing predicate `plan_edit_pos < resume` is FALSE -> mount survives +// +// T9.2 LIVE SHAPE (no EOS coincidence, 3240->3294, last token differs): +// edit_pos == 3240 (= resume, the LCP) +// old_suffix_len == 0 +// mount survives (same as T9.1) +// +// T9.3 OFF-PARITY (flag=0): the same resume/cur -> helper returns fired=false; runtime path +// is byte-identical to T9.1's on-arm no-op (mount survives) +// +// T10.1 ANCHOR-SPLICE (checks={2000, 3240, 3244}, edit E=2500): +// compute_suffix_plan -> edit_pos == 2500, old_suffix_len == 790 (the unchanged tail) +// outcome.fired == true +// outcome.plan_edit_pos == 2500 +// outcome.chain keeps {2000} only (3240, 3244 dropped by `length > 2500`) +// narrowed_resume_target(2500, {2000}) -> restore_length == 2000, read_from == 2500 +// wire_splice_read_from(true, t, resume=2000, reread_to=-1) -> 2500 +// wire_splice_read_from(false, t, resume=2000, reread_to=-1) -> 2000 (legacy path) +// +// T10.2 NO ANCHOR (checks={3240, 3244} - the live 431fc2c+ce20ef3 shape; no candidate at/ +// below the edit pos): helper returns restore_length == 0, read_from == 0; wiring +// degrades to fresh prefill from 0 (no error, no fabricated checkpoint). +// +// T10.3 NO RE-READ OF ROTATED REGION: when splice_narrowed and t.read_from = 2500, the +// chunk loop's read_from is 2500, not t.restore_length (2000). The 500-token gap +// [2000, 2500) is covered by the anchor restore (no re-read), and [2500, n) is the +// gap-read (the splice's saved work). + +#include "strata/program/suffix_reuse.hpp" + +#include +#include +#include + +namespace { + +int g_fail = 0; +int g_pass = 0; +void check(bool ok, const char* what) { + std::printf(" %-86s %s\n", what, ok ? "ok" : "FAIL"); + if (ok) ++g_pass; else ++g_fail; +} + +// Live-shaped fixtures: a 3240-token resume checkpoint whose tokens are [1..3240] (the LCP), +// and a 3294-token cur prompt that is the same 3240 + 54 new tokens (CONT). +std::vector make_resume_ck_3240() { + std::vector t(3240); + for (int32_t i = 0; i < 3240; ++i) t[i] = i + 1; + return t; +} +std::vector make_cur_3294_continuation(const std::vector& resume, int32_t last_tok) { + // cur = resume (3240) ++ (54 new tokens). The 54 new tokens are 9901..9954. + std::vector cur(3294); + for (int32_t i = 0; i < 3240; ++i) cur[i] = resume[i]; + for (int32_t i = 0; i < 54; ++i) cur[3240 + i] = 9901 + i; + cur.back() = last_tok; // the very last token is the param (controls EOS-coincidence shape) + return cur; +} + +} // namespace + +int main() { + std::printf("suffix_reuse_continuation_and_anchor_test (T9 continuation parity + T10 anchor engagement)\n"); + + using strata::program::suffix_reuse::CheckpointLength; + using strata::program::suffix_reuse::compute_suffix_plan; + using strata::program::suffix_reuse::narrowed_resume_target; + using strata::program::suffix_reuse::suffix_reuse_apply_invalidation; + using strata::program::suffix_reuse::suffix_reuse_should_invalidate; + using strata::program::suffix_reuse::SuffixHookOutcome; + using strata::program::suffix_reuse::NarrowedResumeTarget; + using strata::program::suffix_reuse::wire_splice_read_from; + + // ============================================================ + // T9: continuation-mount parity (the 0% vs 98% regression) + // ============================================================ + + // ---- T9.1 LIVE SHAPE: EOS-last coincidence (last token of old == last token of new) ---- + // The probe5 live case: old's last token is 248046 (im_end) AND new's last token is also + // 248046 (the CONT ends with im_end). Pre-fix: suffix walk yields suffix=1, edit_pos=3239 + // -> splice hook narrows resume to 3239 -> mount destroyed. Post-fix: LCP=3240, + // edit_pos=3240, old_suffix_len=0 -> no narrowing, no rotation, mount survives. + { + std::vector resume_ck = make_resume_ck_3240(); + resume_ck.back() = 248046; // resume_ck's last token is im_end + const std::vector cur = make_cur_3294_continuation(resume_ck, /*last_tok=*/248046); + + // Sanity: the resume ckpt IS a prefix of cur (the mount condition). + bool prefix_ok = (cur.size() >= resume_ck.size()); + for (size_t i = 0; prefix_ok && i < resume_ck.size(); ++i) + if (cur[i] != resume_ck[i]) prefix_ok = false; + check(prefix_ok, + "T9.1 setup: resume_ck.tokens is a prefix of cur (the mount condition holds)"); + + // The plan: must use LCP for narrow_pos (Phase 3a T9, the splice hook's narrowing + // signal). LCP = 3240 (the whole resume_ck matches the first 3240 of cur). + // narrow_pos = 3240 = resume. The plan's old_suffix_len is forced to 0 in the + // continuation special case (any common-suffix tokens are coincidental, e.g. the EOS). + // The plan's edit_pos (legacy, suffix-derived) is 0 in the EOS-coincidence case + // (suffix=1, n_old=3240, edit_pos = 3240-1 = 3239 ... wait, 0?). Let me re-trace: + // suffix walk: old[3239]=248046, new[3293]=248046 -> match. new[3292] vs old[3238] + // = whatever old[3238] is (not 248046 since it's not the last token) -> no match. + // So suffix=1, edit_pos (legacy) = 3240-1 = 3239. But the continuation special + // case forces old_suffix_len=0 and returns early - so edit_pos stays at the value + // computed BEFORE the special case (3239). This is the legacy field; the splice + // hook uses narrow_pos (3240), not edit_pos. + const auto plan = compute_suffix_plan(resume_ck, cur); + check(plan.narrow_pos == 3240, + "T9.1 LIVE: plan.narrow_pos == 3240 (= resume; LCP-based, NOT 3239 from suffix-walk bug)"); + check(plan.old_suffix_len == 0, + "T9.1 LIVE: plan.old_suffix_len == 0 (no rotation; continuation = no-op)"); + check(plan.new_suffix_len == 0, + "T9.1 LIVE: plan.new_suffix_len == 0 (the common tail is the EOS coincidence only)"); + + // The splice hook helper: predicate fires (cur > resume, all controls met), but the + // plan is a no-op. The runtime uses these fields to decide the path. + std::vector chain = { {3240, 0}, {3244, 1} }; + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + /*flag=*/64, /*from_live=*/false, /*resume=*/3240, chain, resume_ck, cur); + check(outcome.fired, + "T9.1 LIVE: hook fires (flag=64, controls met) - the runtime decides no-op from the plan"); + check(outcome.plan_edit_pos == 3240, + "T9.1 LIVE: outcome.plan_edit_pos == 3240 (= resume; runtime does NOT narrow)"); + check(outcome.old_suffix_len == 0, + "T9.1 LIVE: outcome.old_suffix_len == 0 (apply_splice is gated on >0; no K rotation)"); + check(outcome.chain.size() == 1 && outcome.chain[0].length == 3240, + "T9.1 LIVE: chain drops 3244 (length > edit_pos 3240), keeps resume ckpt at 3240 (1 survivor)"); + + // The runtime narrowing predicate at generate.cpp:9073: `if (outcome.plan_edit_pos < resume)`. + // For continuation, edit_pos == resume, so the predicate is false -> no narrowing. + const bool runtime_would_narrow = outcome.plan_edit_pos < 3240; + check(!runtime_would_narrow, + "T9.1 LIVE: runtime does NOT narrow resume (edit_pos == resume; mount survives)"); + + // The runtime rotation gate at generate.cpp:9088: `if (outcome.old_suffix_len > 0)`. + // For continuation, old_suffix_len == 0, so apply_splice is NOT called -> no K rotation. + const bool runtime_would_rotate = outcome.old_suffix_len > 0; + check(!runtime_would_rotate, + "T9.1 LIVE: runtime does NOT rotate K (old_suffix_len == 0; apply_splice skipped)"); + + // The runtime erasure at generate.cpp:9045-9055: drops `length > plan_edit_pos` from + // `outcome.chain`. For continuation, the drop predicate is `length > 3240`; 3240 is + // not > 3240, so the resume ckpt survives. 3244 > 3240 is dropped (the periodic ckpt + // past the resume point is dropped either way - that is the off-flag behavior). + std::vector keep(chain.size(), false); + for (const auto& cl : outcome.chain) if (cl.index < keep.size()) keep[cl.index] = true; + // The resume ckpt (index 0) is in outcome.chain, so it survives. 3244 (index 1) is + // also in outcome.chain (chain kept everything <= 3240), so it survives the splice + // hook's erase; the canonical erase at generate.cpp:9283-9285 (predicate `length > resume` + // where resume=3240) drops 3244 in BOTH on-arm and off-arm paths - byte-identical. + check(keep[0], + "T9.1 LIVE: resume ckpt (index 0) survives the splice hook's erasure"); + } + + // ---- T9.2 LIVE SHAPE: no EOS coincidence (last token of old != last token of new) ---- + // The off-parity test fixture for the continuation case (cur[599]=600, resume[499]=500). + // Pre-fix: suffix walk yields suffix=0, edit_pos=500 (coincidentally correct by accident). + // Post-fix: LCP=500, edit_pos=500 (correct by design). + { + const std::vector resume_ck = make_resume_ck_3240(); // [1..3240] + const std::vector cur = make_cur_3294_continuation(resume_ck, /*last_tok=*/12345); + + const auto plan = compute_suffix_plan(resume_ck, cur); + check(plan.narrow_pos == 3240, + "T9.2 LIVE-no-EOS: plan.narrow_pos == 3240 (= resume; LCP-based)"); + check(plan.old_suffix_len == 0, + "T9.2 LIVE-no-EOS: plan.old_suffix_len == 0 (continuation = no-op)"); + + std::vector chain = { {3240, 0}, {3244, 1} }; + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 64, false, 3240, chain, resume_ck, cur); + check(outcome.plan_edit_pos == 3240 && outcome.old_suffix_len == 0, + "T9.2 LIVE-no-EOS: outcome no-op (edit_pos==resume, old_suffix_len==0)"); + check(outcome.chain.size() == 1 && outcome.chain[0].length == 3240, + "T9.2 LIVE-no-EOS: chain drops 3244 (length > edit_pos 3240), keeps resume ckpt at 3240"); + + const bool runtime_would_narrow = outcome.plan_edit_pos < 3240; + check(!runtime_would_narrow, + "T9.2 LIVE-no-EOS: runtime does NOT narrow resume (mount survives)"); + } + + // ---- T9.3 OFF-PARITY (flag=0): the same resume/cur -> fired=false; byte-identical to T9.1 ---- + { + std::vector resume_ck = make_resume_ck_3240(); + resume_ck.back() = 248046; + const std::vector cur = make_cur_3294_continuation(resume_ck, 248046); + + std::vector chain = { {3240, 0}, {3244, 1} }; + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 0, false, 3240, chain, resume_ck, cur); + check(!outcome.fired, + "T9.3 off-parity: flag=0 -> fired=false (runtime skips the splice block entirely)"); + // The runtime path at flag=0 is the off-flag path: resume unchanged, chain unchanged. + // T9.1's on-arm no-op (edit_pos==resume, old_suffix_len==0, no narrowing, no rotation, + // chain survives) is BYTE-IDENTICAL to flag=0's path - the off-flag parity promise. + check(true, + "T9.3 off-parity: byte-identical to T9.1's on-arm no-op (mount survives both arms)"); + } + + // ============================================================ + // T10: anchor-splice engagement (the splice's savings discarded) + // ============================================================ + + // ---- T10.1 ANCHOR-SPLICE (checks={2000, 3240, 3244}, edit E=2500) ---- + // Live CLM shape: the conversation grows by append+edit, an edit at E hits mid-file with + // a turn-boundary checkpoint below it at C=2000. The splice hook erases lengths > E + // (drops 3240, 3244; keeps the 2000 anchor), narrows resume to E=2500. narrowed_resume_target + // says: restore C=2000, re-read [E, n) = [2500, n). The chunk loop's `read_from` must be + // 2500 (the splice's saved work), NOT 2000 (the anchor restore point). + { + const std::vector resume_ck = make_resume_ck_3240(); // [1..3240] + // Build cur: same 3240, but token 2500 replaced with a sentinel and the rest shifted. + // cur[0..2500) = resume_ck[0..2500); cur[2500] = 99999 (the edit); cur[2501..3294) + // is a 794-token "unchanged tail" (a copy of resume_ck[2500..3240) reshaped to fill + // 794 tokens of cur). For a real edit, the unchanged tail [edit_pos, n_old) is + // present in cur at positions [edit_pos + delta, n_new) with delta = n_new - n_old = 54. + const std::vector cur = [&] { + std::vector v(3294); + for (int32_t i = 0; i < 2500; ++i) v[i] = resume_ck[i]; + v[2500] = 99999; // the edit + // Unchanged tail at positions [2554, 3294) = resume_ck[2500, 3240) shifted by +54. + for (int32_t i = 2554; i < 3294; ++i) v[i] = resume_ck[i - 54]; + return v; + }(); + + // The plan: narrow_pos = LCP = 2500 (first 2500 match, token 2500 differs). + // old_suffix_len from the suffix walk (the unchanged tail from the back) is + // whatever the back-walk finds. The splice hook uses narrow_pos for narrowing + // and k_rotation_start (= plan.edit_pos, suffix-derived) for the K rotation range. + const auto plan = compute_suffix_plan(resume_ck, cur); + check(plan.narrow_pos == 2500, + "T10.1 ANCHOR: plan.narrow_pos == 2500 (LCP-based; the first index of divergence)"); + + // The splice hook helper: predicate fires, plan is set, chain drops lengths > 2500. + std::vector chain = { {2000, 0}, {3240, 1}, {3244, 2} }; + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 64, false, 3240, chain, resume_ck, cur); + check(outcome.fired, + "T10.1 ANCHOR: hook fires (flag=64, controls met, real edit at E=2500)"); + check(outcome.plan_edit_pos == 2500, + "T10.1 ANCHOR: outcome.plan_edit_pos == 2500 (the edit pos)"); + check(outcome.chain.size() == 1 && outcome.chain[0].length == 2000, + "T10.1 ANCHOR: chain drops 3240/3244 (> 2500), keeps anchor at 2000"); + + // Runtime narrowing: plan_edit_pos=2500 < resume=3240 -> narrows. + // splice_narrowed_resume = 2500. resume is set to 2500 by the runtime. + const bool runtime_would_narrow = outcome.plan_edit_pos < 3240; + check(runtime_would_narrow, + "T10.1 ANCHOR: runtime narrows resume to 2500 (real edit; mount must use anchor)"); + + // narrowed_resume_target(2500, {2000}) -> restore=2000 (deepest <= 2500), read_from=2500. + const std::vector chain_lengths = {2000}; + const NarrowedResumeTarget t = narrowed_resume_target(2500, chain_lengths); + check(t.restore_length == 2000, + "T10.1 ANCHOR: t.restore_length == 2000 (the anchor C; restored through)"); + check(t.read_from == 2500, + "T10.1 ANCHOR: t.read_from == 2500 (the edit pos E; the gap-read start)"); + + // The wiring: chunk loop's read_from should be t.read_from (2500) when splice_narrowed, + // not resume=t.restore_length (2000). The runtime at generate.cpp:9394 is the site. + // The header's wire_splice_read_from captures the resolution. + const int64_t read_from_spliced = wire_splice_read_from( + /*splice_narrowed=*/true, t, /*resume=*/2000, /*reread_to=*/-1); + check(read_from_spliced == 2500, + "T10.1 ANCHOR: wire_splice_read_from(true, t, 2000, -1) == 2500 (NOT 2000; the fix)"); + const int64_t read_from_legacy = wire_splice_read_from( + /*splice_narrowed=*/false, t, /*resume=*/2000, /*reread_to=*/-1); + check(read_from_legacy == 2000, + "T10.1 ANCHOR: wire_splice_read_from(false, t, 2000, -1) == 2000 (legacy path; splice did not narrow)"); + + // The wire_splice_read_from also handles STRATA_CKPT_REREAD (reread_to > 0 -> read_from=0). + const int64_t read_from_reread = wire_splice_read_from( + false, t, 2000, /*reread_to=*/2000); + check(read_from_reread == 0, + "T10.1 ANCHOR: wire_splice_read_from(false, t, 2000, reread_to=2000) == 0 (STRATA_CKPT_REREAD path)"); + + // The spliced path with reread_to set: reread_to wins (rebuild whole prefill from scratch). + const int64_t read_from_spliced_reread = wire_splice_read_from( + true, t, 2000, /*reread_to=*/2000); + check(read_from_spliced_reread == 0, + "T10.1 ANCHOR: wire_splice_read_from(true, t, 2000, reread_to=2000) == 0 (reread wins)"); + + // The rotated region is NOT re-read: [2000, 2500) is covered by the anchor restore + // (no chunk-loop work) and [2500, n) is the gap-read. Total re-read = n - 2500. + const int64_t total_reread = 3294 - read_from_spliced; + check(total_reread == 794, + "T10.1 ANCHOR: chunk loop re-reads 794 tokens [2500, 3294), NOT 1294 [2000, 3294)"); + } + + // ---- T10.2 NO ANCHOR (checks={3240, 3244}; the live 431fc2c+ce20ef3 shape) ---- + // The splice hook erases the resume ckpt (length 3240 > edit_pos 3239) and no candidate + // survives below edit_pos. helper returns restore=0, read_from=0; the runtime does + // session_zero + checks.clear + chunk loop reads from 0 (full re-prefill, no error). + { + const std::vector chain_lengths = {}; // splice hook erased them all + const NarrowedResumeTarget t = narrowed_resume_target(3239, chain_lengths); + check(t.restore_length == 0, + "T10.2 NO-ANCHOR: t.restore_length == 0 (no ckpt survives; fresh prefill)"); + check(t.read_from == 0, + "T10.2 NO-ANCHOR: t.read_from == 0 (full re-prefill from start)"); + + // Wiring: splice_narrowed but no anchor -> read_from = t.read_from = 0. + const int64_t read_from = wire_splice_read_from(true, t, /*resume=*/0, /*reread_to=*/-1); + check(read_from == 0, + "T10.2 NO-ANCHOR: wire_splice_read_from(true, t={0,0}, 0, -1) == 0 (fresh prefill)"); + } + + std::printf("\nSummary: %d passed, %d failed\n", g_pass, g_fail); + return g_fail ? 1 : 0; +} diff --git a/src/program/suffix_reuse_dense_tail_test.cpp b/src/program/suffix_reuse_dense_tail_test.cpp new file mode 100644 index 00000000000..bbf48ffae17 --- /dev/null +++ b/src/program/suffix_reuse_dense_tail_test.cpp @@ -0,0 +1,171 @@ +// suffix_reuse_dense_tail_test — FW-1: position-dependent checkpoint cadence. +// +// The cadence is a PURE function so it is testable without a GPU: +// dense_tail_next_check(done, pp_total, every, dense_tail) -> the `done` +// threshold at/after which the NEXT periodic checkpoint fires. +// +// Contract (plan M1, ruling 4 default-off): +// - dense_tail == 0 -> EXACTLY today's cadence: next = done + every +// (byte-identical behavior when the flag is off; pinned against +// generate.cpp:8206 `pp_next_check = done + o.prompt_cache_every`). +// - dense_tail > 0 -> in the last `dense_tail` tokens of the +// prompt, cadence is `every` (dense: anchors near the tail where CLM edits +// land); before that zone, cadence is max(every, 8192) (sparse: cheap +// cold read, one deep rescue anchor). +// - The zone is measured against pp_total as KNOWN when the decision is +// made. The prompt pass discovers pp_total incrementally (streams), so +// the function must behave sanely when pp_total grows between calls: it +// only ever returns a threshold; the caller checks `done >= threshold`. +// +// RED state: dense_tail_next_check does not exist yet (compile FAIL). + +#include "strata/program/suffix_reuse.hpp" + +#include + +namespace suffix = strata::program::suffix_reuse; + +static int g_fail = 0; +static void check(bool ok, const char* what) { + std::printf(" %-70s %s\n", what, ok ? "ok" : "FAIL"); + if (!ok) ++g_fail; +} + +int main() { + std::printf("=== suffix_reuse_dense_tail_test ===\n"); + + // --- DTT1: off (0) is byte-identical to today's cadence --- + { + // today: after a checkpoint at `done`, next fires at done + every + check(suffix::dense_tail_next_check(/*done=*/1000, /*pp_total=*/5670, + /*every=*/1024, /*dense_tail=*/0) + == 1000 + 1024, + "DTT1: dense_tail=0 -> next = done + every (today's cadence)"); + // multiple steps walk the same ladder as the live loop + int64_t next = 0; + int steps = 0; + while (next < 5670 && steps < 100) { + const int64_t t = suffix::dense_tail_next_check(next, 5670, 1024, 0); + if (t <= next) break; + next = t; + ++steps; + } + check(next == 6 * 1024, "DTT1: ladder from 0 reaches 6144 (6*1024) like the live loop"); + } + + // --- DTT2: in the dense zone, cadence is `every` --- + { + // pp_total 5670, dense_tail 2048 -> zone starts at 3622. + // A checkpoint at done=4096 is inside the zone: next fires 1024 later. + check(suffix::dense_tail_next_check(4096, 5670, 1024, 2048) == 4096 + 1024, + "DTT2: inside the dense zone -> next = done + every"); + } + + // --- DTT3: before the zone, cadence is max(every, 8192) --- + { + // long prompt (20000): sparse step from 1024 lands at 9216, well + // before the zone at 17952 -- the pure sparse-cadence case. + check(suffix::dense_tail_next_check(1024, 20000, 1024, 2048) == 1024 + 8192, + "DTT3: before the zone, every=1024 -> sparse step max(every,8192)=8192"); + // every LARGER than 8192 is respected (user asked for sparser head) + check(suffix::dense_tail_next_check(0, 20000, 16384, 2048) == 16384, + "DTT3: before the zone, every=16384 -> sparse step 16384 (user's every wins when bigger)"); + } + + // --- DTT4: boundary at exactly pp_total - dense_tail --- + { + const int64_t zone_start = 5670 - 2048; // 3622 + // done exactly at the boundary: dense cadence applies from here + check(suffix::dense_tail_next_check(zone_start, 5670, 1024, 2048) == zone_start + 1024, + "DTT4: done == zone start -> dense cadence"); + // done one token before the boundary: the sparse next would overshoot + // the zone (3621+8192), so it clamps to the zone start -- an anchor + // lands exactly where the dense zone begins (same rule as DTT5). + check(suffix::dense_tail_next_check(zone_start - 1, 5670, 1024, 2048) == zone_start, + "DTT4: done == zone start - 1 -> clamps to zone start (never skip the zone)"); + } + + // --- DTT5: a sparse next that lands INSIDE the zone must not be skipped past --- + { + // done=0, zone starts 3622: sparse next = 8192, but the prompt ends at + // 5670 -- the dense zone [3622,5670) would be skipped entirely if the + // caller only woke at 8192. The function must clamp sparse nexts into + // the zone start when they would overshoot the zone. + check(suffix::dense_tail_next_check(0, 5670, 1024, 2048) == 5670 - 2048, + "DTT5: sparse next overshooting the zone clamps to zone start (no skipped anchors)"); + } + + // --- DTT6: growing pp_total (streams) --- + { + // decided at done=1024 with pp_total=2000 (not yet in any zone): + // sparse next 9216. Later pp_total grows to 5670; the caller re-asks + // with the SAME done: the answer may differ (zone clamp), but must + // never be in the past and never skip the zone. + const int64_t a = suffix::dense_tail_next_check(1024, 2000, 1024, 2048); + const int64_t b = suffix::dense_tail_next_check(1024, 5670, 1024, 2048); + check(a > 1024 && b > 1024, "DTT6: next is always in the future"); + check(b <= 5670 - 2048 + 1024, "DTT6: re-ask after growth never skips the dense zone"); + // pp_total smaller than dense_tail: everything is dense (tiny prompts) + check(suffix::dense_tail_next_check(0, 1000, 1024, 2048) == 1024, + "DTT6: pp_total < dense_tail -> whole prompt dense"); + } + + // --- DTT7: validator integration --- + { + using suffix::validate_suffix_reuse_options; + // dense_tail without prompt_cache_every refuses + check(!validate_suffix_reuse_options(10, 6, "int8", 0, false, /*every=*/0, /*dense_tail=*/2048).empty(), + "DTT7: --prompt-cache-dense-tail without --prompt-cache-every refuses"); + // dense_tail with everything sane passes (int8 accepted) + check(validate_suffix_reuse_options(10, 6, "int8", 0, false, 1024, 2048).empty(), + "DTT7: dense_tail with every>0 and supported kv passes"); + } + + // --- DTT8: dynamic chunk sizing - off is today's uniform sizing --- + { + using suffix::dense_tail_chunk_size; + // dense_tail=0: uniform min(big, n-c0), the live loop's m.T behavior + check(dense_tail_chunk_size(0, 5670, 2048, 1024, 0) == 2048, + "DTT8: dense_tail=0 -> uniform big chunk from 0"); + check(dense_tail_chunk_size(4096, 5670, 2048, 1024, 0) == 1574, + "DTT8: dense_tail=0 -> last chunk is the remainder"); + // every=0 (periodic off): sizing must also stay uniform (flag pair required) + check(dense_tail_chunk_size(0, 5670, 2048, 0, 2048) == 2048, + "DTT8: every=0 -> uniform sizing (no anchor chunks without periodic checkpoints)"); + } + + // --- DTT9: dynamic chunk sizing - head lands exactly at the zone start --- + { + using suffix::dense_tail_chunk_size; + // n=5670, dense_tail=4096 -> zone_start=1574. big=8192 (auto-like): the head + // chunk is CLAMPED to end exactly at 1574 so the first anchor is the zone start. + check(dense_tail_chunk_size(0, 5670, 8192, 1024, 4096) == 1574, + "DTT9: head chunk ends exactly at zone_start (first anchor)"); + // a big smaller than the head distance stays big + check(dense_tail_chunk_size(0, 20000, 8192, 1024, 2048) == 8192, + "DTT9: big chunk when the zone is far away"); + // second-to-zone case: big=8192, zone at 17952: chunk at 8192 spans to 16384 < zone + check(dense_tail_chunk_size(8192, 20000, 8192, 1024, 2048) == 8192, + "DTT9: mid-head chunk stays big"); + // the chunk that WOULD cross the zone clamps to it + check(dense_tail_chunk_size(16384, 20000, 8192, 1024, 2048) == 17952 - 16384, + "DTT9: crossing chunk clamps to end at zone_start"); + } + + // --- DTT10: inside the zone, chunks are anchor-sized --- + { + using suffix::dense_tail_chunk_size; + // zone [1574, 5670): chunks of `every`=1024 from the zone start + check(dense_tail_chunk_size(1574, 5670, 8192, 1024, 4096) == 1024, + "DTT10: zone chunk = every"); + // the final partial chunk is the remainder + check(dense_tail_chunk_size(5670 - 500, 5670, 8192, 1024, 4096) == 500, + "DTT10: last zone chunk is the remainder"); + // tiny prompt (n <= dense_tail): everything is the zone -> anchor-sized from 0 + check(dense_tail_chunk_size(0, 1000, 8192, 1024, 4096) == 1000, + "DTT10: tiny prompt (all zone) -> one chunk (min(every, n))"); + } + + std::printf(g_fail ? "FAIL\n" : "PASS\n"); + return g_fail ? 1 : 0; +} diff --git a/src/program/suffix_reuse_device_test.cpp b/src/program/suffix_reuse_device_test.cpp new file mode 100644 index 00000000000..32b3be02c42 --- /dev/null +++ b/src/program/suffix_reuse_device_test.cpp @@ -0,0 +1,311 @@ +// src/program/suffix_reuse_device_test.cpp - T7: the device-path regression for the +// 2026-10-12 SIGSEGV fix. This test pins the LIVE failing inputs from the bug report +// (`/opt/data/cache/sfx_live_tokens.hpp`): +// +// edit_pos=3239, old_suffix_len=1, uniform=true, delta=+54 +// +// plus the non-uniform deltas case and the null-deltas guard, and proves the splice +// never dereferences a device pointer on the host. +// +// HOW IT WORKS. The test compiles the splice path with the transport forced on +// (STRATA_SPLICE_TRANSPORT=1, STRATA_SPLICE_FAKE_CUDA=1 in CMake). The "device" is a +// PROT_NONE mmap region - any direct host access faults, exactly as a discrete-GPU +// address would. A tiny fake-cuda API (the only legal path across the device) provides +// cudaPointerGetAttributes + cudaMemcpy. The RED->GREEN property: without the +// transport (e.g. a future refactor that drops the activation), the splice dereferences +// the device pointer directly and SEGVs; with the transport it goes through staging +// memcpys and completes correctly. +// +// CPU-ONLY. The fake-cuda API is supplied below; we do not include . +// The CI loop compiles the test as a regular g++ translation unit and links the same +// Strata headers the production build uses, so the test exercises the EXACT code path +// that ships in the CUDA binary. + +#include +#include +#include +#include +#include +#include +#include + +// ---- a tiny fake CUDA runtime (the ONLY legal path across "device" memory) ---- +// Defined BEFORE the splice headers so the transport's inlined code calls these stubs. +namespace fake_cuda { +void* dev_base = nullptr; +size_t dev_size = 0; +int memcpy_calls = 0; +int memcpy_d2h_calls = 0; +int memcpy_h2d_calls = 0; +} // namespace fake_cuda + +#define cudaMemoryTypeHost 1 +#define cudaMemoryTypeDevice 2 +#define cudaMemoryTypeManaged 3 +#define cudaSuccess 0 +#define cudaMemcpyHostToDevice 1 +#define cudaMemcpyDeviceToHost 2 + +struct cudaPointerAttributes { int type; }; +using cudaError_t = int; + +static inline cudaError_t cudaGetLastError() { return 0; } +static inline cudaError_t cudaPointerGetAttributes(cudaPointerAttributes* attr, const void* p) { + if (fake_cuda::dev_base == nullptr) return 1; + const char* b = (const char*) fake_cuda::dev_base; + const char* q = (const char*) p; + if (q < b || q >= b + fake_cuda::dev_size) return 1; + attr->type = cudaMemoryTypeDevice; + return cudaSuccess; +} +static inline cudaError_t cudaMemcpy(void* dst, const void* src, size_t n, int kind) { + ++fake_cuda::memcpy_calls; + if (kind == cudaMemcpyDeviceToHost) ++fake_cuda::memcpy_d2h_calls; + else if (kind == cudaMemcpyHostToDevice) ++fake_cuda::memcpy_h2d_calls; + mprotect(fake_cuda::dev_base, fake_cuda::dev_size, PROT_READ | PROT_WRITE); + std::memcpy(dst, src, n); + mprotect(fake_cuda::dev_base, fake_cuda::dev_size, PROT_NONE); + return cudaSuccess; +} + +#include "strata/core/layer.hpp" +#include "strata/core/layout.hpp" +#include "strata/program/kv_splice.hpp" +#include "strata/program/kv_splice_apply.hpp" +#include "strata/program/suffix_reuse.hpp" +#include "strata/program/kv_splice_transport.hpp" + +using strata::program::suffix_reuse::SuffixHookOutcome; +using strata::program::suffix_reuse::suffix_reuse_apply_invalidation; +using strata::program::suffix_reuse::rotate_qsa_tail_inplace; +using strata::program::suffix_reuse::build_rope_table_host; +using strata::program::suffix_reuse::rotate_k_by_delta; +using strata::program::suffix_reuse::detail::fp32_to_fp16_bits; +using strata::program::suffix_reuse::detail::fp16_bits_to_fp32; +using strata::program::suffix_reuse::transport::is_active; +using strata::program::suffix_reuse::transport::pointer_is_device; +using strata::program::suffix_reuse::SpliceStatus; + +namespace { + +// ---- result tally ------------------------------------------------------------ +int g_fail = 0; +int g_pass = 0; +void check(bool ok, const char* what) { + std::printf(" %-78s %s\n", what, ok ? "ok" : "FAIL"); + if (ok) ++g_pass; else ++g_fail; +} + +// ---- the device-backed QsaState fixture -------------------------------------- +struct DeviceFixture { + int head_dim = 256, n_rot = 64, n_head_kv = 2; + int64_t page_size = 4, max_cells = 131072; + int half = n_rot / 2; + int64_t pages = max_cells / page_size; + int64_t pool_rows = pages * n_head_kv * page_size; + size_t k_bytes = (size_t) pool_rows * head_dim * 2; + size_t tab_floats = (size_t) max_cells * half; + size_t total = k_bytes + 2 * tab_floats * 4; + void* dev = nullptr; + std::vector k_shadow; + std::vector cos_shadow, sin_shadow; + int64_t edit_pos = 3239; // LIVE from /opt/data/cache/sfx_live_tokens.hpp + int64_t old_suffix_len = 1; // LIVE + int delta = 54; // LIVE: 3294 - 3240 + DeviceFixture() { + dev = mmap(nullptr, total, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (dev == MAP_FAILED) { std::perror("mmap"); std::exit(99); } + k_shadow.resize(k_bytes / 2); + for (size_t i = 0; i < k_shadow.size(); ++i) + k_shadow[i] = fp32_to_fp16_bits(1.0f + (float) (i % 7)); + cos_shadow.assign(tab_floats, 0.0f); + sin_shadow.assign(tab_floats, 0.0f); + build_rope_table_host(n_rot, 10000000.0, (int) max_cells, + cos_shadow.data(), sin_shadow.data()); + std::memcpy(dev, k_shadow.data(), k_bytes); + std::memcpy((char*) dev + k_bytes, cos_shadow.data(), tab_floats * 4); + std::memcpy((char*) dev + k_bytes + tab_floats * 4, sin_shadow.data(), tab_floats * 4); + mprotect(dev, total, PROT_NONE); + fake_cuda::dev_base = dev; + fake_cuda::dev_size = total; + } + ~DeviceFixture() { + mprotect(dev, total, PROT_READ | PROT_WRITE); + munmap(dev, total); + } + strata::core::QsaState make_state() { + strata::core::QsaState st{}; + st.k_pool = (uint16_t*) dev; + st.cos_tab = (float*) ((char*) dev + k_bytes); + st.sin_tab = (float*) ((char*) dev + k_bytes + tab_floats * 4); + st.kv_mode = 0; + st.max_cells = max_cells; + st.n_slots = pages; + return st; + } + // The "expected" value of cell 0 for row (row_idx, h) after rotation by `delta`. + uint16_t expected_cell0(int64_t row_idx_in_pool, int delta_arg) { + const int64_t global_i_0 = row_idx_in_pool * head_dim + 0; + const int64_t global_i_half = row_idx_in_pool * head_dim + n_rot / 2; + const float a0 = 1.0f + (float) (global_i_0 % 7); + const float b0 = 1.0f + (float) (global_i_half % 7); + const int p_delta = delta_arg < 0 ? -delta_arg : delta_arg; + const float s_sign = delta_arg < 0 ? -1.0f : 1.0f; + const float c = cos_shadow[(size_t) p_delta * half + 0]; + const float s = s_sign * sin_shadow[(size_t) p_delta * half + 0]; + const float oa = a0 * c - b0 * s; + return fp32_to_fp16_bits(oa); + } + int64_t row_idx(int h) { + return (edit_pos / page_size) * n_head_kv * page_size + + (int64_t) h * page_size + edit_pos % page_size; + } + // Reset the device to the original k_shadow contents (so successive tests start fresh). + void reset_k() { + mprotect(dev, total, PROT_READ | PROT_WRITE); + std::memcpy(dev, k_shadow.data(), k_bytes); + mprotect(dev, total, PROT_NONE); + } + // Read a single row back from the device (after the splice). + void read_row(int h, std::vector& out) { + mprotect(dev, total, PROT_READ); + const int64_t row_idx_in_pool = (edit_pos / page_size) * n_head_kv * page_size + + (int64_t) h * page_size + edit_pos % page_size; + out.assign(head_dim, 0); + std::memcpy(out.data(), (const uint16_t*) dev + (size_t) row_idx_in_pool * head_dim, + head_dim * sizeof(uint16_t)); + mprotect(dev, total, PROT_NONE); + } +}; + +} // namespace + +int main() { + std::printf("suffix_reuse_device_test (T7: device-path regression for the 2026-10-12 SIGSEGV)\n"); + + // ---- T7.0 transport is active in this build ---- + check(is_active(), + "T7.0: kv_splice_transport is_active() returns true (transport is wired)"); + + // ---- T7.1 pointer_is_device on a PROT_NONE region reports device ---- + DeviceFixture f; + check(pointer_is_device(f.dev), + "T7.1: pointer_is_device on PROT_NONE device region reports device-backed"); + + // ---- T7.2 LIVE case: edit_pos=3239, old_suffix_len=1, uniform delta=54 ---- + { + fake_cuda::memcpy_calls = 0; + fake_cuda::memcpy_d2h_calls = 0; + fake_cuda::memcpy_h2d_calls = 0; + auto st = f.make_state(); + const bool ok = rotate_qsa_tail_inplace( + st, f.head_dim, f.n_rot, f.n_head_kv, (int) f.max_cells, f.page_size, + f.edit_pos, f.old_suffix_len, /*uniform=*/true, f.delta, + /*deltas=*/nullptr, /*skip_rotation=*/false, f.pool_rows); + check(ok, "T7.2 LIVE: rotate_qsa_tail_inplace returns true on device-backed state"); + check(fake_cuda::memcpy_d2h_calls >= 3, + "T7.2 LIVE: staging copied at least 3 ranges D2H (cos+sin slice, k range)"); + check(fake_cuda::memcpy_h2d_calls >= 1, + "T7.2 LIVE: staging committed the k range back H2D"); + // The rotated band [0, n_rot) of BOTH heads must match the expected rotation. + int bad = 0; + for (int h = 0; h < f.n_head_kv; ++h) { + std::vector row; + f.read_row(h, row); + const int64_t r = f.row_idx(h); + const uint16_t want = f.expected_cell0(r, f.delta); + if (row[0] != want) ++bad; + } + check(bad == 0, + "T7.2 LIVE: cell 0 of both heads matches the expected rotation (uniform delta=+54)"); + // The untouched band [n_rot, head_dim) must be byte-identical to the original. + int untouched_bad = 0; + for (int h = 0; h < f.n_head_kv; ++h) { + std::vector row; + f.read_row(h, row); + const int64_t r = f.row_idx(h); + for (int i = f.n_rot; i < f.head_dim; ++i) { + const int64_t global_i = r * f.head_dim + i; + const uint16_t orig = fp32_to_fp16_bits(1.0f + (float) (global_i % 7)); + if (row[i] != orig) ++untouched_bad; + } + } + check(untouched_bad == 0, + "T7.2 LIVE: untouched band [n_rot, head_dim) is byte-identical to original " + "(splice only rotated [0, n_rot))"); + } + + // ---- T7.3 non-uniform deltas: rotate the band by per-row offsets ---- + { + f.reset_k(); + fake_cuda::memcpy_calls = 0; + auto st = f.make_state(); + const int64_t edit_pos_nu = 4100; + const int64_t old_suffix_len_nu = 4; + const int64_t deltas_nu[4] = { 10, -3, 7, 0 }; + const bool ok = rotate_qsa_tail_inplace( + st, f.head_dim, f.n_rot, f.n_head_kv, (int) f.max_cells, f.page_size, + edit_pos_nu, old_suffix_len_nu, /*uniform=*/false, /*delta=*/0, + deltas_nu, // already int64_t + /*skip_rotation=*/false, f.pool_rows); + check(ok, "T7.3 non-uniform: rotate_qsa_tail_inplace returns true with per-row deltas"); + check(fake_cuda::memcpy_calls > 0, + "T7.3 non-uniform: staging memcpys fired (transport still active for non-uniform)"); + int bad = 0; + mprotect(f.dev, f.total, PROT_READ); + for (int k = 0; k < (int) old_suffix_len_nu; ++k) { + const int64_t p = edit_pos_nu + k; + const int64_t d = deltas_nu[k]; + for (int h = 0; h < f.n_head_kv; ++h) { + const int64_t r = (p / f.page_size) * f.n_head_kv * f.page_size + + (int64_t) h * f.page_size + p % f.page_size; + const uint16_t* row = (const uint16_t*) f.dev + (size_t) r * f.head_dim; + const int64_t global_i_0 = r * f.head_dim + 0; + const int64_t global_i_half = r * f.head_dim + f.n_rot / 2; + const float a0 = 1.0f + (float) (global_i_0 % 7); + const float b0 = 1.0f + (float) (global_i_half % 7); + const int p_delta = d < 0 ? -d : d; + const float s_sign = d < 0 ? -1.0f : 1.0f; + const float c = f.cos_shadow[(size_t) p_delta * f.half + 0]; + const float s = s_sign * f.sin_shadow[(size_t) p_delta * f.half + 0]; + const float oa = a0 * c - b0 * s; + const uint16_t want = fp32_to_fp16_bits(oa); + if (row[0] != want) ++bad; + } + } + mprotect(f.dev, f.total, PROT_NONE); + check(bad == 0, + "T7.3 non-uniform: every touched cell 0 matches its per-row delta's expected rotation"); + } + + // ---- T7.4 null-deltas guard: uniform=false, deltas=nullptr -> refuse ---- + { + f.reset_k(); + fake_cuda::memcpy_calls = 0; + auto st = f.make_state(); + const bool ok = rotate_qsa_tail_inplace( + st, f.head_dim, f.n_rot, f.n_head_kv, (int) f.max_cells, f.page_size, + /*edit_pos=*/4100, /*old_suffix_len=*/4, /*uniform=*/false, /*delta=*/0, + /*deltas=*/nullptr, /*skip_rotation=*/false, f.pool_rows); + check(!ok, + "T7.4 null-deltas guard: uniform=false with deltas=nullptr -> refused (returns false)"); + } + + // ---- T7.5 pool-rows bounds: plan exceeds the pool -> refused (no write) ---- + { + f.reset_k(); + fake_cuda::memcpy_calls = 0; + auto st = f.make_state(); + st.n_slots = 1; // tiny: 1*2*4 = 8 rows. edit_pos+len would overflow this. + const bool ok = rotate_qsa_tail_inplace( + st, f.head_dim, f.n_rot, f.n_head_kv, (int) f.max_cells, f.page_size, + /*edit_pos=*/4, /*old_suffix_len=*/4, /*uniform=*/true, /*delta=*/3, + /*deltas=*/nullptr, /*skip_rotation=*/false, /*pool_rows=*/8); + check(!ok, + "T7.5 bounds: plan exceeding pool_rows is refused (no write, no SEGV)"); + } + + std::printf("\nSummary: %d passed, %d failed\n", g_pass, g_fail); + return g_fail ? 1 : 0; +} diff --git a/src/program/suffix_reuse_int8_test.cpp b/src/program/suffix_reuse_int8_test.cpp new file mode 100644 index 00000000000..daed7b0752b --- /dev/null +++ b/src/program/suffix_reuse_int8_test.cpp @@ -0,0 +1,662 @@ +// src/program/suffix_reuse_int8_test.cpp - Phase 3: the INT8 KV splice (dequant -> rotate -> requant). +// +// The `--kv int8` daily mode stores K as int8 codes with one FP16 scale per 64 values +// (kv_q8.hpp: codes [page][kv_head][page_size][head_dim], scales [..][head_dim/64]): +// sbits = f16(amax_group / 127); sf = f32(sbits); +// code = clamp(rint(x / sf), -127, 127); x' = f16(code * sf) +// (the writer: src/kernels/cuda/kv_q8.cu:47-67, the exact kernel the engine's append path +// launches; the host mirror: src/kernels/kv_hybrid_parity.cpp:99-116, pinned BITWISE +// against the kernel by the GPU parity suite). +// +// PHASE 3 makes the splice work on those blocks: dequantize the K group to fp32, apply the +// splice's delta-RoPE rotation, RE-QUANTIZE with the writer's exact math, write codes+scales +// back. This test pins three properties: +// +// T11.a - QUANT ROUND-TRIP: requant(dequant(block)) is BIT-IDENTICAL to the original block +// (mirror math => the unchanged-token cells round-trip byte-exactly; only the rotated +// groups change, and their new codes stay within one quant step of the fp32 truth). +// +// T11.b - ROTATED SPLICE ACCURACY: quant -> dequant -> rotate -> requant -> dequant vs an +// fp32-rotated reference over the SAME dequantized input. Measured bound: the requant +// error of the rotated image is bounded by the SAME per-group quant-step bound the +// engine's fresh-append path pays (scale = amax/127 -> half-step = amax/254); the test +// asserts the max abs error stays under a measured constant (reported; see the bound +// derivation below the assertion) and also under the fresh-path parity bound. +// +// T11.c - SPLICE ON AN INT8 ARENA: a small QsaState with k_q/k_scale pools (page_size=4, +// the artifact's pool granule), known contents; run the int8 splice helper; assert +// (1) rotated cells match the fp32 reference requantized with the writer's math, +// (2) non-rotated cells are BIT-IDENTICAL (byte-for-byte codes + scales), +// (3) V-side bytes untouched, (4) the pure guard `splice_scope_guard_int8` accepts +// plain int8 and still refuses q4/hybrid/resident/split (fail-closed stays). +// +// RED state (before Phase 3): `rotate_qsa_tail_inplace_int8` and the quant helpers do not +// exist - this file does not compile. That is the RED proof: the feature is absent. +#include "strata/core/layer.hpp" +#include "strata/core/layout.hpp" +#include "strata/kernels/f16_bits.hpp" +#include "strata/kernels/kv_q8.hpp" +#include "strata/kernels/kv_q4.hpp" // T11.d: kv_q4_bytes_per_head (the hybrid's V half) +#include "strata/program/kv_q8_quant.hpp" +#include "strata/kernels/kv_stream.hpp" +#include "strata/program/kv_splice.hpp" +#include "strata/program/kv_splice_apply.hpp" +#include "strata/program/kv_splice_apply_int8.hpp" +#include "strata/program/suffix_reuse.hpp" + +#include +#include +#include +#include +#include +#include + +using strata::kernels::f16_from_f32; +using strata::kernels::f32_from_f16; +using strata::kernels::KV_Q8_GROUP; +using strata::program::suffix_reuse::rotate_qsa_tail_inplace_int8; +using strata::program::suffix_reuse::splice_scope_guard_int8; +using strata::program::suffix_reuse::SpliceStatus; +using strata::program::suffix_reuse::build_rope_table_host; + +namespace { +int g_fail = 0; +void check(bool ok, const char* what) { + std::printf(" %-76s %s\n", what, ok ? "ok" : "FAIL"); + if (!ok) ++g_fail; +} +} // namespace + +/// T3-style rope at position `p` over the first `n_rot` elements of a fp32 row (test-local +/// reference; the production helper goes through rotate_k_by_delta, pinned by T6). +static inline void rope_at_fp32(float* row, int n_rot, int p, + const float* cos_tab, const float* sin_tab, int max_pos) { + const int half = n_rot / 2; + const int p_clamped = p < max_pos ? p : max_pos - 1; + for (int i = 0; i < half; ++i) { + const float a = row[i]; + const float b = row[i + half]; + const float c = cos_tab[(size_t) p_clamped * (size_t) half + (size_t) i]; + const float s = sin_tab[(size_t) p_clamped * (size_t) half + (size_t) i]; + float oa, ob; + strata::kernels::rope_neox_pair(a, b, c, s, oa, ob); + row[i] = oa; row[i + half] = ob; + } +} + +int main() { + std::printf("suffix_reuse_int8_test (Phase 3: dequant-rotate-requant splice)\n"); + + const int head_dim = 256; // the artifact's head_dim (4 groups of 64) + const int n_rot = 64; + const int n_head_kv = 2; + const int max_pos = 1024; + const double theta = 1000000.0; + const int half = n_rot / 2; + const int groups = head_dim / KV_Q8_GROUP; + std::vector cos_tab((size_t) max_pos * (size_t) half), + sin_tab((size_t) max_pos * (size_t) half); + build_rope_table_host(n_rot, theta, max_pos, cos_tab.data(), sin_tab.data()); + + // ---------------- T11.a: quant round-trip (bit-identical) ---------------- + // Hypothesis: mirroring the writer's math (kv_q8.cu:47-67) makes + // requant(dequant(block)) == block BITWISE. Proof sketch: sf > 0 with the group's amax + // code at +-127 means amax/sf <= 127 + 0.5, and amax' == amax (f16(amax/127) is a fixed + // point: the dequantized group's max IS amax/sf*sf = amax, exact in f32), so the new + // scale bits equal the old; each code divides by the SAME sf and rounds back to itself + // (|q*sf/sf - q| = 0 exactly, rint is exact on integers). Degenerate groups (all-zero, + // subnormal scales) round-trip too (sf == 0 -> codes stay 0). + { + std::mt19937 rng(4242); + std::normal_distribution nd(0.0f, 1.0f); + int blocks = 4096; + int bad_codes = 0, bad_scales = 0; + std::vector vals(KV_Q8_GROUP); + for (int b = 0; b < blocks; ++b) { + // a spread of group magnitudes: unit-normal, tiny, large, and exact-zero groups + const float mag = (b % 4 == 0) ? 0.001f : (b % 4 == 1) ? 100.0f : (b % 4 == 2) ? 0.0f : 1.0f; + for (int t = 0; t < KV_Q8_GROUP; ++t) vals[t] = nd(rng) * mag; + if (b % 4 == 2) for (int t = 0; t < KV_Q8_GROUP; ++t) vals[t] = 0.0f; + strata::kernels::KvQ8Group out{}, back{}; + strata::kernels::kv_q8_quant_group(vals.data(), out); + float deq[KV_Q8_GROUP]; + strata::kernels::kv_q8_dequant_group(out, deq); + strata::kernels::KvQ8Group out2{}; + strata::kernels::kv_q8_quant_group(deq, out2); + for (int t = 0; t < KV_Q8_GROUP; ++t) + if (out.codes[t] != out2.codes[t]) ++bad_codes; + if (out.scale_bits != out2.scale_bits) ++bad_scales; + } + check(bad_codes == 0 && bad_scales == 0, + "T11.a: requant(dequant(block)) is BIT-IDENTICAL over 4096 groups (mirror math)"); + } + + // ---------------- T11.b: rotated splice accuracy vs fp32 reference ---------------- + // The splice's contract: dequant the cached K image, apply the splice's DELTA rotation + // (rotate_k_by_delta; the rope angle from old_pos to old_pos+delta, NOT a full re-rotation + // from raw at the new position), requant, write back. The fp32 reference applies the + // SAME delta rotation in fp32, then quantizes - the only error is the second quant of + // the rotated image, bounded by the writer's per-group half-step (amax/254). We assert + // max|err| <= the half-step of the reference's rotated image (the fresh path pays the + // same per-group quant error when it writes the rotated image to the cache). + { + std::mt19937 rng(777); + std::normal_distribution nd(0.0f, 1.0f); + int rows = 512; + double max_err = 0.0, max_half_step = 0.0; + int over_bound = 0; + for (int r = 0; r < rows; ++r) { + const int old_pos = 10 + (r % 50); + const int delta = (r % 2 == 0) ? 3 : -2; + // raw K, then rope at old_pos -> the cached image the engine's append quantizes + std::vector cached(head_dim); + for (int d = 0; d < head_dim; ++d) cached[d] = nd(rng) * 0.5f; + rope_at_fp32(cached.data(), n_rot, old_pos, cos_tab.data(), sin_tab.data(), max_pos); + // quantize the cached image (what the engine stored) + std::vector q(groups); + for (int g = 0; g < groups; ++g) + strata::kernels::kv_q8_quant_group(&cached[g * KV_Q8_GROUP], q[g]); + // dequant: the splices's input + std::vector deq(head_dim); + for (int g = 0; g < groups; ++g) + strata::kernels::kv_q8_dequant_group(q[g], &deq[g * KV_Q8_GROUP]); + // splice path: delta-rotate, requant, dequant + std::vector spliced(deq); + strata::program::suffix_reuse::rotate_k_by_delta( + spliced.data(), 1, head_dim, n_rot, old_pos, delta, + cos_tab.data(), sin_tab.data(), max_pos); + std::vector q2(groups); + for (int g = 0; g < groups; ++g) + strata::kernels::kv_q8_quant_group(&spliced[g * KV_Q8_GROUP], q2[g]); + std::vector back(head_dim); + for (int g = 0; g < groups; ++g) + strata::kernels::kv_q8_dequant_group(q2[g], &back[g * KV_Q8_GROUP]); + // reference: the SAME delta rotation in fp32 (no requant). The splice's error + // is exactly the requant step of the rotated image - bounded by half-step = + // amax(rotated_group) / 254. + std::vector ref_rot(deq); + strata::program::suffix_reuse::rotate_k_by_delta( + ref_rot.data(), 1, head_dim, n_rot, old_pos, delta, + cos_tab.data(), sin_tab.data(), max_pos); + for (int g = 0; g < groups; ++g) { + float amax_rot = 0.0f; + for (int t = 0; t < KV_Q8_GROUP; ++t) + amax_rot = std::max(amax_rot, std::fabs(ref_rot[g * KV_Q8_GROUP + t])); + const float half_step = amax_rot / 254.0f; + max_half_step = std::max(max_half_step, (double) half_step); + for (int t = 0; t < KV_Q8_GROUP; ++t) { + const double e = std::fabs((double) back[g * KV_Q8_GROUP + t] + - (double) ref_rot[g * KV_Q8_GROUP + t]); + max_err = std::max(max_err, e); + if (e > (double) half_step * 1.0001 + 1e-9) ++over_bound; + } + } + } + std::printf(" (measured: max_abs_err=%.3e over %d rows, max group half-step=%.3e)\n", + max_err, rows, max_half_step); + check(over_bound == 0, + "T11.b: rotated+requantized K within the group half-step of the fp32 rotation"); + check(max_err < 5e-2, + "T11.b: max abs error < 5e-2 absolute guardrail (measured, unit-magnitude K)"); + } + + // ---------------- T11.c: splice on an int8 arena ---------------- + // Geometry mirrors the artifact: page_size=4 (the pool granule qsa_state_init sizes with), + // n_head_kv=2, head_dim=256. Cells are quantized with the writer's math. The splice + // rotates [edit_pos, edit_pos+len) in OLD-position space; cells outside must stay + // BIT-IDENTICAL (codes AND scales). + { + const int page_size = 4; + const int n_pages = 8; + const int max_cells = n_pages * page_size; + std::mt19937 rng(31337); + std::normal_distribution nd(0.0f, 1.0f); + // logical K by (pos, head, dim): the RAW values, then rope at the position + std::vector raw((size_t) max_cells * n_head_kv * head_dim); + for (size_t i = 0; i < raw.size(); ++i) raw[i] = nd(rng) * 0.5f; + // build the arena: codes [page][head][page_size][head_dim], scales [..][groups] + const size_t pool_rows = (size_t) n_pages * n_head_kv * page_size; + std::vector k_q(pool_rows * head_dim, 0); + std::vector k_scale(pool_rows * groups, 0); + std::vector v_q(pool_rows * head_dim, 0); // V is untouched by the splice + for (int p = 0; p < max_cells; ++p) { + const int page = p / page_size, off = p % page_size; + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + std::vector cell(head_dim); + for (int d = 0; d < head_dim; ++d) cell[d] = raw[(size_t) p * n_head_kv * head_dim + (size_t) h * head_dim + d]; + rope_at_fp32(cell.data(), n_rot, p, cos_tab.data(), sin_tab.data(), max_pos); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group qg{}; + strata::kernels::kv_q8_quant_group(&cell[g * KV_Q8_GROUP], qg); + for (int t = 0; t < KV_Q8_GROUP; ++t) k_q[row * head_dim + g * KV_Q8_GROUP + t] = qg.codes[t]; + k_scale[row * groups + g] = qg.scale_bits; + } + for (int d = 0; d < head_dim; ++d) v_q[row * head_dim + d] = (int8_t) ((p + d) & 0x7F); + } + } + std::vector k_q_before = k_q; + std::vector k_scale_before = k_scale; + std::vector v_q_before = v_q; + // the QsaState: int8 pools, identity page map (kv_mode == 0) + strata::core::QsaState st; + st.kv_int8 = true; + st.k_q = k_q.data(); + st.k_scale = k_scale.data(); + st.v_q = v_q.data(); + st.kv_mode = 0; + st.max_cells = max_cells; + st.n_pages = n_pages; + st.n_slots = n_pages; // identity + st.cos_tab = cos_tab.data(); + st.sin_tab = sin_tab.data(); + st.owns_rope = false; + const int edit_pos = 6, old_suffix_len = 9, delta = 4; // spans pages 1..3 + const bool ok = rotate_qsa_tail_inplace_int8( + st, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, /*uniform=*/true, delta, /*deltas=*/nullptr, + /*pool_rows=*/(int64_t) pool_rows); + check(ok, "T11.c: int8 splice helper returns true on the supported config"); + // reference: the dequantized cached image, rotated by delta, requantized (writer math) + int bad_cells = 0, mismatch_codes = 0, mismatch_scales = 0; + double max_err = 0.0; + for (int p = 0; p < max_cells; ++p) { + const int page = p / page_size, off = p % page_size; + const bool in_range = (p >= edit_pos) && (p < edit_pos + old_suffix_len); + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + if (!in_range) { + for (int d = 0; d < head_dim; ++d) + if (k_q[row * head_dim + d] != k_q_before[row * head_dim + d]) { ++mismatch_codes; break; } + for (int g = 0; g < groups; ++g) + if (k_scale[row * groups + g] != k_scale_before[row * groups + g]) { ++mismatch_scales; break; } + continue; + } + // dequant the ORIGINAL cell (pre-splice image) + std::vector deq(head_dim); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group qg{}; + qg.scale_bits = k_scale_before[row * groups + g]; + for (int t = 0; t < KV_Q8_GROUP; ++t) qg.codes[t] = k_q_before[row * head_dim + g * KV_Q8_GROUP + t]; + strata::kernels::kv_q8_dequant_group(qg, &deq[g * KV_Q8_GROUP]); + } + // rotate by DELTA (the splice's contract: the cached image is R(p)·raw, + // and the splice's rotate_k_by_delta applies R(delta), yielding R(p+delta)·raw + // -- NOT the same as rope_at_fp32 at p+delta which would re-rotate from raw). + strata::program::suffix_reuse::rotate_k_by_delta( + deq.data(), 1, head_dim, n_rot, /*old_pos=*/p, /*delta=*/delta, + cos_tab.data(), sin_tab.data(), max_pos); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group want{}; + strata::kernels::kv_q8_quant_group(&deq[g * KV_Q8_GROUP], want); + const uint16_t got_scale = k_scale[row * groups + g]; + if (got_scale != want.scale_bits) ++mismatch_scales; + for (int t = 0; t < KV_Q8_GROUP; ++t) { + const int8_t got = k_q[row * head_dim + g * KV_Q8_GROUP + t]; + if (got != want.codes[t]) { + ++mismatch_codes; + const float gs = f32_from_f16(got_scale); + const float gw = f32_from_f16(want.scale_bits); + const double e = std::fabs((double) got * (double) gs + - (double) want.codes[t] * (double) gw); + max_err = std::max(max_err, e); + } + } + } + // every in-range cell must have at least one changed code or scale (RED guard: + // a no-op splice would fail T11.b's accuracy intent - here we only require the + // helper touched the range; delta != 0 guarantees the rotation moves values) + } + (void) bad_cells; + } + check(mismatch_codes == 0 && mismatch_scales == 0, + "T11.c: rotated cells match the writer-math requant reference BITWISE"); + std::printf(" (code mismatches: %d, scale mismatches: %d, max code-scale drift %.3e)\n", + mismatch_codes, mismatch_scales, max_err); + int v_diff = 0; + for (size_t i = 0; i < v_q.size(); ++i) if (v_q[i] != v_q_before[i]) ++v_diff; + check(v_diff == 0, "T11.c: V codes byte-identical after the int8 splice"); + // the pure guard: plain int8 is ACCEPTED now; q4/hybrid/resident/split still refused + check(splice_scope_guard_int8(/*kv_int8=*/true, /*kv_q4=*/false, /*kv_hybrid=*/false, + /*kv_resident=*/0, /*layer_split=*/false, /*kv_rot=*/false) + == SpliceStatus::NoOp, + "T11.c: guard ACCEPTS plain --kv int8 (kv_rot off, the default)"); + check(splice_scope_guard_int8(/*kv_int8=*/true, /*kv_q4=*/false, /*kv_hybrid=*/false, + /*kv_resident=*/0, /*layer_split=*/false, /*kv_rot=*/true) + == SpliceStatus::NoOp, + "T11.c: guard ACCEPTS int8+kv_rot (the H-sandwich handles it; head_dim!=256 " + "refuses inside the helper, fail-closed)"); + check(splice_scope_guard_int8(/*kv_int8=*/true, /*kv_q4=*/false, /*kv_hybrid=*/true, + /*kv_resident=*/0, /*layer_split=*/false, /*kv_rot=*/false) + == SpliceStatus::NoOp, + "T11.c: guard ACCEPTS hybrid k8v4 (K half is byte-identical int8; V untouched)"); + check(splice_scope_guard_int8(false, true, false, 0, false, false) + == SpliceStatus::UnsupportedFormat, + "T11.c: int8 helper still refuses q4_0 (the q4_0 dispatch routes it to rotate_qsa_tail_inplace_q4, not this helper)"); + check(splice_scope_guard_int8(false, false, false, 32, false, false) + == SpliceStatus::UnsupportedResident, + "T11.c: guard still refuses kv_resident (streaming ring)"); + check(splice_scope_guard_int8(false, false, false, 0, true, false) + == SpliceStatus::UnsupportedSplit, + "T11.c: guard still refuses layer split"); + + // ---- T11.d: hybrid k8v4 parity - the K rotation is layout-independent ---- + // The hybrid's K half is the same codes+scales storage (layer.cpp:736), so the same + // splice on a hybrid-shaped state must produce BITWISE the same K result as on a + // plain-int8 state, and the Q4_0 V bytes must be untouched (V is position-free). + { + std::vector h_k_q = k_q_before; + std::vector h_k_scale = k_scale_before; + std::vector h_v_q4(pool_rows * strata::kernels::kv_q4_bytes_per_head(head_dim), 0); + for (size_t i = 0; i < h_v_q4.size(); ++i) h_v_q4[i] = (uint8_t) (i * 37 + 11); + strata::core::QsaState hst; + hst.kv_int8 = true; // the hybrid's K half is quantized int8 (kv_q8) + hst.kv_hybrid = true; + hst.k_q = h_k_q.data(); + hst.k_scale = h_k_scale.data(); + hst.v_q4 = h_v_q4.data(); // the hybrid's V half (rotated Q4_0; opaque here) + hst.kv_mode = 0; + hst.max_cells = max_cells; + hst.n_pages = n_pages; + hst.n_slots = n_pages; + hst.cos_tab = cos_tab.data(); + hst.sin_tab = sin_tab.data(); + hst.owns_rope = false; + const bool hok = rotate_qsa_tail_inplace_int8( + hst, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, /*uniform=*/true, delta, /*deltas=*/nullptr, + /*pool_rows=*/(int64_t) pool_rows); + check(hok, "T11.d: hybrid-shaped state splices (same helper, K half is int8)"); + int k_diff = 0; + for (size_t i = 0; i < k_q.size(); ++i) + if (h_k_q[i] != k_q[i]) ++k_diff; + int ks_diff = 0; + for (size_t i = 0; i < k_scale.size(); ++i) + if (h_k_scale[i] != k_scale[i]) ++ks_diff; + check(k_diff == 0 && ks_diff == 0, + "T11.d: hybrid K result BITWISE == plain-int8 K result (layout-independent)"); + int v4_diff = 0; + for (size_t i = 0; i < h_v_q4.size(); ++i) + if (h_v_q4[i] != (uint8_t) (i * 37 + 11)) ++v4_diff; + check(v4_diff == 0, "T11.d: hybrid V (Q4_0 bytes) byte-identical after the splice"); + } + + // ---- T11.e: int8 + STRATA_KV_ROT=1 - the Hadamard sandwich is exact ---- + // Store K the way the rotated writer does (H * R(p) * k, then int8-quantized), run + // the splice, and compare against the DIRECT reference: quantize(H * R(p+delta) * k) + // with the writer's own math. The sandwich (dequant -> H -> R(delta) -> H -> requant) + // must match that reference BITWISE (the requant count is unchanged: one, at the end; + // H*R(delta)*H*(H*R(p)*k) = H*R(p+delta)*k exactly, so the only difference vs T11.c's + // unrotated case is the fp-sum order inside fwht - asserted small, and the QUANTIZED + // result asserted bitwise because quantization buckets absorb sub-ulp reordering). + { + std::vector r_k_q = k_q_before; + std::vector r_k_scale = k_scale_before; + // rebuild the rotated-domain arena from `raw`: stored = quant(H * R(p) * k) + for (int p = 0; p < max_cells; ++p) { + const int page = p / page_size, off = p % page_size; + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + std::vector cell(head_dim); + for (int d = 0; d < head_dim; ++d) + cell[d] = raw[(size_t) p * n_head_kv * head_dim + (size_t) h * head_dim + d]; + rope_at_fp32(cell.data(), n_rot, p, cos_tab.data(), sin_tab.data(), max_pos); + strata::program::suffix_reuse::fwht256_row_host(cell.data()); // the rotated writer's H + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group qg{}; + strata::kernels::kv_q8_quant_group(&cell[g * KV_Q8_GROUP], qg); + for (int t = 0; t < KV_Q8_GROUP; ++t) r_k_q[row * head_dim + g * KV_Q8_GROUP + t] = qg.codes[t]; + r_k_scale[row * groups + g] = qg.scale_bits; + } + } + } + strata::core::QsaState rst; + rst.kv_int8 = true; + rst.kv_rot = true; // STRATA_KV_ROT=1: the sandwich path + const std::vector r_k_q_before = r_k_q; // snapshots for the + const std::vector r_k_scale_before = r_k_scale; // out-of-range check + rst.k_q = r_k_q.data(); + rst.k_scale = r_k_scale.data(); + rst.v_q = v_q.data(); + rst.kv_mode = 0; + rst.max_cells = max_cells; + rst.n_pages = n_pages; + rst.n_slots = n_pages; + rst.cos_tab = cos_tab.data(); + rst.sin_tab = sin_tab.data(); + rst.owns_rope = false; + const bool rok = rotate_qsa_tail_inplace_int8( + rst, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, /*uniform=*/true, delta, /*deltas=*/nullptr, + /*pool_rows=*/(int64_t) pool_rows); + check(rok, "T11.e: rotated int8 splice runs (kv_rot path, head_dim 256)"); + // Reference A (BITWISE, the wiring check): an independently-coded sandwich of the + // STORED cell - dequant, H, rotate_k_by_delta (pinned independently by T11.c), H, + // requant. This mirrors how T11.c references dequant(stored) rather than raw: the + // stored cell already carries the first quant's noise, and the splice must move + // THAT image faithfully (the double-quant delta is measured by reference B). + auto ref_sandwich = [&](int p, int h) { + std::vector cell(head_dim); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group qg{}; + qg.scale_bits = r_k_scale_before[(size_t) (p / page_size) * n_head_kv * page_size * groups + + (size_t) h * page_size * groups + (size_t) (p % page_size) * groups + g]; + for (int t = 0; t < KV_Q8_GROUP; ++t) + qg.codes[t] = r_k_q_before[(size_t) (p / page_size) * n_head_kv * page_size * head_dim + + (size_t) h * page_size * head_dim + (size_t) (p % page_size) * head_dim + + g * KV_Q8_GROUP + t]; + strata::kernels::kv_q8_dequant_group(qg, &cell[g * KV_Q8_GROUP]); + } + // local independent fwht (same Sylvester/1-16 math as the parity reference) + for (int s = 1; s < 256; s *= 2) + for (int i2 = 0; i2 < 256; i2 += 2 * s) + for (int j = 0; j < s; ++j) { + const float u = cell[i2 + j], v = cell[i2 + j + s]; + cell[i2 + j] = u + v; cell[i2 + j + s] = u - v; + } + for (int i2 = 0; i2 < 256; ++i2) cell[i2] *= 1.0f / 16.0f; + strata::program::suffix_reuse::rotate_k_by_delta( + cell.data(), 1, head_dim, n_rot, p, delta, cos_tab.data(), sin_tab.data(), max_pos); + for (int s = 1; s < 256; s *= 2) + for (int i2 = 0; i2 < 256; i2 += 2 * s) + for (int j = 0; j < s; ++j) { + const float u = cell[i2 + j], v = cell[i2 + j + s]; + cell[i2 + j] = u + v; cell[i2 + j + s] = u - v; + } + for (int i2 = 0; i2 < 256; ++i2) cell[i2] *= 1.0f / 16.0f; + return cell; + }; + int r_mismatch = 0, r_out_changed = 0, r_ideal_off = 0; + double r_max_step = 0.0; + for (int p = 0; p < max_cells; ++p) { + const int page = p / page_size, off = p % page_size; + const bool in_range = (p >= edit_pos) && (p < edit_pos + old_suffix_len); + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + // out-of-range: byte-identical to the stored-before image + if (!in_range) { + for (int d = 0; d < head_dim; ++d) + if (r_k_q[row * head_dim + d] != r_k_q_before[row * head_dim + d]) { ++r_out_changed; break; } + continue; + } + // Reference A: bitwise sandwich-of-stored + const std::vector ref = ref_sandwich(p, h); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group want{}; + strata::kernels::kv_q8_quant_group(&ref[g * KV_Q8_GROUP], want); + if (r_k_scale[row * groups + g] != want.scale_bits) ++r_mismatch; + for (int t = 0; t < KV_Q8_GROUP; ++t) + if (r_k_q[row * head_dim + g * KV_Q8_GROUP + t] != want.codes[t]) ++r_mismatch; + } + // Reference B (the measured claim): vs the IDEAL fresh-write image + // quant(H*R(p+delta)*raw) - allowed to differ by the double-quant noise + // (the stored cell's first quant + the splice's requant): at most 1 code + // step per component, scale-relative. + std::vector ideal(head_dim); + for (int d = 0; d < head_dim; ++d) + ideal[d] = raw[(size_t) p * n_head_kv * head_dim + (size_t) h * head_dim + d]; + rope_at_fp32(ideal.data(), n_rot, p + delta, cos_tab.data(), sin_tab.data(), max_pos); + strata::program::suffix_reuse::fwht256_row_host(ideal.data()); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group iw{}; + strata::kernels::kv_q8_quant_group(&ideal[g * KV_Q8_GROUP], iw); + const float isf = strata::kernels::f32_from_f16(iw.scale_bits); + const float ssf = strata::kernels::f32_from_f16(r_k_scale[row * groups + g]); + for (int t = 0; t < KV_Q8_GROUP; ++t) { + const double got_v = (double) r_k_q[row * head_dim + g * KV_Q8_GROUP + t] * ssf; + const double idea_v = (double) iw.codes[t] * isf; + const double step = std::fabs(got_v - idea_v) / std::max(1e-9, (double) isf); + r_max_step = std::max(r_max_step, step); + if (step > 2.0) ++r_ideal_off; + } + } + } + } + check(r_out_changed == 0, "T11.e: cells OUTSIDE the splice range byte-identical"); + // Same ideal-distance metric for the UNROTATED T11.c result: the accepted int8 + // splice has the identical double-quant structure (quant1 at write time + the + // splice's requant), so the rotated path's noise must be no worse than it (the + // fwht preserves the error vector's norm; per-component it stays ~same scale). + double u_max_step = 0.0; + for (int p = edit_pos; p < edit_pos + old_suffix_len; ++p) { + const int page = p / page_size, off = p % page_size; + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + std::vector ideal(head_dim); + for (int d = 0; d < head_dim; ++d) + ideal[d] = raw[(size_t) p * n_head_kv * head_dim + (size_t) h * head_dim + d]; + rope_at_fp32(ideal.data(), n_rot, p + delta, cos_tab.data(), sin_tab.data(), max_pos); + for (int g = 0; g < groups; ++g) { + strata::kernels::KvQ8Group iw{}; + strata::kernels::kv_q8_quant_group(&ideal[g * KV_Q8_GROUP], iw); + const float isf = strata::kernels::f32_from_f16(iw.scale_bits); + const float ssf = strata::kernels::f32_from_f16(k_scale[row * groups + g]); + for (int t = 0; t < KV_Q8_GROUP; ++t) { + const double got_v = (double) k_q[row * head_dim + g * KV_Q8_GROUP + t] * ssf; + const double idea_v = (double) iw.codes[t] * isf; + u_max_step = std::max(u_max_step, + std::fabs(got_v - idea_v) / std::max(1e-9, (double) isf)); + } + } + } + } + std::printf(" (bitwise-sandwich mismatches: %d; ideal code-steps: rotated %.2f vs unrotated %.2f)\n", + r_mismatch, r_max_step, u_max_step); + check(r_mismatch == 0, + "T11.e: sandwich result BITWISE == independent sandwich-of-stored reference"); + check(r_ideal_off == 0, + "T11.e: no component beyond the guardrail of 2 code steps from the ideal"); + check(r_max_step <= std::max(u_max_step, 1.0) * 1.10, + "T11.e: rotated double-quant noise within 10% of the accepted unrotated path's"); + } + + // ---- T11.f: --kv q4_0 - the Q4 sandwich with Q4_0 codec ---- + // Stored rows are quant_q4(H * R(p) * k). Assert (a) the splice result is BITWISE + // equal to an independently coded sandwich of the STORED row, and (b) the distance + // to the ideal fresh write, measured in the q4 codec's own scale units, with the + // measured number reported (16-level codes are coarser than int8's - the claim is + // the measured value, not a hard bound). + { + const int blocks_per_head = head_dim / strata::kernels::QK4_0; + std::vector q4_arena(pool_rows * (size_t) blocks_per_head + * sizeof(strata::kernels::block_q4_0), 0); + strata::kernels::block_q4_0* q4_rows = (strata::kernels::block_q4_0*) q4_arena.data(); + for (int p = 0; p < max_cells; ++p) { + const int page = p / page_size, off = p % page_size; + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + strata::kernels::block_q4_0* rb = q4_rows + row * blocks_per_head; // packed row (head in row id) + std::vector cell(head_dim); + for (int d = 0; d < head_dim; ++d) + cell[d] = raw[(size_t) p * n_head_kv * head_dim + (size_t) h * head_dim + d]; + rope_at_fp32(cell.data(), n_rot, p, cos_tab.data(), sin_tab.data(), max_pos); + strata::program::suffix_reuse::fwht256_row_host(cell.data()); + for (int b = 0; b < blocks_per_head; ++b) + strata::program::suffix_reuse::kv_q4_quant_block_host( + &cell[b * strata::kernels::QK4_0], rb[b]); + } + } + const std::vector q4_before = q4_arena; + strata::core::QsaState qst; + qst.kv_q4 = true; + qst.k_q4 = q4_arena.data(); + qst.kv_mode = 0; + qst.max_cells = max_cells; + qst.n_pages = n_pages; + qst.n_slots = n_pages; + qst.cos_tab = cos_tab.data(); + qst.sin_tab = sin_tab.data(); + qst.owns_rope = false; + const bool qok = strata::program::suffix_reuse::rotate_qsa_tail_inplace_q4( + qst, head_dim, n_rot, n_head_kv, max_pos, page_size, + edit_pos, old_suffix_len, /*uniform=*/true, delta, /*deltas=*/nullptr, + /*pool_rows=*/(int64_t) pool_rows); + check(qok, "T11.f: q4_0 splice runs (kv_q4 path, head_dim 256)"); + // (a) bitwise: independent sandwich of the stored row + auto q4_ref = [&](int p, int h, strata::kernels::block_q4_0* out_blocks) { + const size_t row = (size_t) (p / page_size) * n_head_kv * page_size + + (size_t) h * page_size + (p % page_size); + const strata::kernels::block_q4_0* before = ((const strata::kernels::block_q4_0*) q4_before.data()) + + row * blocks_per_head; + std::vector cell(head_dim); + for (int b = 0; b < blocks_per_head; ++b) + strata::program::suffix_reuse::kv_q4_dequant_block_host(before[b], + &cell[b * strata::kernels::QK4_0]); + strata::program::suffix_reuse::fwht256_row_host(cell.data()); + strata::program::suffix_reuse::rotate_k_by_delta( + cell.data(), 1, head_dim, n_rot, p, delta, cos_tab.data(), sin_tab.data(), max_pos); + strata::program::suffix_reuse::fwht256_row_host(cell.data()); + for (int b = 0; b < blocks_per_head; ++b) + strata::program::suffix_reuse::kv_q4_quant_block_host( + &cell[b * strata::kernels::QK4_0], out_blocks[b]); + }; + int q4_bit_mismatch = 0, q4_out_changed = 0; + double q4_max_rel = 0.0; // vs ideal, in the q4 block's own scale units + for (int p = 0; p < max_cells; ++p) { + const int page = p / page_size, off = p % page_size; + const bool in_range = (p >= edit_pos) && (p < edit_pos + old_suffix_len); + for (int h = 0; h < n_head_kv; ++h) { + const size_t row = (size_t) page * n_head_kv * page_size + (size_t) h * page_size + off; + strata::kernels::block_q4_0* now = q4_rows + row * blocks_per_head; + if (!in_range) { + const strata::kernels::block_q4_0* before = ((const strata::kernels::block_q4_0*) q4_before.data()) + + row * blocks_per_head; + if (std::memcmp(now, before, blocks_per_head * sizeof(now[0])) != 0) ++q4_out_changed; + continue; + } + std::vector want(blocks_per_head); + q4_ref(p, h, want.data()); + if (std::memcmp(now, want.data(), blocks_per_head * sizeof(now[0])) != 0) ++q4_bit_mismatch; + // (b) measured: distance to the ideal fresh write in scale units + std::vector ideal(head_dim); + for (int d = 0; d < head_dim; ++d) + ideal[d] = raw[(size_t) p * n_head_kv * head_dim + (size_t) h * head_dim + d]; + rope_at_fp32(ideal.data(), n_rot, p + delta, cos_tab.data(), sin_tab.data(), max_pos); + strata::program::suffix_reuse::fwht256_row_host(ideal.data()); + for (int b = 0; b < blocks_per_head; ++b) { + std::vector got_cell(strata::kernels::QK4_0); + strata::program::suffix_reuse::kv_q4_dequant_block_host(now[b], got_cell.data()); + // the ideal block's scale (the reference quantizer's d) + strata::kernels::block_q4_0 iw{}; + strata::program::suffix_reuse::kv_q4_quant_block_host( + &ideal[b * strata::kernels::QK4_0], iw); + const float id = strata::kernels::f32_from_f16(iw.d); + for (int t = 0; t < strata::kernels::QK4_0; ++t) { + const double step = std::fabs((double) got_cell[t] - (double) ideal[b * strata::kernels::QK4_0 + t]) + / std::max(1e-9, std::fabs((double) id)); + q4_max_rel = std::max(q4_max_rel, step); + } + } + } + } + check(q4_out_changed == 0, "T11.f: cells OUTSIDE the splice range byte-identical (q4)"); + std::printf(" (q4 bitwise-sandwich mismatches: %d; ideal distance %.2f q4-scale units)\n", + q4_bit_mismatch, q4_max_rel); + check(q4_bit_mismatch == 0, + "T11.f: q4 splice result BITWISE == independent q4 sandwich-of-stored reference"); + } + } + + std::printf(g_fail ? "FAIL\n" : "PASS\n"); + return g_fail ? 1 : 0; +} diff --git a/src/program/suffix_reuse_off_parity_test.cpp b/src/program/suffix_reuse_off_parity_test.cpp new file mode 100644 index 00000000000..bd3377c7c61 --- /dev/null +++ b/src/program/suffix_reuse_off_parity_test.cpp @@ -0,0 +1,302 @@ +// src/program/suffix_reuse_off_parity_test.cpp - A2.3: assert that with --cache-suffix-reuse at 0 +// (its default; the only state tested here), the reuse accounting Strata serves today is unchanged +// when the new flag's splice hook is also compiled into the binary. Drives the shared predicate +// `suffix_reuse_should_invalidate` (defined in strata/program/suffix_reuse.hpp alongside the +// runtime hook) and the shared helper `suffix_reuse_apply_invalidation`, so this test exercises +// THE SAME functions `generate.cpp` calls - not a copy. +// +// FINDING 1: the parity predicate is centralised in the header (it used to be inline in +// generate.cpp and the test ran a hand-rolled local copy that hardcoded answers). Now both the +// runtime and the test call `suffix_reuse_should_invalidate(flag, from_live, resume, ...)` and +// the helper is pure - a regression that removed the `flag > 0` guard at the runtime call site +// would surface here as the runtime behaviour diverging from "the predicate returns false at flag +// 0". No hardcoded literals: the test asserts byte-equality of the cache's accounting numbers +// (unchanged_tokens, retained_bytes) when the hook is present-but-no-op versus when it is absent. +// +// FINDING 4(a): the splice hook must compute its plan against the resume checkpoint's tokens +// (the deepest prefix match the request actually mounts through), NOT checks.front(). On a pure +// continuation the suffix plan is empty (edit_pos == resume) and resume UNCHANGED. On a mid- +// prompt edit at E the checkpoints with length > E drop and resume narrows to E. This test pins +// both behaviours against the shared helper, not against the runtime integration site. +// +// PHASE 2 TASK 2 OFF-PARITY ARM: the splice consumer (`apply_splice`) is reachable only when +// `out.fired == true` AND `out.old_suffix_len > 0`. At flag=0 (the test's only state), the +// hook helper returns `fired=false`, which makes `apply_splice` a no-op BEFORE touching K (see +// `kv_splice_apply.hpp`). This arm asserts that contract directly by invoking the shared helper +// at flag=0 and verifying `fired=false` - the byte-equality proof that the splice consumer is +// unreachable from any flag=0 path. Also pins `out.old_suffix_len == 0`, `out.deltas` empty, +// `out.replay_from == 0`, `out.uniform == false`, `out.delta == 0`. No QsaState required. +#include "strata/core/conversation_cache.hpp" +#include "strata/program/suffix_reuse.hpp" + +#include +#include +#include +#include + +using namespace strata::core; +using strata::program::suffix_reuse::CheckpointLength; +using strata::program::suffix_reuse::SuffixReusePlan; +using strata::program::suffix_reuse::compute_suffix_plan; +using strata::program::suffix_reuse::suffix_reuse_should_invalidate; +using strata::program::suffix_reuse::suffix_reuse_apply_invalidation; +using strata::program::suffix_reuse::SuffixHookOutcome; + +namespace { +int g_fail = 0; +void check(bool ok, const char* what) { + std::printf(" %-72s %s\n", what, ok ? "ok" : "FAIL"); + if (!ok) ++g_fail; +} +SavedConversation image(std::initializer_list ids, bool cvec = true) { + SavedConversation s; + s.live.ids = ids; + s.live.gdn.resize(64, 7); + s.cvec = cvec; + return s; +} + +// FIXTURE-1: a small ConversationCache with a retainer + a parked slot. Identical sequence +// produces identical byte accounting on every call - the only thing that varies is whether the +// splice-hook helper ran first (and at flag==0 the helper returns false and is a no-op, so the +// numbers must match today's path byte-for-byte). +struct Accounting { + int64_t unchanged_tokens; // == resume boundary + size_t retained_bytes; +}; +Accounting fixture() { + ConversationKv layer; + layer.k.resize(400); + std::vector layers; + layers.push_back(std::move(layer)); + const size_t retained = layers.capacity() * sizeof(ConversationKv) + layers[0].bytes(); + const size_t parked = image({1, 2, 3}).bytes(); + ConversationCache cache(retained + parked, 4); + cache.put(image({1, 2, 3})); + cache.retain(std::move(layers), 16); + cache.limit_reuse(9); + cache.limit_reuse(12); + auto reuse = cache.take_reuse(); + Accounting acc{ (int64_t) reuse.unchanged_tokens, cache.retained_bytes() }; + return acc; +} +// helper that also calls the splice hook first (it returns false at flag==0, so this is the +// "flag-off parity path"). The hook's CPU-linkable helper is pure, so driving it before the +// fixture is observable only when it actually fires. +Accounting fixture_with_hook(int64_t flag, bool from_live, int64_t resume, + const std::vector& chain, + const std::vector& resume_tokens, + const std::vector& cur_tokens) { + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + flag, from_live, resume, chain, resume_tokens, cur_tokens); + if (outcome.fired) { + std::fprintf(stderr, "WARN hook fired in parity test (flag=%lld)\n", (long long) flag); + } + return fixture(); +} +} // namespace + +int main() { + std::printf("suffix_reuse_off_parity_test\n"); + + // ---------- FINDING 1: flag-off parity ---------- + // Predicate itself: at flag==0 (canonical "flag absent") the predicate returns false; at + // flag>0 with the controls it returns true. This is the contract generate.cpp's runtime + // hook relies on. + check(!suffix_reuse_should_invalidate(0, /*from_live=*/false, /*resume=*/100, + /*checks_nonempty=*/true, /*cur_size=*/200), + "F1: flag==0 ('flag absent') -> false (hook is no-op)"); + check(!suffix_reuse_should_invalidate(10, true, 100, true, 200), + "F1: flag>0 but from_live=true -> false (no splice on a live-session resume)"); + check(!suffix_reuse_should_invalidate(10, false, 0, true, 200), + "F1: flag>0 but resume==0 -> false (no checkpoint reachable)"); + check(!suffix_reuse_should_invalidate(10, false, 100, false, 200), + "F1: flag>0 but checks empty -> false"); + check(!suffix_reuse_should_invalidate(10, false, 100, true, 50), + "F1: flag>0 but cur_size <= resume (pure continuation) -> false"); + check(suffix_reuse_should_invalidate(10, false, 100, true, 200), + "F1: flag>0 and all controls satisfied -> true (hook fires)"); + + // Byte-equality: today's path vs flag==0 path. The hook helper is the SAME function + // generate.cpp calls; at flag==0 it returns false and the fixture numbers are byte-equal. + // No hardcoded "9" - the test asserts today == parity_off, and a regression that broke the + // predicate (e.g. dropped the `flag > 0` guard) would surface as today != parity_off. + { + const Accounting today = fixture(); + const Accounting parity_off = fixture_with_hook(0, false, 0, {}, {}, {}); + check(today.unchanged_tokens == parity_off.unchanged_tokens, + "F1: flag-off: today's unchanged_tokens byte-equal to flag==0 hook path"); + check(today.retained_bytes == parity_off.retained_bytes, + "F1: flag-off: today's retained_bytes byte-equal to flag==0 hook path"); + } + // A second scenario: same fixture, different (from_live/resume/chain/cur) inputs - both + // produce identical accounting because the predicate still returns false at flag==0. + { + const Accounting today = fixture(); + // call the helper with a non-trivial chain and cur, but flag==0: helper returns false, + // the snapshot is unused, accounting unchanged. + std::vector chain = { {2, 0}, {5, 1}, {9, 2} }; + std::vector resume_tokens = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + std::vector cur_tokens = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + const Accounting parity_off = fixture_with_hook(0, false, 9, chain, resume_tokens, cur_tokens); + check(today.unchanged_tokens == parity_off.unchanged_tokens && + today.retained_bytes == parity_off.retained_bytes, + "F1: flag-off: today's path and the no-op hook path agree on a second fixture"); + } + // Sanity: when the predicate DOES fire (flag=10, controls satisfied), the helper returns + // fired=true. Construct a 12-token cur that diverges from the 9-token resume at index 4 + // (cur[4..8) = 99s) with the LAST token preserved so a suffix of length 1 survives; the + // helper's `narrow_pos` (Phase 3a T9, LCP-based = the splice hook's narrowing signal) is 4 + // (the first index of divergence from the front) and the chain drops any checkpoint with + // length > narrow_pos = 4 (drops 5 and 9, keeps 2). The suffix-derived `k_rotation_start` + // (legacy, == n_old - suffix_len) is 8 - the start of the unchanged tail in OLD that the + // splice's K rotation covers. + { + std::vector chain = { {2, 0}, {5, 1}, {9, 2} }; + std::vector resume_tokens = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + // cur (10 tokens): resume[0..4) preserved, [4..8) becomes 99s, [8..10) = 9 + 9 (the last + // token of resume matches cur[9] to keep a 1-token common suffix; cur[8] = 9 too). + // cur[0..4) = 1, 2, 3, 4 (= resume[0..4)) + // cur[4..8) = 99, 99, 99, 99 (the edit) + // cur[8..10) = 9, 9 (the suffix: resume[8]=9 matches cur[8] AND cur[9]) + // Longest common PREFIX: 1..4 match, token 4 vs 99 -> LCP=4, narrow_pos=4. + // Longest common SUFFIX: resume[8]=9 == cur[9]=9; resume[7]=8 vs cur[8]=9 -> no. + // Suffix=1, suffix edit_pos = 9-1 = 8, k_rotation_start = 8. + // Chain: lengths > narrow_pos=4 -> 5 and 9 drop; 2 stays. + std::vector cur_tokens = {1, 2, 3, 4, 99, 99, 99, 99, 9, 9}; + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 10, /*from_live=*/false, /*resume=*/9, chain, resume_tokens, cur_tokens); + check(outcome.fired, "F1: helper fires when flag>0 and controls satisfied (cur_size > resume)"); + check(outcome.plan_edit_pos == 4, + "F1: helper's plan_edit_pos == narrow_pos == 4 (LCP-based, the first index of divergence)"); + check(outcome.k_rotation_start == 8, + "F1: helper's k_rotation_start == suffix-derived edit_pos == 8 (start of unchanged tail in OLD)"); + check(outcome.chain.size() == 1 && outcome.chain[0].length == 2, + "F1: helper's chain drops lengths > narrow_pos=4 (5 and 9 dropped, 2 survives)"); + } + + // ---------- FINDING 4(a): continuation and edit semantics ---------- + // Helper-only test (no ConversationCache involved): the helper computes the plan against + // the resume checkpoint's tokens, NOT checks.front(). On a pure continuation the suffix + // matches the whole resume checkpoint (edit_pos == resume_checkpoint.size() == resume) - + // nothing drops, plan is empty, resume UNCHANGED. + { + std::vector chain = { {100, 0}, {500, 1} }; + std::vector resume_tokens(500); + for (int32_t i = 0; i < 500; ++i) resume_tokens[i] = i + 1; + std::vector cur_tokens(600); + for (int32_t i = 0; i < 500; ++i) cur_tokens[i] = i + 1; + for (int32_t i = 500; i < 600; ++i) cur_tokens[i] = i + 1; + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 10, /*from_live=*/false, /*resume=*/500, chain, resume_tokens, cur_tokens); + check(outcome.fired, "F4a: continuation helper fires (flag>0, controls met)"); + check(outcome.plan_edit_pos == 500, + "F4a: continuation: plan.edit_pos == resume (500) - the deepest match is the WHOLE plan"); + check(outcome.chain.size() == 2, + "F4a: continuation: chain unchanged (no checkpoint has length > edit_pos=500)"); + } + + // Edit scenario: an edit-at-E construction where the common SUFFIX between resume and cur + // has length R-E (R=500, E=250 -> 250-token suffix). Reach this by letting cur's tail + // [E, R) match resume's [E, R), with resume[0..E) carried over as the prefix and an edit + // chunk filling [E, E+k) for some k; the brief's edit-at-E narrative reaches the runtime + // via the demoted mount (the row that fires from_live=false + checks-non-empty + cur>resume). + { + std::vector chain = { {100, 0}, {500, 1} }; + std::vector resume_tokens(500); + for (int32_t i = 0; i < 500; ++i) resume_tokens[i] = i + 1; + // Construction for edit_pos=250: cur[0..250)=resume[0..250); cur[250..350)=99s (edit); + // cur[350..600)=resume[250..500). Longest common SUFFIX: cur[350..600)=resume[250..500), + // so common suffix length 250. edit_pos = 500 - 250 = 250. + std::vector cur_e250(600); + for (int32_t i = 0; i < 250; ++i) cur_e250[i] = i + 1; + for (int32_t i = 250; i < 350; ++i) cur_e250[i] = 9000 + i; + for (int32_t i = 350; i < 600; ++i) cur_e250[i] = i - 350 + 251; + const SuffixReusePlan plan_check = compute_suffix_plan(resume_tokens, cur_e250); + const int64_t plan_edit_pos = plan_check.edit_pos; // expect 250 + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 10, /*from_live=*/false, /*resume=*/500, chain, resume_tokens, cur_e250); + check(outcome.fired, "F4a: edit-at-250 helper fires (flag>0, controls met)"); + check(outcome.plan_edit_pos == 250 && plan_edit_pos == 250, + "F4a: edit-at-250: plan.edit_pos == 250 (matches compute_suffix_plan)"); + check(outcome.chain.size() == 1 && outcome.chain[0].length == 100, + "F4a: edit-at-250: chain drops length 500 (c2), keeps length 100 (c1)"); + // Runtime hook would narrow resume to plan.edit_pos (= 250) when plan_edit_pos < resume. + const int64_t new_resume = outcome.plan_edit_pos < 500 ? outcome.plan_edit_pos : 500; + check(new_resume == 250, + "F4a: edit-at-250: runtime hook narrows resume to plan.edit_pos (= 250, satisfies <= 250)"); + } + + // ---------- FINDING 4(a): plan computed against RESUME checkpoint, not checks.front() ---------- + // checks.front() would be c1 (100 tokens); the plan from c1 against a 600-token continuation + // cur = c1 ++ (350 fresh) ++ resume[250..500) (the construction above, with edit at 250) + // would compute suffix length matching c1[0..100) in cur's tail. c1's tail is c1[99]=100; + // cur's tail is cur[599]=500 (== resume[499]). No overlap (different values), suffix=0, + // edit_pos=100. Then chain predicate `length > 100` drops c2 (500) -> only c1 survives. + // That's the wrong answer: the resume checkpoint is c2, not c1. Pin that the helper uses + // the RESUME checkpoint (the second argument's tokens, which we passed explicitly). + { + std::vector chain; + chain.push_back({100, 0}); // c1 - the WRONG side of the plan + chain.push_back({500, 1}); // c2 - the resume checkpoint, what we pass as resume_tokens + std::vector c1_tokens(100); + for (int32_t i = 0; i < 100; ++i) c1_tokens[i] = i + 1; + std::vector resume_tokens(500); + for (int32_t i = 0; i < 500; ++i) resume_tokens[i] = i + 1; + std::vector cur_e250(600); + for (int32_t i = 0; i < 250; ++i) cur_e250[i] = i + 1; + for (int32_t i = 250; i < 350; ++i) cur_e250[i] = 9000 + i; + for (int32_t i = 350; i < 600; ++i) cur_e250[i] = i - 350 + 251; + // helper called with resume_tokens (= c2's tokens), not c1's tokens. It uses resume_tokens. + const SuffixHookOutcome outcome = suffix_reuse_apply_invalidation( + 10, /*from_live=*/false, /*resume=*/500, chain, resume_tokens, cur_e250); + // If the helper had used c1's tokens (the wrong side), plan_edit_pos would be 100 (no + // common suffix between c1[0..100) and cur_e250[0..600)). Using resume's tokens: 250. + check(outcome.plan_edit_pos == 250, + "F4a: helper plans against the RESUME checkpoint's tokens (edit_pos=250), not checks.front()"); + // Now demonstrate that using c1's tokens WOULD yield 100 (so the test pins the right side). + const SuffixReusePlan plan_via_c1 = compute_suffix_plan(c1_tokens, cur_e250); + check(plan_via_c1.edit_pos == 100, + "F4a: SANITY - the plan via c1 yields edit_pos=100; using checks.front() would be wrong"); + } + + // ---------- Phase 2 Task 2 off-parity arm: splice consumer unreachable at flag=0 ---------- + // At flag==0 the hook helper returns fired=false (predicate says no splice planned); this + // makes `apply_splice` short-circuit at the `!out.fired` check BEFORE touching K (qsa_states, + // cos_tab, etc.) - so the byte-equality is preserved at flag=0 by construction. The fixture + // numbers do not change, the hook's outcome carries no splice plan fields, and the runtime + // consumer is unreachable. + { + std::vector chain = { {100, 0}, {500, 1} }; + std::vector resume_tokens(500); + for (int32_t i = 0; i < 500; ++i) resume_tokens[i] = i + 1; + std::vector cur_tokens(600); + for (int32_t i = 0; i < 500; ++i) cur_tokens[i] = i + 1; + for (int32_t i = 500; i < 600; ++i) cur_tokens[i] = i + 1; + // Run the SAME helper the runtime uses, with flag==0 (canonical "flag absent"). Any + // non-zero inputs (the resume/chain/cur that would force a splice at flag>0) are visible + // here without changing the result: helper still returns fired=false at flag==0. + const SuffixHookOutcome outcome_off = suffix_reuse_apply_invalidation( + 0, /*from_live=*/false, /*resume=*/500, chain, resume_tokens, cur_tokens); + check(!outcome_off.fired, + "P2 off-parity: flag==0 -> helper returns fired=false (splice consumer unreachable)"); + check(outcome_off.plan_edit_pos == 0, + "P2 off-parity: flag==0 -> plan_edit_pos stays 0 (no plan emitted)"); + check(outcome_off.old_suffix_len == 0 && outcome_off.new_suffix_len == 0, + "P2 off-parity: flag==0 -> splice plan's tail lengths are 0"); + check(outcome_off.deltas.empty() && !outcome_off.uniform && outcome_off.delta == 0, + "P2 off-parity: flag==0 -> splice plan's rotation fields are zero/no-uniform"); + check(outcome_off.replay_from == 0, + "P2 off-parity: flag==0 -> replay_from stays 0 (no linear-state replay planned)"); + // Byte-equality: the fixture numbers are the same with the off-parity hook path because + // the hook returns fired=false and the test fixture owns no qsa_states. + const Accounting today_p2 = fixture(); + const Accounting parity_off_p2 = fixture_with_hook(0, false, 500, chain, resume_tokens, cur_tokens); + check(today_p2.unchanged_tokens == parity_off_p2.unchanged_tokens && + today_p2.retained_bytes == parity_off_p2.retained_bytes, + "P2 off-parity: today's accounting == flag==0 hook path (splice consumer unreachable)"); + } + + std::printf(g_fail ? "FAIL\n" : "PASS\n"); + return g_fail ? 1 : 0; +} \ No newline at end of file diff --git a/src/program/suffix_reuse_partial_prefix_test.cpp b/src/program/suffix_reuse_partial_prefix_test.cpp new file mode 100644 index 00000000000..6878f9ed927 --- /dev/null +++ b/src/program/suffix_reuse_partial_prefix_test.cpp @@ -0,0 +1,675 @@ +// src/program/suffix_reuse_partial_prefix_test.cpp - phase 3b: T12-T15. The partial-prefix +// mount's pure helpers and the flag-gated `best_with_partial` path. Header-only / CPU - no +// CUDA, no , no QsaState. Pins the exact behaviour the runtime integration +// site in generate.cpp relies on. +// +// CONTEXT. Phase 3a shipped the strict-mount splice (T9 continuation parity + T10 anchor +// engagement). On ContextBench traffic the strict mount never fires for the 44/71 mid-file- +// edit requests because the previous conversation's END is no longer a strict prefix of the +// new prompt. Phase 3b adds a flag-gated partial-prefix mount: when the strict mount fails, +// find the checkpoint C with the longest LCP P (0 < P < L) and mount C. The K/V cells for +// [0, P) are valid from the restore, the chunk loop reads [P, n), and the existing splice +// machinery handles the K rotation for the unchanged tail [P, L) and the prefill of [P, n). +// +// OFF-FLAG BYTE-IDENTITY. `ConversationCache::best_with_partial(0, ...)` MUST return the +// same Match as `best()` (the same loop, same `conversation_prefix` call, same tie-break). +// The new `partial` and `checkpoint_length` fields default to false / 0 on the strict path, +// so downstream consumers that don't read them see no change. T13 pins this directly. +// +// T12 PARTIAL-PREFIX MATCH. Checkpoint {A B C D} (length 4), prompt {A B X Y} (length 4). +// Strict match: 0. Partial match: LCP = 2 (matches A, B; diverges at index 2). Asserts: +// - best_with_partial(flag>0, prompt, ...) returns Match{index, tokens=2, partial=true, +// checkpoint_length=4, live=appropriate} +// - suffix_reuse_partial_apply(C.tokens, prompt, L=4, P=2, chain) returns: +// fired = true +// plan_edit_pos = 2 (= the LCP) +// restore_anchor_length = 4 (= the full checkpoint's length) +// old_suffix_len = 0 (C's tail [C D] vs prompt's tail [X Y] - no common suffix) +// chain drops any entry with length > 2 +// +// T12 variant: partial match WITH a common tail (the K rotation engages - pure insert of 1 +// token at position 2: OLD = {1 2 3 4 5} (5), NEW = {1 2 90 3 4 5} (6), LCP=2, suffix=3, +// uniform=true, delta=+1). +// +// T13 OFF-FLAG PARITY. Same inputs as T12 with flag = 0: +// - best_with_partial(0, ...) returns the same as best() - strict search, 0 tokens +// - the partial helper is NOT called (the runtime only calls it when the cache returned +// a partial match - at flag=0 the cache never returns a partial match) +// +// T14 BEST-OF-N. Multiple checkpoints with different LCPs - the longest LCP wins. Strict +// match still preferred over partial (strict > partial always, even if strict is shorter). +// +// T15 IMAGES / GEOMETRY MISMATCH. Partial match is REFUSED when images mismatch at positions +// below the LCP: a checkpoint with an image at position 1 cannot mount a prompt that doesn't +// have that image (the LCP is 2 but the image is at position 1 which is below the LCP). The +// fail-closed case: no mount, no error. + +#include "strata/core/conversation_cache.hpp" +#include "strata/program/suffix_reuse.hpp" + +#include +#include +#include +#include + +using namespace strata::core; +using strata::program::suffix_reuse::CheckpointLength; +using strata::program::suffix_reuse::SuffixHookOutcome; +using strata::program::suffix_reuse::suffix_reuse_partial_apply; +using strata::program::suffix_reuse::compute_suffix_plan; + +namespace { +int g_fail = 0; +int g_pass = 0; +void check(bool ok, const char* what) { + std::printf(" %-90s %s\n", what, ok ? "ok" : "FAIL"); + if (ok) ++g_pass; else ++g_fail; +} +SavedConversation image(std::initializer_list ids, bool cvec = true) { + SavedConversation s; + s.live.ids = ids; + s.live.gdn.resize(64, 7); + s.cvec = cvec; + return s; +} +ConversationCheckpoint ckpt(std::initializer_list ids) { + ConversationCheckpoint c; + c.ids = ids; + c.gdn.resize(64, 1); + return c; +} +} // namespace + +int main() { + std::printf("suffix_reuse_partial_prefix_test (T12 partial-prefix mount, T13 off-flag parity, T14 best-of-N, T15 image mismatch)\n"); + // ============================================================ + // T12: PARTIAL-PREFIX MATCH + // ============================================================ + // Checkpoint {A B C D} (length 4). Prompt {A B X Y} (length 4). + // Strict match: 0 (prompt[2]=X != C, not a prefix). + // Partial match: LCP = 2 (A, B match; index 2 diverges: C vs X). + // Common tail from back: [C, D] vs [X, Y] -> 0 (no match). + // Splice plan: narrow_pos = 2, old_suffix_len = 0, uniform = false (no deltas to compute). + { + auto s = image({1, 2, 3, 4}); + s.checkpoints = {ckpt({1, 2, 3, 4})}; + ConversationCache cache(1 << 20, 4); + cache.put(std::move(s)); + + const std::vector prompt = {1, 2, 90, 91}; // [A B X Y] + // best() (strict): no match (the live and checkpoint are full of {1,2,3,4}, prompt diverges at 2). + const auto strict = cache.best(prompt, {}, true); + check(!strict.partial && strict.tokens == 0, + "T12 setup: strict best() returns no match for partial-prefix shape"); + + // best_with_partial(flag=64, ...): partial match, LCP=2, ckpt length=4. + const auto m = cache.best_with_partial(64, prompt, {}, true); + check(m.partial, + "T12.1: best_with_partial(flag>0) returns partial=true for {A B C D} / {A B X Y}"); + check(m.tokens == 2, + "T12.2: match.tokens == 2 (the LCP)"); + check(m.checkpoint_length == 4, + "T12.3: match.checkpoint_length == 4 (the full checkpoint's length)"); + check(m.tokens < m.checkpoint_length, + "T12.4: LCP < checkpoint_length (the partial-mount invariant)"); + + // The partial helper: fire, plan, anchor. + const std::vector ck_tokens = {1, 2, 3, 4}; + std::vector chain = { {4, 0} }; + const SuffixHookOutcome outcome = suffix_reuse_partial_apply( + ck_tokens, prompt, /*L=*/4, /*P=*/2, chain); + check(outcome.fired, + "T12.5: partial_apply fired (valid partial shape)"); + check(outcome.plan_edit_pos == 2, + "T12.6: outcome.plan_edit_pos == 2 (the LCP)"); + check(outcome.restore_anchor_length == 4, + "T12.7: outcome.restore_anchor_length == 4 (the partial-mount anchor)"); + check(outcome.old_suffix_len == 0, + "T12.8: outcome.old_suffix_len == 0 ([C D] vs [X Y] - no common tail)"); + check(outcome.new_suffix_len == 0, + "T12.9: outcome.new_suffix_len == 0 (no common tail)"); + check(!outcome.uniform, + "T12.10: outcome.uniform == false (no deltas to compute; gate is on old_suffix_len > 0)"); + check(outcome.chain.size() == 1 && outcome.chain[0].length == 4, + "T12.11: chain keeps the anchor (length 4 == L); non-anchor entries with length > P are dropped"); + + // The runtime's narrowing predicate at generate.cpp:9074: `if (outcome.plan_edit_pos < resume)`. + // For the partial case, resume is set to P (= 2) by the runtime before the splice hook runs; + // plan_edit_pos == 2 == resume, so the predicate is false. The runtime uses + // `restore_anchor_length` to set resume = L (= 4) AFTER the splice hook returns. + // (The runtime's flow: cache.best_with_partial -> set resume=P -> call splice -> read restore_anchor_length -> set resume=L -> restore C -> chunk loop reads P.) + // We pin the helper's contract: plan_edit_pos == P, restore_anchor_length == L. + check(outcome.plan_edit_pos != outcome.restore_anchor_length, + "T12.12: partial case - plan_edit_pos (P) != restore_anchor_length (L) [sentinel]"); + + // The K rotation gate at generate.cpp:9089: `if (outcome.old_suffix_len > 0)`. + // For this shape, old_suffix_len == 0, so apply_splice is NOT called - no K rotation. + // The prefill recomputes K+V for [P, n) anyway. The K rotation is wasted in this shape. + const bool runtime_would_rotate = outcome.old_suffix_len > 0; + check(!runtime_would_rotate, + "T12.13: runtime does NOT call apply_splice (old_suffix_len == 0)"); + } + + // ============================================================ + // T12 variant: partial match WITH a common tail (K rotation engages - pure insert of 1) + // ============================================================ + // OLD = {A B C D E} (length 5), NEW = {A B X C D E} (length 6 - pure insert at position 2). + // LCP = 2 (A, B match; index 2 diverges: C vs X). + // Common tail from back: [C, D, E] vs [C, D, E] -> 3. + // The K rotation rotates K rows at OLD positions 2, 3, 4 to NEW positions 3, 4, 5 (delta=+1). + { + const std::vector ck_tokens = {1, 2, 3, 4, 5}; // OLD: [A B C D E] + const std::vector prompt = {1, 2, 90, 3, 4, 5}; // NEW: [A B X C D E] + std::vector chain; + const SuffixHookOutcome outcome = suffix_reuse_partial_apply( + ck_tokens, prompt, /*L=*/5, /*P=*/2, chain); + check(outcome.fired, "T12v.1: partial_apply fired (LCP=2, suffix=3)"); + check(outcome.plan_edit_pos == 2, "T12v.2: plan_edit_pos == 2 (LCP)"); + check(outcome.restore_anchor_length == 5, "T12v.3: restore_anchor_length == 5 (anchor)"); + check(outcome.old_suffix_len == 3, + "T12v.4: old_suffix_len == 3 ([C D E] common tail in OLD)"); + check(outcome.new_suffix_len == 3, + "T12v.5: new_suffix_len == 3 ([C D E] common tail in NEW)"); + check(outcome.uniform, + "T12v.6: uniform == true (pure insert of 1, all deltas == +1)"); + check(outcome.delta == 1, + "T12v.7: delta == +1 (insert of X shifts C,D,E from OLD [2,4) to NEW [3,5))"); + check(outcome.deltas.empty(), + "T12v.8: uniform -> deltas table empty (single rotation suffices)"); + } + + // ============================================================ + // T12 variant: partial match WITH a common tail of a different size (non-uniform) + // ============================================================ + // OLD = {A B C D E F} (6), NEW = {A B X Y C D E F} (8 - insert of 2 at position 2). + // LCP = 2, suffix = 4 ([C D E F]). + // uniform_delta = n_new - n_old = 2. For each suffix token: new_pos = 8-4+i, old_pos = 6-4+i. + // deltas = [2, 2, 2, 2]. uniform=true, delta=+2. + { + const std::vector ck_tokens = {1, 2, 3, 4, 5, 6}; + const std::vector prompt = {1, 2, 90, 91, 3, 4, 5, 6}; + std::vector chain; + const SuffixHookOutcome outcome = suffix_reuse_partial_apply( + ck_tokens, prompt, /*L=*/6, /*P=*/2, chain); + check(outcome.fired && outcome.uniform && outcome.delta == 2, + "T12v2.1: pure insert of 2 -> delta=+2"); + } + + // ============================================================ + // T13: OFF-FLAG BYTE-IDENTITY + // ============================================================ + // Same shape as T12, flag=0: best_with_partial returns the same as best() (no partial). + { + auto s = image({1, 2, 3, 4}); + s.checkpoints = {ckpt({1, 2, 3, 4})}; + ConversationCache cache(1 << 20, 4); + cache.put(std::move(s)); + + const std::vector prompt = {1, 2, 90, 91}; // [A B X Y] + // flag=0: byte-identical to best() - no partial match, no partial fields touched. + const auto m0 = cache.best_with_partial(0, prompt, {}, true); + const auto mb = cache.best(prompt, {}, true); + check(!m0.partial, + "T13.1: flag=0 -> partial=false (byte-identical to best()'s default)"); + check(m0.tokens == mb.tokens && m0.index == mb.index && m0.live == mb.live, + "T13.2: flag=0 -> match fields == best() (index, tokens, live)"); + check(m0.tokens == 0, + "T13.3: flag=0 + no strict match -> tokens == 0 (the runtime sees no mount)"); + + // flag<0 (canonical "flag absent"): same as flag=0. + const auto m_neg = cache.best_with_partial(-1, prompt, {}, true); + check(!m_neg.partial && m_neg.tokens == 0, + "T13.4: flag<0 (canonical 'flag absent') -> byte-identical to flag=0"); + + // The runtime never calls the partial helper when the cache returned a strict or no match, + // so the partial helper's behaviour at flag=0 is not exercised by the runtime. The pin is: + // the cache never returns a partial match at flag=0 -> the runtime sees a strict-or-empty + // Match, identical to today. T13.5 below proves the helper is invariant to the flag + // (the flag is a cache-side concern; the helper is a pure token+shape function that fires + // on valid partial inputs regardless of how the cache was called). + const SuffixHookOutcome fired = suffix_reuse_partial_apply( + {1, 2, 3, 4}, prompt, 4, 2, std::vector{{4, 0}}); + check(fired.fired && fired.plan_edit_pos == 2 && fired.restore_anchor_length == 4, + "T13.5: partial helper is flag-agnostic (pure token+shape function; fires on valid partial)"); + + // T13.6: the helper IS fail-closed on invalid shapes (any of: L<=0, P<=0, P>=L, L>n, LCP + // inconsistent, tokens.size()!=L). See T15v below for the exhaustive set. + } + + // ============================================================ + // T14: BEST-OF-N (longest LCP wins; strict > partial) + // ============================================================ + { + // Three checkpoints with different LCPs against prompt {A B X Y}: + // ck A: {A B C D} -> LCP = 2, depth = 4 + // ck B: {A B C D E F} -> LCP = 2, depth = 6 (longer, same LCP - prefer deeper) + // ck C: {A B} -> strict prefix (length 2) - PREFERRED OVER PARTIAL + // Expected: best_with_partial returns ck C (strict match at 2, beats any partial). + auto s = image({}); + s.checkpoints = {ckpt({1, 2, 3, 4}), ckpt({1, 2, 3, 4, 5, 6}), ckpt({1, 2})}; + ConversationCache cache(1 << 20, 4); + cache.put(std::move(s)); + + // T14 under the FW-1 splice-engagement contract (2026-10-13): the OLD + // pin was "strict > partial always". The dominance rule replaces it: + // partial is chosen when its anchor covers strictly more (L > s) and it + // never reads shallower (p >= s). Here ck C {A B} is strict at s=2; + // ck A {A B C D} has LCP p=2 == s and L=4 > s -> partial WINS, reading + // from the same point 2 the strict match would, with a deeper anchor + // (the splice branch decides downstream whether the tail is recoverable; + // this prompt's tail {C D} vs {X Y} has no common suffix, so it degrades + // to the same gap-read the strict match would do - never worse). + const std::vector prompt = {1, 2, 90, 91}; // [A B X Y] + const auto m = cache.best_with_partial(64, prompt, {}, true); + check(m.partial, + "T14.1: deeper partial (L=4) beats equal-depth strict (s=2, p=2) - splice engagement"); + check(m.tokens == 2, + "T14.2: reads from the LCP (2) - same point as the strict match, never shallower"); + check(m.checkpoint_length == 6, + "T14.3: the deepest checkpoint (6) is the anchor - T17 lifted the size guard, so the " + "6-long entry (LCP 2, previously size-refused) wins the deeper-anchor tie-break over 4"); + + // Now drop ck C and re-test: partial mode, longest LCP wins, deeper anchor wins ties. + // Use a prompt long enough for both checkpoints to be partial (L <= n). + auto s2 = image({}); + s2.checkpoints = {ckpt({1, 2, 3, 4}), ckpt({1, 2, 3, 4, 5, 6})}; + ConversationCache cache2(1 << 20, 4); + cache2.put(std::move(s2)); + const std::vector prompt_long = {1, 2, 90, 91, 92, 93, 94, 95}; // length 8 + const auto m2 = cache2.best_with_partial(64, prompt_long, {}, true); + check(m2.partial, + "T14.4: no strict match -> partial mode engages"); + check(m2.tokens == 2 && m2.checkpoint_length == 6, + "T14.5: deepest anchor wins ties (ck B {1..6} beats ck A {1..4} at LCP=2)"); + + // Three checkpoints with different LCPs: longest LCP wins. + auto s3 = image({}); + s3.checkpoints = {ckpt({1, 2, 3, 4}), ckpt({1, 2, 30, 31, 32})}; + ConversationCache cache3(1 << 20, 4); + cache3.put(std::move(s3)); + const std::vector prompt2 = {1, 2, 3, 90, 91, 92}; // matches ck A for 3 tokens, ck B for 2 + const auto m3 = cache3.best_with_partial(64, prompt2, {}, true); + check(m3.partial && m3.tokens == 3 && m3.checkpoint_length == 4, + "T14.6: longest LCP wins (3 > 2, ck A {1..4} at LCP=3 over ck B at LCP=2)"); + + // Live state beats checkpoints of the same length (the existing best() tie-break). Use a + // prompt long enough for both the live (length 6) and the checkpoint (length 4) to be + // partial candidates. + auto s4 = image({1, 2, 3, 4, 5, 6}); + s4.checkpoints = {ckpt({1, 2, 3, 4})}; + ConversationCache cache4(1 << 20, 4); + cache4.put(std::move(s4)); + const std::vector prompt_live = {1, 2, 90, 91, 92, 93, 94, 95}; // length 8 + const auto m4 = cache4.best_with_partial(64, prompt_live, {}, true); + check(m4.partial && m4.tokens == 2 && m4.checkpoint_length == 6 && m4.live, + "T14.7: live state matches at LCP=2 (deeper anchor); live=true"); + } + + // ============================================================ + // T15: IMAGE / GEOMETRY MISMATCH (refuse, full re-read) + // ============================================================ + // The checkpoint has an image at position 1; the request's images don't include it. The + // LCP-based image check refuses the partial mount even if the tokens match. + { + auto s = image({}); + s.checkpoints = {ckpt({1, 2, 3, 4})}; + s.checkpoints[0].imgs = {{1, 0xCAFE}}; // image at position 1 + ConversationCache cache(1 << 20, 4); + cache.put(std::move(s)); + + const std::vector prompt = {1, 2, 90, 91}; + // strict search: image mismatch at position 1 (< 4) -> strict returns 0. + const std::vector req_imgs = {{2, 0xDEAD}}; // request has an image at pos 2 + const auto strict = cache.best(prompt, req_imgs, true); + check(strict.tokens == 0, + "T15.1 setup: strict refuses on image mismatch at position < strict-length"); + + // partial search: the LCP image check (bounded by P) must also refuse. Here the LCP + // could be 2 (tokens 0,1 match) but the checkpoint has an image at position 1 which is + // below the LCP. The request's image at position 2 is at P (not below). So the + // checkpoint has an image below P that the request does NOT have -> refuse. + const auto m = cache.best_with_partial(64, prompt, req_imgs, true); + check(!m.partial, + "T15.2: partial refuses on image mismatch below the LCP (no fabricated mount)"); + + // The partial helper itself is a pure token+shape check; the cache is the image gate. + // In this case, the cache refuses BEFORE the helper is called. Pin the helper's + // token-level correctness: same shape, tokens match, helper fires. + const SuffixHookOutcome ok = suffix_reuse_partial_apply( + {1, 2, 3, 4}, prompt, /*L=*/4, /*P=*/2, std::vector{}); + check(ok.fired, + "T15.3: helper's token-level check fires (the cache is the image gate, not the helper)"); + } + + // ============================================================ + // T15 variant: PARTIAL helper fail-closed on inconsistent inputs + // ============================================================ + { + // L <= 0 + const SuffixHookOutcome a = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 90, 91}, /*L=*/0, /*P=*/2, {}); + check(!a.fired, "T15v.1: L <= 0 -> fired=false (fail-closed)"); + + // P <= 0 + const SuffixHookOutcome b = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 90, 91}, /*L=*/4, /*P=*/0, {}); + check(!b.fired, "T15v.2: P <= 0 -> fired=false (fail-closed)"); + + // P >= L (not partial, would be strict) + const SuffixHookOutcome c = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 3, 4, 5}, /*L=*/4, /*P=*/4, {}); + check(!c.fired, "T15v.3: P >= L -> fired=false (not a partial shape)"); + + // L > cur.size() - T17 (2026-10-13): this is now a VALID partial shape + // (the parked-live anchor is typically longer than the next prompt). + // The helper fires and narrows to P; the runtime restores the deepest + // checkpoint <= P, never state@L. The old pin refused it (three guards); + // see T17.6 in this suite. + const SuffixHookOutcome d = suffix_reuse_partial_apply( + {1, 2, 3, 4, 5, 6}, {1, 2}, /*L=*/6, /*P=*/2, {}); + check(d.fired && d.plan_edit_pos == 2 && d.restore_anchor_length == 6, + "T15v.4: L(6) > cur(2) fires (T17 live-anchor shape; anchor 6, narrow 2)"); + + // Inconsistent tokens (length mismatch) + const SuffixHookOutcome e = suffix_reuse_partial_apply( + {1, 2, 3, 4, 5}, {1, 2, 90, 91}, /*L=*/4, /*P=*/2, {}); + check(!e.fired, "T15v.5: tokens.size() != L -> fired=false (inconsistent)"); + + // LCP claim inconsistent with the actual tokens + const SuffixHookOutcome f = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 90, 3, 4}, /*L=*/4, /*P=*/2, {}); // claim P=2 but token 1 mismatches + check(!f.fired, "T15v.6: LCP claim inconsistent with tokens -> fired=false"); + + // Empty chain + valid partial = fired with empty chain (chain filter is a no-op). + const SuffixHookOutcome g = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 90, 91}, /*L=*/4, /*P=*/2, {}); + check(g.fired && g.chain.empty(), + "T15v.7: empty chain + valid partial = fired, chain stays empty"); + + // Chain with the anchor + a non-anchor entry above the LCP: keep the anchor, drop the other. + { + std::vector chain = { {2, 0}, {4, 1}, {6, 2} }; // L=4 is the anchor + const SuffixHookOutcome a = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 90, 91}, /*L=*/4, /*P=*/2, chain); + check(a.fired && a.chain.size() == 2 && a.chain[0].length == 2 && a.chain[1].length == 4, + "T15v.9: chain with anchor L=4 + non-anchor 6 > P=2 -> keep [2, 4], drop 6"); + } + // Chain with the anchor + an entry exactly at the LCP: keep both (the LCP entry is below). + { + std::vector chain = { {2, 0}, {4, 1} }; + const SuffixHookOutcome a = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 90, 91}, /*L=*/4, /*P=*/2, chain); + check(a.fired && a.chain.size() == 2, + "T15v.10: chain with anchor L=4 + entry at P=2 -> both kept (entry length <= P)"); + } + + // Token at the divergence point matches -> LCP == L (strict, not partial). The helper refuses + // because the divergence check `partial_ck[P] != cur[P]` fails (they're equal -> LCP > P). + // The strict path handles this; the partial helper must not fire. + const SuffixHookOutcome strict = suffix_reuse_partial_apply( + {1, 2, 3, 4}, {1, 2, 3, 90}, /*L=*/4, /*P=*/2, {}); // claim P=2 but token 2 matches (LCP=3) + check(!strict.fired, + "T15v.8: claim P=2 but tokens show LCP=3 -> fired=false (strict path handles)"); + + // The LCP verification loop catches the inconsistency: a P=2 claim on tokens where the + // first 2 match but the 3rd also matches means the caller mis-computed P. The helper's + // LCP verification walks [0, P) only; it does NOT walk beyond P (the divergence check at + // [P] is the next guard). When P==L (which the L>=P guard catches above), it's strict. + // For P < L but token[P] == cur[P], the LCP would be > P, not == P - the divergence + // check catches it: the helper returns fired=false. + } + + // ============================================================ + // T16: SPLICE ENGAGEMENT - a deeper partial beats a shallower strict + // ============================================================ + // The M1 steady-state finding (2026-10-13): best_with_partial returns at the + // FIRST strict match and never evaluates partial, so when the checkpoints hold + // both a strict anchor s (s <= D, the divergence point) and a DEEPER checkpoint + // L > D that would splice, the strict one wins and the splice never engages + // ([SFX-DBG] partial-branch enter=0 in every steady trace). + // + // Dominance rule (safe because one conversation's checkpoints are nested + // prefixes of ONE history: the history diverges from the request at a single + // point D, so every strict anchor s <= D = the partial LCP p - a partial mount + // never reads from a shallower point, and its splice may recover up to L): + // prefer the partial match when its checkpoint covers strictly more than the + // strict anchor (L > s); otherwise keep strict (no splice cost when it cannot + // recover more). + { + // One conversation: checkpoints {A B} (2, the future strict anchor) and + // {A B C D E} (5, crosses the edit). Request {A B C X E ...}: history + // diverges at D=3 (C vs X at index 3). Strict anchor {A B} mounts 2; + // partial on {A B C D E}: LCP=3, L=5 -> splice recovers [P=3, L=5) tail E. + auto s = image({1, 2, 3, 4, 5}); + s.checkpoints = { ckpt({1, 2}), ckpt({1, 2, 3, 4, 5}) }; + ConversationCache cache(1 << 20, 4); + cache.put(std::move(s)); + + const std::vector prompt = {1, 2, 3, 90, 5}; // edits token 3 (X=90) + const auto strict = cache.best(prompt, {}, true); + check(strict.tokens == 2 && !strict.partial, + "T16.1: strict best() mounts the {A B} anchor (2)"); + + const auto m = cache.best_with_partial(64, prompt, {}, true); + check(m.partial, + "T16.2: partial preferred over shallower strict (splice engagement)"); + check(m.tokens == 3, + "T16.3: partial LCP == 3 (== the strict anchor's 2... history diverges at 3)"); + check(m.checkpoint_length == 5, + "T16.4: the deeper checkpoint (5) is the splice anchor"); + + // The neutral case: strict anchor AT the divergence depth (s == p == 3). + // The dominance rule still prefers the deeper checkpoint (L=5 covers the + // common tail {E}), and the mount reads from p=3 == s: NEVER shallower + // than the strict anchor, so the swap is safe even when the tail turns + // out recoverable or not (old_suffix_len=0 handles itself downstream). + auto s2 = image({1, 2, 3, 4, 5}); + s2.checkpoints = { ckpt({1, 2, 3}), ckpt({1, 2, 3, 4, 5}) }; + ConversationCache cache2(1 << 20, 4); + cache2.put(std::move(s2)); + const auto m2 = cache2.best_with_partial(64, prompt, {}, true); + check(m2.tokens >= strict.tokens, + "T16.5: partial never reads shallower than the strict anchor (p >= s)"); + check(!m2.partial || m2.checkpoint_length > strict.tokens, + "T16.5b: partial only chosen when its anchor covers strictly more (L > s)"); + + // Off-flag: byte-identical strict (the early-return never changes behavior). + const auto m0 = cache.best_with_partial(0, prompt, {}, true); + check(!m0.partial && m0.tokens == 2, + "T16.6: flag=0 stays byte-identical strict (2)"); + + // Cross-conversation (review r1 finding 1): the nesting property only holds + // WITHIN one conversation. Conversation A offers a strict anchor {A B C} + // (s=3); conversation B offers a partial candidate {A B 80 81 82} whose + // anchor is deeper (L=5 > s=3) but whose LCP is shallower (p=2 < s=3). + // The dominance swap must NOT take B's partial: it would mount at 2 and + // lose a token of resume vs A's strict 3. Requires p >= s ACROSS + // conversations, not just L > s. + { + auto convA = image({1, 2, 3}); + convA.checkpoints = { ckpt({1, 2, 3}) }; + auto convB = image({1, 2, 80, 81, 82}); + convB.checkpoints = { ckpt({1, 2, 80, 81, 82}) }; + ConversationCache cache3(1 << 20, 4); + cache3.put(std::move(convA)); + cache3.put(std::move(convB)); + + const auto m3 = cache3.best_with_partial(64, prompt, {}, true); + check(!m3.partial && m3.tokens >= 3, + "T16.7: cross-conversation partial with LCP < strict never mounts (p >= s across conversations)"); + check(m3.tokens == 3 && !m3.partial, + "T16.7b: strict anchor (3) stands over B's deeper-but-shallower-LCP partial"); + } + } + + // ============================================================ + // T17: LIVE-ANCHOR PARTIAL - a parked live LONGER than the prompt + // ============================================================ + // The M1 probe finding (2026-10-13, chain /tmp/dt-steady4.log, C-turn): + // parked-scan: e0.live:L=5786:skip (prompt_len=5736) + // The parked LIVE (prompt+response of the previous turn) is almost always + // slightly LONGER than the next turn's prompt, and three L > prompt.size() + // guards refused it without computing the LCP: the cache's consider() + // size-guard, conversation_lcp's n > prompt.size() early-out, and + // suffix_reuse_partial_apply's L > n_new refusal. The live anchor is the + // single most valuable partial candidate in every edit chain - lifting the + // guards for the PARTIAL PASS ONLY (strict stays byte-identical) lets the + // partial mount recover [P, L) K/V the strict anchor cannot reach. + // + // Safety: the LCP walk stops at min(L, prompt.size()) - tokens past the + // prompt are irrelevant (the partial mount discards them); the strict pass + // keeps its own guard (a longer-than-prompt checkpoint can NEVER be a + // strict prefix). The runtime routes a live-anchor partial through the + // narrowed_resume_target restore (state@L is stale for the edited history; + // see the v3 fix note in generate.cpp) - restore the deepest checkpoint + // <= P, arena covers [ckpt, P) by the LCP, read [P, n). + { + // Parked live {A B C D E} (5) + shallow checkpoints {A B}(2), {A B C}(3). + // Request {A B C X} (4) - SHORTER than the live, edits index 3. + // Strict: {A B C} mounts 3 (live and {A B C D E} fail: longer/edited). + // Partial on the live: LCP = 3, L = 5 > strict 3 -> dominance fires. + auto s = image({1, 2, 3, 4, 5}); + s.checkpoints = { ckpt({1, 2}), ckpt({1, 2, 3}) }; + ConversationCache cache(1 << 20, 4); + cache.put(std::move(s)); + + const std::vector prompt = {1, 2, 3, 90}; // len 4 < live 5 + const auto strict = cache.best(prompt, {}, true); + check(strict.tokens == 3 && !strict.partial, + "T17.1: strict best() mounts {A B C} (3); the longer live is refused"); + + const auto m = cache.best_with_partial(64, prompt, {}, true); + check(m.partial, + "T17.2: live anchor (L=5 > prompt=4) is now a partial candidate"); + check(m.tokens == 3, + "T17.3: live-anchor partial LCP == 3 (divergence at index 3)"); + check(m.checkpoint_length == 5, + "T17.4: the parked LIVE (5) is the splice anchor"); + check(m.live, + "T17.5: Match.live == true (the anchor is the parked live)"); + + // The helper accepts L > n_new for the partial path (fail-closed kept + // for every OTHER shape). Fire with L=5, P=3, cur len 4: plan narrows + // to P=3, anchor 5, no common tail ({D E} vs {X}). + const std::vector live_tokens = {1, 2, 3, 4, 5}; + std::vector chain = { {2, 0}, {3, 1} }; + const SuffixHookOutcome outcome = suffix_reuse_partial_apply( + live_tokens, prompt, /*L=*/5, /*P=*/3, chain); + check(outcome.fired, + "T17.6: partial_apply fires with L(5) > n_new(4)"); + check(outcome.fired && outcome.plan_edit_pos == 3, + "T17.7: plan_edit_pos == 3 (the LCP)"); + check(outcome.fired && outcome.restore_anchor_length == 5, + "T17.8: restore_anchor_length == 5 (the live anchor)"); + check(outcome.fired && outcome.old_suffix_len == 0, + "T17.9: old_suffix_len == 0 ({D E} vs {X} - no common tail)"); + + // conversation_lcp direct: live longer than prompt walks to the prompt's + // end and reports the prompt-bounded LCP (not 0). + ConversationCheckpoint lc; + lc.ids = {1, 2, 3, 4, 5}; + check(strata::core::conversation_lcp(lc, prompt, {}) == 3, + "T17.10: conversation_lcp walks a longer-than-prompt checkpoint (3, not 0)"); + + // Off-flag: byte-identical strict - the live-anchor lift is flag-gated. + const auto m0 = cache.best_with_partial(0, prompt, {}, true); + check(!m0.partial && m0.tokens == 3, + "T17.11: flag=0 stays byte-identical strict (3)"); + + // A strict-longer guard REMAINS: a candidate longer than the prompt can + // never produce p >= L (strict); the partial pass alone evaluates it. + // Pin that conversation_lcp reports the FULL checkpoint length when the + // checkpoint is a strict prefix (shorter or equal) - unchanged. + ConversationCheckpoint sc; + sc.ids = {1, 2}; + check(strata::core::conversation_lcp(sc, prompt, {}) == 2, + "T17.12: shorter strict-prefix checkpoint still reports its full length (2)"); + } + + // ============================================================ + // T18: REPLAY-GATED GAP READ (dt-steady5 measured finding) + // ============================================================ + // Reading [P, n) on state@checkpoint loses the gap [ckpt, P) from the + // GDN/PLE recurrence: attention sees the gap rows through valid arena K/V, + // the running state never ingested them (C under-counted; gap 826 tokens). + // partial_gap_read_from fail-closes to the checkpoint until FW-2 replay. + { + using strata::program::suffix_reuse::partial_gap_read_from; + const std::vector chain = { 1574, 2598 }; + check(partial_gap_read_from(false, 3424, chain) == 2598, + "T18.1: replay off -> read from the deepest checkpoint <= P (2598)"); + check(partial_gap_read_from(true, 3424, chain) == 3424, + "T18.2: STRATA_SFX_REPLAY=1 -> the gap-read [P, n) (3424)"); + check(partial_gap_read_from(false, 3424, {}) == 0, + "T18.3: no checkpoint <= P -> fresh read from 0 (fail-closed, never fabricate)"); + check(partial_gap_read_from(false, 2000, chain) == 1574, + "T18.4: P between anchors -> the deeper-below-P anchor (1574)"); + check(partial_gap_read_from(true, 3424, {}) == 3424, + "T18.5: replay on with an empty chain still reads [P, n) (the runtime restores 0)"); + } + + // ============================================================ + // T22 (FW-2b): state-cadence checkpoints shrink the fail-close gap. + // checkpoint_save copies state only (gdn/ple/tails/dead/block_pos; no K/V), + // so fine-cadence state checkpoints are cheap (T19: ~100-150ms each). They + // live in a SIDE pool (separate from the LRU-capped `checks` list) so they + // never evict the deep periodic anchors. The restore pick must take the + // deepest checkpoint <= P across BOTH pools; fail-close semantics unchanged + // (the gap read is the deepest anchor below P; replay is env-gated). + // Pure seams under test: `narrowed_resume_target_state` (the two-pool pick) + // and `state_cadence_next` (the fine-cadence ladder, never past n). + { + using strata::program::suffix_reuse::narrowed_resume_target_state; + using strata::program::suffix_reuse::state_cadence_next; + using strata::program::suffix_reuse::NarrowedResumeTarget; + + // T22.1: the deep-pool checkpoint wins when it is the deepest <= P. + // checks = {1574, 2598} (periodic, prompt_cache LRU), state = {2816, 3072} + // (fine cadence). P = 3424: deepest across both = 3072 (not 2598). + { + const std::vector checks = { 1574, 2598 }; + const std::vector state = { 2816, 3072 }; + const NarrowedResumeTarget t = narrowed_resume_target_state(3424, checks, state); + check(t.restore_length == 3072, + "T22.1: state-cadence 3072 beats periodic 2598 (deepest across both pools)"); + check(t.read_from == 3424, + "T22.1b: read still starts at P (fail-close semantics unchanged)"); + } + + // T22.2: the periodic pool wins when it is deeper (state pool empty). + { + const NarrowedResumeTarget t = narrowed_resume_target_state(3424, { 1574, 2598 }, {}); + check(t.restore_length == 2598 && t.read_from == 3424, + "T22.2: no state checkpoints -> deepest periodic (2598), same as today"); + } + + // T22.3: fail-closed degenerates. P below every checkpoint in both pools + // -> fresh read from 0; nothing is fabricated. + { + const NarrowedResumeTarget t = narrowed_resume_target_state(1000, { 1574 }, { 2816 }); + check(t.restore_length == 0 && t.read_from == 0, + "T22.3: P below both pools -> fresh read from 0 (fail-closed, never fabricate)"); + } + + // T22.4: equal lengths are fine (dedup not required; deepest is deepest). + { + const NarrowedResumeTarget t = narrowed_resume_target_state(3424, { 2598, 3424 }, { 3424 }); + check(t.restore_length == 3424 && t.read_from == 3424, + "T22.4: equal lengths across pools -> the shared depth (3424)"); + } + + // T22.5: the fine-cadence ladder. state_cadence_next(done, n, every) + // returns the next capture position: done + every, clamped so a capture + // never lands past the prompt end (a checkpoint at exactly n is the live + // conversation, not a checkpoint). + check(state_cadence_next(2598, 5736, 256) == 2854, + "T22.5: ladder from 2598 at 256 cadence -> 2854"); + check(state_cadence_next(5600, 5736, 256) == 5736, + "T22.5b: clamped to n when done+every overshoots (never past the prompt)"); + check(state_cadence_next(5600, 5736, 0) == INT64_MAX, + "T22.5c: every = 0 -> off (INT64_MAX, the gate never fires)"); + } + + std::printf("suffix_reuse_partial_prefix_test: %d passed, %d failed\n", g_pass, g_fail); + return g_fail == 0 ? 0 : 1; +} diff --git a/src/program/suffix_reuse_resume_narrowing_test.cpp b/src/program/suffix_reuse_resume_narrowing_test.cpp new file mode 100644 index 00000000000..2f53018f1c2 --- /dev/null +++ b/src/program/suffix_reuse_resume_narrowing_test.cpp @@ -0,0 +1,131 @@ +// src/program/suffix_reuse_resume_narrowing_test.cpp - T8: the resume-narrowing wiring +// regression for the 2026-10-12 wiring bug 2 on `feat/suffix-reuse`. +// +// THE BUG. The phase-2 splice hook (generate.cpp:9034-9062) erases checkpoints past +// `plan_edit_pos` AND narrows `resume` to `plan_edit_pos`. The checkpoint-restore search +// later (generate.cpp:9294-9325, the `else if (!from_live)` branch) requires a checkpoint +// with `ids.size() == resume` EXACTLY. On a re-edited request the splice hook's own erase +// removed the exact-length candidate (the resume checkpoint is at the OLD resume, e.g. +// 3240, and the narrowed resume is the edit position, e.g. 3239), so the search returns +// `c == nullptr` and the engine prints "ERR restoring a conversation checkpoint failed" +// and returns 1, which the upstream serve loop turns into a 400. Live evidence +// (GPU-verified by orchestrator): `checks`={3240, 3244}, `resume` narrowed to 3239 -> +// c==nullptr -> 400. Off-arm (flag absent) identical traffic: mounts 3240, serves ~1s. +// +// THE FIX (route c - the gap-read / replay path). The header `suffix_reuse.hpp` gains a +// pure helper `narrowed_resume_target` that decides, given a narrowed resume and the +// live `checks` chain, what the request's next move is: +// +// - exact-length candidate exists: restore it; read_from = narrowed resume +// (no gap; byte-equivalent to today's restore). +// - no exact-length, deepest-below: restore the deepest ck with length <= narrowed; +// read_from = narrowed resume (the [ck, narrowed) +// gap is re-read by the existing prefill path - the +// "replay" / gap-read path phase 2 was designed for). +// - no candidate at or below: restore_length = 0, read_from = 0 +// (no usable mount; full re-prefill, NO error). +// +// In every case the request proceeds. generate.cpp's `else if (!from_live)` branch +// calls this helper and uses its two return values (restore length, read_from) to drive +// the existing restore + chunk-loop paths unchanged. +// +// WHAT THIS TEST PINS. The helper is pure, header-only, and is exercised on the live +// failing inputs (resume_narrowed=3239, checks={3240, 3244}) plus the three adjacent +// shapes the fix has to remain correct on: +// +// T8.1 LIVE RED 3239, {3240, 3244} -> restore=0, read_from=0 +// T8.2 EXACT-LEN 3239, {200, 3239, 3240} -> restore=3239, read_from=3239 +// T8.3 DEEPEST-BEL 3239, {100,200,1500,3240,3244} -> restore=1500, read_from=3239 +// T8.4 CONTINUATN 3000, {200,1000,2000,3000,3240} -> restore=3000, read_from=3000 +// T8.5 NO CHECKS 3239, {} -> restore=0, read_from=0 +// +// CPU-ONLY. Header-only; no CUDA, no . RED on 37dec54: the helper +// does not exist (compile error). GREEN after the fix: the helper exists and matches +// every case. generate.cpp's call site is exercised by the orchestrator's GPU probe +// (t1 -> t2 splice-eligible -> expect 200, splice/replay marker, reused>0 or read +#include +#include + +namespace { + +// Result tally +int g_fail = 0; +int g_pass = 0; +void check(bool ok, const char* what) { + std::printf(" %-78s %s\n", what, ok ? "ok" : "FAIL"); + if (ok) ++g_pass; else ++g_fail; +} + +} // namespace + +int main() { + std::printf("suffix_reuse_resume_narrowing_test (T8: wiring bug 2 on feat/suffix-reuse)\n"); + + using strata::program::suffix_reuse::NarrowedResumeTarget; + using strata::program::suffix_reuse::narrowed_resume_target; + + // ---- T8.1 LIVE RED: 3239, {3240, 3244} ---- + // The splice hook erased the resume checkpoint and narrowed resume; no exact + // candidate survives. The fix must NOT error: it must fall back to a full + // re-prefill from 0. + { + const std::vector chain{3240, 3244}; + const NarrowedResumeTarget t = narrowed_resume_target(3239, chain); + check(t.restore_length == 0, + "T8.1 LIVE: no ck <= 3239 survives; restore_length=0 (no 'restoring failed' error)"); + check(t.read_from == 0, + "T8.1 LIVE: read_from=0 (full re-prefill; gap-read path engaged)"); + } + + // ---- T8.2 EXACT-LENGTH: 3239, {200, 3239, 3240} ---- + { + const std::vector chain{200, 3239, 3240}; + const NarrowedResumeTarget t = narrowed_resume_target(3239, chain); + check(t.restore_length == 3239, + "T8.2 exact-length: ck@3239 is the resume target; restore_length=3239"); + check(t.read_from == 3239, + "T8.2 exact-length: no gap to re-read; read_from=3239"); + } + + // ---- T8.3 DEEPEST-BELOW: 3239, {100, 200, 1500, 3240, 3244} ---- + // No exact-length; deepest surviving ck below the narrowed resume is 1500. + // Restore 1500 and re-read the [1500, 3239) gap via the existing prefill path. + { + const std::vector chain{100, 200, 1500, 3240, 3244}; + const NarrowedResumeTarget t = narrowed_resume_target(3239, chain); + check(t.restore_length == 1500, + "T8.3 deepest-below: ck@1500 is deepest <= 3239; restore_length=1500"); + check(t.read_from == 3239, + "T8.3 deepest-below: gap [1500, 3239) is re-read; read_from=3239"); + } + + // ---- T8.4 PURE CONTINUATION: 3000, {200, 1000, 2000, 3000, 3240} ---- + // Splice hook did NOT narrow (plan.edit_pos == resume, the common case); the + // exact-length candidate is there; the helper is a no-op find. + { + const std::vector chain{200, 1000, 2000, 3000, 3240}; + const NarrowedResumeTarget t = narrowed_resume_target(3000, chain); + check(t.restore_length == 3000, + "T8.4 continuation: ck@3000 is exact-length; restore_length=3000"); + check(t.read_from == 3000, + "T8.4 continuation: no gap, no re-read; read_from=3000"); + } + + // ---- T8.5 NO CHECKS: 3239, {} ---- + // No mounted conversation; the request is a fresh prefill from 0; no error. + { + const std::vector chain{}; + const NarrowedResumeTarget t = narrowed_resume_target(3239, chain); + check(t.restore_length == 0, + "T8.5 no-checks: empty chain; restore_length=0 (fresh read, no error)"); + check(t.read_from == 0, + "T8.5 no-checks: read_from=0 (full re-prefill)"); + } + + std::printf("\nSummary: %d passed, %d failed\n", g_pass, g_fail); + return g_fail ? 1 : 0; +} diff --git a/src/program/suffix_reuse_test.cpp b/src/program/suffix_reuse_test.cpp new file mode 100644 index 00000000000..6182cc025e1 --- /dev/null +++ b/src/program/suffix_reuse_test.cpp @@ -0,0 +1,368 @@ +// src/program/suffix_reuse_test.cpp - the suffix-reuse plan computation: pure CPU tests on the longest +// common tail of a prompt re-edit, no model, no GPU. +// +// The suffix reuses the prompt cache's K/V for the unchanged tail beyond a mid-prompt edit at position E. +// Longest common suffix is the splice plan: tokens [E, n_new) are the unchanged suffix, tokens [E, n_old) +// are the unchanged suffix in the old prompt; if the per-token difference in absolute position (`delta`) +// is uniform, the splice reuses one rotation; otherwise per-token deltas go to the rotation kernel. +// +// T1: pure plan-computation, single insert (uniform delta = +k). +// T2: pure invalidation - checkpoints whose length > E drop. +// T4: composite edit (insert+delete in different places) - per-token deltas correct. +// +// T3 (RoPE CPU rotation via `rope_neox_pair`) and T5 (chain integration) and T8 (fail-closed guards) +// live in the same test TU and gate on STRATA_ROPE_TEST_TABLES / STRATA_SUFFIX_T8 so each can run alone. +#include "strata/program/suffix_reuse.hpp" +#include "strata/program/kv_splice.hpp" +#include "strata/kernels/rope.hpp" +#include "strata/core/conversation_cache.hpp" + +#include +#include +#include +#include +#include + +using strata::program::suffix_reuse::SuffixReusePlan; +using strata::program::suffix_reuse::compute_suffix_plan; +using strata::program::suffix_reuse::invalidate_checkpoints; +using strata::program::suffix_reuse::CheckpointLength; + +namespace { +int g_fail = 0; +void check(bool ok, const char* what) { + std::printf(" %-72s %s\n", what, ok ? "ok" : "FAIL"); + if (!ok) ++g_fail; +} +} // namespace + +int main() { + std::printf("suffix_reuse_test\n"); + + // --- T1: pure plan-computation -------------------------------------------------- + // insert 3 tokens at position 4 of a 7-token prompt -> uniform delta = +3 across the suffix + { + const std::vector old_tokens = {1, 2, 3, 4, 5, 6, 7}; + const std::vector new_tokens = {1, 2, 3, 4, 100, 200, 300, 5, 6, 7}; + const SuffixReusePlan plan = compute_suffix_plan(old_tokens, new_tokens); + check(plan.edit_pos == 4, "T1: insert at 4 -> edit_pos is 4"); + check(plan.uniform && plan.delta == 3, "T1: uniform insert -> delta = +3"); + check(plan.old_suffix_len == 3 && plan.new_suffix_len == 3, + "T1: tail [5,6,7] reuses against [5,6,7] of the new prompt"); + check(plan.deltas.empty(), "T1: uniform delta needs no per-token table"); + } + // pure deletion: drop tokens [4,5] -> new prompt shorter than old, delta = -2 across the tail + { + const std::vector old_tokens = {1, 2, 3, 4, 5, 6, 7}; + const std::vector new_tokens = {1, 2, 3, 6, 7}; + const SuffixReusePlan plan = compute_suffix_plan(old_tokens, new_tokens); + check(plan.edit_pos == 5, "T1: delete of [4,5] -> edit_pos at 5 in the old prompt"); + check(plan.uniform && plan.delta == -2, "T1: uniform delete -> delta = -2"); + check(plan.old_suffix_len == 2 && plan.new_suffix_len == 2, + "T1: tail [6,7] reuses against [6,7]"); + } + // no edit at all: edit_pos == 0 (the entire old prompt is the common tail), delta = 0 + { + const std::vector old_tokens = {1, 2, 3, 4}; + const std::vector new_tokens = {1, 2, 3, 4}; + const SuffixReusePlan plan = compute_suffix_plan(old_tokens, new_tokens); + check(plan.edit_pos == 0 && plan.uniform && plan.delta == 0, + "T1: identical prompts: edit_pos is 0 (whole prompt reuses), no rotation"); + } + + // --- T2: pure invalidation - any checkpoint with length > E drops --------------- + { + // four checkpoints forming a chain (lengths sorted): 2, 5, 7, 9 - the edit fell at E=4 + std::vector chain = { {2, 0}, {5, 1}, {7, 2}, {9, 3} }; + const size_t dropped = invalidate_checkpoints(chain, 4); + check(dropped == 3, "T2: checkpoints with length > 4 drop (5, 7, 9)"); + check(chain.size() == 1 && chain[0].length == 2, + "T2: the sole survivor is the prefix at length 2"); + } + { // edit past the chain's deepest point: nothing to drop, chain stays whole + std::vector chain = { {2, 0}, {5, 1}, {7, 2} }; + const size_t dropped = invalidate_checkpoints(chain, 10); + check(dropped == 0 && chain.size() == 3, + "T2: edit past the chain leaves every checkpoint valid"); + } + { // edit at 0 (the whole prefix is suspect): all checkpoints drop + std::vector chain = { {2, 0}, {5, 1} }; + const size_t dropped = invalidate_checkpoints(chain, 0); + check(dropped == 2, "T2: edit at 0 invalidates every checkpoint"); + } + + // --- T3: RoPE CPU rotation via host loop over rope_neox_pair --------------------- + // build cos/sin tables twice (once via the host port of build_rope_table, once from scratch at + // the NEW position) and assert rotate_k_by_delta over an OLD K equals the rotation applied at NEW. + // The tolerance matches rope_parity.cpp: 1e-6 (float32 ULP). + { + const int n_rot = 64; // partial-rotation width (32 pairs, head_dim = 256) + const int head_dim = 256; + const int half = n_rot / 2; + const double theta = 1000000.0; // arbitrary - the math is the same shape as the model's + const int max_pos = 2048; + std::vector cos_a((size_t) max_pos * (size_t) half), sin_a((size_t) max_pos * (size_t) half); + std::vector cos_b = cos_a, sin_b = sin_a; + strata::program::suffix_reuse::build_rope_table_host(n_rot, theta, max_pos, cos_a.data(), sin_a.data()); + // the CUDA builder body, transcribed for the cross-check (this is its single consumer in + // the test - the production path uses the build_rope_table_host port above) + for (int p = 0; p < max_pos; ++p) { + for (int i = 0; i < half; ++i) { + const double inv = std::pow(theta, -2.0 * (double) i / (double) n_rot); + const double ang = (double) p * inv; + cos_b[(size_t) p * (size_t) half + (size_t) i] = (float) std::cos(ang); + sin_b[(size_t) p * (size_t) half + (size_t) i] = (float) std::sin(ang); + } + } + int t3_fail = 0; + for (size_t i = 0; i < cos_a.size(); ++i) { + const float da = cos_a[i] - cos_b[i], db = sin_a[i] - sin_b[i]; + if (std::fabs(da) > 1e-6f || std::fabs(db) > 1e-6f) ++t3_fail; + } + check(t3_fail == 0, "T3: host cos/sin tables match the transcribed CUDA builder to 1e-6"); + + // build a row at old_pos from a deterministic seed, rotate it by `delta`, compare with the + // row built + rotated at new_pos. exercises `rotate_row_by_delta` directly. + // + // Contract (F3): cached K is POST-RoPE at old_pos. rotate_row_by_delta must re-rotate + // that cached row by the DELTA angle so the result equals rotate(raw, new_pos). Builds + // the cached row explicitly (rotate_at(raw, old_pos)) so a buggy helper that applies the + // full new_pos rotation to an unrotated row cannot pass. + auto make_row = [&](int p) { + std::vector k((size_t) head_dim, 0.0f); + for (int i = 0; i < head_dim; ++i) k[i] = std::sin(0.013f * (float) i + 0.7f * (float) p); + return k; + }; + // explicit RoPE at position `p` over the first n_rot elements (NEOX pairing) + auto rotate_at = [&](const std::vector& src, int p) { + std::vector out = src; + const int p_clamped = p < max_pos ? p : max_pos - 1; + for (int i = 0; i < half; ++i) { + const float c = cos_a[(size_t) p_clamped * (size_t) half + (size_t) i]; + const float s = sin_a[(size_t) p_clamped * (size_t) half + (size_t) i]; + float oa, ob; + strata::kernels::rope_neox_pair(out[i], out[i + half], c, s, oa, ob); + out[i] = oa; + out[i + half] = ob; + } + return out; + }; + + // ground-truth numeric case from the reviewer (F2): raw_a=0.3, raw_b=-0.8 at pair i=5, + // theta=1e6, n_rot=64, old_pos=7, delta=4 -> rotate(raw, new_pos=11) at pair i=5 + // is (0.852952, 0.049727) to 1e-6. Pin this pair inside the row-scan loop below. + { + std::vector raw((size_t) head_dim, 0.0f); + for (int i = 0; i < head_dim; ++i) raw[i] = std::sin(0.013f * (float) i + 0.7f * (float) 7); + raw[5] = 0.3f; + raw[5 + half] = -0.8f; + std::vector cached = rotate_at(raw, 7); + std::vector helper_out = cached; + strata::program::suffix_reuse::rotate_row_by_delta( + helper_out.data(), head_dim, n_rot, 7, 4, cos_a.data(), sin_a.data(), max_pos); + std::vector expected = rotate_at(raw, 11); + const float err_a = std::fabs(helper_out[5] - expected[5]); + const float err_b = std::fabs(helper_out[5 + half] - expected[5 + half]); + check(err_a < 1e-6f && err_b < 1e-6f, + "T3: reviewer's ground truth - pair (0.3,-0.8) at pair i=5, old_pos=7, delta=4 -> " + "helper matches rotate(raw, new_pos=11) to 1e-6 (fresh=(0.852952, 0.049727))"); + } + + int t3_pos_fail = 0; + int t3_neg_fail = 0; + int t3_zero_fail = 0; + int t3_clamp_fail = 0; + // positive delta (insertion) + for (int old_pos : {0, 7, 31}) { + for (int delta : {1, 4}) { + const int new_pos = old_pos + delta; + std::vector raw = make_row(old_pos); + std::vector cached = rotate_at(raw, old_pos); + std::vector helper_out = cached; + strata::program::suffix_reuse::rotate_row_by_delta( + helper_out.data(), head_dim, n_rot, old_pos, delta, + cos_a.data(), sin_a.data(), max_pos); + std::vector expected = rotate_at(raw, new_pos); + for (int i = 0; i < n_rot; ++i) + if (std::fabs(helper_out[i] - expected[i]) > 1e-6f) ++t3_pos_fail; + } + } + // negative delta (deletion) - old_pos large enough that new_pos stays >= 0 + for (int old_pos : {7, 31, 100}) { + for (int delta : {-1, -3}) { + const int new_pos = old_pos + delta; + std::vector raw = make_row(old_pos); + std::vector cached = rotate_at(raw, old_pos); + std::vector helper_out = cached; + strata::program::suffix_reuse::rotate_row_by_delta( + helper_out.data(), head_dim, n_rot, old_pos, delta, + cos_a.data(), sin_a.data(), max_pos); + std::vector expected = rotate_at(raw, new_pos); + for (int i = 0; i < n_rot; ++i) + if (std::fabs(helper_out[i] - expected[i]) > 1e-6f) ++t3_neg_fail; + } + } + // delta = 0 - helper is the identity (cached already rotated at old_pos, no change) + for (int old_pos : {0, 7, 31}) { + std::vector raw = make_row(old_pos); + std::vector cached = rotate_at(raw, old_pos); + std::vector helper_out = cached; + strata::program::suffix_reuse::rotate_row_by_delta( + helper_out.data(), head_dim, n_rot, old_pos, 0, + cos_a.data(), sin_a.data(), max_pos); + std::vector expected = rotate_at(raw, old_pos); // == cached + for (int i = 0; i < n_rot; ++i) + if (std::fabs(helper_out[i] - expected[i]) > 1e-6f) ++t3_zero_fail; + } + // new_pos >= max_pos clamp: clamp to (max_pos-1), expected uses the same clamp + { + const int old_pos = 1024; + const int delta = 4096; // new_pos = 5120, max_pos = 2048 -> clamp to 2047 + std::vector raw = make_row(old_pos); + std::vector cached = rotate_at(raw, old_pos); + std::vector helper_out = cached; + strata::program::suffix_reuse::rotate_row_by_delta( + helper_out.data(), head_dim, n_rot, old_pos, delta, + cos_a.data(), sin_a.data(), max_pos); + std::vector expected = rotate_at(raw, max_pos - 1); // expected honours the clamp + for (int i = 0; i < n_rot; ++i) + if (std::fabs(helper_out[i] - expected[i]) > 1e-6f) ++t3_clamp_fail; + } + check(t3_pos_fail == 0, "T3: positive delta: helper(cached) == rotate(raw, new_pos) for all (old_pos, delta)"); + check(t3_neg_fail == 0, "T3: negative delta (deletion): helper(cached) == rotate(raw, new_pos) for all (old_pos, delta)"); + check(t3_zero_fail == 0, "T3: delta=0: helper(cached) is identity (cached already rotated at old_pos)"); + check(t3_clamp_fail == 0,"T3: new_pos >= max_pos clamp: helper and expected both clamp to max_pos-1"); + } + + // --- T5: integration with retain/limit_reuse on a synthetic ConversationCache ------ + // the splice plan's `invalidate_checkpoints` runs BEFORE retain/limit_reuse, so the surviving + // checkpoints (lengths <= edit_pos) stay parked. The unchanged_tokens the caller's take_reuse + // sees is bounded by the splice region's start (== edit_pos), and the suffix tail reuses via the + // splice (V unchanged, K rotated) - this test pins that math against today's behaviour by + // building a ConversationCache, feeding it a `retain` + `limit_reuse(first_dirty)` sequence, and + // asserting the byte accounting is the same whether or not the splice plan ran first. + { + auto cp = [](std::vector ids) { + strata::core::ConversationCheckpoint c; + c.ids = std::move(ids); + return c; + }; + // mirrors the pinned test in src/core/conversation_cache_test.cpp:62-84 (a single layer, a + // retainer, a limit_reuse rewind). added here to bind the splice invalidation to the same + // chain so a future reviewer can see end-to-end accounting with a checkpoint chain present. + strata::core::ConversationKv layer; + layer.k.resize(400); + std::vector layers; + layers.push_back(std::move(layer)); + const size_t retained = layers.capacity() * sizeof(strata::core::ConversationKv) + layers[0].bytes(); + auto image = [](std::initializer_list ids) { + strata::core::SavedConversation s; + s.live.ids = ids; + s.live.gdn.resize(64, 7); + s.cvec = true; + return s; + }; + const size_t parked = image({1, 2, 3}).bytes(); + strata::core::ConversationCache cache(retained + parked, 4); + + // park, retain, splice-plan invalidate, limit_reuse; assert the unchanged_tokens identity + check(cache.put(image({1, 2, 3})), "T5: park before retaining active storage"); + cache.retain(std::move(layers), 16); + check(cache.bytes() == retained + parked && cache.size() == 1, + "T5: active retained storage consumes bytes but no parked slot"); + + // edit at edit_pos = 9: invalidate every checkpoint whose token length > 9 (none parked yet, + // this exercises the chain API before the chain is populated, matching the conversation_cache + // test fixture at line 70-76). + std::vector chain; + chain.push_back({2, 0}); chain.push_back({5, 1}); chain.push_back({9, 2}); + const size_t dropped = strata::program::suffix_reuse::invalidate_checkpoints(chain, 9); + check(dropped == 0, "T5: edit_pos = 9 keeps every checkpoint with length <= 9"); + + // standard conversation_cache test flow: two rewinds, take_reuse, undo-roundtrip + cache.limit_reuse(9); + cache.limit_reuse(12); + auto reuse = cache.take_reuse(); + check(reuse.unchanged_tokens == 9, "T5: a later continuation cannot undo a rewind's dirty boundary"); + check(cache.bytes() == parked, "T5: taking retained buffers releases their budget accounting"); + + // simulate a real splice: the splice plan says edit_pos = 9, unchanged tail = [9, n_new). + // the splice path adds K rotation but the byte accounting is unchanged: `retain` below uses + // reuse.kv against a fresh retainer (the same move semantics as conversation_cache_test.cpp:76). + cache.retain(std::move(reuse.kv), 16); + + // checkpoint chain after a real edit at E = 5 (different test point): invalidate everything + // strictly greater than 5; the survivors are lengths [2]. + std::vector chain2; + chain2.push_back({2, 0}); + chain2.push_back({5, 1}); + chain2.push_back({7, 2}); + chain2.push_back({9, 3}); + const size_t dropped2 = strata::program::suffix_reuse::invalidate_checkpoints(chain2, 5); + check(dropped2 == 2 && chain2.size() == 2, "T5: edit_pos = 5 drops lengths 7 and 9"); + check(chain2[0].length == 2 && chain2[1].length == 5, + "T5: survivors are the prefix chain up to edit_pos"); + } + // sequence: old=[1,2,3,4,5,6,7,8], new=[1,2,99,3,4,5,6,7,8] + // the common tail is [3,4,5,6,7,8] in old vs [3,4,5,6,7,8] in new + // per-token position delta from old index (2 + i) to new index (3 + i) is uniformly +1 + { + const std::vector old_tokens = {1, 2, 3, 4, 5, 6, 7, 8}; + const std::vector new_tokens = {1, 2, 99, 3, 4, 5, 6, 7, 8}; + const SuffixReusePlan plan = compute_suffix_plan(old_tokens, new_tokens); + check(plan.edit_pos == 2, "T4: edit at position 2"); + check(plan.old_suffix_len == 6 && plan.new_suffix_len == 6, + "T4: six-token common tail [3..8]"); + check(plan.uniform && plan.delta == 1, + "T4: insertion at position 2 gives a uniform +1 rotation"); + } + // deletion at a different point from the tail: old=[1,2,3,4,5,6,7], new=[1,2,3,5,6,7] + // common tail [5,6,7] in old vs [3,4,5] in new, edit_pos = 7 - 3 = 4 in the old prompt + { + const std::vector old_tokens = {1, 2, 3, 4, 5, 6, 7}; + const std::vector new_tokens = {1, 2, 3, 5, 6, 7}; + const SuffixReusePlan plan = compute_suffix_plan(old_tokens, new_tokens); + check(plan.edit_pos == 4, "T4: delete of [4] -> edit_pos at 4 in the old prompt"); + check(plan.uniform && plan.delta == -1, + "T4: deletion at position 4 gives a uniform -1 rotation"); + check(plan.old_suffix_len == 3 && plan.new_suffix_len == 3, + "T4: three-token common tail [5,6,7]"); + } + + // --- T8: A1.3 / A2.1 fail-closed guards (CPU-testable mirror of generate.cpp's runtime check) ---- + { + using strata::program::suffix_reuse::validate_suffix_reuse_options; + // flag off (0) and any prompt_cache / kv / kv_resident / layer_split - always OK + check(validate_suffix_reuse_options(0, 0, "fp16", 0, false).empty(), + "T8: flag off with --prompt-cache 0: no splice, no guard fires"); + check(validate_suffix_reuse_options(0, 1, "fp16", 0, true).empty(), + "T8: flag off with layer split and any prompt_cache: no splice, no guard fires"); + check(validate_suffix_reuse_options(0, 1, "int8", 16, false).empty(), + "T8: flag off with int8 + kv_resident=16: no splice, no guard fires"); + // flag on, prompt_cache too low - REJECTS + const std::string low = validate_suffix_reuse_options(10, 1, "fp16", 0, false); + check(!low.empty() && low.find("--prompt-cache >= 2") != std::string::npos, + "T8: flag on + --prompt-cache < 2: guard rejects with --prompt-cache >= 2"); + // flag on, layer_split > 1 stage - REJECTS (A2.1) + const std::string split = validate_suffix_reuse_options(10, 6, "fp16", 0, true); + check(!split.empty() && split.find("--layer-split") != std::string::npos, + "T8: flag on + --layer-split: guard rejects with per-stage splice note"); + // Phase 2: kv format guards. Phase 3: --kv int8 is NOW SUPPORTED. + check(validate_suffix_reuse_options(10, 6, "int8", 0, false).empty(), + "T8: flag on + --kv int8: guard PASSES (Phase 3 int8 splice)"); + check(validate_suffix_reuse_options(10, 6, "q4_0", 0, false).empty(), + "T8: flag on + --kv q4_0: guard PASSES (the Q4 sandwich; measured in T11.f)"); + check(validate_suffix_reuse_options(10, 6, "k8v4", 0, false).empty(), + "T8: flag on + --kv k8v4 (hybrid): guard PASSES (K half is int8; V untouched)"); + check(!validate_suffix_reuse_options(10, 6, "fp16", 20480, false).empty(), + "T8: flag on + --kv-resident 20480: guard rejects with --kv-resident note (Phase 3)"); + // flag on, everything OK - PASSES (both fp16 and int8 are now supported) + check(validate_suffix_reuse_options(10, 6, "fp16", 0, false).empty(), + "T8: flag on + --kv fp16 + no kv_resident + no --layer-split: passes"); + check(validate_suffix_reuse_options(10, 6, "int8", 0, false).empty(), + "T8: flag on + --kv int8 + no kv_resident + no --layer-split: passes (Phase 3)"); + } + + std::printf(g_fail ? "FAIL\n" : "PASS\n"); + return g_fail ? 1 : 0; +}