diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c592e81c..9c3d2c35 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -163,8 +163,9 @@ keep the score matrix in registers/L1. A `geist_model` is immutable, shared, read-only weights. A `geist_session` owns the mutable per-conversation state: KV cache, pending logits, sampler config, stats. Multiple sessions can share one model. The KV cache supports quantized -modes (`INT8`, packed `INT4` — half the INT8 footprint, near-lossless with the -optional Hadamard rotation `GEIST_KV_ROT`; and `KIVI` 2-bit) and prefix pinning +modes (`INT8`, packed `INT4` — half the INT8 footprint, near-lossless via a +default-on Hadamard rotation, `GEIST_KV_ROT=0` to opt out; and `KIVI` 2-bit) +and prefix pinning (`geist_session_pin_prefix`) to amortize a constant system prompt across chat turns. Speculative decode drafts via an n-gram lookup over history and verifies in one batched forward. diff --git a/include/geist.h b/include/geist.h index b4cc16bb..89e53206 100644 --- a/include/geist.h +++ b/include/geist.h @@ -197,9 +197,9 @@ enum geist_kv_mode { GEIST_KV_KIVI = 3, GEIST_KV_F16 = 4, /* Packed symmetric 4-bit KV cache (2 values/byte, per-token per-head - * scale). Half the INT8 footprint, near-lossless — especially with the - * Hadamard rotation (GEIST_KV_ROT=1, issue #61). No per-channel/group - * bookkeeping (unlike KIVI). Env: GEIST_KV_INT4=1. */ + * scale). Half the INT8 footprint, near-lossless. Enables the Hadamard + * rotation by default (issue #61) — GEIST_KV_ROT=0 opts out. No + * per-channel/group bookkeeping (unlike KIVI). Env: GEIST_KV_INT4=1. */ GEIST_KV_INT4 = 5, }; diff --git a/src/archs/transformer/arch_state.c b/src/archs/transformer/arch_state.c index 6629ecd0..dbfa858e 100644 --- a/src/archs/transformer/arch_state.c +++ b/src/archs/transformer/arch_state.c @@ -1031,10 +1031,15 @@ struct transformer_arch_session *transformer_session_alloc(struct transformer_ar sess->kv_kivi_enabled = false; } } - /* Issue #61: opt-in Hadamard rotation, only meaningful on the INT8 path. */ + /* Issue #61: Hadamard rotation, only meaningful on the INT8 storage path. + * Packed INT4 rotates by DEFAULT (INT4-without-rotation is a real quality + * cliff and the rotation is ~free / a net win at long context); plain + * INT8 stays opt-in. GEIST_KV_ROT=0 opts out, =1 forces on. */ { - const char *env_rot = getenv("GEIST_KV_ROT"); - sess->kv_rot_enabled = sess->kv_int8_enabled && env_rot != nullptr && env_rot[0] == '1'; + const char *env_rot = getenv("GEIST_KV_ROT"); + const bool rot_default = sess->kv_int4_packed_enabled; + const bool rot_on = (env_rot != nullptr) ? (env_rot[0] == '1') : rot_default; + sess->kv_rot_enabled = sess->kv_int8_enabled && rot_on; } /* F16 cache: explicit request, or AUTO-resolved FP32 upgraded when the * backend has the fused converting append (env GEIST_KV_F16=0 forces