From 9d7e3419dcc1fec3e9768560178408b822bea88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=A7Tetto=E2=9C=A7?= <122175170+Tetto-chan@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:52:33 +0700 Subject: [PATCH 1/2] vulkan: optimize dequant_ptq1_0 shader (remove dynamic loop & divergence) ### Summary Optimizes `dequant_ptq1_0.comp` by eliminating the dynamic loop, removing per-element branch divergence, and reducing redundant global memory fetches. ### Details & Key Changes 1. **$O(1)$ Trit Unpacking:** The original implementation used a dynamic loop `for (uint i = 0u; i < n; ++i) v = (v * 3u) & 0xFFu;`. In modular arithmetic $\pmod{256}$, this is equivalent to $v_n = (b \cdot 3^n) \pmod{256}$. Replaced the loop with a compile-time lookup table `pow3 = {1, 3, 9, 27, 81}`, eliminating dynamic loop execution and severe warp/wavefront divergence. 2. **Hoisted Branch Invariants:** Each thread processes 8 consecutive elements (`8 * il + l`). Thread element ranges never cross block boundaries (`il < 10` is strictly $< 80$, `10 <= il < 15` is strictly $[80..119]$, and `il == 15` is strictly $\ge 120$). Hoisted the `if/else` checks out of the unrolled 8-element loop, evaluating conditions once per thread instead of 8 times. 3. **Memory Access Optimization:** For `il == 15`, `qh[0]` and `qh[1]` are now loaded into scalar registers once rather than repeatedly fetched from global memory for every trit. ### Verification Mathematically verified to produce bit-exact identical dequantized values compared to the previous CPU/GPU codec reference. Drastically improves Vulkan decode token rate, particularly on AMD (RADV) and non-CUDA hardware. --- .../vulkan-shaders/dequant_ptq1_0.comp | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp index 53baf41f65b9..047ed0630941 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp @@ -7,48 +7,57 @@ layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {block_ptq1_0 data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; -// One invocation per 8 elements: 16 invocations cover a 128-weight block. Element -// order follows the CPU codec (16-byte qs chunk, 8-byte qs chunk, then qh), so the -// index maths lives in one place rather than being reproduced per shader. -float ptq1_0_trit_l(uint ib, uint e) { - uint b; - uint n; - if (e < 80u) { - b = uint(data_a[ib].qs[e & 15u]); - n = e >> 4u; - } else if (e < 120u) { - const uint t = e - 80u; - b = uint(data_a[ib].qs[16u + (t & 7u)]); - n = t >> 3u; - } else { - const uint t = e - 120u; - b = uint(data_a[ib].qh[t & 1u]); - n = t >> 1u; - } - - uint v = b; - for (uint i = 0u; i < n; ++i) { - v = (v * 3u) & 0xFFu; - } - return float(int((v * 3u) >> 8u) - 1); -} +// Powers of 3: 3^0, 3^1, 3^2, 3^3, 3^4 (mod 256) +const uint pow3[5] = uint[5](1u, 3u, 9u, 27u, 81u); void main() { const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; const uint tid = gl_LocalInvocationID.x % 64; - const uint il = tid / 4; - const uint ir = tid % 4; + const uint il = tid / 4; + const uint ir = tid % 4; const uint ib = 4*i + ir; if (ib >= p.nel / 128) { return; } const uint b_idx = 512*i + 128*ir + 8*il; - const float d = float(data_a[ib].d); - [[unroll]] for (uint l = 0; l < 8; ++l) { - data_b[b_idx + l] = D_TYPE(ptq1_0_trit_l(ib, 8*il + l) * d); + if (il < 10u) { + // e in [0..79]: qs[0..15], n is uniform for all 8 elements + const uint n = il >> 1u; + const uint p_val = pow3[n]; + const uint qs_offset = (il & 1u) != 0u ? 8u : 0u; + + [[unroll]] for (uint l = 0u; l < 8u; ++l) { + const uint b = uint(data_a[ib].qs[qs_offset + l]); + const uint v = (b * p_val) & 0xFFu; + const float trit = float(int((v * 3u) >> 8u) - 1); + data_b[b_idx + l] = D_TYPE(trit * d); + } + } else if (il < 15u) { + // e in [80..119]: qs[16..23], n is uniform for all 8 + const uint n = il - 10u; + const uint p_val = pow3[n]; + + [[unroll]] for (uint l = 0u; l < 8u; ++l) { + const uint b = uint(data_a[ib].qs[16u + l]); + const uint v = (b * p_val) & 0xFFu; + const float trit = float(int((v * 3u) >> 8u) - 1); + data_b[b_idx + l] = D_TYPE(trit * d); + } + } else { + // il == 15 (e in [120..127]): read both qh bytes once into registers + const uint qh0 = uint(data_a[ib].qh[0]); + const uint qh1 = uint(data_a[ib].qh[1]); + + [[unroll]] for (uint l = 0u; l < 8u; ++l) { + const uint b = (l & 1u) != 0u ? qh1 : qh0; + const uint n = l >> 1u; + const uint v = (b * pow3[n]) & 0xFFu; + const float trit = float(int((v * 3u) >> 8u) - 1); + data_b[b_idx + l] = D_TYPE(trit * d); + } } } From 51730614cbde3c294973da4dac7e9d8845fba717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=A7Tetto=E2=9C=A7?= <122175170+Tetto-chan@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:13:19 +0700 Subject: [PATCH 2/2] vulkan: make mul_mat_vec_ptq1_0 compatible with GPUs lacking DP4A and optimize ALU ### Summary Enables `mul_mat_vec_ptq1_0` to run universally across all Vulkan-capable GPUs (including AMD Polaris GCN 4.0 / Vega / older hardware lacking `VK_KHR_shader_integer_dot_product`) while optimizing dot product ALU throughput. ### Problem Previously, the shader enforced `#extension GL_EXT_integer_dot_product : require`. On GPUs without hardware DP4A (such as AMD RX 480/580/590, Vega 56/64, Radeon VII), shader pipeline compilation failed at runtime. This caused the engine to silently fall back to the slow CPU-style `dequant_ptq1_0` path, reducing decode speed to <1 token/second. Additionally, the default path enforced saturating dot products (`dotPacked4x8AccSatEXT`), adding overhead on AMD architectures. ### Solution 1. Changed extension declaration to `#extension GL_EXT_integer_dot_product : enable`. 2. Added an emulated `DOT4` fallback using core GLSL 450 `bitfieldExtract`. On AMD GCN/Polaris, this compiles cleanly to native `v_bfe_i32` instructions without pipeline failure. 3. Switched the hardware path to non-saturating `dotPacked4x8EXT` by default: the maximum accumulation across 32 elements is $32 \times 2 \times 127 = 8128 \ll 2^{31}-1$, making integer overflow impossible and allowing direct emission of single-cycle hardware DP4A (`v_dot4_i32_i8`). 4. Set default `MUL3` to single-cycle shift-add `((v << 1u) + (v))` mapping to `v_lshl_add_u32`. --- .../vulkan-shaders/mul_mat_vecq_ptq1_0.comp | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_ptq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_ptq1_0.comp index fd21537bf58a..a8aa65ea7ecd 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_ptq1_0.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_ptq1_0.comp @@ -12,9 +12,11 @@ // Several columns (batched decode, ALU bound): passes of CG columns; per block decode all rows once per level, pack the trits and dot them with every column of the pass. #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require -#extension GL_EXT_integer_dot_product : require #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +// Enable integer dot product optionally: allows compilation on GPUs without hardware DP4A (e.g. AMD Polaris / Vega) +#extension GL_EXT_integer_dot_product : enable + #define B_TYPE block_q8_1_x4 #include "mul_mat_vec_base.glsl" @@ -38,18 +40,27 @@ layout (constant_id = 3) const uint CG = 3; uint a_offset, b_offset, d_offset; -#if defined(PTQ1_0_MUL3_SHIFT) -#define MUL3(v) (((v) << 1u) + (v)) -#elif defined(PTQ1_0_MUL3_PK16) +#if defined(PTQ1_0_MUL3_PK16) #define MUL3(v) pack32(unpack16(v) * u16vec2(3us, 3us)) #else -#define MUL3(v) ((v) * 3u) +// (v << 1u) + (v) maps to a single-cycle shift-add (v_lshl_add_u32) on AMD GCN/RDNA +#define MUL3(v) (((v) << 1u) + (v)) #endif -#ifdef PTQ1_0_DOT_NOSAT +#if defined(GL_EXT_integer_dot_product) +// Hardware DP4A: saturation is mathematically impossible (max sum in 32 trits is 8128 << 2^31-1), +// so non-saturating dotPacked4x8EXT maps directly to single-cycle native instructions (v_dot4_i32_i8). #define DOT4(a, b, acc) ((acc) + dotPacked4x8EXT((a), (b))) #else -#define DOT4(a, b, acc) dotPacked4x8AccSatEXT((a), (b), (acc)) +// Universal fallback for GPUs without hardware DP4A (AMD Polaris RX 400/500, Vega, older Intel/Nvidia): +// bitfieldExtract maps directly to AMD GCN's v_bfe_i32 instruction. +int32_t dot4_emulated(const int32_t a, const int32_t b, const int32_t acc) { + return acc + bitfieldExtract(a, 0, 8) * bitfieldExtract(b, 0, 8) + + bitfieldExtract(a, 8, 8) * bitfieldExtract(b, 8, 8) + + bitfieldExtract(a, 16, 8) * bitfieldExtract(b, 16, 8) + + bitfieldExtract(a, 24, 8) * bitfieldExtract(b, 24, 8); +} +#define DOT4(a, b, acc) dot4_emulated((a), (b), (acc)) #endif // One trit level from the 16-bit lanes vl (bytes 0,2) and vh (bytes 1,3): after w = v*3 the trit is in byte 1/3 and the residue in byte 0/2.