Skip to content

Commit 7e74b82

Browse files
jinlongsongsjl623
jinlongsong
authored and
sjl623
committed
ggml-cpu : fix STQ1_0 CI failures
1 parent b1e5559 commit 7e74b82

7 files changed

Lines changed: 41 additions & 57 deletions

File tree

‎ggml/src/ggml-cpu/arch-fallback.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,6 @@
212212
// quants.c
213213
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
214214
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
215-
#define ggml_vec_dot_stq_0_q8_K_generic ggml_vec_dot_stq_0_q8_K
216215
#define ggml_vec_dot_stq1_0_q8_K_generic ggml_vec_dot_stq1_0_q8_K
217216
// repack.cpp
218217
#define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1

‎ggml/src/ggml-cpu/arch/arm/quants.c‎

Lines changed: 16 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1682,21 +1682,6 @@ void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
16821682
#endif
16831683
}
16841684

1685-
// repack q8_K qs from contiguous to planar layout (in-place, per 64-byte chunk)
1686-
// after repack, vld1q_s8 reads produce the same values as vld4q_s8 de-interleave
1687-
void stq1_0_repack_q8_K_inplace(void * vy, int nb) {
1688-
block_q8_K * y = (block_q8_K *) vy;
1689-
for (int i = 0; i < nb; ++i) {
1690-
for (int j = 0; j < QK_K; j += 64) {
1691-
const int8x16x4_t v = vld4q_s8(y[i].qs + j);
1692-
vst1q_s8(y[i].qs + j + 0, v.val[0]);
1693-
vst1q_s8(y[i].qs + j + 16, v.val[1]);
1694-
vst1q_s8(y[i].qs + j + 32, v.val[2]);
1695-
vst1q_s8(y[i].qs + j + 48, v.val[3]);
1696-
}
1697-
}
1698-
}
1699-
17001685
void ggml_vec_dot_stq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
17011686
assert(nrc == 1);
17021687
UNUSED(nrc);
@@ -1714,7 +1699,7 @@ void ggml_vec_dot_stq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
17141699

17151700
const uint8x16_t m3 = vdupq_n_u8(3);
17161701
const uint8x16_t mask_0f = vdupq_n_u8(0x0F);
1717-
const uint8_t (*sign_lut_16)[8] = (const uint8_t (*)[8]) table_b2b_0;
1702+
const uint8_t * sign_lut_16 = (const uint8_t *)(const void *) table_b2b_0;
17181703

17191704
#if defined(__ARM_FEATURE_DOTPROD)
17201705
// dotprod path: single 32-byte tbl lookup + native vdotq.
@@ -1736,20 +1721,20 @@ void ggml_vec_dot_stq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
17361721
#endif
17371722

17381723
// Each half processes 16 bytes of x.qs (32 codes), 4 bytes of x.sign,
1739-
// and 128 bytes of y.qs (4 lanes × 16 bytes × 2 wide-blocks).
1740-
// sumi0..sumi3, mask_0f, sign_lut_16, m3, STQ1_0_LOOKUP, STQ1_0_DOT are
1741-
// captured from the enclosing scope.
1724+
// and 128 bytes of y.qs (2 chunks of 64 B). Stride-16 grouping lets each
1725+
// lane plane be a contiguous 16-byte slice of y (offsets 0/16/32/48 within
1726+
// a chunk), so we read it with plain vld1q_s8 — no deinterleave needed.
17421727
#define STQ1_0_DOT_HALF(QS_PTR, SIGN_PTR, YP_PTR) do { \
17431728
const uint8x16_t packed = vld1q_u8(QS_PTR); \
17441729
const uint8x16_t lo = vandq_u8(packed, mask_0f); \
17451730
const uint8x16_t hi = vshrq_n_u8(packed, 4); \
17461731
const uint8x16_t idx0 = vzip1q_u8(lo, hi); \
17471732
const uint8x16_t idx1 = vzip2q_u8(lo, hi); \
17481733
const uint8_t * sp = (SIGN_PTR); \
1749-
const uint8x16_t s0 = vcombine_u8(vld1_u8(sign_lut_16[sp[0]]), \
1750-
vld1_u8(sign_lut_16[sp[1]])); \
1751-
const uint8x16_t s1 = vcombine_u8(vld1_u8(sign_lut_16[sp[2]]), \
1752-
vld1_u8(sign_lut_16[sp[3]])); \
1734+
const uint8x16_t s0 = vcombine_u8(vld1_u8(sign_lut_16 + 8*sp[0]), \
1735+
vld1_u8(sign_lut_16 + 8*sp[1])); \
1736+
const uint8x16_t s1 = vcombine_u8(vld1_u8(sign_lut_16 + 8*sp[2]), \
1737+
vld1_u8(sign_lut_16 + 8*sp[3])); \
17531738
const uint8x16_t sel_0 = STQ1_0_LOOKUP(vorrq_u8(idx0, s0)); \
17541739
const uint8x16_t sel_1 = STQ1_0_LOOKUP(vorrq_u8(idx1, s1)); \
17551740
const int8x16_t sqx0 = vreinterpretq_s8_u8(vandq_u8(sel_0, m3)); \
@@ -1761,14 +1746,14 @@ void ggml_vec_dot_stq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
17611746
const int8x16_t sqx6 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(sel_1, 4), m3)); \
17621747
const int8x16_t sqx7 = vreinterpretq_s8_u8(vshrq_n_u8(sel_1, 6)); \
17631748
const int8_t * yp = (YP_PTR); \
1764-
sumi0 = STQ1_0_DOT(sumi0, sqx0, vld1q_s8(yp + 0)); \
1765-
sumi1 = STQ1_0_DOT(sumi1, sqx1, vld1q_s8(yp + 16)); \
1766-
sumi2 = STQ1_0_DOT(sumi2, sqx2, vld1q_s8(yp + 32)); \
1767-
sumi3 = STQ1_0_DOT(sumi3, sqx3, vld1q_s8(yp + 48)); \
1768-
sumi0 = STQ1_0_DOT(sumi0, sqx4, vld1q_s8(yp + 64)); \
1769-
sumi1 = STQ1_0_DOT(sumi1, sqx5, vld1q_s8(yp + 80)); \
1770-
sumi2 = STQ1_0_DOT(sumi2, sqx6, vld1q_s8(yp + 96)); \
1771-
sumi3 = STQ1_0_DOT(sumi3, sqx7, vld1q_s8(yp + 112)); \
1749+
sumi0 = STQ1_0_DOT(sumi0, sqx0, vld1q_s8(yp + 0)); \
1750+
sumi1 = STQ1_0_DOT(sumi1, sqx1, vld1q_s8(yp + 16)); \
1751+
sumi2 = STQ1_0_DOT(sumi2, sqx2, vld1q_s8(yp + 32)); \
1752+
sumi3 = STQ1_0_DOT(sumi3, sqx3, vld1q_s8(yp + 48)); \
1753+
sumi0 = STQ1_0_DOT(sumi0, sqx4, vld1q_s8(yp + 64)); \
1754+
sumi1 = STQ1_0_DOT(sumi1, sqx5, vld1q_s8(yp + 80)); \
1755+
sumi2 = STQ1_0_DOT(sumi2, sqx6, vld1q_s8(yp + 96)); \
1756+
sumi3 = STQ1_0_DOT(sumi3, sqx7, vld1q_s8(yp + 112)); \
17721757
} while (0)
17731758

17741759
for (int i = 0; i < nb; ++i) {

‎ggml/src/ggml-cpu/ggml-cpu.c‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1356,17 +1356,6 @@ UseGgmlGemm1:;
13561356
from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
13571357
(void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
13581358
(ne10_block_end - ne10_block_start) * bs);
1359-
1360-
#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
1361-
// STQ1_0 NEON vec_dot wants Q8_K activations in planar
1362-
// layout (vld1q reads); deinterleave once here so the
1363-
// per-row dot loop amortizes it across all M weight rows.
1364-
if (src0->type == GGML_TYPE_STQ1_0) {
1365-
stq1_0_repack_q8_K_inplace(
1366-
(void *)(wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1367-
(int)(ne10_block_end - ne10_block_start));
1368-
}
1369-
#endif
13701359
}
13711360
}
13721361
}

‎ggml/src/ggml-cpu/quants.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -583,14 +583,18 @@ void ggml_vec_dot_stq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs,
583583
for (int i = 0; i < nb; ++i) {
584584
int32_t sumi = 0;
585585

586+
// Groups are stride-16 within each 64-byte chunk: group g (chunk-local
587+
// in 0..15) pairs with {y[c*64 + g + p*16] : p in 0..3}.
586588
for (int g = 0; g < QK_K/4; ++g) {
587589
const uint8_t code = (x[i].qs[g/2] >> (4 * (g & 1))) & 0x0F;
588590
const uint8_t sign = (x[i].sign[g/8] >> (g % 8)) & 0x01;
589591
const uint8_t qpack = stq1_0_codebook[((uint32_t) sign << 4) | code];
590592

593+
const int chunk = g / 16;
594+
const int gloc = g % 16;
591595
for (int p = 0; p < 4; ++p) {
592596
const int q = (qpack >> (2*p)) & 0x3;
593-
sumi += (q - 1) * y[i].qs[g*4 + p];
597+
sumi += (q - 1) * y[i].qs[chunk*64 + gloc + p*16];
594598
}
595599
}
596600

‎ggml/src/ggml-cpu/quants.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,6 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
5959
void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
6060
void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
6161
void ggml_vec_dot_stq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
62-
#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
63-
// STQ1_0 ARM NEON path uses planar Q8_K activation; the in-place repack
64-
// is only declared on ARM and called from ggml_compute_forward_mul_mat.
65-
void stq1_0_repack_q8_K_inplace(void * vy, int nb);
66-
#endif
6762

6863
void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
6964
void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);

‎ggml/src/ggml-quants.c‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2426,15 +2426,19 @@ void quantize_row_stq1_0_ref(const float * GGML_RESTRICT x, block_stq1_0 * GGML_
24262426
}
24272427
y[i].d = GGML_FP32_TO_FP16(amax);
24282428

2429-
// STQ1_0 forces exactly one zero per group of 4. Pick the smallest-|x|
2430-
// lane as that zero; project the other 3 onto {-d, +d} via sign.
2429+
// STQ1_0 forces exactly one zero per group of 4. Groups are stride-16
2430+
// within each 64-weight chunk: group g (chunk-local in 0..15) covers
2431+
// {x[c*64 + g + p*16] : p in 0..3}. Pick the smallest-|x| lane as zero;
2432+
// project the other 3 onto {-d, +d} via sign.
24312433
for (int g = 0; g < QK_K/4; ++g) {
2432-
const float * xv = x + g*4;
2434+
const int chunk = g / 16;
2435+
const int gloc = g % 16;
2436+
const float * base = x + chunk*64 + gloc;
24332437

24342438
int zero_pos = 0;
2435-
float min_abs = fabsf(xv[0]);
2439+
float min_abs = fabsf(base[0]);
24362440
for (int p = 1; p < 4; ++p) {
2437-
const float a = fabsf(xv[p]);
2441+
const float a = fabsf(base[p*16]);
24382442
if (a < min_abs) { min_abs = a; zero_pos = p; }
24392443
}
24402444

@@ -2445,7 +2449,7 @@ void quantize_row_stq1_0_ref(const float * GGML_RESTRICT x, block_stq1_0 * GGML_
24452449
if (p == zero_pos) {
24462450
lane = 0x1;
24472451
} else {
2448-
lane = (xv[p] < 0.0f) ? 0x0 : 0x2;
2452+
lane = (base[p*16] < 0.0f) ? 0x0 : 0x2;
24492453
}
24502454
qpack |= (uint8_t)(lane << (2*p));
24512455
}
@@ -2548,16 +2552,22 @@ void dequantize_row_stq1_0(const block_stq1_0 * GGML_RESTRICT x, float * GGML_RE
25482552
for (int64_t i = 0; i < nb; ++i) {
25492553
const float d = GGML_FP16_TO_FP32(x[i].d);
25502554

2555+
// Groups are stride-16 within each 64-weight chunk: group g (chunk-local
2556+
// in 0..15) writes to {y[c*64 + g + p*16] : p in 0..3}.
25512557
for (int g = 0; g < QK_K/4; ++g) {
25522558
const uint8_t code = (x[i].qs[g/2] >> (4 * (g & 1))) & 0x0F;
25532559
const uint8_t sign = (x[i].sign[g/8] >> (g % 8)) & 0x01;
25542560
const uint8_t qpack = stq1_0_codebook[((uint32_t) sign << 4) | code];
25552561

2562+
const int chunk = g / 16;
2563+
const int gloc = g % 16;
25562564
for (int p = 0; p < 4; ++p) {
25572565
const int q = (qpack >> (2*p)) & 0x3;
2558-
*y++ = (float) (q - 1) * d;
2566+
y[chunk*64 + gloc + p*16] = (float) (q - 1) * d;
25592567
}
25602568
}
2569+
2570+
y += QK_K;
25612571
}
25622572
}
25632573

‎tests/test-quantize-fns.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ static int test_vec_dot_q(bool verbose) {
156156
const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
157157
const float max_quantization_error =
158158
type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :
159+
// STQ1_0 is QAT-based: inputs sit on a {-a, 0, +a} grid; binary bound fits.
160+
type == GGML_TYPE_STQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :
159161
type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
160162
type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
161163
type == GGML_TYPE_Q2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
@@ -182,7 +184,7 @@ static int test_vec_dot_q(bool verbose) {
182184
const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||
183185
type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S
184186
? MAX_DOT_PRODUCT_ERROR_LOWBIT
185-
: type == GGML_TYPE_Q1_0
187+
: type == GGML_TYPE_Q1_0 || type == GGML_TYPE_STQ1_0
186188
? MAX_DOT_PRODUCT_ERROR_BINARY
187189
: type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0
188190
? MAX_DOT_PRODUCT_ERROR_TERNARY

0 commit comments

Comments
 (0)