From 0e3dbf0f23cf0ba00832a0697909e9eb8b214caf Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Tue, 13 Oct 2026 09:48:03 -0700 Subject: [PATCH 1/3] llama : fix on-device state copy sizes for quantized KV caches The on-device sequence state copies (LLAMA_STATE_SEQ_FLAGS_ON_DEVICE) computed their element count as size / ggml_element_size(). For a quantized type such as q8_0 or q4_0 that is the number of blocks, not elements, so only part of each range was copied: the restored state was wrong, or the copy crashed. Compute the count as size / ggml_type_size() * ggml_blck_size() for the save, the restore and the copy between differently chunked ranges, and assert that each range covers whole blocks. --- src/llama-context.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 9ee64b566779..a1205bb99047 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3090,7 +3090,9 @@ class llama_io_write_device : public llama_io_write_i { for (const auto & winfo : winfos) { auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer); - const int64_t n = winfo.size/ggml_element_size(winfo.tensor); + // number of elements, not blocks: a quantized type stores ggml_blck_size() elements per ggml_type_size() bytes + GGML_ASSERT(winfo.size % ggml_type_size(winfo.tensor->type) == 0); + const int64_t n = winfo.size/ggml_type_size(winfo.tensor->type)*ggml_blck_size(winfo.tensor->type); auto & mbuf = mbufs_new[buft]; @@ -3221,7 +3223,9 @@ class llama_io_read_device : public llama_io_read_i { for (const auto & rinfo : rinfos) { auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer); - const int64_t n = rinfo.size/ggml_element_size(rinfo.tensor); + // number of elements, not blocks: a quantized type stores ggml_blck_size() elements per ggml_type_size() bytes + GGML_ASSERT(rinfo.size % ggml_type_size(rinfo.tensor->type) == 0); + const int64_t n = rinfo.size/ggml_type_size(rinfo.tensor->type)*ggml_blck_size(rinfo.tensor->type); auto & mbuf = mbufs_new[buft]; @@ -3288,8 +3292,9 @@ class llama_io_read_device : public llama_io_read_i { const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off); - const size_t el = ggml_element_size(src_t); - const int64_t n_el = (int64_t) (n_copy / el); + // number of elements, not blocks (see above) + GGML_ASSERT(n_copy % ggml_type_size(src_t->type) == 0); + const int64_t n_el = (int64_t) (n_copy / ggml_type_size(src_t->type) * ggml_blck_size(src_t->type)); auto * src_v = ggml_view_1d(ctx_scratch, src_t, n_el, src_off); ggml_backend_view_init(src_v); From 429ffaf326e316a97a2f55cce304cd5603bb4002 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Tue, 3 Nov 2026 09:50:14 -0800 Subject: [PATCH 2/3] llama : drop the one-line comments --- src/llama-context.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index a1205bb99047..e42340d8884a 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3090,7 +3090,6 @@ class llama_io_write_device : public llama_io_write_i { for (const auto & winfo : winfos) { auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer); - // number of elements, not blocks: a quantized type stores ggml_blck_size() elements per ggml_type_size() bytes GGML_ASSERT(winfo.size % ggml_type_size(winfo.tensor->type) == 0); const int64_t n = winfo.size/ggml_type_size(winfo.tensor->type)*ggml_blck_size(winfo.tensor->type); @@ -3223,7 +3222,6 @@ class llama_io_read_device : public llama_io_read_i { for (const auto & rinfo : rinfos) { auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer); - // number of elements, not blocks: a quantized type stores ggml_blck_size() elements per ggml_type_size() bytes GGML_ASSERT(rinfo.size % ggml_type_size(rinfo.tensor->type) == 0); const int64_t n = rinfo.size/ggml_type_size(rinfo.tensor->type)*ggml_blck_size(rinfo.tensor->type); @@ -3292,7 +3290,6 @@ class llama_io_read_device : public llama_io_read_i { const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off); - // number of elements, not blocks (see above) GGML_ASSERT(n_copy % ggml_type_size(src_t->type) == 0); const int64_t n_el = (int64_t) (n_copy / ggml_type_size(src_t->type) * ggml_blck_size(src_t->type)); From bd1c3d36acb9f5366969ba393adee749b3f8d9a0 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Wed, 4 Nov 2026 15:43:55 -0800 Subject: [PATCH 3/3] llama : drop the redundant cast in the state copy size --- src/llama-context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index e42340d8884a..d76503d524ba 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3291,7 +3291,7 @@ class llama_io_read_device : public llama_io_read_i { const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off); GGML_ASSERT(n_copy % ggml_type_size(src_t->type) == 0); - const int64_t n_el = (int64_t) (n_copy / ggml_type_size(src_t->type) * ggml_blck_size(src_t->type)); + const int64_t n_el = n_copy / ggml_type_size(src_t->type) * ggml_blck_size(src_t->type); auto * src_v = ggml_view_1d(ctx_scratch, src_t, n_el, src_off); ggml_backend_view_init(src_v);