From 5d35bd4e6acbf76327f08a1cd49d5bc8030600fb Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 6 Sep 2026 17:52:30 +0800 Subject: [PATCH] ggml : add FP8 types and CPU conversions --- ggml/include/ggml.h | 4 +- ggml/src/ggml-cpu/ggml-cpu.cpp | 7 ++++ ggml/src/ggml-cpu/ops.cpp | 41 ++++++++++++++++++++ ggml/src/ggml-quants.c | 47 +++++++++++++++++++++++ ggml/src/ggml-quants.h | 5 +++ ggml/src/ggml.c | 14 +++++++ tests/test-backend-ops.cpp | 30 ++++++++++++++- tests/test-quantize-fns.cpp | 69 +++++++++++++++++++++++++++++++++- 8 files changed, 214 insertions(+), 3 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index b88b7e54a5ea..cd1775c99dac 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -430,7 +430,9 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + GGML_TYPE_F8_E4M3 = 43, // E4M3FN + GGML_TYPE_F8_E5M2 = 44, + GGML_TYPE_COUNT = 45, }; // precision diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 8cece71f186f..1225b48c0247 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -439,6 +439,13 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st } } + const bool src0_fp8 = src0 && (src0->type == GGML_TYPE_F8_E4M3 || src0->type == GGML_TYPE_F8_E5M2); + const bool dst_fp8 = op->type == GGML_TYPE_F8_E4M3 || op->type == GGML_TYPE_F8_E5M2; + if (src0_fp8 || dst_fp8) { + return (op->op == GGML_OP_CPY || op->op == GGML_OP_DUP || op->op == GGML_OP_CONT) && + (src0->type == op->type || (src0_fp8 && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16))); + } + switch (op->op) { case GGML_OP_CPY: case GGML_OP_SET_ROWS: diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 266261c5e5a4..231664c50626 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -2,6 +2,7 @@ #include "ggml-cpu.h" #include "ggml-impl.h" +#include "ggml-quants.h" #include "binary-ops.h" #include "simd-gemm.h" #include "ggml.h" @@ -12,6 +13,32 @@ #include #include +struct ggml_fp8_e4m3_cpu { + uint8_t value; +}; + +struct ggml_fp8_e5m2_cpu { + uint8_t value; +}; + +static float fp8_e4m3_to_f32(ggml_fp8_e4m3_cpu value) { + return ggml_fp8_e4m3_to_fp32(value.value); +} + +static float fp8_e5m2_to_f32(ggml_fp8_e5m2_cpu value) { + return ggml_fp8_e5m2_to_fp32(value.value); +} + +template<> +struct type_conversion_table { + static constexpr float (*to_f32)(ggml_fp8_e4m3_cpu) = fp8_e4m3_to_f32; +}; + +template<> +struct type_conversion_table { + static constexpr float (*to_f32)(ggml_fp8_e5m2_cpu) = fp8_e5m2_to_f32; +}; + // ggml_compute_forward_dup static void ggml_compute_forward_dup_same_cont( @@ -542,6 +569,20 @@ void ggml_compute_forward_dup( else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst); else ggml_compute_forward_dup_to_q(params, dst); } break; + case GGML_TYPE_F8_E4M3: + { + /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt(params, dst); + else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt(params, dst); + else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst); + else GGML_ABORT("unsupported FP8 E4M3 conversion"); + } break; + case GGML_TYPE_F8_E5M2: + { + /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt(params, dst); + else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt(params, dst); + else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst); + else GGML_ABORT("unsupported FP8 E5M2 conversion"); + } break; case GGML_TYPE_BF16: { /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt(params, dst); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 1ebc50a763f1..b3362e4390f8 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -25,6 +25,41 @@ #define UNUSED GGML_UNUSED +float ggml_fp8_e4m3_to_fp32(uint8_t x) { + const uint32_t sign = x >> 7; + const uint32_t exponent = (x >> 3) & 0x0f; + const uint32_t mantissa = x & 0x07; + + if (exponent == 0x0f && mantissa == 0x07) { + return sign ? -NAN : NAN; + } + + float value; + if (exponent == 0) { + value = ldexpf((float) mantissa, -9); + } else { + value = ldexpf(1.0f + (float) mantissa / 8.0f, (int) exponent - 7); + } + return sign ? -value : value; +} + +float ggml_fp8_e5m2_to_fp32(uint8_t x) { + const ggml_fp16_t value = (ggml_fp16_t) x << 8; + return GGML_FP16_TO_FP32(value); +} + +void dequantize_row_f8_e4m3(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + for (int64_t i = 0; i < k; ++i) { + y[i] = ggml_fp8_e4m3_to_fp32(x[i]); + } +} + +void dequantize_row_f8_e5m2(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + for (int64_t i = 0; i < k; ++i) { + y[i] = ggml_fp8_e5m2_to_fp32(x[i]); + } +} + static inline int best_index_int8(int n, const int8_t * val, float x) { if (x <= val[0]) return 0; if (x >= val[n-1]) return n-1; @@ -5420,6 +5455,18 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte const size_t nb = nbytes/ggml_type_size(type); switch (type) { + case GGML_TYPE_F8_E4M3: + case GGML_TYPE_F8_E5M2: + { + const uint8_t * f = (const uint8_t *) data; + const uint8_t limit = type == GGML_TYPE_F8_E4M3 ? 0x7f : 0x7c; + for (size_t i = 0; i < nb; ++i) { + if ((f[i] & 0x7f) >= limit) { + fprintf(stderr, "%s: found non-finite %s value at %zu\n", __func__, ggml_type_name(type), i); + return false; + } + } + } break; case GGML_TYPE_BF16: { int nans = 0; diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af180..ec32b74be99c 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -43,6 +43,11 @@ GGML_API void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k); // Dequantization +GGML_API float ggml_fp8_e4m3_to_fp32(uint8_t x); +GGML_API float ggml_fp8_e5m2_to_fp32(uint8_t x); +GGML_API void dequantize_row_f8_e4m3(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_f8_e5m2(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 6257cdbe5821..9b8434009121 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -674,6 +674,20 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) ggml_fp16_to_fp32_row, .from_float_ref = (ggml_from_float_t) ggml_fp32_to_fp16_row, }, + [GGML_TYPE_F8_E4M3] = { + .type_name = "f8_e4m3", + .blck_size = 1, + .type_size = sizeof(uint8_t), + .is_quantized = false, + .to_float = (ggml_to_float_t) dequantize_row_f8_e4m3, + }, + [GGML_TYPE_F8_E5M2] = { + .type_name = "f8_e5m2", + .blck_size = 1, + .type_size = sizeof(uint8_t), + .is_quantized = false, + .to_float = (ggml_to_float_t) dequantize_row_f8_e5m2, + }, [GGML_TYPE_Q1_0] = { .type_name = "q1_0", .blck_size = QK1_0, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c93e41b2e253..2687467d3d9c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -53,6 +53,18 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { size_t nels = ggml_nelements(tensor); + if (tensor->type == GGML_TYPE_F8_E4M3 || tensor->type == GGML_TYPE_F8_E5M2) { + std::vector data(nels); + const uint8_t limit = tensor->type == GGML_TYPE_F8_E4M3 ? 0x7f : 0x7c; + for (size_t i = 0; i < nels; ++i) { + data[i] = uint8_t(i); + if ((data[i] & 0x7f) >= limit) { + data[i] = 0; + } + } + ggml_backend_tensor_set(tensor, data.data(), 0, data.size()); + return; + } std::vector data(nels); { // parallel initialization @@ -280,7 +292,7 @@ static std::vector tensor_to_float(const ggml_tensor * t) { tv.push_back((float)*(int16_t *) &buf[i]); } else if (t->type == GGML_TYPE_I8) { tv.push_back((float)*(int8_t *) &buf[i]); - } else if (quantized) { + } else if (quantized || t->type == GGML_TYPE_F8_E4M3 || t->type == GGML_TYPE_F8_E5M2) { tt->to_float(&buf[i], vq.data(), bs); tv.insert(tv.end(), vq.begin(), vq.end()); } else { @@ -3054,6 +3066,13 @@ struct test_cpy : public test_case { return ne_src[0] * ne_src[1] * ne_src[2] * ne_src[3]; } + double err(const float * a, const float * b, size_t n) override { + if (type_src == GGML_TYPE_F8_E4M3 || type_src == GGML_TYPE_F8_E5M2) { + return memcmp(a, b, n * sizeof(float)) == 0 ? 0.0 : 1.0; + } + return test_case::err(a, b, n); + } + double max_nmse_err() override { if (type_src == type_dst) { return 0.0; @@ -9260,6 +9279,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous } } + for (ggml_type type_src : {GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E5M2}) { + for (ggml_type type_dst : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_cpy(type_src, type_dst, {257, 2, 3, 2})); + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1, -1, -1, -1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1, -1, -1, -1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 1}, {256, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {256, 4, 3, 1})); + } + } + // quant block count not a multiple of the kernel block size test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {96, 1, 1, 1})); test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {96, 1, 1, 1})); diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 9510ac14ce00..6a1ebe8481f5 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -2,10 +2,11 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "ggml-backend.h" #undef NDEBUG #include -#include +#include #include #include #include @@ -200,6 +201,71 @@ static int test_vec_dot_q(bool verbose) { return num_failed; } +static int test_fp8(bool verbose) { + int num_failed = 0; + ggml_backend_t backend = ggml_backend_cpu_init(); + for (ggml_type type : {GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E5M2}) { + ggml_init_params params = {16 * ggml_tensor_overhead() + ggml_graph_overhead() + 16384, nullptr, false}; + ggml_context * ctx = ggml_init(params); + ggml_tensor * src = ggml_new_tensor_2d(ctx, type, 256, 1); + uint8_t * data = (uint8_t *) src->data; + float expected[256]; + for (int i = 0; i < 256; ++i) { + data[i] = uint8_t(i); + if (type == GGML_TYPE_F8_E4M3) { + // Decode through F16 with an exponent bias adjustment. + const ggml_fp16_t bits = ((i & 0x80) << 8) | ((i & 0x7f) << 7); + expected[i] = (i & 0x7f) == 0x7f ? NAN : 256.0f * ggml_fp16_to_fp32(bits); + } else { + expected[i] = ggml_fp16_to_fp32(i << 8); + } + } + + float decoded[256]; + ggml_get_type_traits(type)->to_float(data, decoded, 256); + ggml_cgraph * graph = ggml_new_graph(ctx); + std::vector outputs; + for (ggml_type dst_type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { + outputs.push_back(ggml_cast(ctx, src, dst_type)); + ggml_build_forward_expand(graph, outputs.back()); + } + assert(ggml_graph_compute_with_ctx(ctx, graph, 2) == GGML_STATUS_SUCCESS); + + bool failed = ggml_type_size(type) != 1 || ggml_blck_size(type) != 1 || ggml_nbytes(src) != 256; + for (int i = 0; i < 256; ++i) { + failed |= ggml_validate_row_data(type, data + i, 1) != std::isfinite(expected[i]); + std::vector actual = {decoded[i]}; + for (ggml_tensor * out : outputs) { + if (out->type == GGML_TYPE_F32) { + actual.push_back(((float *) out->data)[i]); + } else if (out->type == GGML_TYPE_F16) { + actual.push_back(ggml_fp16_to_fp32(((ggml_fp16_t *) out->data)[i])); + } else { + actual.push_back(ggml_bf16_to_fp32(((ggml_bf16_t *) out->data)[i])); + } + } + for (float value : actual) { + if (std::isnan(expected[i])) { + failed |= !std::isnan(value); + } else { + failed |= value != expected[i] || std::signbit(value) != std::signbit(expected[i]); + } + } + } + + ggml_tensor * activation = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 256, 1); + failed |= ggml_backend_supports_op(backend, ggml_mul_mat(ctx, src, activation)); + failed |= ggml_backend_supports_op(backend, ggml_cast(ctx, activation, type)); + num_failed += failed; + if (failed || verbose) { + printf("%s decoding and casts: %s\n", ggml_type_name(type), RESULT_STR[failed]); + } + ggml_free(ctx); + } + ggml_backend_free(backend); + return num_failed; +} + int main(int argc, char * argv[]) { bool verbose = false; @@ -219,6 +285,7 @@ int main(int argc, char * argv[]) { int num_failed = 0; + num_failed += test_fp8(verbose); num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose);