ggml : add FP8 types and CPU conversions

ggml/include/ggml.h

@@ -430,7 +430,9 @@ extern "C" {
         GGML_TYPE_NVFP4   = 40, // NVFP4 (4 blocks, E4M3 scale)
         GGML_TYPE_Q1_0    = 41,
         GGML_TYPE_Q2_0    = 42,
-        GGML_TYPE_COUNT   = 43,
+        GGML_TYPE_F8_E4M3 = 43, // E4M3FN
+        GGML_TYPE_F8_E5M2 = 44,
+        GGML_TYPE_COUNT   = 45,
     };
 
     // precision

ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -439,6 +439,13 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
         }
     }
 
+    const bool src0_fp8 = src0 && (src0->type == GGML_TYPE_F8_E4M3 || src0->type == GGML_TYPE_F8_E5M2);
+    const bool dst_fp8 = op->type == GGML_TYPE_F8_E4M3 || op->type == GGML_TYPE_F8_E5M2;
+    if (src0_fp8 || dst_fp8) {
+        return (op->op == GGML_OP_CPY || op->op == GGML_OP_DUP || op->op == GGML_OP_CONT) &&
+            (src0->type == op->type || (src0_fp8 && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16)));
+    }
+
     switch (op->op) {
         case GGML_OP_CPY:
         case GGML_OP_SET_ROWS:

ggml/src/ggml-cpu/ops.cpp

@@ -2,6 +2,7 @@
 
 #include "ggml-cpu.h"
 #include "ggml-impl.h"
+#include "ggml-quants.h"
 #include "binary-ops.h"
 #include "simd-gemm.h"
 #include "ggml.h"
@@ -12,6 +13,32 @@
 #include <cfloat>
 #include <cmath>
 
+struct ggml_fp8_e4m3_cpu {
+    uint8_t value;
+};
+
+struct ggml_fp8_e5m2_cpu {
+    uint8_t value;
+};
+
+static float fp8_e4m3_to_f32(ggml_fp8_e4m3_cpu value) {
+    return ggml_fp8_e4m3_to_fp32(value.value);
+}
+
+static float fp8_e5m2_to_f32(ggml_fp8_e5m2_cpu value) {
+    return ggml_fp8_e5m2_to_fp32(value.value);
+}
+
+template<>
+struct type_conversion_table<ggml_fp8_e4m3_cpu> {
+    static constexpr float (*to_f32)(ggml_fp8_e4m3_cpu) = fp8_e4m3_to_f32;
+};
+
+template<>
+struct type_conversion_table<ggml_fp8_e5m2_cpu> {
+    static constexpr float (*to_f32)(ggml_fp8_e5m2_cpu) = fp8_e5m2_to_f32;
+};
+
 // ggml_compute_forward_dup
 
 static void ggml_compute_forward_dup_same_cont(
@@ -542,6 +569,20 @@ void ggml_compute_forward_dup(
                 else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_fp16_t, float      >(params, dst);
                 else ggml_compute_forward_dup_to_q<ggml_fp16_t>(params, dst);
             } break;
+        case GGML_TYPE_F8_E4M3:
+            {
+                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_fp8_e4m3_cpu, ggml_fp16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_cpu, ggml_bf16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_fp8_e4m3_cpu, float>(params, dst);
+                else GGML_ABORT("unsupported FP8 E4M3 conversion");
+            } break;
+        case GGML_TYPE_F8_E5M2:
+            {
+                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_fp8_e5m2_cpu, ggml_fp16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp8_e5m2_cpu, ggml_bf16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_fp8_e5m2_cpu, float>(params, dst);
+                else GGML_ABORT("unsupported FP8 E5M2 conversion");
+            } break;
         case GGML_TYPE_BF16:
             {
                 /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);

ggml/src/ggml-quants.c

@@ -25,6 +25,41 @@
 
 #define UNUSED GGML_UNUSED
 
+float ggml_fp8_e4m3_to_fp32(uint8_t x) {
+    const uint32_t sign     = x >> 7;
+    const uint32_t exponent = (x >> 3) & 0x0f;
+    const uint32_t mantissa = x & 0x07;
+
+    if (exponent == 0x0f && mantissa == 0x07) {
+        return sign ? -NAN : NAN;
+    }
+
+    float value;
+    if (exponent == 0) {
+        value = ldexpf((float) mantissa, -9);
+    } else {
+        value = ldexpf(1.0f + (float) mantissa / 8.0f, (int) exponent - 7);
+    }
+    return sign ? -value : value;
+}
+
+float ggml_fp8_e5m2_to_fp32(uint8_t x) {
+    const ggml_fp16_t value = (ggml_fp16_t) x << 8;
+    return GGML_FP16_TO_FP32(value);
+}
+
+void dequantize_row_f8_e4m3(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+    for (int64_t i = 0; i < k; ++i) {
+        y[i] = ggml_fp8_e4m3_to_fp32(x[i]);
+    }
+}
+
+void dequantize_row_f8_e5m2(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+    for (int64_t i = 0; i < k; ++i) {
+        y[i] = ggml_fp8_e5m2_to_fp32(x[i]);
+    }
+}
+
 static inline int best_index_int8(int n, const int8_t * val, float x) {
     if (x <= val[0]) return 0;
     if (x >= val[n-1]) return n-1;
@@ -5420,6 +5455,18 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte
     const size_t nb = nbytes/ggml_type_size(type);
 
     switch (type) {
+        case GGML_TYPE_F8_E4M3:
+        case GGML_TYPE_F8_E5M2:
+            {
+                const uint8_t * f = (const uint8_t *) data;
+                const uint8_t limit = type == GGML_TYPE_F8_E4M3 ? 0x7f : 0x7c;
+                for (size_t i = 0; i < nb; ++i) {
+                    if ((f[i] & 0x7f) >= limit) {
+                        fprintf(stderr, "%s: found non-finite %s value at %zu\n", __func__, ggml_type_name(type), i);
+                        return false;
+                    }
+                }
+            } break;
         case GGML_TYPE_BF16:
             {
                 int nans = 0;

ggml/src/ggml-quants.h

@@ -43,6 +43,11 @@ GGML_API void quantize_row_iq3_s_ref  (const float * GGML_RESTRICT x, block_iq3_
 GGML_API void quantize_row_iq2_s_ref  (const float * GGML_RESTRICT x, block_iq2_s   * GGML_RESTRICT y, int64_t k);
 
 // Dequantization
+GGML_API float ggml_fp8_e4m3_to_fp32(uint8_t x);
+GGML_API float ggml_fp8_e5m2_to_fp32(uint8_t x);
+GGML_API void dequantize_row_f8_e4m3(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+GGML_API void dequantize_row_f8_e5m2(const uint8_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
 GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
 GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
 GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);

ggml/src/ggml.c

@@ -674,6 +674,20 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {
         .to_float                 = (ggml_to_float_t) ggml_fp16_to_fp32_row,
         .from_float_ref           = (ggml_from_float_t) ggml_fp32_to_fp16_row,
     },
+    [GGML_TYPE_F8_E4M3] = {
+        .type_name                = "f8_e4m3",
+        .blck_size                = 1,
+        .type_size                = sizeof(uint8_t),
+        .is_quantized             = false,
+        .to_float                 = (ggml_to_float_t) dequantize_row_f8_e4m3,
+    },
+    [GGML_TYPE_F8_E5M2] = {
+        .type_name                = "f8_e5m2",
+        .blck_size                = 1,
+        .type_size                = sizeof(uint8_t),
+        .is_quantized             = false,
+        .to_float                 = (ggml_to_float_t) dequantize_row_f8_e5m2,
+    },
     [GGML_TYPE_Q1_0] = {
         .type_name                = "q1_0",
         .blck_size                = QK1_0,

tests/test-backend-ops.cpp

@@ -53,6 +53,18 @@
 
 static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {
     size_t nels = ggml_nelements(tensor);
+    if (tensor->type == GGML_TYPE_F8_E4M3 || tensor->type == GGML_TYPE_F8_E5M2) {
+        std::vector<uint8_t> data(nels);
+        const uint8_t limit = tensor->type == GGML_TYPE_F8_E4M3 ? 0x7f : 0x7c;
+        for (size_t i = 0; i < nels; ++i) {
+            data[i] = uint8_t(i);
+            if ((data[i] & 0x7f) >= limit) {
+                data[i] = 0;
+            }
+        }
+        ggml_backend_tensor_set(tensor, data.data(), 0, data.size());
+        return;
+    }
     std::vector<float> data(nels);
     {
         // parallel initialization
@@ -280,7 +292,7 @@ static std::vector<float> tensor_to_float(const ggml_tensor * t) {
                         tv.push_back((float)*(int16_t *) &buf[i]);
                     } else if (t->type == GGML_TYPE_I8) {
                         tv.push_back((float)*(int8_t *) &buf[i]);
-                    } else if (quantized) {
+                    } else if (quantized || t->type == GGML_TYPE_F8_E4M3 || t->type == GGML_TYPE_F8_E5M2) {
                         tt->to_float(&buf[i], vq.data(), bs);
                         tv.insert(tv.end(), vq.begin(), vq.end());
                     } else {
@@ -3054,6 +3066,13 @@ struct test_cpy : public test_case {
         return ne_src[0] * ne_src[1] * ne_src[2] * ne_src[3];
     }
 
+    double err(const float * a, const float * b, size_t n) override {
+        if (type_src == GGML_TYPE_F8_E4M3 || type_src == GGML_TYPE_F8_E5M2) {
+            return memcmp(a, b, n * sizeof(float)) == 0 ? 0.0 : 1.0;
+        }
+        return test_case::err(a, b, n);
+    }
+
     double max_nmse_err() override {
         if (type_src == type_dst) {
             return 0.0;
@@ -9260,6 +9279,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
             test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous
         }
     }
+    for (ggml_type type_src : {GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E5M2}) {
+        for (ggml_type type_dst : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) {
+            test_cases.emplace_back(new test_cpy(type_src, type_dst, {257, 2, 3, 2}));
+            test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1, -1, -1, -1}, {0, 2, 1, 3}));
+            test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1, -1, -1, -1}, {1, 0, 2, 3}));
+            test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 1}, {256, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {256, 4, 3, 1}));
+        }
+    }
+
     // quant block count not a multiple of the kernel block size
     test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {96, 1, 1, 1}));
     test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {96, 1, 1, 1}));

tests/test-quantize-fns.cpp

@@ -2,10 +2,11 @@
 
 #include "ggml.h"
 #include "ggml-cpu.h"
+#include "ggml-backend.h"
 
 #undef NDEBUG
 #include <assert.h>
-#include <math.h>
+#include <cmath>
 #include <stdio.h>
 #include <string>
 #include <vector>
@@ -200,6 +201,71 @@ static int test_vec_dot_q(bool verbose) {
     return num_failed;
 }
 
+static int test_fp8(bool verbose) {
+    int num_failed = 0;
+    ggml_backend_t backend = ggml_backend_cpu_init();
+    for (ggml_type type : {GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E5M2}) {
+        ggml_init_params params = {16 * ggml_tensor_overhead() + ggml_graph_overhead() + 16384, nullptr, false};
+        ggml_context * ctx = ggml_init(params);
+        ggml_tensor * src = ggml_new_tensor_2d(ctx, type, 256, 1);
+        uint8_t * data = (uint8_t *) src->data;
+        float expected[256];
+        for (int i = 0; i < 256; ++i) {
+            data[i] = uint8_t(i);
+            if (type == GGML_TYPE_F8_E4M3) {
+                // Decode through F16 with an exponent bias adjustment.
+                const ggml_fp16_t bits = ((i & 0x80) << 8) | ((i & 0x7f) << 7);
+                expected[i] = (i & 0x7f) == 0x7f ? NAN : 256.0f * ggml_fp16_to_fp32(bits);
+            } else {
+                expected[i] = ggml_fp16_to_fp32(i << 8);
+            }
+        }
+
+        float decoded[256];
+        ggml_get_type_traits(type)->to_float(data, decoded, 256);
+        ggml_cgraph * graph = ggml_new_graph(ctx);
+        std::vector<ggml_tensor *> outputs;
+        for (ggml_type dst_type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
+            outputs.push_back(ggml_cast(ctx, src, dst_type));
+            ggml_build_forward_expand(graph, outputs.back());
+        }
+        assert(ggml_graph_compute_with_ctx(ctx, graph, 2) == GGML_STATUS_SUCCESS);
+
+        bool failed = ggml_type_size(type) != 1 || ggml_blck_size(type) != 1 || ggml_nbytes(src) != 256;
+        for (int i = 0; i < 256; ++i) {
+            failed |= ggml_validate_row_data(type, data + i, 1) != std::isfinite(expected[i]);
+            std::vector<float> actual = {decoded[i]};
+            for (ggml_tensor * out : outputs) {
+                if (out->type == GGML_TYPE_F32) {
+                    actual.push_back(((float *) out->data)[i]);
+                } else if (out->type == GGML_TYPE_F16) {
+                    actual.push_back(ggml_fp16_to_fp32(((ggml_fp16_t *) out->data)[i]));
+                } else {
+                    actual.push_back(ggml_bf16_to_fp32(((ggml_bf16_t *) out->data)[i]));
+                }
+            }
+            for (float value : actual) {
+                if (std::isnan(expected[i])) {
+                    failed |= !std::isnan(value);
+                } else {
+                    failed |= value != expected[i] || std::signbit(value) != std::signbit(expected[i]);
+                }
+            }
+        }
+
+        ggml_tensor * activation = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 256, 1);
+        failed |= ggml_backend_supports_op(backend, ggml_mul_mat(ctx, src, activation));
+        failed |= ggml_backend_supports_op(backend, ggml_cast(ctx, activation, type));
+        num_failed += failed;
+        if (failed || verbose) {
+            printf("%s decoding and casts: %s\n", ggml_type_name(type), RESULT_STR[failed]);
+        }
+        ggml_free(ctx);
+    }
+    ggml_backend_free(backend);
+    return num_failed;
+}
+
 int main(int argc, char * argv[]) {
     bool verbose = false;
 
@@ -219,6 +285,7 @@ int main(int argc, char * argv[]) {
 
     int num_failed = 0;
 
+    num_failed += test_fp8(verbose);
     num_failed += test_vec_dot_f32(verbose);
     num_failed += test_vec_dot_q(verbose);