From bbf3ff4ebbb3d79410cbd14f7c17b9028cae2d96 Mon Sep 17 00:00:00 2001 From: velonica Date: Wed, 29 Apr 2026 13:08:14 +0800 Subject: [PATCH 1/2] rvv_q1_0_dot --- ggml/src/ggml-cpu/arch-fallback.h | 1 - ggml/src/ggml-cpu/arch/riscv/quants.c | 50 +++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 595ded09f03f..b0391a67c88d 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -203,7 +203,6 @@ #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 diff --git a/ggml/src/ggml-cpu/arch/riscv/quants.c b/ggml/src/ggml-cpu/arch/riscv/quants.c index d3278d6489fd..003251b91de6 100644 --- a/ggml/src/ggml-cpu/arch/riscv/quants.c +++ b/ggml/src/ggml-cpu/arch/riscv/quants.c @@ -274,6 +274,56 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__riscv_v) + const int qk = QK1_0; + const int nb = n / qk; + const size_t vl = 8; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + float sumi = 0.0f; + + for (int k = 0; k < 4; ++k) { + const block_q8_0 * GGML_RESTRICT yb = &y[ib * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + int sumi_block = 0; + const uint8_t * GGML_RESTRICT bits = &x[ib].qs[k * 4]; + + for (int b = 0; b < 4; ++b) { + vbool8_t m_positive = __riscv_vlm_v_b8(bits + b, vl); + vint8m1_t v_q8 = __riscv_vle8_v_i8m1(yb->qs + b * 8, vl); + vint8m1_t v_q8_neg = __riscv_vsub_vv_i8m1(__riscv_vmv_v_x_i8m1(0, vl), v_q8, vl); + vint8m1_t v_q8_signed = __riscv_vmerge_vvm_i8m1(v_q8_neg, v_q8, m_positive, vl); + + vint16m1_t v_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_q8_signed, __riscv_vmv_v_x_i16m1(0, 1), vl); + sumi_block += __riscv_vmv_x_s_i16m1_i16(v_sum); + } + + sumi += d1 * sumi_block; + } + + sumf += d0 * sumi; + } + + *s = sumf; +#else + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { #if defined(__riscv_v) const int qk = QK8_1; From bd184c0697388be96d09d771c96ff34546bd9f1b Mon Sep 17 00:00:00 2001 From: velonica Date: Wed, 6 May 2026 20:08:45 +0800 Subject: [PATCH 2/2] lmul=4 --- ggml/src/ggml-cpu/arch/riscv/quants.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/riscv/quants.c b/ggml/src/ggml-cpu/arch/riscv/quants.c index 003251b91de6..3d04c6eb915d 100644 --- a/ggml/src/ggml-cpu/arch/riscv/quants.c +++ b/ggml/src/ggml-cpu/arch/riscv/quants.c @@ -278,9 +278,12 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #if defined(__riscv_v) const int qk = QK1_0; const int nb = n / qk; - const size_t vl = 8; + // A single q8_0 block always contains 32 int8 values. With e8,m4 this + // maps to vl=32 on any spec-compliant RVV target (minimum VLEN is 64-bit). + const size_t vl = __riscv_vsetvl_e8m4(32); assert(n % qk == 0); + assert(vl == 32); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -289,6 +292,7 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const block_q1_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; + const vint16m1_t v_zero_sum = __riscv_vmv_v_x_i16m1(0, 1); float sumf = 0.0f; @@ -299,18 +303,13 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi for (int k = 0; k < 4; ++k) { const block_q8_0 * GGML_RESTRICT yb = &y[ib * 4 + k]; const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); - int sumi_block = 0; const uint8_t * GGML_RESTRICT bits = &x[ib].qs[k * 4]; - - for (int b = 0; b < 4; ++b) { - vbool8_t m_positive = __riscv_vlm_v_b8(bits + b, vl); - vint8m1_t v_q8 = __riscv_vle8_v_i8m1(yb->qs + b * 8, vl); - vint8m1_t v_q8_neg = __riscv_vsub_vv_i8m1(__riscv_vmv_v_x_i8m1(0, vl), v_q8, vl); - vint8m1_t v_q8_signed = __riscv_vmerge_vvm_i8m1(v_q8_neg, v_q8, m_positive, vl); - - vint16m1_t v_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_q8_signed, __riscv_vmv_v_x_i16m1(0, 1), vl); - sumi_block += __riscv_vmv_x_s_i16m1_i16(v_sum); - } + vbool2_t m_positive = __riscv_vlm_v_b2(bits, vl); + vint8m4_t v_q8 = __riscv_vle8_v_i8m4(yb->qs, vl); + vint8m4_t v_q8_neg = __riscv_vsub_vv_i8m4(__riscv_vmv_v_x_i8m4(0, vl), v_q8, vl); + vint8m4_t v_q8_signed = __riscv_vmerge_vvm_i8m4(v_q8_neg, v_q8, m_positive, vl); + vint16m1_t v_sum = __riscv_vwredsum_vs_i8m4_i16m1(v_q8_signed, v_zero_sum, vl); + const int sumi_block = __riscv_vmv_x_s_i16m1_i16(v_sum); sumi += d1 * sumi_block; }