From 27c36da94464d2eab8c3d1f5bbfa178ec0ba1813 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 01/21] power-mma: Q1_0/Q2_0 MMA GEMM + no-packing GEMV for POWER10/11 MMA outer-product kernels for the 1-bit and ternary Bonsai formats, with a select-and-sum GEMV that reads raw 1/2-bit weights for token generation. Dispatch via llamafile_sgemm behind #if __MMA__. --- ggml/src/ggml-cpu/CMakeLists.txt | 2 + ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp | 316 +++++++++++++++++++ ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h | 18 ++ ggml/src/ggml-cpu/llamafile/sgemm.cpp | 27 ++ 4 files changed, 363 insertions(+) create mode 100644 ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 8c735a045b35..f3a421f986a9 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -82,6 +82,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name) list(APPEND GGML_CPU_SOURCES ggml-cpu/llamafile/sgemm.cpp + ggml-cpu/llamafile/qbit_ppc_mma.cpp + ggml-cpu/llamafile/qbit_ppc_mma.h ggml-cpu/llamafile/sgemm.h) endif() diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp new file mode 100644 index 000000000000..f9562625c51f --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp @@ -0,0 +1,316 @@ +// qbit_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for Q1_0/Q2_0 x Q8_0. +// Imported from https://github.com/mavin2009/ppc-mma-kernels (v3 kernel, +// correctness-verified against double references under qemu -cpu power10). +// See that repo's docs/DESIGN.md for the derivation. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "qbit_ppc_mma.h" + +#include +#include +#include + +#if defined(__MMA__) && defined(__powerpc64__) +#include + + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_BLKS 16 // 2048-element K slab +#define KC_CHUNKS (KC_BLKS * 4) +#define MR 16 // weight rows per tile +#define NR 8 // activation cols per tile + +// Packed weights: unsigned codes in MMA layout. +// v[ch][4*x + b]: depth step x (0..7), row group b (0..3). +typedef struct { + vuc v[KC_CHUNKS][32]; + vfl dA[KC_BLKS][4]; // per-block scales, 4 rows per vfl +} apack_t; + +// Packed activations: signed int8, untouched. +// v[ch][2*x + a]: depth step x, col group a (0..1). +typedef struct { + vuc v[KC_CHUNKS][16]; + vfl dB[KC_CHUNKS][2]; // per-chunk scales, cols 0-3 / 4-7 + vfl E[KC_BLKS][2]; // sum_c dB(j,c)*sum(y_c) per block +} bpack_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// ---- weight unpack: raw UNSIGNED codes (no sign mapping needed) ---- + +// Q1_0: chunk c from the block's 16 qs bytes -> bits 0/1 as bytes. +static inline void q1_codes32(vuc raw, int c, vuc v[2]) { + const vuc rep0 = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; + const vuc rep1 = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; + const vuc sh = { 0,1,2,3,4,5,6,7, 0,1,2,3,4,5,6,7 }; + const vuc m1 = vec_splats((unsigned char)1); + const vuc off = vec_splats((unsigned char)(4*c)); + v[0] = vec_and(vec_sr(vec_perm(raw, raw, vec_add(rep0, off)), sh), m1); + v[1] = vec_and(vec_sr(vec_perm(raw, raw, vec_add(rep1, off)), sh), m1); +} + +// Q2_0: chunk c from the block's 32 qs bytes (lo/hi) -> codes 0..3. +static inline void q2_codes32(vuc lo, vuc hi, int c, vuc v[2]) { + const vuc rep0 = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 }; + const vuc rep1 = { 4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7 }; + const vuc sh = { 0,2,4,6, 0,2,4,6, 0,2,4,6, 0,2,4,6 }; + const vuc m3 = vec_splats((unsigned char)3); + const vuc off = vec_splats((unsigned char)(8*c)); + v[0] = vec_and(vec_sr(vec_perm(lo, hi, vec_add(rep0, off)), sh), m3); + v[1] = vec_and(vec_sr(vec_perm(lo, hi, vec_add(rep1, off)), sh), m3); +} + +// ---- packers ---- + +static void pack_A16_q1(const block_q1_0 * A, int64_t lda, int64_t rows, + int64_t blk0, int64_t nblk, apack_t * P) { + for (int64_t b = 0; b < nblk; b++) { + vuc raw[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = r < rows ? r : rows - 1; + const block_q1_0 * bp = &A[rr*lda + blk0 + b]; + d[r] = GGML_FP16_TO_FP32(bp->d); + raw[r] = vec_xl(0, (const unsigned char *)bp->qs); + } + for (int g = 0; g < 4; g++) + P->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; + for (int c = 0; c < 4; c++) { + const int ch = 4*b + c; + vuc t[MR][2]; + for (int r = 0; r < MR; r++) q1_codes32(raw[r], c, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &P->v[ch][16*h + g], 4); + } + } + } +} + +static void pack_A16_q2(const block_q2_0 * A, int64_t lda, int64_t rows, + int64_t blk0, int64_t nblk, apack_t * P) { + for (int64_t b = 0; b < nblk; b++) { + vuc lo[MR], hi[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = r < rows ? r : rows - 1; + const block_q2_0 * bp = &A[rr*lda + blk0 + b]; + d[r] = GGML_FP16_TO_FP32(bp->d); + lo[r] = vec_xl(0, (const unsigned char *)bp->qs); + hi[r] = vec_xl(16, (const unsigned char *)bp->qs); + } + for (int g = 0; g < 4; g++) + P->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; + for (int c = 0; c < 4; c++) { + const int ch = 4*b + c; + vuc t[MR][2]; + for (int r = 0; r < MR; r++) q2_codes32(lo[r], hi[r], c, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &P->v[ch][16*h + g], 4); + } + } + } +} + +// Activations: no flip; also builds per-chunk scales and the separable +// correction accumulators E. +static void pack_B8(const block_q8_0 * B, int64_t ldb, int64_t cols, + int64_t q8blk0, int64_t nblk, bpack_t * P) { + for (int64_t b = 0; b < nblk; b++) { + vfl E0 = vec_splats(0.0f), E1 = vec_splats(0.0f); + for (int c = 0; c < 4; c++) { + const int64_t ch = 4*b + c; + const block_q8_0 * yb[NR]; + float dB[NR], S[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = j < cols ? j : cols - 1; + yb[j] = &B[jj*ldb + q8blk0 + ch]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + } + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = vec_xl(0, (const unsigned char *)yb[4*a + j]->qs); + q[j][1] = vec_xl(16, (const unsigned char *)yb[4*a + j]->qs); + vsi z = vec_splats(0); + vsi s = vec_sum4s((vsc)q[j][0], z); + s = vec_sum4s((vsc)q[j][1], s); + S[4*a + j] = (float)(s[0] + s[1] + s[2] + s[3]); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &P->v[ch][8*h + a], 2); + } + } + vfl dB0 = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + vfl dB1 = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + P->dB[ch][0] = dB0; + P->dB[ch][1] = dB1; + E0 = vec_madd(dB0, (vfl){ S[0], S[1], S[2], S[3] }, E0); + E1 = vec_madd(dB1, (vfl){ S[4], S[5], S[6], S[7] }, E1); + } + P->E[b][0] = E0; + P->E[b][1] = E1; + } +} + +// ---- 16x8 microkernel on all 8 accumulators ---- +// fin[j][g]: activation col j (0..7), weight row group g (0..3). +static void kernel_16x8(const apack_t * PA, const bpack_t * PB, + int64_t nblk, float alpha, vfl fin[NR][4]) { + const vfl valpha = vec_splats(alpha); + for (int64_t b = 0; b < nblk; b++) { + for (int c = 0; c < 4; c++) { + const int64_t ch = 4*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 4*nblk) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // fixup: fin += alpha*dB_j * dA_g * P + const vfl dBa0 = vec_mul(PB->dB[ch][0], valpha); + const vfl dBa1 = vec_mul(PB->dB[ch][1], valpha); + for (int i = 0; i < 2; i++) { + const vfl dBa = i ? dBa1 : dBa0; + for (int g = 0; g < 4; g++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[i][g]); + const vfl s0 = vec_mul(vec_splat(dBa, 0), PA->dA[b][g]); + const vfl s1 = vec_mul(vec_splat(dBa, 1), PA->dA[b][g]); + const vfl s2 = vec_mul(vec_splat(dBa, 2), PA->dA[b][g]); + const vfl s3 = vec_mul(vec_splat(dBa, 3), PA->dA[b][g]); + fin[4*i + 0][g] = vec_madd(vec_ctf(rowsP[0], 0), s0, fin[4*i + 0][g]); + fin[4*i + 1][g] = vec_madd(vec_ctf(rowsP[1], 0), s1, fin[4*i + 1][g]); + fin[4*i + 2][g] = vec_madd(vec_ctf(rowsP[2], 0), s2, fin[4*i + 2][g]); + fin[4*i + 3][g] = vec_madd(vec_ctf(rowsP[3], 0), s3, fin[4*i + 3][g]); + } + } + } + } + // separable correction: fin[j][g] -= sum_b dA[b][g] * E(j,b) + for (int64_t b = 0; b < nblk; b++) { + for (int g = 0; g < 4; g++) { + const vfl dA = PA->dA[b][g]; + fin[0][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 0), fin[0][g]); + fin[1][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 1), fin[1][g]); + fin[2][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 2), fin[2][g]); + fin[3][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 3), fin[3][g]); + fin[4][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 0), fin[4][g]); + fin[5][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 1), fin[5][g]); + fin[6][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 2), fin[6][g]); + fin[7][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 3), fin[7][g]); + } + } +} + +// ---- driver, templated over weight type ---- +template +static void gemm_qbit(int64_t m, int64_t n, int64_t k, + const BLK * A, int64_t lda, + const block_q8_0 * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + assert(k % 128 == 0); + const int64_t kb = k / 128; + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t i0 = MR * (ith * tpt); + const int64_t i1 = m < MR * ((ith + 1) * tpt) ? m : MR * ((ith + 1) * tpt); + if (i0 >= m) return; + + const int64_t njt = (n + NR - 1) / NR; + apack_t * PA = (apack_t*)aligned_alloc(64, sizeof(apack_t)); + bpack_t * PB = (bpack_t*)aligned_alloc(64, njt * sizeof(bpack_t)); + vfl (*fin)[NR][4] = (vfl(*)[NR][4])aligned_alloc(64, njt * sizeof(vfl[NR][4])); + + for (int64_t i = i0; i < i1; i += MR) { + const int64_t rows = (i1 - i) < MR ? (i1 - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) + fin[jt][j][g] = vec_splats(0.0f); + + for (int64_t blk0 = 0; blk0 < kb; blk0 += KC_BLKS) { + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + PACK_A(A + i*lda, lda, rows, blk0, nblk, PA); + for (int64_t jt = 0; jt < njt; jt++) { + const int64_t j = NR*jt; + const int64_t cols = (n - j) < NR ? (n - j) : NR; + if (i == i0 || kb > KC_BLKS) + pack_B8(B + j*ldb, ldb, cols, 4*blk0, nblk, &PB[jt]); + kernel_16x8(PA, &PB[jt], nblk, (float)ALPHA, fin[jt]); + } + } + + for (int64_t jt = 0; jt < njt; jt++) { + const int64_t j = NR*jt; + const int64_t cols = (n - j) < NR ? (n - j) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) + vec_xst(fin[jt][cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[jt][cj][r >> 2][r & 3]; + } + } + } + } + free(PA); free(PB); free(fin); +} + + +extern "C" void gemm_q1_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + gemm_qbit(m, n, k, + (const block_q1_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth); +} + +extern "C" void gemm_q2_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + gemm_qbit(m, n, k, + (const block_q2_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth); +} + +#endif // __MMA__ && __powerpc64__ diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h new file mode 100644 index 000000000000..349e75520251 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h @@ -0,0 +1,18 @@ +#pragma once +// POWER10/POWER11 MMA GEMM for PrismML Q1_0/Q2_0 weights x Q8_0 activations. +// k is in ELEMENTS (multiple of 128); lda in weight blocks, ldb in q8 blocks; +// C column-major float. A/B are block_q1_0|block_q2_0 / block_q8_0 arrays. +// From github.com/mavin2009/ppc-mma-kernels. +#include +#ifdef __cplusplus +extern "C" { +#endif +void gemm_q1_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q2_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index e13828e3be6f..d921cb95202d 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -49,6 +49,7 @@ #endif #include "sgemm.h" +#include "qbit_ppc_mma.h" #include "ggml-impl.h" #include "ggml-cpu-impl.h" #include "ggml-quants.h" @@ -3962,6 +3963,32 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_Q1_0: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + gemm_q1_0_q8_0_ppc_v3(m, n, k * QK1_0, + A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q2_0: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + gemm_q2_0_q8_0_ppc_v3(m, n, k * QK2_0, + A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + case GGML_TYPE_Q4_0: { if (Btype != GGML_TYPE_Q8_0) return false; From 2f9b4a267e02110e495583cc3d09339eed8d131e Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 02/21] power-mma: Q4_K/Q5_K/Q2_K/Q6_K MMA sgemm K-quant GEMM kernels: raw unsigned codes ride the unsigned GER operand, offsets handled algebraically (separable rank-one update from the activation quantizer's existing sums). --- ggml/src/ggml-cpu/CMakeLists.txt | 5 + ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 23 ++ ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp | 280 +++++++++++++++ ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 313 +++++++++++++++++ ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp | 321 ++++++++++++++++++ ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp | 271 +++++++++++++++ ggml/src/ggml-cpu/llamafile/sgemm.cpp | 49 +++ 7 files changed, 1262 insertions(+) create mode 100644 ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h create mode 100644 ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index f3a421f986a9..c08f09e14d49 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -83,6 +83,11 @@ function(ggml_add_cpu_backend_variant_impl tag_name) list(APPEND GGML_CPU_SOURCES ggml-cpu/llamafile/sgemm.cpp ggml-cpu/llamafile/qbit_ppc_mma.cpp + ggml-cpu/llamafile/q4k_ppc_mma.cpp + ggml-cpu/llamafile/q5k_ppc_mma.cpp + ggml-cpu/llamafile/q6k_ppc_mma.cpp + ggml-cpu/llamafile/q2k_ppc_mma.cpp + ggml-cpu/llamafile/kquants_ppc_mma.h ggml-cpu/llamafile/qbit_ppc_mma.h ggml-cpu/llamafile/sgemm.h) endif() diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h new file mode 100644 index 000000000000..981d888b1267 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -0,0 +1,23 @@ +#pragma once +// POWER10/POWER11 MMA GEMM one-shot entries for K-quant weights x Q8_K +// activations. k in ELEMENTS (multiple of 256); lda/ldb in superblocks; +// C column-major float. From github.com/mavin2009/ppc-mma-kernels. +#include +#ifdef __cplusplus +extern "C" { +#endif +void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp new file mode 100644 index 000000000000..8ada5d25f25c --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -0,0 +1,280 @@ +// q2k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q2_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + +static_assert(sizeof(block_q2_K) == QK_K/16 + QK_K/4 + 2*sizeof(ggml_half), "bad q2_K"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_SB 8 // superblocks per slab (2048 elems) +#define KC_CH16 (KC_SB * 16) // 16-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH16][16]; // per chunk: 4 depth-steps x 4 rowgroups + vfl dsc[KC_CH16][4]; // d*(sc&0xF) per chunk per rowgroup + vfl dm [KC_CH16][4]; // dmin*(sc>>4) +} a2k_t; + +typedef struct { + vuc v[KC_CH16][8]; // per chunk: 4 depth-steps x 2 colgroups + vfl dB[KC_SB][2]; + vfl TS[KC_CH16][2]; // dB * bsums[c] per column +} b2k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 16 unsigned 2-bit codes of chunk c (0..15) of one superblock. +static inline vuc q2k_codes16(const uint8_t * qs, int c) { + const int h = c >> 3, idx = c & 7, j = idx >> 1, lo = idx & 1; + vuc v = vec_xl(32*h + 16*lo, (const unsigned char *)qs); + return vec_and(vec_sr(v, vec_splats((unsigned char)(2*j))), + vec_splats((unsigned char)3)); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q2k_apack_size(int64_t m, int64_t k) { + return (size_t)(rt(m) * sl(k)) * sizeof(a2k_t); +} +extern "C" size_t q2k_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct(n) * sl(k)) * sizeof(b2k_t); +} + +extern "C" void q2k_repack_a(const block_q2_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a2k_t * P = (a2k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + a2k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q2_K * bp[MR]; float d[MR], dmin[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + dmin[r] = GGML_FP16_TO_FP32(bp[r]->dmin); + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + for (int g = 0; g < 4; g++) { + float sc[4], mn[4]; + for (int r = 0; r < 4; r++) { + const uint8_t s8 = bp[4*g + r]->scales[c]; + sc[r] = d[4*g + r] * (float)(s8 & 0xF); + mn[r] = dmin[4*g + r] * (float)(s8 >> 4); + } + T->dsc[ch][g] = (vfl){ sc[0], sc[1], sc[2], sc[3] }; + T->dm [ch][g] = (vfl){ mn[0], mn[1], mn[2], mn[3] }; + } + vui rows4[4]; + for (int g = 0; g < 4; g++) { + for (int r = 0; r < 4; r++) + rows4[r] = (vui)q2k_codes16(bp[4*g + r]->qs, c); + mma_transpose4(rows4, &T->v[ch][g], 4); + } + } + } + } +} + +extern "C" void q2k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b2k_t * P = (b2k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b2k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * (float)yb[j]->bsums[c]; + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xl(16*c, (const unsigned char *)yb[4*a + j]->qs); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel2k_16x8(const a2k_t * PA, const b2k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 16*nsl) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 4; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Disassemble all accumulators to a stack buffer first: + // frees the acc-aliased VSRs (0-31) before the fixup runs, so + // fin + scale vectors fit the register file without spills. + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += dsc*(dB_j*P) ; fin -= dm*TS_j (mins term) + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + const vfl dm = PA->dm [ch][g]; + const vfl s0 = vec_mul(vec_splat(dB, 0), dsc); + const vfl s1 = vec_mul(vec_splat(dB, 1), dsc); + const vfl s2 = vec_mul(vec_splat(dB, 2), dsc); + const vfl s3 = vec_mul(vec_splat(dB, 3), dsc); + fin[4*i+0][g] = vec_madd(vec_ctf(rowsP[0],0), s0, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(vec_ctf(rowsP[1],0), s1, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(vec_ctf(rowsP[2],0), s2, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(vec_ctf(rowsP[3],0), s3, fin[4*i+3][g]); + fin[4*i+0][g] = vec_nmsub(dm, vec_splat(TS, 0), fin[4*i+0][g]); + fin[4*i+1][g] = vec_nmsub(dm, vec_splat(TS, 1), fin[4*i+1][g]); + fin[4*i+2][g] = vec_nmsub(dm, vec_splat(TS, 2), fin[4*i+2][g]); + fin[4*i+3][g] = vec_nmsub(dm, vec_splat(TS, 3), fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q2k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a2k_t * PA = (const a2k_t *)packedA; + const b2k_t * PB = (const b2k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel2k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q2_K * A = (const block_q2_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + void * PA = aligned_alloc(64, q2k_apack_size(MR, k)); + void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + q2k_pack_b(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q2k_repack_a(A + i*lda, lda, rows, k, PA); + q2k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp new file mode 100644 index 000000000000..f8bedcf9b2a6 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -0,0 +1,313 @@ +// q4k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q4_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + +static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "bad q4_K"); +static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + (QK_K/16)*sizeof(int16_t), "bad q8_K"); + + +// ggml's 6-bit scale/min decode +static inline void q4k_get_scale_min(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; *m = q[j + 4] & 63; + } else { + *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); + *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); + } +} + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_SB 8 // superblocks per K slab (2048 elems) +#define KC_CHUNKS (KC_SB * 8) // 32-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CHUNKS][32]; // MMA-layout unsigned nibbles + vfl dsc[KC_CHUNKS][4]; // d*sc per chunk, 4 rows per vfl + vfl dm [KC_CHUNKS][4]; // dmin*m per chunk +} a4k_t; + +typedef struct { + vuc v[KC_CHUNKS][16]; // signed activations + vfl dB[KC_SB][2]; // Q8_K d per column, per superblock + vfl TS[KC_CHUNKS][2]; // dB * S_sub per (col, chunk) +} b4k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// nibble codes of sub-block `sub` (0..7) of one superblock +static inline void q4k_codes32(const uint8_t * qs, int sub, vuc v[2]) { + const int g = sub >> 1; + vuc lo = vec_xl(32*g, (const unsigned char *)qs); + vuc hi = vec_xl(32*g + 16, (const unsigned char *)qs); + if ((sub & 1) == 0) { + const vuc mF = vec_splats((unsigned char)0xF); + v[0] = vec_and(lo, mF); + v[1] = vec_and(hi, mF); + } else { + v[0] = vec_sr(lo, vec_splats((unsigned char)4)); + v[1] = vec_sr(hi, vec_splats((unsigned char)4)); + } +} + +// ---- one-time packing ---- + +static inline int64_t rt4k(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct4k(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl4k(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q4k_apack_size(int64_t m, int64_t k) { + return (size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t); +} +extern "C" size_t q4k_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t); +} + +extern "C" void q4k_repack_a(const block_q4_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a4k_t * P = (a4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t it = 0; it < rt4k(m); it++) + for (int64_t s = 0; s < ns; s++) { + a4k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q4_K * bp[MR]; + float dsc[MR][8], dm[MR][8]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + const float d = GGML_FP16_TO_FP32(bp[r]->d); + const float dmin = GGML_FP16_TO_FP32(bp[r]->dmin); + for (int sub = 0; sub < 8; sub++) { + uint8_t sc, mn; + q4k_get_scale_min(sub, bp[r]->scales, &sc, &mn); + dsc[r][sub] = d * sc; + dm [r][sub] = dmin * mn; + } + } + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + for (int g = 0; g < 4; g++) { + T->dsc[ch][g] = (vfl){ dsc[4*g][sub], dsc[4*g+1][sub], + dsc[4*g+2][sub], dsc[4*g+3][sub] }; + T->dm [ch][g] = (vfl){ dm[4*g][sub], dm[4*g+1][sub], + dm[4*g+2][sub], dm[4*g+3][sub] }; + } + vuc t[MR][2]; + for (int r = 0; r < MR; r++) q4k_codes32(bp[r]->qs, sub, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][16*h + g], 4); + } + } + } + } +} + +extern "C" void q4k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b4k_t * P = (b4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t jt = 0; jt < ct4k(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b4k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; + float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * (float)(yb[j]->bsums[2*sub] + yb[j]->bsums[2*sub + 1]); + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = vec_xl(32*sub, (const unsigned char *)yb[4*a + j]->qs); + q[j][1] = vec_xl(32*sub + 16, (const unsigned char *)yb[4*a + j]->qs); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } + } +} + +// ---- 16x8 microkernel ---- +static void kernel4k_16x8(const a4k_t * PA, const b4k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 8*nsl) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Stage all accumulators to the stack first (frees VSRs 0-31 + // for the fixup; see DESIGN.md register-pressure note). + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += P * (dB_j * dsc_g) ; then fin -= TS_j * dm_g + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + const vfl dm = PA->dm [ch][g]; + const vfl s0 = vec_mul(vec_splat(dB, 0), dsc); + const vfl s1 = vec_mul(vec_splat(dB, 1), dsc); + const vfl s2 = vec_mul(vec_splat(dB, 2), dsc); + const vfl s3 = vec_mul(vec_splat(dB, 3), dsc); + fin[4*i+0][g] = vec_madd(vec_ctf(rowsP[0],0), s0, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(vec_ctf(rowsP[1],0), s1, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(vec_ctf(rowsP[2],0), s2, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(vec_ctf(rowsP[3],0), s3, fin[4*i+3][g]); + fin[4*i+0][g] = vec_nmsub(dm, vec_splat(TS, 0), fin[4*i+0][g]); + fin[4*i+1][g] = vec_nmsub(dm, vec_splat(TS, 1), fin[4*i+1][g]); + fin[4*i+2][g] = vec_nmsub(dm, vec_splat(TS, 2), fin[4*i+2][g]); + fin[4*i+3][g] = vec_nmsub(dm, vec_splat(TS, 3), fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q4k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a4k_t * PA = (const a4k_t *)packedA; + const b4k_t * PB = (const b4k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl4k(k), mt = rt4k(m), njt = ct4k(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel4k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q4_K * A = (const block_q4_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + void * PA = aligned_alloc(64, q4k_apack_size(MR, k)); + void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + q4k_pack_b(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q4k_repack_a(A + i*lda, lda, rows, k, PA); + q4k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp new file mode 100644 index 000000000000..167ff6a0ecae --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -0,0 +1,321 @@ +// q5k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q5_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + +static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/8 + QK_K/2, "bad q5_K"); +static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + (QK_K/16)*sizeof(int16_t), "bad q8_K"); + + +// ggml's 6-bit scale/min decode +static inline void q5k_get_scale_min(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; *m = q[j + 4] & 63; + } else { + *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); + *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); + } +} + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_SB 8 // superblocks per K slab (2048 elems) +#define KC_CHUNKS (KC_SB * 8) // 32-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CHUNKS][32]; // MMA-layout unsigned nibbles + vfl dsc[KC_CHUNKS][4]; // d*sc per chunk, 4 rows per vfl + vfl dm [KC_CHUNKS][4]; // dmin*m per chunk +} a4k_t; + +typedef struct { + vuc v[KC_CHUNKS][16]; // signed activations + vfl dB[KC_SB][2]; // Q8_K d per column, per superblock + vfl TS[KC_CHUNKS][2]; // dB * S_sub per (col, chunk) +} b4k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 5-bit codes of sub-block `sub` (0..7): Q4_K nibble + qh bit `sub`. +static inline void q5k_codes32(const uint8_t * qs, const uint8_t * qh, + int sub, vuc v[2]) { + const int g = sub >> 1; + vuc lo = vec_xl(32*g, (const unsigned char *)qs); + vuc hi = vec_xl(32*g + 16, (const unsigned char *)qs); + if ((sub & 1) == 0) { + const vuc mF = vec_splats((unsigned char)0xF); + lo = vec_and(lo, mF); + hi = vec_and(hi, mF); + } else { + lo = vec_sr(lo, vec_splats((unsigned char)4)); + hi = vec_sr(hi, vec_splats((unsigned char)4)); + } + const vuc hsh = vec_splats((unsigned char)sub); + const vuc one = vec_splats((unsigned char)1); + const vuc four = vec_splats((unsigned char)4); + vuc h0 = vec_sl(vec_and(vec_sr(vec_xl(0, (const unsigned char *)qh), hsh), one), four); + vuc h1 = vec_sl(vec_and(vec_sr(vec_xl(16, (const unsigned char *)qh), hsh), one), four); + v[0] = vec_or(lo, h0); + v[1] = vec_or(hi, h1); +} + +// ---- one-time packing ---- + +static inline int64_t rt4k(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct4k(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl4k(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q5k_apack_size(int64_t m, int64_t k) { + return (size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t); +} +extern "C" size_t q5k_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t); +} + +extern "C" void q5k_repack_a(const block_q5_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a4k_t * P = (a4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t it = 0; it < rt4k(m); it++) + for (int64_t s = 0; s < ns; s++) { + a4k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q5_K * bp[MR]; + float dsc[MR][8], dm[MR][8]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + const float d = GGML_FP16_TO_FP32(bp[r]->d); + const float dmin = GGML_FP16_TO_FP32(bp[r]->dmin); + for (int sub = 0; sub < 8; sub++) { + uint8_t sc, mn; + q5k_get_scale_min(sub, bp[r]->scales, &sc, &mn); + dsc[r][sub] = d * sc; + dm [r][sub] = dmin * mn; + } + } + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + for (int g = 0; g < 4; g++) { + T->dsc[ch][g] = (vfl){ dsc[4*g][sub], dsc[4*g+1][sub], + dsc[4*g+2][sub], dsc[4*g+3][sub] }; + T->dm [ch][g] = (vfl){ dm[4*g][sub], dm[4*g+1][sub], + dm[4*g+2][sub], dm[4*g+3][sub] }; + } + vuc t[MR][2]; + for (int r = 0; r < MR; r++) q5k_codes32(bp[r]->qs, bp[r]->qh, sub, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][16*h + g], 4); + } + } + } + } +} + +extern "C" void q5k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b4k_t * P = (b4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t jt = 0; jt < ct4k(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b4k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; + float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * (float)(yb[j]->bsums[2*sub] + yb[j]->bsums[2*sub + 1]); + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = vec_xl(32*sub, (const unsigned char *)yb[4*a + j]->qs); + q[j][1] = vec_xl(32*sub + 16, (const unsigned char *)yb[4*a + j]->qs); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } + } +} + +// ---- 16x8 microkernel ---- +static void kernel4k_16x8(const a4k_t * PA, const b4k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 8*nsl) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Stage all accumulators to the stack first (frees VSRs 0-31 + // for the fixup; see q6_k for rationale). + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += P * (dB_j * dsc_g) ; then fin -= TS_j * dm_g + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + const vfl dm = PA->dm [ch][g]; + const vfl s0 = vec_mul(vec_splat(dB, 0), dsc); + const vfl s1 = vec_mul(vec_splat(dB, 1), dsc); + const vfl s2 = vec_mul(vec_splat(dB, 2), dsc); + const vfl s3 = vec_mul(vec_splat(dB, 3), dsc); + fin[4*i+0][g] = vec_madd(vec_ctf(rowsP[0],0), s0, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(vec_ctf(rowsP[1],0), s1, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(vec_ctf(rowsP[2],0), s2, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(vec_ctf(rowsP[3],0), s3, fin[4*i+3][g]); + fin[4*i+0][g] = vec_nmsub(dm, vec_splat(TS, 0), fin[4*i+0][g]); + fin[4*i+1][g] = vec_nmsub(dm, vec_splat(TS, 1), fin[4*i+1][g]); + fin[4*i+2][g] = vec_nmsub(dm, vec_splat(TS, 2), fin[4*i+2][g]); + fin[4*i+3][g] = vec_nmsub(dm, vec_splat(TS, 3), fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q5k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a4k_t * PA = (const a4k_t *)packedA; + const b4k_t * PB = (const b4k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl4k(k), mt = rt4k(m), njt = ct4k(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel4k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q5_K * A = (const block_q5_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + void * PA = aligned_alloc(64, q5k_apack_size(MR, k)); + void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + q5k_pack_b(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q5k_repack_a(A + i*lda, lda, rows, k, PA); + q5k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp new file mode 100644 index 000000000000..c05e01f3815b --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -0,0 +1,271 @@ +// q6k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q6_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + +static_assert(sizeof(block_q6_K) == QK_K/2 + QK_K/4 + QK_K/16 + sizeof(ggml_half), "bad q6_K"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_SB 8 // superblocks per slab (2048 elems) +#define KC_CH16 (KC_SB * 16) // 16-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH16][16]; // per chunk: 4 depth-steps x 4 rowgroups + vfl dsc[KC_CH16][4]; // d*sc per chunk per rowgroup +} a6k_t; + +typedef struct { + vuc v[KC_CH16][8]; // per chunk: 4 depth-steps x 2 colgroups + vfl dB[KC_SB][2]; + vfl TS[KC_CH16][2]; // dB * 32 * bsums[c] per column +} b6k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 16 unsigned 6-bit codes of chunk c (0..15) of one superblock. +static inline vuc q6k_codes16(const uint8_t * ql, const uint8_t * qh, int c) { + const int h = c >> 3, idx = c & 7, qq = idx >> 1, lo = idx & 1; + vuc nib = vec_xl(64*h + 32*(qq & 1) + 16*lo, (const unsigned char *)ql); + nib = (qq < 2) ? vec_and(nib, vec_splats((unsigned char)0xF)) + : vec_sr (nib, vec_splats((unsigned char)4)); + vuc hb = vec_xl(32*h + 16*lo, (const unsigned char *)qh); + hb = vec_sl(vec_and(vec_sr(hb, vec_splats((unsigned char)(2*qq))), + vec_splats((unsigned char)3)), + vec_splats((unsigned char)4)); + return vec_or(nib, hb); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q6k_apack_size(int64_t m, int64_t k) { + return (size_t)(rt(m) * sl(k)) * sizeof(a6k_t); +} +extern "C" size_t q6k_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct(n) * sl(k)) * sizeof(b6k_t); +} + +extern "C" void q6k_repack_a(const block_q6_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a6k_t * P = (a6k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + a6k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q6_K * bp[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + for (int g = 0; g < 4; g++) + T->dsc[ch][g] = (vfl){ + d[4*g+0]*bp[4*g+0]->scales[c], d[4*g+1]*bp[4*g+1]->scales[c], + d[4*g+2]*bp[4*g+2]->scales[c], d[4*g+3]*bp[4*g+3]->scales[c] }; + vui rows4[4]; + for (int g = 0; g < 4; g++) { + for (int r = 0; r < 4; r++) + rows4[r] = (vui)q6k_codes16(bp[4*g + r]->ql, bp[4*g + r]->qh, c); + mma_transpose4(rows4, &T->v[ch][g], 4); + } + } + } + } +} + +extern "C" void q6k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b6k_t * P = (b6k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b6k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * 32.0f * (float)yb[j]->bsums[c]; + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xl(16*c, (const unsigned char *)yb[4*a + j]->qs); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel6k_16x8(const a6k_t * PA, const b6k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 16*nsl) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 4; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Disassemble all accumulators to a stack buffer first: + // frees the acc-aliased VSRs (0-31) before the fixup runs, so + // fin + scale vectors fit the register file without spills. + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += dsc * (dB_j*P - TS_j) (offset correction folded) + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + vfl t0 = vec_msub(vec_ctf(rowsP[0],0), vec_splat(dB,0), vec_splat(TS,0)); + vfl t1 = vec_msub(vec_ctf(rowsP[1],0), vec_splat(dB,1), vec_splat(TS,1)); + vfl t2 = vec_msub(vec_ctf(rowsP[2],0), vec_splat(dB,2), vec_splat(TS,2)); + vfl t3 = vec_msub(vec_ctf(rowsP[3],0), vec_splat(dB,3), vec_splat(TS,3)); + fin[4*i+0][g] = vec_madd(t0, dsc, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(t1, dsc, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(t2, dsc, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(t3, dsc, fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q6k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a6k_t * PA = (const a6k_t *)packedA; + const b6k_t * PB = (const b6k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel6k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q6_K * A = (const block_q6_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + void * PA = aligned_alloc(64, q6k_apack_size(MR, k)); + void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + q6k_pack_b(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q6k_repack_a(A + i*lda, lda, rows, k, PA); + q6k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index d921cb95202d..0acb43d49a4f 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -50,6 +50,7 @@ #include "sgemm.h" #include "qbit_ppc_mma.h" +#include "kquants_ppc_mma.h" #include "ggml-impl.h" #include "ggml-cpu-impl.h" #include "ggml-quants.h" @@ -3963,6 +3964,54 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_Q4_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_q4_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q5_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_q5_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q6_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_q6_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q2_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_q2_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + case GGML_TYPE_Q1_0: { if (Btype != GGML_TYPE_Q8_0) return false; From c5c8a2dec831f90c08bb87ce6957bb5b09c22529 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 03/21] power-mma: Q3_K and IQ4_NL/IQ4_XS MMA sgemm Q3_K completes the K-quants; the IQ4 family flips operand orientation (signed codebook values on the signed side, activations XOR-flipped) with a per-row 128*codesum correction at repack. --- ggml/src/ggml-cpu/CMakeLists.txt | 2 + ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 350 ++++++++++++++++++ ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 9 + ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp | 276 ++++++++++++++ ggml/src/ggml-cpu/llamafile/sgemm.cpp | 28 ++ 5 files changed, 665 insertions(+) create mode 100644 ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index c08f09e14d49..31fffa5735d7 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -87,6 +87,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/llamafile/q5k_ppc_mma.cpp ggml-cpu/llamafile/q6k_ppc_mma.cpp ggml-cpu/llamafile/q2k_ppc_mma.cpp + ggml-cpu/llamafile/q3k_ppc_mma.cpp + ggml-cpu/llamafile/iq4_ppc_mma.cpp ggml-cpu/llamafile/kquants_ppc_mma.h ggml-cpu/llamafile/qbit_ppc_mma.h ggml-cpu/llamafile/sgemm.h) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp new file mode 100644 index 000000000000..2b56b8d259f5 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -0,0 +1,350 @@ +// iq4_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified vs exact double references under qemu -cpu power10). + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + + +static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "bad iq4_nl"); +static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + 2 + QK_K/64 + QK_K/2, "bad iq4_xs"); + +static const int8_t iq4_kvalues_local[16] = { + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +}; + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_ELEMS 2048 // K slab +#define KC_CH (KC_ELEMS / 32) // 32-element chunks per slab +#define MR 8 +#define NR 8 + +// packed weights: per chunk, 8 depth-steps x 2 rowgroups of signed values +typedef struct { + vuc v[KC_CH][16]; + vfl sA [KC_CH][2]; // per-row scale (dA or d*(ls-32)) + vfl C128[KC_CH][2]; // 128 * W * scale, pre-folded +} aiq4_t; + +// packed activations: per chunk, 8 depth-steps x 2 colgroups, flipped +typedef struct { + vuc v[KC_CH][16]; + vfl dB[KC_CH][2]; // per-chunk column scales +} biq4_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 32 signed codebook values from 16 packed bytes; elements j / j+16 +// come from qs[j] low / high nibble. One vec_perm each. +static inline void iq4_lookup32(const uint8_t * qs, vsc v[2]) { + vsc tbl; memcpy(&tbl, iq4_kvalues_local, 16); + vuc raw = vec_xl(0, (const unsigned char *)qs); + vuc lo = vec_and(raw, vec_splats((unsigned char)0xF)); + vuc hi = vec_sr(raw, vec_splats((unsigned char)4)); + v[0] = vec_perm(tbl, tbl, lo); + v[1] = vec_perm(tbl, tbl, hi); +} + +static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } + +// pack one chunk's 8 weight rows into T at chunk ch; w[r] = row values, +// scale[r] = per-row scale for this chunk. +static void iq4_place_chunk(aiq4_t * T, int64_t ch, + const vsc w[MR][2], const float scale[MR]) { + for (int g = 0; g < 2; g++) { + float W[4]; + vui rows4[4]; + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)w[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][8*h + g], 2); + } + for (int r = 0; r < 4; r++) { + vsi z = vec_splats(0); + vsi s = vec_sum4s(w[4*g + r][0], z); + s = vec_sum4s(w[4*g + r][1], s); + W[r] = (float)hsum(s); + } + T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0]*scale[4*g], 128.0f*W[1]*scale[4*g+1], + 128.0f*W[2]*scale[4*g+2], 128.0f*W[3]*scale[4*g+3] }; + } +} + +static inline int64_t rt8(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct8(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl32(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } + +extern "C" size_t iq4_apack_size(int64_t m, int64_t k) { + return (size_t)(rt8(m) * sl32(k)) * sizeof(aiq4_t); +} +extern "C" size_t iq4_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct8(n) * sl32(k)) * sizeof(biq4_t); +} + +// ---- weight repack: IQ4_NL ---- +extern "C" void iq4nl_repack_a(const block_iq4_nl * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const block_iq4_nl * bp = &A[rr*lda + b0 + b]; + sc[r] = GGML_FP16_TO_FP32(bp->d); + iq4_lookup32(bp->qs, w[r]); + } + iq4_place_chunk(T, b, w, sc); + } + } +} + +// ---- weight repack: IQ4_XS ---- +extern "C" void iq4xs_repack_a(const block_iq4_xs * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t nsb = k/QK_K, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_iq4_xs * bp[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + sb]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + } + for (int ib = 0; ib < 8; ib++) { // 32-groups + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + const int ls = ((bp[r]->scales_l[ib/2] >> 4*(ib%2)) & 0xF) + | (((bp[r]->scales_h >> 2*ib) & 3) << 4); + sc[r] = d[r] * (float)(ls - 32); + iq4_lookup32(bp[r]->qs + 16*ib, w[r]); + } + iq4_place_chunk(T, 8*sb + ib, w, sc); + } + } + } +} + +// ---- activation packs ---- +extern "C" void iq4_pack_b_q8_0(const block_q8_0 * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + biq4_t * P = (biq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct8(n); jt++) + for (int64_t s = 0; s < ns; s++) { + biq4_t * T = &P[jt*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + const block_q8_0 * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + b0 + b]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + vec_xl(16*h, (const unsigned char *)yb[4*a + j]->qs), flip); + mma_transpose4(rows4, &T->v[b][8*h + a], 2); + } + } + } +} + +extern "C" void iq4_pack_b_q8_K(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + biq4_t * P = (biq4_t *)packed; + const int64_t nsb = k/QK_K, ns = sl32(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct8(n); jt++) + for (int64_t s = 0; s < ns; s++) { + biq4_t * T = &P[jt*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + sb]; + dB[j] = yb[j]->d; + } + for (int ib = 0; ib < 8; ib++) { + const int64_t ch = 8*sb + ib; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + vec_xl(32*ib + 16*h, (const unsigned char *)yb[4*a + j]->qs), flip); + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } +} + +// ---- 8x8 microkernel on 4 accumulators (weights on the signed operand; +// acc rows = weight rows) ---- +static void kernel_iq4_8x8(const aiq4_t * PA, const biq4_t * PB, + int64_t nch, vfl fin[MR][2]) { + for (int64_t ch = 0; ch < nch; ch++) { + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < nch) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][2]; + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 8; x++) { + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + // fin += dB ⊙ (P*sA_r - C128_r) per weight row r + for (int r = 0; r < 4; r++) { + vfl t = vec_msub(vec_ctf(rowsP[r],0), + vec_splats(sA[r]), vec_splats(C128[r])); + fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + } + } + } + } +} + +static void iq4_gemm_core(int64_t m, int64_t n, int64_t k, + const aiq4_t * PA, const biq4_t * PB, + float * C, int64_t ldc, int ith, int nth) { + const int64_t kb = k/32, ns = sl32(k), mt = rt8(m), njt = ct8(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[MR][2]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int r = 0; r < MR; r++) fin[r][0] = fin[r][1] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t b0 = s*KC_CH; + const int64_t nch = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + kernel_iq4_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t r = 0; r < rows; r++) + for (int64_t cj = 0; cj < cols; cj++) + C[(i + r) + (j0 + cj)*ldc] = fin[r][cj >> 2][cj & 3]; + } + } +} + +extern "C" void iq4_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + iq4_gemm_core(m, n, k, (const aiq4_t *)packedA, (const biq4_t *)packedB, + C, ldc, ith, nth); +} + + +extern "C" void gemm_iq4_nl_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_iq4_nl * A = (const block_iq4_nl *)Av; + const block_q8_0 * B = (const block_q8_0 *)Bv; + void * PA = aligned_alloc(64, iq4_apack_size(MR, k)); + void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + iq4_pack_b_q8_0(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + iq4nl_repack_a(A + i*lda, lda, rows, k, PA); + iq4_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +extern "C" void gemm_iq4_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_iq4_xs * A = (const block_iq4_xs *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + void * PA = aligned_alloc(64, iq4_apack_size(MR, k)); + void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + iq4_pack_b_q8_K(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + iq4xs_repack_a(A + i*lda, lda, rows, k, PA); + iq4_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index 981d888b1267..834071248f56 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -18,6 +18,15 @@ void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, const void * A, int64_t lda, const void * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth); +void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq4_nl_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq4_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp new file mode 100644 index 000000000000..0cb87bb2b468 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -0,0 +1,276 @@ +// q3k_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified vs exact double references under qemu -cpu power10). + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + +static_assert(sizeof(block_q3_K) == QK_K/8 + QK_K/4 + 12 + sizeof(ggml_half), "bad q3_K"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_SB 8 // superblocks per slab (2048 elems) +#define KC_CH16 (KC_SB * 16) // 16-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH16][16]; // per chunk: 4 depth-steps x 4 rowgroups + vfl dsc[KC_CH16][4]; // d*sc per chunk per rowgroup +} a3k_t; + +typedef struct { + vuc v[KC_CH16][8]; // per chunk: 4 depth-steps x 2 colgroups + vfl dB[KC_SB][2]; + vfl TS[KC_CH16][2]; // dB * 4 * bsums[c] per column +} b3k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 16 unsigned 3-bit codes q' = code | (hbit << 2) of chunk c (0..15). +static inline vuc q3k_codes16(const uint8_t * qs, const uint8_t * hmask, int c) { + const int h = c >> 3, idx = c & 7, j = idx >> 1, lo = idx & 1; + vuc code = vec_and(vec_sr(vec_xl(32*h + 16*lo, (const unsigned char *)qs), + vec_splats((unsigned char)(2*j))), + vec_splats((unsigned char)3)); + vuc hb = vec_sl(vec_and(vec_sr(vec_xl(16*lo, (const unsigned char *)hmask), + vec_splats((unsigned char)(4*h + j))), + vec_splats((unsigned char)1)), + vec_splats((unsigned char)2)); + return vec_or(code, hb); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q3k_apack_size(int64_t m, int64_t k) { + return (size_t)(rt(m) * sl(k)) * sizeof(a3k_t); +} +extern "C" size_t q3k_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct(n) * sl(k)) * sizeof(b3k_t); +} + +extern "C" void q3k_repack_a(const block_q3_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a3k_t * P = (a3k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + a3k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q3_K * bp[MR]; float d[MR]; int8_t sc6[MR][16]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + // decode 16 x 6-bit scales from the 12-byte packing + const uint32_t kmask1 = 0x03030303, kmask2 = 0x0f0f0f0f; + uint32_t aux[4]; + memcpy(aux, bp[r]->scales, 12); + uint32_t tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + memcpy(sc6[r], aux, 16); + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + for (int g = 0; g < 4; g++) + T->dsc[ch][g] = (vfl){ + d[4*g+0]*(sc6[4*g+0][c]-32), d[4*g+1]*(sc6[4*g+1][c]-32), + d[4*g+2]*(sc6[4*g+2][c]-32), d[4*g+3]*(sc6[4*g+3][c]-32) }; + vui rows4[4]; + for (int g = 0; g < 4; g++) { + for (int r = 0; r < 4; r++) + rows4[r] = (vui)q3k_codes16(bp[4*g + r]->qs, bp[4*g + r]->hmask, c); + mma_transpose4(rows4, &T->v[ch][g], 4); + } + } + } + } +} + +extern "C" void q3k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b3k_t * P = (b3k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b3k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * 4.0f * (float)yb[j]->bsums[c]; + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xl(16*c, (const unsigned char *)yb[4*a + j]->qs); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel3k_16x8(const a3k_t * PA, const b3k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 16*nsl) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 4; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Disassemble all accumulators to a stack buffer first: + // frees the acc-aliased VSRs (0-31) before the fixup runs, so + // fin + scale vectors fit the register file without spills. + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += dsc * (dB_j*P - TS_j) (offset correction folded) + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + vfl t0 = vec_msub(vec_ctf(rowsP[0],0), vec_splat(dB,0), vec_splat(TS,0)); + vfl t1 = vec_msub(vec_ctf(rowsP[1],0), vec_splat(dB,1), vec_splat(TS,1)); + vfl t2 = vec_msub(vec_ctf(rowsP[2],0), vec_splat(dB,2), vec_splat(TS,2)); + vfl t3 = vec_msub(vec_ctf(rowsP[3],0), vec_splat(dB,3), vec_splat(TS,3)); + fin[4*i+0][g] = vec_madd(t0, dsc, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(t1, dsc, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(t2, dsc, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(t3, dsc, fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q3k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a3k_t * PA = (const a3k_t *)packedA; + const b3k_t * PB = (const b3k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel3k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q3_K * A = (const block_q3_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + void * PA = aligned_alloc(64, q3k_apack_size(MR, k)); + void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); + if (!PA || !PB) { free(PA); free(PB); return; } + q3k_pack_b(B, ldb, n, k, PB); + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q3k_repack_a(A + i*lda, lda, rows, k, PA); + q3k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + } + free(PA); free(PB); +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 0acb43d49a4f..980cf98e0a9b 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -4012,6 +4012,30 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_Q3_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_q3_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ4_XS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq4_xs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + case GGML_TYPE_Q1_0: { if (Btype != GGML_TYPE_Q8_0) return false; @@ -4102,6 +4126,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__MMA__) + gemm_iq4_nl_q8_0_ppc(m, n, k * QK4_NL, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; #else return false; #endif From 60e9367eb53efecd05c9030feb35bfbf6b2d09b0 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 04/21] power-mma: Q4_1/Q5_0/Q5_1 legacy-format MMA sgemm The Q4_1/Q5_1 min term reduces to one multiply-add because Q8_1 blocks already store the scaled activation sum. --- ggml/src/ggml-cpu/CMakeLists.txt | 1 + ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 9 + .../src/ggml-cpu/llamafile/legacy_ppc_mma.cpp | 315 ++++++++++++++++++ ggml/src/ggml-cpu/llamafile/sgemm.cpp | 28 ++ 4 files changed, 353 insertions(+) create mode 100644 ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 31fffa5735d7..999c9be96448 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -89,6 +89,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/llamafile/q2k_ppc_mma.cpp ggml-cpu/llamafile/q3k_ppc_mma.cpp ggml-cpu/llamafile/iq4_ppc_mma.cpp + ggml-cpu/llamafile/legacy_ppc_mma.cpp ggml-cpu/llamafile/kquants_ppc_mma.h ggml-cpu/llamafile/qbit_ppc_mma.h ggml-cpu/llamafile/sgemm.h) diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index 834071248f56..fd539d585e64 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -27,6 +27,15 @@ void gemm_iq4_nl_q8_0_ppc(int64_t m, int64_t n, int64_t k, void gemm_iq4_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, const void * A, int64_t lda, const void * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth); +void gemm_q4_1_q8_1_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q5_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q5_1_q8_1_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp new file mode 100644 index 000000000000..0eb1064af8b9 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -0,0 +1,315 @@ +// legacy_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified vs exact double references under qemu -cpu power10). +// Q4_1 x Q8_1, Q5_0 x Q8_0, Q5_1 x Q8_1 on POWER10/POWER11 MMA. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + + + +static_assert(sizeof(block_q4_1) == 4 + 16, "bad q4_1"); +static_assert(sizeof(block_q5_0) == 2 + 4 + 16, "bad q5_0"); +static_assert(sizeof(block_q5_1) == 4 + 4 + 16, "bad q5_1"); +static_assert(sizeof(block_q8_1) == 4 + 32, "bad q8_1"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_CH 64 // 32-elem chunks per slab (2048) +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH][32]; // 8 depth-steps x 4 rowgroups + vfl dA[KC_CH][4]; // per-block scale per rowgroup + vfl mA[KC_CH][4]; // per-block min (Q4_1/Q5_1; 0 for Q5_0) +} aleg_t; + +typedef struct { + vuc v[KC_CH][16]; // 8 depth-steps x 2 colgroups + vfl dB[KC_CH][2]; // per-block activation scale + vfl SB[KC_CH][2]; // Q8_1: s = dB*sum(y); Q8_0: 16*dB*sum(y) +} bleg_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +static inline void nibbles32(const uint8_t * qs, vuc v[2]) { + vuc raw = vec_xl(0, (const unsigned char *)qs); + v[0] = vec_and(raw, vec_splats((unsigned char)0xF)); + v[1] = vec_sr(raw, vec_splats((unsigned char)4)); +} + +// merge qh bits: elems 0..15 use bits 0..15 (bytes 0-1), elems 16..31 +// use bits 16..31 (bytes 2-3); bit -> position 4. +static inline void qh_merge(const uint8_t * qh, vuc v[2]) { + vuc raw = vec_xl(0, (const unsigned char *)qh); // first 4 bytes used + const vuc repL = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; + const vuc repH = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; + const vuc sh = { 0,1,2,3,4,5,6,7, 0,1,2,3,4,5,6,7 }; + const vuc one = vec_splats((unsigned char)1); + const vuc four = vec_splats((unsigned char)4); + vuc b0 = vec_sl(vec_and(vec_sr(vec_perm(raw, raw, repL), sh), one), four); + vuc b1 = vec_sl(vec_and(vec_sr(vec_perm(raw, raw, repH), sh), one), four); + v[0] = vec_or(v[0], b0); + v[1] = vec_or(v[1], b1); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } + +extern "C" size_t leg_apack_size(int64_t m, int64_t k) { + return (size_t)(rt(m) * sl(k)) * sizeof(aleg_t); +} +extern "C" size_t leg_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct(n) * sl(k)) * sizeof(bleg_t); +} + +// generic weight repack over a per-block "codes + d + m" extractor +template +static void repack_generic(const BLK * A, int64_t lda, int64_t m, int64_t k, aleg_t * P) { + const int64_t kb = k/32, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + aleg_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vuc t[MR][2]; float d[MR], mm[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const BLK * bp = &A[rr*lda + b0 + b]; + d[r] = GGML_FP16_TO_FP32(bp->d); + mm[r] = HAS_M ? GGML_FP16_TO_FP32(((const ggml_half *)bp)[1]) : 0.0f; + CODES(bp, t[r]); + } + for (int g = 0; g < 4; g++) { + T->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; + T->mA[b][g] = (vfl){ mm[4*g], mm[4*g+1], mm[4*g+2], mm[4*g+3] }; + } + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[b][16*h + g], 4); + } + } + } +} + +static void codes_q4_1(const block_q4_1 * bp, vuc v[2]) { nibbles32(bp->qs, v); } +static void codes_q5_0(const block_q5_0 * bp, vuc v[2]) { nibbles32(bp->qs, v); qh_merge(bp->qh, v); } +static void codes_q5_1(const block_q5_1 * bp, vuc v[2]) { nibbles32(bp->qs, v); qh_merge(bp->qh, v); } + +extern "C" void leg_repack_q4_1(const block_q4_1 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} +extern "C" void leg_repack_q5_0(const block_q5_0 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} +extern "C" void leg_repack_q5_1(const block_q5_1 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} + +// activation packs: SB = dB*sum(y) computed (Q8_0, x16 for the Q5_0 +// offset) or read from the precomputed s field (Q8_1). +template +static void pack_b_generic(const YBLK * B, int64_t ldb, int64_t n, int64_t k, bleg_t * P) { + const int64_t kb = k/32, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bleg_t * T = &P[jt*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + const YBLK * yb[NR]; float dB[NR], SB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + b0 + b]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + if (READ_S) { + SB[j] = GGML_FP16_TO_FP32(((const ggml_half *)yb[j])[1]); + } else { + vsi z = vec_splats(0); + vsi sm = vec_sum4s((vsc)vec_xl(0, (const unsigned char *)yb[j]->qs), z); + sm = vec_sum4s((vsc)vec_xl(16, (const unsigned char *)yb[j]->qs), sm); + SB[j] = (float)SFACT * dB[j] * (float)(sm[0]+sm[1]+sm[2]+sm[3]); + } + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + T->SB[b][0] = (vfl){ SB[0], SB[1], SB[2], SB[3] }; + T->SB[b][1] = (vfl){ SB[4], SB[5], SB[6], SB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xl(16*h, (const unsigned char *)yb[4*a + j]->qs); + mma_transpose4(rows4, &T->v[b][8*h + a], 2); + } + } + } +} + +extern "C" void leg_pack_b_q8_0(const block_q8_0 * B, int64_t ldb, int64_t n, int64_t k, void * p) { + pack_b_generic(B, ldb, n, k, (bleg_t *)p); +} +extern "C" void leg_pack_b_q8_1(const block_q8_1 * B, int64_t ldb, int64_t n, int64_t k, void * p) { + pack_b_generic(B, ldb, n, k, (bleg_t *)p); +} + +// MODE 0 (offset, Q5_0): fin += dA ⊙ (dB_j*P - SB_j) [SB = 16*dB*S] +// MODE 1 (affine, Q4_1/Q5_1): fin += dA*(dB_j*P) + mA*SB_j [SB = dB*sum(y)] +template +static void kernel_leg_16x8(const aleg_t * PA, const bleg_t * PB, + int64_t nb, vfl fin[NR][4]) { + for (int64_t b = 0; b < nb; b++) { + const vuc * a = PA->v[b]; + const vuc * y = PB->v[b]; + if (b + 1 < nb) { + __builtin_prefetch(PA->v[b + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[b + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[b + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + for (int i = 0; i < 2; i++) { + const vfl dB = PB->dB[b][i]; + const vfl SB = PB->SB[b][i]; + for (int g = 0; g < 4; g++) { + const vfl dA = PA->dA[b][g]; + const vfl mA = PA->mA[b][g]; + for (int r = 0; r < 4; r++) { + const vfl res = vec_ctf(pr[i][g][r], 0); + if (MODE == 0) { + vfl t = vec_msub(res, vec_splat(dB, r), vec_splat(SB, r)); + fin[4*i + r][g] = vec_madd(t, dA, fin[4*i + r][g]); + } else { + vfl vs = vec_mul(vec_splat(dB, r), dA); + fin[4*i + r][g] = vec_madd(res, vs, fin[4*i + r][g]); + fin[4*i + r][g] = vec_madd(mA, vec_splat(SB, r), fin[4*i + r][g]); + } + } + } + } + } +} + +template +static void leg_gemm_core(int64_t m, int64_t n, int64_t k, + const aleg_t * PA, const bleg_t * PB, + float * C, int64_t ldc, int ith, int nth) { + const int64_t kb = k/32, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + kernel_leg_16x8(&PA[it*ns + s], &PB[jt*ns + s], nb, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + +extern "C" void leg_gemm_offset(int64_t m, int64_t n, int64_t k, + const void * PA, const void * PB, float * C, int64_t ldc, int ith, int nth) { + leg_gemm_core<0>(m, n, k, (const aleg_t *)PA, (const bleg_t *)PB, C, ldc, ith, nth); +} +extern "C" void leg_gemm_affine(int64_t m, int64_t n, int64_t k, + const void * PA, const void * PB, float * C, int64_t ldc, int ith, int nth) { + leg_gemm_core<1>(m, n, k, (const aleg_t *)PA, (const bleg_t *)PB, C, ldc, ith, nth); +} + + +// one-shot drivers (pack per call; see K-quant note in INTEGRATION.md) +#define LEG_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, GEMM) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const YBLK * B = (const YBLK *)Bv; \ + void * PA = aligned_alloc(64, leg_apack_size(MR, k)); \ + void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ + if (!PA || !PB) { free(PA); free(PB); return; } \ + PACKB(B, ldb, n, k, PB); \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PA); \ + GEMM(rows, n, k, PA, PB, C + i, ldc, 0, 1); \ + } \ + free(PA); free(PB); \ +} +LEG_ONESHOT(gemm_q4_1_q8_1_ppc, block_q4_1, leg_repack_q4_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine) +LEG_ONESHOT(gemm_q5_0_q8_0_ppc, block_q5_0, leg_repack_q5_0, block_q8_0, leg_pack_b_q8_0, leg_gemm_offset) +LEG_ONESHOT(gemm_q5_1_q8_1_ppc, block_q5_1, leg_repack_q5_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine) + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 980cf98e0a9b..6c8fe3bfe75d 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -4036,6 +4036,30 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_Q4_1: { + if (Btype != GGML_TYPE_Q8_1) + return false; +#if defined(__MMA__) + gemm_q4_1_q8_1_ppc(m, n, k * QK4_1, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q5_1: { + if (Btype != GGML_TYPE_Q8_1) + return false; +#if defined(__MMA__) + gemm_q5_1_q8_1_ppc(m, n, k * QK5_1, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + case GGML_TYPE_Q1_0: { if (Btype != GGML_TYPE_Q8_0) return false; @@ -4110,6 +4134,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__MMA__) + gemm_q5_0_q8_0_ppc(m, n, k * QK5_0, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; #else return false; #endif From 547dd51c9a2617a0b9cc71e0ea0efcd1f9f53a26 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 05/21] power-mma: grid/ternary formats via two shared signed kernels TQ1_0, TQ2_0, IQ2_XXS/XS/S, IQ3_XXS/S, IQ1_S/M collapse into shared 32-deep and 16-deep chunk kernels plus per-format scalar decoders; decode runs at repack time, off the hot path. --- ggml/src/ggml-cpu/CMakeLists.txt | 2 + .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 415 +++++++++++ ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h | 701 ++++++++++++++++++ ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 18 + ggml/src/ggml-cpu/llamafile/sgemm.cpp | 72 ++ 5 files changed, 1208 insertions(+) create mode 100644 ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 999c9be96448..c86341d80347 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -90,6 +90,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/llamafile/q3k_ppc_mma.cpp ggml-cpu/llamafile/iq4_ppc_mma.cpp ggml-cpu/llamafile/legacy_ppc_mma.cpp + ggml-cpu/llamafile/iq_grid_ppc_mma.cpp + ggml-cpu/llamafile/iq_grids_ppc.h ggml-cpu/llamafile/kquants_ppc_mma.h ggml-cpu/llamafile/qbit_ppc_mma.h ggml-cpu/llamafile/sgemm.h) diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp new file mode 100644 index 000000000000..851b44f82400 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -0,0 +1,415 @@ +// iq_grid_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified under qemu -cpu power10). Grid-codebook and +// ternary formats on POWER10/POWER11 MMA via decode-at-repack + the +// shared signed 8x8 kernel. Uses a local copy of the grid tables +// (iq_grids_ppc.h) to avoid depending on GGML_COMMON_IMPL linkage. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include +#include "iq_grids_ppc.h" + +#define QK_K 256 +#define IQ1S_DELTA 0.125f + +typedef uint16_t ppc_half; + +typedef struct { uint8_t qs[(QK_K - 4*QK_K/64)/5]; uint8_t qh[QK_K/64]; ppc_half d; } block_tq1_0_ppc; +typedef struct { uint8_t qs[QK_K/4]; ppc_half d; } block_tq2_0_ppc; +typedef struct { ppc_half d; uint16_t qs[QK_K/8]; } block_iq2_xxs_ppc; +typedef struct { ppc_half d; uint8_t qs[3*QK_K/8]; } block_iq3_xxs_ppc; +typedef struct { ppc_half d; uint8_t qs[QK_K/4]; uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; uint8_t scales[QK_K/64]; } block_iq3_s_ppc; +typedef struct { ppc_half d; uint8_t qs[QK_K/8]; uint16_t qh[QK_K/32]; } block_iq1_s_ppc; +typedef struct { float d; int8_t qs[QK_K]; int16_t bsums[QK_K/16]; } block_q8_K_ppc; + +static_assert(sizeof(block_tq1_0_ppc) == 2 + 4 + 48, "bad tq1_0"); +static_assert(sizeof(block_tq2_0_ppc) == 2 + 64, "bad tq2_0"); +static_assert(sizeof(block_iq2_xxs_ppc) == 2 + QK_K/4, "bad iq2_xxs"); +static_assert(sizeof(block_iq3_xxs_ppc) == 2 + 3*QK_K/8, "bad iq3_xxs"); +static_assert(sizeof(block_iq3_s_ppc) == 2 + 13*(QK_K/32) + QK_K/64, "bad iq3_s"); +static_assert(sizeof(block_iq1_s_ppc) == 2 + QK_K/8 + QK_K/16, "bad iq1_s"); + +static inline float fp16_to_fp32(ppc_half h) { + const uint32_t sign = (uint32_t)(h >> 15) << 31; + uint32_t exp = (h >> 10) & 0x1f; + uint32_t mant = h & 0x3ff; + uint32_t bits; + if (exp == 0) { + if (mant == 0) bits = sign; + else { + exp = 127 - 15 + 1; + while (!(mant & 0x400)) { mant <<= 1; exp--; } + mant &= 0x3ff; + bits = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1f) bits = sign | 0x7f800000u | (mant << 13); + else bits = sign | ((exp + 127 - 15) << 23) | (mant << 13); + float f; memcpy(&f, &bits, 4); + return f; +} + +// ---- scalar decoders: superblock -> 256 signed int8 codes + 8 scales ---- +// (one per 32-chunk). Direct ports of dequantize_row semantics. + +static void dec_tq2_0(const block_tq2_0_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + int8_t * y = code; + for (size_t j = 0; j < sizeof(b->qs); j += 32) + for (size_t l = 0; l < 4; ++l) + for (size_t m = 0; m < 32; ++m) + *y++ = (int8_t)(((b->qs[j + m] >> (l*2)) & 3) - 1); + for (int c = 0; c < 8; c++) sc[c] = d; +} + +static void dec_tq1_0(const block_tq1_0_ppc * b, int8_t code[QK_K], float sc[8]) { + static const uint8_t pow3[6] = { 1, 3, 9, 27, 81, 243 }; + const float d = fp16_to_fp32(b->d); + int8_t * y = code; + for (size_t j = 0; j < sizeof(b->qs) - sizeof(b->qs) % 32; j += 32) + for (size_t n = 0; n < 5; ++n) + for (size_t m = 0; m < 32; ++m) { + uint8_t q = (uint8_t)(b->qs[j + m] * pow3[n]); + *y++ = (int8_t)((((uint16_t)q * 3) >> 8) - 1); + } + for (size_t j = sizeof(b->qs) - sizeof(b->qs) % 32; j < sizeof(b->qs); j += 16) + for (size_t n = 0; n < 5; ++n) + for (size_t m = 0; m < 16; ++m) { + uint8_t q = (uint8_t)(b->qs[j + m] * pow3[n]); + *y++ = (int8_t)((((uint16_t)q * 3) >> 8) - 1); + } + for (size_t n = 0; n < 4; ++n) + for (size_t j = 0; j < sizeof(b->qh); ++j) { + uint8_t q = (uint8_t)(b->qh[j] * pow3[n]); + *y++ = (int8_t)((((uint16_t)q * 3) >> 8) - 1); + } + for (int c = 0; c < 8; c++) sc[c] = d; +} + +static void dec_iq2_xxs(const block_iq2_xxs_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + uint32_t aux32[2]; + const uint8_t * aux8 = (const uint8_t *)aux32; + for (int ib32 = 0; ib32 < 8; ++ib32) { + memcpy(aux32, b->qs + 4*ib32, 8); + sc[ib32] = d * (0.5f + (aux32[1] >> 28)) * 0.25f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]); + const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127]; + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)((signs & kmask_iq2xs[j]) ? -(int)grid[j] : (int)grid[j]); + } + } +} + +static void dec_iq3_xxs(const block_iq3_xxs_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + const uint8_t * sas = b->qs + QK_K/4; + uint32_t aux32; + for (int ib32 = 0; ib32 < 8; ++ib32) { + memcpy(&aux32, sas + 4*ib32, 4); + sc[ib32] = d * (0.5f + (aux32 >> 28)) * 0.5f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127]; + const uint8_t * g1 = (const uint8_t *)(iq3xxs_grid + qs[8*ib32 + 2*l + 0]); + const uint8_t * g2 = (const uint8_t *)(iq3xxs_grid + qs[8*ib32 + 2*l + 1]); + for (int j = 0; j < 4; ++j) { + y[8*l + j + 0] = (int8_t)((signs & kmask_iq2xs[j+0]) ? -(int)g1[j] : (int)g1[j]); + y[8*l + j + 4] = (int8_t)((signs & kmask_iq2xs[j+4]) ? -(int)g2[j] : (int)g2[j]); + } + } + } +} + +static void dec_iq3_s(const block_iq3_s_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + const uint8_t * qh = b->qh; + const uint8_t * signs = b->signs; + for (int ib32 = 0; ib32 < 8; ib32 += 2) { + sc[ib32 + 0] = d * (1 + 2*(b->scales[ib32/2] & 0xF)); + sc[ib32 + 1] = d * (1 + 2*(b->scales[ib32/2] >> 4)); + for (int half = 0; half < 2; half++) { + int8_t * y = code + 32*(ib32 + half); + for (int l = 0; l < 4; ++l) { + const uint8_t * g1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[half] << (8-2*l)) & 256))); + const uint8_t * g2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[half] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + y[8*l + j + 0] = (int8_t)((signs[l] & kmask_iq2xs[j+0]) ? -(int)g1[j] : (int)g1[j]); + y[8*l + j + 4] = (int8_t)((signs[l] & kmask_iq2xs[j+4]) ? -(int)g2[j] : (int)g2[j]); + } + } + qs += 8; signs += 4; + } + qh += 2; + } +} + +// IQ1_S: value = dl*(g + delta), g in {-1,+1}, delta = +/-1/8 per 32. +// Exact integer form: code = 8*g + (delta<0 ? -1 : +1), scale = dl/8. +static void dec_iq1_s(const block_iq1_s_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + for (int ib = 0; ib < 8; ++ib) { + const float dl = d * (2*((b->qh[ib] >> 12) & 7) + 1); + sc[ib] = dl * 0.125f; + const int dsign = (b->qh[ib] & 0x8000) ? -1 : +1; + int8_t * y = code + 32*ib; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((b->qh[ib] >> 3*l) & 7) << 8))); + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)(8*grid[j] + dsign); + } + qs += 4; + } +} + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +#define KC_ELEMS 2048 +#define KC_CH (KC_ELEMS / 32) +#define MR 8 +#define NR 8 + +typedef struct { + vuc v[KC_CH][16]; + vfl sA [KC_CH][2]; + vfl C128[KC_CH][2]; +} agrid_t; + +typedef struct { + vuc v[KC_CH][16]; + vfl dB[KC_CH][2]; +} bgrid_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } + +static void grid_place_chunk(agrid_t * T, int64_t ch, + const vsc w[MR][2], const float scale[MR]) { + for (int g = 0; g < 2; g++) { + float W[4]; + vui rows4[4]; + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)w[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][8*h + g], 2); + } + for (int r = 0; r < 4; r++) { + vsi z = vec_splats(0); + vsi s = vec_sum4s(w[4*g + r][0], z); + s = vec_sum4s(w[4*g + r][1], s); + W[r] = (float)hsum(s); + } + T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0]*scale[4*g], 128.0f*W[1]*scale[4*g+1], + 128.0f*W[2]*scale[4*g+2], 128.0f*W[3]*scale[4*g+3] }; + } +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } + +extern "C" size_t grid_apack_size(int64_t m, int64_t k) { + return (size_t)(rt(m) * sl(k)) * sizeof(agrid_t); +} +extern "C" size_t grid_bpack_size(int64_t n, int64_t k) { + return (size_t)(ct(n) * sl(k)) * sizeof(bgrid_t); +} + +// generic repack over any superblock decoder +template +static void repack_grid(const BLK * A, int64_t lda, int64_t m, int64_t k, agrid_t * P) { + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + agrid_t * T = &P[it*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + int8_t code[MR][QK_K]; float sc[MR][8]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + DEC(&A[rr*lda + sb0 + sb], code[r], sc[r]); + } + for (int c = 0; c < 8; c++) { + vsc w[MR][2]; float scale[MR]; + for (int r = 0; r < MR; r++) { + memcpy(&w[r][0], code[r] + 32*c, 16); + memcpy(&w[r][1], code[r] + 32*c + 16, 16); + scale[r] = sc[r][c]; + } + grid_place_chunk(T, 8*sb + c, w, scale); + } + } + } +} + +extern "C" void grid_repack_tq2_0(const block_tq2_0_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_tq1_0(const block_tq1_0_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq2_xxs(const block_iq2_xxs_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq3_xxs(const block_iq3_xxs_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq3_s(const block_iq3_s_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq1_s(const block_iq1_s_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } + +extern "C" void grid_pack_b_q8_K(const block_q8_K_ppc * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bgrid_t * P = (bgrid_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bgrid_t * T = &P[jt*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_q8_K_ppc * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + sb]; + dB[j] = yb[j]->d; + } + for (int ib = 0; ib < 8; ib++) { + const int64_t ch = 8*sb + ib; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + vec_xl(32*ib + 16*h, (const unsigned char *)yb[4*a + j]->qs), flip); + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } +} + +static void kernel_grid_8x8(const agrid_t * PA, const bgrid_t * PB, + int64_t nch, vfl fin[MR][2]) { + for (int64_t ch = 0; ch < nch; ch++) { + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < nch) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][2]; + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 8; x++) { + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + for (int r = 0; r < 4; r++) { + vfl t = vec_msub(vec_ctf(rowsP[r],0), + vec_splats(sA[r]), vec_splats(C128[r])); + fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + } + } + } + } +} + +extern "C" void grid_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const agrid_t * PA = (const agrid_t *)packedA; + const bgrid_t * PB = (const bgrid_t *)packedB; + const int64_t kb = k/32, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[MR][2]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int r = 0; r < MR; r++) fin[r][0] = fin[r][1] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t b0 = s*KC_CH; + const int64_t nch = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + kernel_grid_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t r = 0; r < rows; r++) + for (int64_t cj = 0; cj < cols; cj++) + C[(i + r) + (j0 + cj)*ldc] = fin[r][cj >> 2][cj & 3]; + } + } +} + + +#define GRID_ONESHOT(NAME, BLKA, REPACK) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const block_q8_K_ppc * B = (const block_q8_K_ppc *)Bv; \ + void * PA = aligned_alloc(64, grid_apack_size(MR, k)); \ + void * PB = aligned_alloc(64, grid_bpack_size(n, k)); \ + if (!PA || !PB) { free(PA); free(PB); return; } \ + grid_pack_b_q8_K(B, ldb, n, k, PB); \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PA); \ + grid_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); \ + } \ + free(PA); free(PB); \ +} +GRID_ONESHOT(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0) +GRID_ONESHOT(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0) +GRID_ONESHOT(gemm_iq2_xxs_q8_K_ppc, block_iq2_xxs_ppc, grid_repack_iq2_xxs) +GRID_ONESHOT(gemm_iq3_xxs_q8_K_ppc, block_iq3_xxs_ppc, grid_repack_iq3_xxs) +GRID_ONESHOT(gemm_iq3_s_q8_K_ppc, block_iq3_s_ppc, grid_repack_iq3_s) +GRID_ONESHOT(gemm_iq1_s_q8_K_ppc, block_iq1_s_ppc, grid_repack_iq1_s) + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h b/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h new file mode 100644 index 000000000000..4797197a9111 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h @@ -0,0 +1,701 @@ +// iq_grids.h - grid/sign tables extracted from ggml-common.h (MIT, ggml authors) +#pragma once +#include + +static const uint64_t iq2xxs_grid[256] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, + 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, + 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, + 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, + 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, + 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, + 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, + 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, + 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, + 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, + 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, + 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, + 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, + 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, + 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, + 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, + 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, + 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, + 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, + 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, + 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, + 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, + 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, + 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, + 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, + 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, + 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, + 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, + 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, + 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, + 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, + 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, + 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, + 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, + 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, + 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, + 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, + 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, + 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, + 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, + 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, + 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, + 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, + 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, + 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, + 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, + 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, + 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, + 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, + 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, + 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, + 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, + 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, + 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, +}; + +static const uint32_t iq3xxs_grid[256] = { + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, +}; + +static const uint32_t iq3s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, +}; + +static const uint8_t ksigns_iq2xs[128] = { + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, + 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, + 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, + 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, + 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, + 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, + 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, + 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, +}; + +static const uint8_t kmask_iq2xs[8] = { + 1, 2, 4, 8, 16, 32, 64, 128 +}; +static const uint64_t iq1s_grid[2048] = { + 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, + 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff, + 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000, + 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000, + 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101, + 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff, + 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00, + 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101, + 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100, + 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00, + 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000, + 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000, + 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101, + 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff, + 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100, + 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01, + 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000, + 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff, + 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001, + 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000, + 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00, + 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff, + 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff, + 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000, + 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff, + 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff, + 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101, + 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff, + 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000, + 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100, + 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01, + 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00, + 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00, + 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01, + 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff, + 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000, + 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff, + 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000, + 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101, + 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100, + 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff, + 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff, + 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001, + 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01, + 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff, + 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000, + 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000, + 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff, + 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01, + 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00, + 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000, + 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000, + 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000, + 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001, + 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff, + 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01, + 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff, + 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff, + 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000, + 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000, + 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00, + 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001, + 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100, + 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101, + 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00, + 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00, + 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000, + 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001, + 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff, + 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000, + 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000, + 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff, + 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100, + 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000, + 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101, + 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001, + 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000, + 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff, + 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01, + 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, + 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000, + 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01, + 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101, + 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000, + 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff, + 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff, + 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001, + 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001, + 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000, + 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000, + 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00, + 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100, + 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01, + 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00, + 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff, + 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000, + 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00, + 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000, + 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff, + 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00, + 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000, + 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000, + 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff, + 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00, + 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00, + 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001, + 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001, + 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000, + 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100, + 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101, + 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101, + 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000, + 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00, + 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff, + 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000, + 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff, + 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff, + 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff, + 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101, + 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001, + 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100, + 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101, + 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01, + 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00, + 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00, + 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000, + 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101, + 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100, + 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001, + 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff, + 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00, + 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000, + 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100, + 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00, + 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01, + 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff, + 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101, + 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100, + 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100, + 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000, + 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff, + 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff, + 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000, + 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101, + 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000, + 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101, + 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101, + 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100, + 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01, + 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001, + 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001, + 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff, + 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff, + 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000, + 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000, + 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101, + 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff, + 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001, + 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000, + 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff, + 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00, + 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff, + 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000, + 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00, + 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff, + 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000, + 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000, + 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff, + 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000, + 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000, + 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000, + 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101, + 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff, + 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100, + 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101, + 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001, + 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff, + 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100, + 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff, + 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01, + 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff, + 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff, + 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff, + 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff, + 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001, + 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff, + 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01, + 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00, + 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100, + 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101, + 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff, + 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00, + 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01, + 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00, + 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100, + 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00, + 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000, + 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000, + 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000, + 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000, + 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001, + 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff, + 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00, + 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff, + 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00, + 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000, + 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff, + 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff, + 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101, + 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000, + 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff, + 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff, + 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff, + 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff, + 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000, + 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000, + 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100, + 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00, + 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100, + 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100, + 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00, + 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff, + 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff, + 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100, + 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff, + 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000, + 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00, + 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00, + 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100, + 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff, + 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff, + 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001, + 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00, + 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101, + 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001, + 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000, + 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100, + 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000, + 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001, + 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000, + 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff, + 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101, + 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101, + 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000, + 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff, + 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000, + 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff, + 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01, + 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100, + 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101, + 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001, + 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01, + 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff, + 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00, + 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff, + 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff, + 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff, + 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, + 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff, + 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff, + 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001, + 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff, + 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff, + 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000, + 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00, + 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100, + 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01, + 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff, + 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff, + 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000, + 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101, + 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01, + 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100, + 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100, + 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00, + 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101, + 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001, + 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01, + 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100, + 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff, + 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01, + 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff, + 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000, + 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001, + 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01, + 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff, + 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff, + 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00, + 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff, + 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100, + 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000, + 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001, + 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100, + 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff, + 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff, + 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01, + 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101, + 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001, + 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff, + 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101, + 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100, + 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00, + 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01, + 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001, + 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00, + 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000, + 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101, + 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001, + 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100, + 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000, + 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000, + 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00, + 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101, + 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff, + 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101, + 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001, + 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01, + 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100, + 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000, + 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00, + 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff, + 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001, + 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001, + 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff, + 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00, + 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100, + 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00, + 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101, + 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff, + 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff, + 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff, + 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00, + 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff, + 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff, + 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101, + 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100, + 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101, + 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00, + 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00, + 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff, + 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff, + 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001, + 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000, + 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff, + 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff, + 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff, + 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff, + 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001, + 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100, + 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101, + 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001, + 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001, + 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101, + 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101, + 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff, + 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff, + 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000, + 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101, + 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001, + 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff, + 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000, + 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff, + 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100, + 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000, + 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101, + 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff, + 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100, + 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff, + 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01, + 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100, + 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000, + 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100, + 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100, + 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001, + 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000, + 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000, + 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000, + 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00, + 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff, + 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001, + 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000, + 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101, + 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101, + 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000, + 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff, + 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000, + 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101, + 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff, + 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff, + 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000, + 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101, + 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff, + 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff, + 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01, + 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff, + 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000, + 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101, + 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff, + 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff, + 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff, + 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01, + 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00, + 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000, + 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000, + 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101, + 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001, + 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff, + 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000, + 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff, + 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000, + 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000, + 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000, + 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000, + 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001, + 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff, + 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001, + 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000, + 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000, + 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00, + 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000, + 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101, + 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001, + 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00, + 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001, + 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff, + 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000, + 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00, + 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100, + 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101, + 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001, + 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01, + 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff, + 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff, + 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00, + 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01, + 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100, + 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000, + 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff, + 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff, + 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff, + 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000, + 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff, + 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101, + 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff, + 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001, + 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001, + 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001, + 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000, + 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000, + 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff, + 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101, + 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001, + 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff, + 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000, + 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000, + 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff, + 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101, + 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000, + 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00, + 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000, + 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff, + 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01, + 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00, + 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff, + 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000, + 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101, + 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff, + 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001, + 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff, + 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000, + 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100, + 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff, + 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101, + 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100, + 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff, + 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01, + 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000, + 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff, + 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00, + 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100, + 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff, + 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001, + 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00, + 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100, + 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff, + 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01, + 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00, + 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000, + 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01, + 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff, + 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000, + 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff, + 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff, + 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff, + 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001, + 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff, + 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01, + 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff, + 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00, + 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00, + 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001, + 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101, + 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101, + 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff, + 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000, + 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101, +}; diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index fd539d585e64..d5a624c4359a 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -36,6 +36,24 @@ void gemm_q5_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, void gemm_q5_1_q8_1_ppc(int64_t m, int64_t n, int64_t k, const void * A, int64_t lda, const void * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth); +void gemm_tq2_0_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_tq1_0_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq2_xxs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq3_xxs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq3_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq1_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 6c8fe3bfe75d..57cb23a75e8e 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -4060,6 +4060,78 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_TQ2_0: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_tq2_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_TQ1_0: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_tq1_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ2_XXS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq2_xxs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ3_XXS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq3_xxs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ3_S: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq3_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ1_S: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq1_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + case GGML_TYPE_Q1_0: { if (Btype != GGML_TYPE_Q8_0) return false; From d5cf352402c7116282d6dbd3fd64797478e84779 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 06/21] power-mma: abort loudly on pack-buffer allocation failure Replaces silent skips in the one-shot drivers with GGML_ABORT. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 19 +++++++++------ .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 13 +++++++---- .../src/ggml-cpu/llamafile/legacy_ppc_mma.cpp | 21 ++++++++++------- ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp | 15 ++++++++---- ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp | 17 +++++++++----- ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 19 +++++++++------ ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp | 23 +++++++++++-------- ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp | 17 +++++++++----- ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp | 16 +++++++++---- 9 files changed, 103 insertions(+), 57 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index 2b56b8d259f5..3bc298ece69e 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -35,6 +35,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_ELEMS 2048 // K slab #define KC_CH (KC_ELEMS / 32) // 32-element chunks per slab #define MR 8 @@ -68,7 +73,7 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { // come from qs[j] low / high nibble. One vec_perm each. static inline void iq4_lookup32(const uint8_t * qs, vsc v[2]) { vsc tbl; memcpy(&tbl, iq4_kvalues_local, 16); - vuc raw = vec_xl(0, (const unsigned char *)qs); + vuc raw = load16u((const uint8_t *)(qs) + (0)); vuc lo = vec_and(raw, vec_splats((unsigned char)0xF)); vuc hi = vec_sr(raw, vec_splats((unsigned char)4)); v[0] = vec_perm(tbl, tbl, lo); @@ -105,10 +110,10 @@ static inline int64_t ct8(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl32(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } extern "C" size_t iq4_apack_size(int64_t m, int64_t k) { - return (size_t)(rt8(m) * sl32(k)) * sizeof(aiq4_t); + return (((size_t)(rt8(m) * sl32(k)) * sizeof(aiq4_t)) + 63) & ~(size_t)63; } extern "C" size_t iq4_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct8(n) * sl32(k)) * sizeof(biq4_t); + return (((size_t)(ct8(n) * sl32(k)) * sizeof(biq4_t)) + 63) & ~(size_t)63; } // ---- weight repack: IQ4_NL ---- @@ -190,7 +195,7 @@ extern "C" void iq4_pack_b_q8_0(const block_q8_0 * B, int64_t ldb, for (int h = 0; h < 2; h++) { for (int j = 0; j < 4; j++) rows4[j] = (vui)vec_xor( - vec_xl(16*h, (const unsigned char *)yb[4*a + j]->qs), flip); + load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*h)), flip); mma_transpose4(rows4, &T->v[b][8*h + a], 2); } } @@ -223,7 +228,7 @@ extern "C" void iq4_pack_b_q8_K(const block_q8_K * B, int64_t ldb, for (int h = 0; h < 2; h++) { for (int j = 0; j < 4; j++) rows4[j] = (vui)vec_xor( - vec_xl(32*ib + 16*h, (const unsigned char *)yb[4*a + j]->qs), flip); + load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*ib + 16*h)), flip); mma_transpose4(rows4, &T->v[ch][8*h + a], 2); } } @@ -313,7 +318,7 @@ extern "C" void gemm_iq4_nl_q8_0_ppc(int64_t m, int64_t n, int64_t k, const block_q8_0 * B = (const block_q8_0 *)Bv; void * PA = aligned_alloc(64, iq4_apack_size(MR, k)); void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } iq4_pack_b_q8_0(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; @@ -333,7 +338,7 @@ extern "C" void gemm_iq4_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, const block_q8_K * B = (const block_q8_K *)Bv; void * PA = aligned_alloc(64, iq4_apack_size(MR, k)); void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } iq4_pack_b_q8_K(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 851b44f82400..3632348cc85d 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -181,6 +181,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_ELEMS 2048 #define KC_CH (KC_ELEMS / 32) #define MR 8 @@ -236,10 +241,10 @@ static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } extern "C" size_t grid_apack_size(int64_t m, int64_t k) { - return (size_t)(rt(m) * sl(k)) * sizeof(agrid_t); + return (((size_t)(rt(m) * sl(k)) * sizeof(agrid_t)) + 63) & ~(size_t)63; } extern "C" size_t grid_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct(n) * sl(k)) * sizeof(bgrid_t); + return (((size_t)(ct(n) * sl(k)) * sizeof(bgrid_t)) + 63) & ~(size_t)63; } // generic repack over any superblock decoder @@ -309,7 +314,7 @@ extern "C" void grid_pack_b_q8_K(const block_q8_K_ppc * B, int64_t ldb, for (int h = 0; h < 2; h++) { for (int j = 0; j < 4; j++) rows4[j] = (vui)vec_xor( - vec_xl(32*ib + 16*h, (const unsigned char *)yb[4*a + j]->qs), flip); + load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*ib + 16*h)), flip); mma_transpose4(rows4, &T->v[ch][8*h + a], 2); } } @@ -392,7 +397,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const block_q8_K_ppc * B = (const block_q8_K_ppc *)Bv; \ void * PA = aligned_alloc(64, grid_apack_size(MR, k)); \ void * PB = aligned_alloc(64, grid_bpack_size(n, k)); \ - if (!PA || !PB) { free(PA); free(PB); return; } \ + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } \ grid_pack_b_q8_K(B, ldb, n, k, PB); \ const int64_t mt = (m + MR - 1) / MR; \ const int64_t tpt = (mt + nth - 1) / nth; \ diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp index 0eb1064af8b9..bde49f21df34 100644 --- a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -35,6 +35,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_CH 64 // 32-elem chunks per slab (2048) #define MR 16 #define NR 8 @@ -63,7 +68,7 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { } static inline void nibbles32(const uint8_t * qs, vuc v[2]) { - vuc raw = vec_xl(0, (const unsigned char *)qs); + vuc raw = load16u((const uint8_t *)(qs) + (0)); v[0] = vec_and(raw, vec_splats((unsigned char)0xF)); v[1] = vec_sr(raw, vec_splats((unsigned char)4)); } @@ -71,7 +76,7 @@ static inline void nibbles32(const uint8_t * qs, vuc v[2]) { // merge qh bits: elems 0..15 use bits 0..15 (bytes 0-1), elems 16..31 // use bits 16..31 (bytes 2-3); bit -> position 4. static inline void qh_merge(const uint8_t * qh, vuc v[2]) { - vuc raw = vec_xl(0, (const unsigned char *)qh); // first 4 bytes used + vuc raw = load16u((const uint8_t *)(qh) + (0)); // first 4 bytes used const vuc repL = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; const vuc repH = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; const vuc sh = { 0,1,2,3,4,5,6,7, 0,1,2,3,4,5,6,7 }; @@ -88,10 +93,10 @@ static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } extern "C" size_t leg_apack_size(int64_t m, int64_t k) { - return (size_t)(rt(m) * sl(k)) * sizeof(aleg_t); + return (((size_t)(rt(m) * sl(k)) * sizeof(aleg_t)) + 63) & ~(size_t)63; } extern "C" size_t leg_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct(n) * sl(k)) * sizeof(bleg_t); + return (((size_t)(ct(n) * sl(k)) * sizeof(bleg_t)) + 63) & ~(size_t)63; } // generic weight repack over a per-block "codes + d + m" extractor @@ -160,8 +165,8 @@ static void pack_b_generic(const YBLK * B, int64_t ldb, int64_t n, int64_t k, bl SB[j] = GGML_FP16_TO_FP32(((const ggml_half *)yb[j])[1]); } else { vsi z = vec_splats(0); - vsi sm = vec_sum4s((vsc)vec_xl(0, (const unsigned char *)yb[j]->qs), z); - sm = vec_sum4s((vsc)vec_xl(16, (const unsigned char *)yb[j]->qs), sm); + vsi sm = vec_sum4s((vsc)load16u((const uint8_t *)(yb[j]->qs) + (0)), z); + sm = vec_sum4s((vsc)load16u((const uint8_t *)(yb[j]->qs) + (16)), sm); SB[j] = (float)SFACT * dB[j] * (float)(sm[0]+sm[1]+sm[2]+sm[3]); } } @@ -173,7 +178,7 @@ static void pack_b_generic(const YBLK * B, int64_t ldb, int64_t n, int64_t k, bl for (int a = 0; a < 2; a++) for (int h = 0; h < 2; h++) { for (int j = 0; j < 4; j++) - rows4[j] = (vui)vec_xl(16*h, (const unsigned char *)yb[4*a + j]->qs); + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*h)); mma_transpose4(rows4, &T->v[b][8*h + a], 2); } } @@ -295,7 +300,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const YBLK * B = (const YBLK *)Bv; \ void * PA = aligned_alloc(64, leg_apack_size(MR, k)); \ void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ - if (!PA || !PB) { free(PA); free(PB); return; } \ + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } \ PACKB(B, ldb, n, k, PB); \ const int64_t mt = (m + MR - 1) / MR; \ const int64_t tpt = (mt + nth - 1) / nth; \ diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp index 8ada5d25f25c..2bf8a5d6de42 100644 --- a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -33,6 +33,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_SB 8 // superblocks per slab (2048 elems) #define KC_CH16 (KC_SB * 16) // 16-element chunks per slab #define MR 16 @@ -64,7 +69,7 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { // 16 unsigned 2-bit codes of chunk c (0..15) of one superblock. static inline vuc q2k_codes16(const uint8_t * qs, int c) { const int h = c >> 3, idx = c & 7, j = idx >> 1, lo = idx & 1; - vuc v = vec_xl(32*h + 16*lo, (const unsigned char *)qs); + vuc v = load16u((const uint8_t *)(qs) + (32*h + 16*lo)); return vec_and(vec_sr(v, vec_splats((unsigned char)(2*j))), vec_splats((unsigned char)3)); } @@ -74,10 +79,10 @@ static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } extern "C" size_t q2k_apack_size(int64_t m, int64_t k) { - return (size_t)(rt(m) * sl(k)) * sizeof(a2k_t); + return (((size_t)(rt(m) * sl(k)) * sizeof(a2k_t)) + 63) & ~(size_t)63; } extern "C" size_t q2k_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct(n) * sl(k)) * sizeof(b2k_t); + return (((size_t)(ct(n) * sl(k)) * sizeof(b2k_t)) + 63) & ~(size_t)63; } extern "C" void q2k_repack_a(const block_q2_K * A, int64_t lda, @@ -148,7 +153,7 @@ extern "C" void q2k_pack_b(const block_q8_K * B, int64_t ldb, vui rows4[4]; for (int a = 0; a < 2; a++) { for (int j = 0; j < 4; j++) - rows4[j] = (vui)vec_xl(16*c, (const unsigned char *)yb[4*a + j]->qs); + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*c)); mma_transpose4(rows4, &T->v[ch][a], 2); } } @@ -263,7 +268,7 @@ extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, const block_q8_K * B = (const block_q8_K *)Bv; void * PA = aligned_alloc(64, q2k_apack_size(MR, k)); void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } q2k_pack_b(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp index 0cb87bb2b468..f840e06d0c71 100644 --- a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -29,6 +29,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_SB 8 // superblocks per slab (2048 elems) #define KC_CH16 (KC_SB * 16) // 16-element chunks per slab #define MR 16 @@ -59,10 +64,10 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { // 16 unsigned 3-bit codes q' = code | (hbit << 2) of chunk c (0..15). static inline vuc q3k_codes16(const uint8_t * qs, const uint8_t * hmask, int c) { const int h = c >> 3, idx = c & 7, j = idx >> 1, lo = idx & 1; - vuc code = vec_and(vec_sr(vec_xl(32*h + 16*lo, (const unsigned char *)qs), + vuc code = vec_and(vec_sr(load16u((const uint8_t *)(qs) + (32*h + 16*lo)), vec_splats((unsigned char)(2*j))), vec_splats((unsigned char)3)); - vuc hb = vec_sl(vec_and(vec_sr(vec_xl(16*lo, (const unsigned char *)hmask), + vuc hb = vec_sl(vec_and(vec_sr(load16u((const uint8_t *)(hmask) + (16*lo)), vec_splats((unsigned char)(4*h + j))), vec_splats((unsigned char)1)), vec_splats((unsigned char)2)); @@ -74,10 +79,10 @@ static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } extern "C" size_t q3k_apack_size(int64_t m, int64_t k) { - return (size_t)(rt(m) * sl(k)) * sizeof(a3k_t); + return (((size_t)(rt(m) * sl(k)) * sizeof(a3k_t)) + 63) & ~(size_t)63; } extern "C" size_t q3k_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct(n) * sl(k)) * sizeof(b3k_t); + return (((size_t)(ct(n) * sl(k)) * sizeof(b3k_t)) + 63) & ~(size_t)63; } extern "C" void q3k_repack_a(const block_q3_K * A, int64_t lda, @@ -151,7 +156,7 @@ extern "C" void q3k_pack_b(const block_q8_K * B, int64_t ldb, vui rows4[4]; for (int a = 0; a < 2; a++) { for (int j = 0; j < 4; j++) - rows4[j] = (vui)vec_xl(16*c, (const unsigned char *)yb[4*a + j]->qs); + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*c)); mma_transpose4(rows4, &T->v[ch][a], 2); } } @@ -259,7 +264,7 @@ extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, const block_q8_K * B = (const block_q8_K *)Bv; void * PA = aligned_alloc(64, q3k_apack_size(MR, k)); void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } q3k_pack_b(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp index f8bedcf9b2a6..d0cf08e8d7a7 100644 --- a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -44,6 +44,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_SB 8 // superblocks per K slab (2048 elems) #define KC_CHUNKS (KC_SB * 8) // 32-element chunks per slab #define MR 16 @@ -75,8 +80,8 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { // nibble codes of sub-block `sub` (0..7) of one superblock static inline void q4k_codes32(const uint8_t * qs, int sub, vuc v[2]) { const int g = sub >> 1; - vuc lo = vec_xl(32*g, (const unsigned char *)qs); - vuc hi = vec_xl(32*g + 16, (const unsigned char *)qs); + vuc lo = load16u((const uint8_t *)(qs) + (32*g)); + vuc hi = load16u((const uint8_t *)(qs) + (32*g + 16)); if ((sub & 1) == 0) { const vuc mF = vec_splats((unsigned char)0xF); v[0] = vec_and(lo, mF); @@ -94,10 +99,10 @@ static inline int64_t ct4k(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl4k(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } extern "C" size_t q4k_apack_size(int64_t m, int64_t k) { - return (size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t); + return (((size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t)) + 63) & ~(size_t)63; } extern "C" size_t q4k_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t); + return (((size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t)) + 63) & ~(size_t)63; } extern "C" void q4k_repack_a(const block_q4_K * A, int64_t lda, @@ -175,8 +180,8 @@ extern "C" void q4k_pack_b(const block_q8_K * B, int64_t ldb, for (int a = 0; a < 2; a++) { vuc q[4][2]; for (int j = 0; j < 4; j++) { - q[j][0] = vec_xl(32*sub, (const unsigned char *)yb[4*a + j]->qs); - q[j][1] = vec_xl(32*sub + 16, (const unsigned char *)yb[4*a + j]->qs); + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub + 16)); } for (int h = 0; h < 2; h++) { for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; @@ -296,7 +301,7 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, const block_q8_K * B = (const block_q8_K *)Bv; void * PA = aligned_alloc(64, q4k_apack_size(MR, k)); void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } q4k_pack_b(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp index 167ff6a0ecae..bcbb04c92c8a 100644 --- a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -44,6 +44,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_SB 8 // superblocks per K slab (2048 elems) #define KC_CHUNKS (KC_SB * 8) // 32-element chunks per slab #define MR 16 @@ -76,8 +81,8 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { static inline void q5k_codes32(const uint8_t * qs, const uint8_t * qh, int sub, vuc v[2]) { const int g = sub >> 1; - vuc lo = vec_xl(32*g, (const unsigned char *)qs); - vuc hi = vec_xl(32*g + 16, (const unsigned char *)qs); + vuc lo = load16u((const uint8_t *)(qs) + (32*g)); + vuc hi = load16u((const uint8_t *)(qs) + (32*g + 16)); if ((sub & 1) == 0) { const vuc mF = vec_splats((unsigned char)0xF); lo = vec_and(lo, mF); @@ -89,8 +94,8 @@ static inline void q5k_codes32(const uint8_t * qs, const uint8_t * qh, const vuc hsh = vec_splats((unsigned char)sub); const vuc one = vec_splats((unsigned char)1); const vuc four = vec_splats((unsigned char)4); - vuc h0 = vec_sl(vec_and(vec_sr(vec_xl(0, (const unsigned char *)qh), hsh), one), four); - vuc h1 = vec_sl(vec_and(vec_sr(vec_xl(16, (const unsigned char *)qh), hsh), one), four); + vuc h0 = vec_sl(vec_and(vec_sr(load16u((const uint8_t *)(qh) + (0)), hsh), one), four); + vuc h1 = vec_sl(vec_and(vec_sr(load16u((const uint8_t *)(qh) + (16)), hsh), one), four); v[0] = vec_or(lo, h0); v[1] = vec_or(hi, h1); } @@ -102,10 +107,10 @@ static inline int64_t ct4k(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl4k(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } extern "C" size_t q5k_apack_size(int64_t m, int64_t k) { - return (size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t); + return (((size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t)) + 63) & ~(size_t)63; } extern "C" size_t q5k_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t); + return (((size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t)) + 63) & ~(size_t)63; } extern "C" void q5k_repack_a(const block_q5_K * A, int64_t lda, @@ -183,8 +188,8 @@ extern "C" void q5k_pack_b(const block_q8_K * B, int64_t ldb, for (int a = 0; a < 2; a++) { vuc q[4][2]; for (int j = 0; j < 4; j++) { - q[j][0] = vec_xl(32*sub, (const unsigned char *)yb[4*a + j]->qs); - q[j][1] = vec_xl(32*sub + 16, (const unsigned char *)yb[4*a + j]->qs); + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub + 16)); } for (int h = 0; h < 2; h++) { for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; @@ -304,7 +309,7 @@ extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, const block_q8_K * B = (const block_q8_K *)Bv; void * PA = aligned_alloc(64, q5k_apack_size(MR, k)); void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } q5k_pack_b(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp index c05e01f3815b..a68b53a36f70 100644 --- a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -32,6 +32,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_SB 8 // superblocks per slab (2048 elems) #define KC_CH16 (KC_SB * 16) // 16-element chunks per slab #define MR 16 @@ -62,10 +67,10 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { // 16 unsigned 6-bit codes of chunk c (0..15) of one superblock. static inline vuc q6k_codes16(const uint8_t * ql, const uint8_t * qh, int c) { const int h = c >> 3, idx = c & 7, qq = idx >> 1, lo = idx & 1; - vuc nib = vec_xl(64*h + 32*(qq & 1) + 16*lo, (const unsigned char *)ql); + vuc nib = load16u((const uint8_t *)(ql) + (64*h + 32*(qq & 1) + 16*lo)); nib = (qq < 2) ? vec_and(nib, vec_splats((unsigned char)0xF)) : vec_sr (nib, vec_splats((unsigned char)4)); - vuc hb = vec_xl(32*h + 16*lo, (const unsigned char *)qh); + vuc hb = load16u((const uint8_t *)(qh) + (32*h + 16*lo)); hb = vec_sl(vec_and(vec_sr(hb, vec_splats((unsigned char)(2*qq))), vec_splats((unsigned char)3)), vec_splats((unsigned char)4)); @@ -77,10 +82,10 @@ static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } extern "C" size_t q6k_apack_size(int64_t m, int64_t k) { - return (size_t)(rt(m) * sl(k)) * sizeof(a6k_t); + return (((size_t)(rt(m) * sl(k)) * sizeof(a6k_t)) + 63) & ~(size_t)63; } extern "C" size_t q6k_bpack_size(int64_t n, int64_t k) { - return (size_t)(ct(n) * sl(k)) * sizeof(b6k_t); + return (((size_t)(ct(n) * sl(k)) * sizeof(b6k_t)) + 63) & ~(size_t)63; } extern "C" void q6k_repack_a(const block_q6_K * A, int64_t lda, @@ -144,7 +149,7 @@ extern "C" void q6k_pack_b(const block_q8_K * B, int64_t ldb, vui rows4[4]; for (int a = 0; a < 2; a++) { for (int j = 0; j < 4; j++) - rows4[j] = (vui)vec_xl(16*c, (const unsigned char *)yb[4*a + j]->qs); + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*c)); mma_transpose4(rows4, &T->v[ch][a], 2); } } @@ -254,7 +259,7 @@ extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, const block_q8_K * B = (const block_q8_K *)Bv; void * PA = aligned_alloc(64, q6k_apack_size(MR, k)); void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); - if (!PA || !PB) { free(PA); free(PB); return; } + if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } q6k_pack_b(B, ldb, n, k, PB); const int64_t mt = (m + MR - 1) / MR; const int64_t tpt = (mt + nth - 1) / nth; diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp index f9562625c51f..43e11db8d1e1 100644 --- a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp @@ -22,6 +22,11 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + #define KC_BLKS 16 // 2048-element K slab #define KC_CHUNKS (KC_BLKS * 4) #define MR 16 // weight rows per tile @@ -87,7 +92,7 @@ static void pack_A16_q1(const block_q1_0 * A, int64_t lda, int64_t rows, int64_t rr = r < rows ? r : rows - 1; const block_q1_0 * bp = &A[rr*lda + blk0 + b]; d[r] = GGML_FP16_TO_FP32(bp->d); - raw[r] = vec_xl(0, (const unsigned char *)bp->qs); + raw[r] = load16u((const uint8_t *)(bp->qs) + (0)); } for (int g = 0; g < 4; g++) P->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; @@ -113,8 +118,8 @@ static void pack_A16_q2(const block_q2_0 * A, int64_t lda, int64_t rows, int64_t rr = r < rows ? r : rows - 1; const block_q2_0 * bp = &A[rr*lda + blk0 + b]; d[r] = GGML_FP16_TO_FP32(bp->d); - lo[r] = vec_xl(0, (const unsigned char *)bp->qs); - hi[r] = vec_xl(16, (const unsigned char *)bp->qs); + lo[r] = load16u((const uint8_t *)(bp->qs) + (0)); + hi[r] = load16u((const uint8_t *)(bp->qs) + (16)); } for (int g = 0; g < 4; g++) P->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; @@ -151,8 +156,8 @@ static void pack_B8(const block_q8_0 * B, int64_t ldb, int64_t cols, for (int a = 0; a < 2; a++) { vuc q[4][2]; for (int j = 0; j < 4; j++) { - q[j][0] = vec_xl(0, (const unsigned char *)yb[4*a + j]->qs); - q[j][1] = vec_xl(16, (const unsigned char *)yb[4*a + j]->qs); + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (0)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (16)); vsi z = vec_splats(0); vsi s = vec_sum4s((vsc)q[j][0], z); s = vec_sum4s((vsc)q[j][1], s); @@ -260,6 +265,7 @@ static void gemm_qbit(int64_t m, int64_t n, int64_t k, apack_t * PA = (apack_t*)aligned_alloc(64, sizeof(apack_t)); bpack_t * PB = (bpack_t*)aligned_alloc(64, njt * sizeof(bpack_t)); vfl (*fin)[NR][4] = (vfl(*)[NR][4])aligned_alloc(64, njt * sizeof(vfl[NR][4])); + if (!PA || !PB || !fin) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } for (int64_t i = i0; i < i1; i += MR) { const int64_t rows = (i1 - i) < MR ? (i1 - i) : MR; From 8116a708dd991fca526d43e0a02ddec85b1674dd Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 07/21] power-mma: tensor-keyed pack cache + MXFP4/NVFP4 coverage First llamafile_sgemm call for a tensor packs the whole matrix once; subsequent calls reuse it for the model lifetime. Capacity-bounded (PPC_MMA_PACK_CACHE_MB). --- ggml/src/ggml-cpu/CMakeLists.txt | 1 + ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 117 +++-- .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 438 ++++++++++++++++- ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h | 393 +++++++++++++++ ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 20 + .../src/ggml-cpu/llamafile/legacy_ppc_mma.cpp | 39 +- .../src/ggml-cpu/llamafile/ppc_pack_cache.cpp | 111 +++++ ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp | 30 +- ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp | 30 +- ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 30 +- ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp | 30 +- ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp | 30 +- ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp | 456 +++++++++++------- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 60 +++ 14 files changed, 1486 insertions(+), 299 deletions(-) create mode 100644 ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index c86341d80347..ed98b262cc08 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -91,6 +91,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/llamafile/iq4_ppc_mma.cpp ggml-cpu/llamafile/legacy_ppc_mma.cpp ggml-cpu/llamafile/iq_grid_ppc_mma.cpp + ggml-cpu/llamafile/ppc_pack_cache.cpp ggml-cpu/llamafile/iq_grids_ppc.h ggml-cpu/llamafile/kquants_ppc_mma.h ggml-cpu/llamafile/qbit_ppc_mma.h diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index 3bc298ece69e..edd7eb4063a9 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -1,5 +1,6 @@ // iq4_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels // (standalone-verified vs exact double references under qemu -cpu power10). +// IQ4_NL x Q8_0, IQ4_XS x Q8_K, MXFP4 x Q8_0 on POWER10/POWER11 MMA. #include "ggml-impl.h" #include "ggml-cpu-impl.h" @@ -19,12 +20,21 @@ + static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "bad iq4_nl"); +static_assert(sizeof(block_mxfp4) == 1 + QK_MXFP4/2, "bad mxfp4"); static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + 2 + QK_K/64 + QK_K/2, "bad iq4_xs"); static const int8_t iq4_kvalues_local[16] = { -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, }; +// MXFP4 e2m1 values, pre-doubled to integers (the E8M0 "half" scale +// supplies the /2): {0,1,2,3,4,6,8,12} with sign. +static const int8_t kvalues_mxfp4_[16] = { + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +}; +#include +static inline float e8m0_half_to_fp32(uint8_t e) { return ldexpf(1.0f, (int)e - 128); } #if defined(__MMA__) && defined(__powerpc64__) @@ -35,8 +45,9 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; -// Unaligned 16-byte load via memcpy: single lxv, well-defined for the -// odd struct offsets several block formats use. +// Unaligned 16-byte load via memcpy: compiles to a single lxv (which is +// alignment-agnostic on POWER) while staying well-defined C++ for any +// source alignment -- several block structs place qs at odd offsets. static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } @@ -71,8 +82,8 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { // 32 signed codebook values from 16 packed bytes; elements j / j+16 // come from qs[j] low / high nibble. One vec_perm each. -static inline void iq4_lookup32(const uint8_t * qs, vsc v[2]) { - vsc tbl; memcpy(&tbl, iq4_kvalues_local, 16); +static inline void iq4_lookup32t(const uint8_t * qs, const int8_t * table, vsc v[2]) { + vsc tbl; memcpy(&tbl, table, 16); vuc raw = load16u((const uint8_t *)(qs) + (0)); vuc lo = vec_and(raw, vec_splats((unsigned char)0xF)); vuc hi = vec_sr(raw, vec_splats((unsigned char)4)); @@ -132,7 +143,30 @@ extern "C" void iq4nl_repack_a(const block_iq4_nl * A, int64_t lda, int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; const block_iq4_nl * bp = &A[rr*lda + b0 + b]; sc[r] = GGML_FP16_TO_FP32(bp->d); - iq4_lookup32(bp->qs, w[r]); + iq4_lookup32t(bp->qs, iq4_kvalues_local, w[r]); + } + iq4_place_chunk(T, b, w, sc); + } + } +} + +// ---- weight repack: MXFP4 (same shape as IQ4_NL, different table+scale) ---- +extern "C" void mxfp4_repack_a(const block_mxfp4 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const block_mxfp4 * bp = &A[rr*lda + b0 + b]; + sc[r] = e8m0_half_to_fp32(bp->e); + iq4_lookup32t(bp->qs, kvalues_mxfp4_, w[r]); } iq4_place_chunk(T, b, w, sc); } @@ -162,7 +196,7 @@ extern "C" void iq4xs_repack_a(const block_iq4_xs * A, int64_t lda, const int ls = ((bp[r]->scales_l[ib/2] >> 4*(ib%2)) & 0xF) | (((bp[r]->scales_h >> 2*ib) & 3) << 4); sc[r] = d[r] * (float)(ls - 32); - iq4_lookup32(bp[r]->qs + 16*ib, w[r]); + iq4_lookup32t(bp[r]->qs + 16*ib, iq4_kvalues_local, w[r]); } iq4_place_chunk(T, 8*sb + ib, w, sc); } @@ -311,45 +345,40 @@ extern "C" void iq4_gemm_packed(int64_t m, int64_t n, int64_t k, } -extern "C" void gemm_iq4_nl_q8_0_ppc(int64_t m, int64_t n, int64_t k, - const void * Av, int64_t lda, const void * Bv, int64_t ldb, - float * C, int64_t ldc, int ith, int nth) { - const block_iq4_nl * A = (const block_iq4_nl *)Av; - const block_q8_0 * B = (const block_q8_0 *)Bv; - void * PA = aligned_alloc(64, iq4_apack_size(MR, k)); - void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } - iq4_pack_b_q8_0(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - iq4nl_repack_a(A + i*lda, lda, rows, k, PA); - iq4_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); - } - free(PA); free(PB); -} - -extern "C" void gemm_iq4_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, - const void * Av, int64_t lda, const void * Bv, int64_t ldb, - float * C, int64_t ldc, int ith, int nth) { - const block_iq4_xs * A = (const block_iq4_xs *)Av; - const block_q8_K * B = (const block_q8_K *)Bv; - void * PA = aligned_alloc(64, iq4_apack_size(MR, k)); - void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } - iq4_pack_b_q8_K(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - iq4xs_repack_a(A + i*lda, lda, rows, k, PA); - iq4_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); - } - free(PA); free(PB); +#define IQ4_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, VARIANT) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const YBLK * B = (const YBLK *)Bv; \ + void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ + iq4_apack_size(m, k), &fresh); \ + if (PA) { \ + if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ + ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + iq4_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); \ + } else { \ + void * PT = aligned_alloc(64, iq4_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + iq4_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ + } \ + free(PB); \ } +IQ4_ONESHOT(gemm_iq4_nl_q8_0_ppc, block_iq4_nl, iq4nl_repack_a, block_q8_0, iq4_pack_b_q8_0, 1) +IQ4_ONESHOT(gemm_iq4_xs_q8_K_ppc, block_iq4_xs, iq4xs_repack_a, block_q8_K, iq4_pack_b_q8_K, 2) +IQ4_ONESHOT(gemm_mxfp4_q8_0_ppc, block_mxfp4, mxfp4_repack_a, block_q8_0, iq4_pack_b_q8_0, 3) #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 3632348cc85d..57e52517adb3 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -1,8 +1,7 @@ // iq_grid_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels -// (standalone-verified under qemu -cpu power10). Grid-codebook and -// ternary formats on POWER10/POWER11 MMA via decode-at-repack + the -// shared signed 8x8 kernel. Uses a local copy of the grid tables -// (iq_grids_ppc.h) to avoid depending on GGML_COMMON_IMPL linkage. +// (standalone-verified under qemu -cpu power10). Grid-codebook, ternary, +// and NVFP4 formats via decode-at-repack + shared signed kernels (32- +// and 16-deep chunk variants). Local grid tables in iq_grids_ppc.h. #include "ggml-impl.h" #include "ggml-cpu-impl.h" @@ -27,7 +26,15 @@ typedef struct { ppc_half d; uint8_t qs[3*QK_K/8]; } block_iq3_xxs_ppc; typedef struct { ppc_half d; uint8_t qs[QK_K/4]; uint8_t qh[QK_K/32]; uint8_t signs[QK_K/8]; uint8_t scales[QK_K/64]; } block_iq3_s_ppc; typedef struct { ppc_half d; uint8_t qs[QK_K/8]; uint16_t qh[QK_K/32]; } block_iq1_s_ppc; +typedef struct { ppc_half d; uint16_t qs[QK_K/8]; uint8_t scales[QK_K/32]; } block_iq2_xs_ppc; +typedef struct { ppc_half d; uint8_t qs[QK_K/4]; uint8_t qh[QK_K/32]; uint8_t scales[QK_K/32]; } block_iq2_s_ppc; +typedef struct { uint8_t qs[QK_K/8]; uint8_t qh[QK_K/16]; uint8_t scales[QK_K/32]; } block_iq1_m_ppc; typedef struct { float d; int8_t qs[QK_K]; int16_t bsums[QK_K/16]; } block_q8_K_ppc; +#define QK_NVFP4 64 +#define QK_NVFP4_SUB 16 +typedef struct { uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; uint8_t qs[QK_NVFP4/2]; } block_nvfp4_ppc; +#define QK8_0 32 +typedef struct { ppc_half d; int8_t qs[QK8_0]; } block_q8_0_ppc; static_assert(sizeof(block_tq1_0_ppc) == 2 + 4 + 48, "bad tq1_0"); static_assert(sizeof(block_tq2_0_ppc) == 2 + 64, "bad tq2_0"); @@ -35,6 +42,10 @@ static_assert(sizeof(block_iq2_xxs_ppc) == 2 + QK_K/4, "bad iq2_xxs"); static_assert(sizeof(block_iq3_xxs_ppc) == 2 + 3*QK_K/8, "bad iq3_xxs"); static_assert(sizeof(block_iq3_s_ppc) == 2 + 13*(QK_K/32) + QK_K/64, "bad iq3_s"); static_assert(sizeof(block_iq1_s_ppc) == 2 + QK_K/8 + QK_K/16, "bad iq1_s"); +static_assert(sizeof(block_iq2_xs_ppc) == 2 + QK_K/4 + QK_K/32, "bad iq2_xs"); +static_assert(sizeof(block_iq2_s_ppc) == 2 + QK_K/4 + QK_K/32 + QK_K/32, "bad iq2_s"); +static_assert(sizeof(block_iq1_m_ppc) == QK_K/8 + QK_K/16 + QK_K/32, "bad iq1_m"); +static_assert(sizeof(block_nvfp4_ppc) == 4 + QK_NVFP4/2, "bad nvfp4"); static inline float fp16_to_fp32(ppc_half h) { const uint32_t sign = (uint32_t)(h >> 15) << 31; @@ -173,6 +184,104 @@ static void dec_iq1_s(const block_iq1_s_ppc * b, int8_t code[QK_K], float sc[8]) } } + +// NVFP4 (64-element blocks, four 16-element sub-blocks with UE4M3 +// scales, MXFP4's pre-doubled e2m1 codebook -- the *0.5 inside the +// UE4M3 conversion compensates the doubling, matching ggml's dequant). +static const int8_t nvfp4_kvalues[16] = { + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +}; +#include +static inline float ue4m3_to_fp32(uint8_t x) { + if (x == 0 || x == 0x7F) return 0.0f; + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float raw = (exp == 0) ? ldexpf((float)man, -9) + : ldexpf(1.0f + (float)man / 8.0f, exp - 7); + return raw * 0.5f; +} + +// one 64-block -> 64 signed codes + 4 per-16 scales +static void dec_nvfp4(const block_nvfp4_ppc * b, int8_t code[QK_NVFP4], float sc[4]) { + for (int s = 0; s < 4; s++) { + sc[s] = ue4m3_to_fp32(b->d[s]); + for (int j = 0; j < 8; j++) { + code[16*s + j + 0] = nvfp4_kvalues[b->qs[8*s + j] & 0xF]; + code[16*s + j + 8] = nvfp4_kvalues[b->qs[8*s + j] >> 4]; + } + } +} + +// ---- per-16-scale decoders (codes + 16 scales per superblock) ---- + +static void dec16_iq2_xs(const block_iq2_xs_ppc * b, int8_t code[QK_K], float sc[16]) { + const float d = fp16_to_fp32(b->d); + for (int ib32 = 0; ib32 < 8; ++ib32) { + sc[2*ib32 + 0] = d * (0.5f + (b->scales[ib32] & 0xF)) * 0.25f; + sc[2*ib32 + 1] = d * (0.5f + (b->scales[ib32] >> 4)) * 0.25f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint16_t w = b->qs[4*ib32 + l]; + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (w & 511)); + const uint8_t signs = ksigns_iq2xs[w >> 9]; + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)((signs & kmask_iq2xs[j]) ? -(int)grid[j] : (int)grid[j]); + } + } +} + +static void dec16_iq2_s(const block_iq2_s_ppc * b, int8_t code[QK_K], float sc[16]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + const uint8_t * signs = b->qs + QK_K/8; + for (int ib32 = 0; ib32 < 8; ++ib32) { + sc[2*ib32 + 0] = d * (0.5f + (b->scales[ib32] & 0xF)) * 0.25f; + sc[2*ib32 + 1] = d * (0.5f + (b->scales[ib32] >> 4)) * 0.25f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2s_grid + + (qs[l] | ((b->qh[ib32] << (8 - 2*l)) & 0x300))); + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)((signs[l] & kmask_iq2xs[j]) ? -(int)grid[j] : (int)grid[j]); + } + qs += 4; signs += 4; + } +} + +// IQ1_M: fp16 superscale reassembled from scale-nibble high bits; +// per-16 3-bit scales, per-8 deltas. Exact integer form as IQ1_S: +// codes = 8*grid + sign(delta), scale = dl/8. +static void dec16_iq1_m(const block_iq1_m_ppc * b, int8_t code[QK_K], float sc[16]) { + const uint16_t * scw = (const uint16_t *)b->scales; + uint16_t du16 = (uint16_t)((scw[0] >> 12) | ((scw[1] >> 8) & 0x00f0) | + ((scw[2] >> 4) & 0x0f00) | (scw[3] & 0xf000)); + const float d = fp16_to_fp32(du16); + const uint8_t * qs = b->qs; + const uint8_t * qh = b->qh; + for (int ib = 0; ib < 8; ++ib) { + const float dl1 = d * (2*((scw[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1); + const float dl2 = d * (2*((scw[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1); + sc[2*ib + 0] = dl1 * 0.125f; + sc[2*ib + 1] = dl2 * 0.125f; + uint16_t idx[4]; int dsg[4]; + idx[0] = (uint16_t)(qs[0] | ((qh[0] << 8) & 0x700)); + idx[1] = (uint16_t)(qs[1] | ((qh[0] << 4) & 0x700)); + idx[2] = (uint16_t)(qs[2] | ((qh[1] << 8) & 0x700)); + idx[3] = (uint16_t)(qs[3] | ((qh[1] << 4) & 0x700)); + dsg[0] = (qh[0] & 0x08) ? -1 : +1; + dsg[1] = (qh[0] & 0x80) ? -1 : +1; + dsg[2] = (qh[1] & 0x08) ? -1 : +1; + dsg[3] = (qh[1] & 0x80) ? -1 : +1; + int8_t * y = code + 32*ib; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)(8*grid[j] + dsg[l]); + } + qs += 4; qh += 2; + } +} + #if defined(__MMA__) && defined(__powerpc64__) typedef vector unsigned char vuc; @@ -181,8 +290,9 @@ typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; -// Unaligned 16-byte load via memcpy: single lxv, well-defined for the -// odd struct offsets several block formats use. +// Unaligned 16-byte load via memcpy: compiles to a single lxv (which is +// alignment-agnostic on POWER) while staying well-defined C++ for any +// source alignment -- several block structs place qs at odd offsets. static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } @@ -389,32 +499,310 @@ extern "C" void grid_gemm_packed(int64_t m, int64_t n, int64_t k, } -#define GRID_ONESHOT(NAME, BLKA, REPACK) \ +// ---- 16-deep chunk variant for per-16-scale formats ---- + +#define KC_CH16 (KC_ELEMS / 16) + +typedef struct { + vuc v[KC_CH16][8]; // 4 depth-steps x 2 rowgroups + vfl sA [KC_CH16][2]; + vfl C128[KC_CH16][2]; +} agrid16_t; + +typedef struct { + vuc v[KC_CH16][8]; // 4 depth-steps x 2 colgroups + vfl dB[KC_CH16][2]; +} bgrid16_t; + +extern "C" size_t grid16_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(agrid16_t)) + 63) & ~(size_t)63; +} +extern "C" size_t grid16_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(bgrid16_t)) + 63) & ~(size_t)63; +} + +static void grid16_place_chunk(agrid16_t * T, int64_t ch, + const vsc w[MR], const float scale[MR]) { + for (int g = 0; g < 2; g++) { + float W[4]; + vui rows4[4]; + for (int r = 0; r < 4; r++) rows4[r] = (vui)w[4*g + r]; + mma_transpose4(rows4, &T->v[ch][g], 2); + for (int r = 0; r < 4; r++) { + vsi z = vec_splats(0); + vsi sm = vec_sum4s(w[4*g + r], z); + W[r] = (float)hsum(sm); + } + T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0]*scale[4*g], 128.0f*W[1]*scale[4*g+1], + 128.0f*W[2]*scale[4*g+2], 128.0f*W[3]*scale[4*g+3] }; + } +} + +template +static void repack_grid16(const BLK * A, int64_t lda, int64_t m, int64_t k, agrid16_t * P) { + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + agrid16_t * T = &P[it*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + int8_t code[MR][QK_K]; float sc[MR][16]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + DEC(&A[rr*lda + sb0 + sb], code[r], sc[r]); + } + for (int c = 0; c < 16; c++) { + vsc w[MR]; float scale[MR]; + for (int r = 0; r < MR; r++) { + memcpy(&w[r], code[r] + 16*c, 16); + scale[r] = sc[r][c]; + } + grid16_place_chunk(T, 16*sb + c, w, scale); + } + } + } +} + +extern "C" void grid16_repack_iq2_xs(const block_iq2_xs_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid16(A, lda, m, k, (agrid16_t *)p); } +extern "C" void grid16_repack_iq2_s(const block_iq2_s_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid16(A, lda, m, k, (agrid16_t *)p); } +extern "C" void grid16_repack_iq1_m(const block_iq1_m_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid16(A, lda, m, k, (agrid16_t *)p); } + +extern "C" void grid16_repack_nvfp4(const block_nvfp4_ppc * A, int64_t lda, + int64_t m, int64_t k, void * p) { + agrid16_t * P = (agrid16_t *)p; + const int64_t nb = k/QK_NVFP4, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + agrid16_t * T = &P[it*ns + s]; + const int64_t b0 = (s*KC_CH16)/4; // 64-blocks per slab start + const int64_t nbl = (nb - b0) < KC_CH16/4 ? (nb - b0) : KC_CH16/4; + for (int64_t b = 0; b < nbl; b++) { + int8_t code[MR][QK_NVFP4]; float sc[MR][4]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + dec_nvfp4(&A[rr*lda + b0 + b], code[r], sc[r]); + } + for (int c = 0; c < 4; c++) { + vsc w[MR]; float scale[MR]; + for (int r = 0; r < MR; r++) { + memcpy(&w[r], code[r] + 16*c, 16); + scale[r] = sc[r][c]; + } + grid16_place_chunk(T, 4*b + c, w, scale); + } + } + } +} + +// q8_0 activations for the 16-deep framework: each 32-block feeds two +// chunks; its dB is replicated to both. +extern "C" void grid16_pack_b_q8_0(const block_q8_0_ppc * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bgrid16_t * P = (bgrid16_t *)packed; + const int64_t kb = k/32, ns = sl(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bgrid16_t * T = &P[jt*ns + s]; + const int64_t b0 = (s*KC_CH16)/2; + const int64_t nbl = (kb - b0) < KC_CH16/2 ? (kb - b0) : KC_CH16/2; + for (int64_t b = 0; b < nbl; b++) { + const block_q8_0_ppc * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + b0 + b]; + dB[j] = fp16_to_fp32(yb[j]->d); + } + for (int h = 0; h < 2; h++) { + const int64_t ch = 2*b + h; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + 16*h), flip); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +extern "C" void grid16_pack_b_q8_K(const block_q8_K_ppc * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bgrid16_t * P = (bgrid16_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bgrid16_t * T = &P[jt*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_q8_K_ppc * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + sb]; + dB[j] = yb[j]->d; + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*sb + c; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + 16*c), flip); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel_grid16_8x8(const agrid16_t * PA, const bgrid16_t * PB, + int64_t nch, vfl fin[MR][2]) { + for (int64_t ch = 0; ch < nch; ch++) { + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < nch) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][2]; + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 4; x++) { + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + for (int r = 0; r < 4; r++) { + vfl t = vec_msub(vec_ctf(rowsP[r],0), + vec_splats(sA[r]), vec_splats(C128[r])); + fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + } + } + } + } +} + +extern "C" void grid16_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const agrid16_t * PA = (const agrid16_t *)packedA; + const bgrid16_t * PB = (const bgrid16_t *)packedB; + const int64_t kc16 = k/16, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[MR][2]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int r = 0; r < MR; r++) fin[r][0] = fin[r][1] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t c0 = s*KC_CH16; + const int64_t nch = (kc16 - c0) < KC_CH16 ? (kc16 - c0) : KC_CH16; + kernel_grid16_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t r = 0; r < rows; r++) + for (int64_t cj = 0; cj < cols; cj++) + C[(i + r) + (j0 + cj)*ldc] = fin[r][cj >> 2][cj & 3]; + } + } +} + + +#define GRID_ONESHOT_C(NAME, BLKA, REPACK, GEMM, ASZ, BSZ, PACKB, VARIANT) \ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ float * C, int64_t ldc, int ith, int nth) { \ const BLKA * A = (const BLKA *)Av; \ const block_q8_K_ppc * B = (const block_q8_K_ppc *)Bv; \ - void * PA = aligned_alloc(64, grid_apack_size(MR, k)); \ - void * PB = aligned_alloc(64, grid_bpack_size(n, k)); \ - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } \ - grid_pack_b_q8_K(B, ldb, n, k, PB); \ - const int64_t mt = (m + MR - 1) / MR; \ - const int64_t tpt = (mt + nth - 1) / nth; \ - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ - const int64_t i = it*MR; \ - const int64_t rows = (m - i) < MR ? (m - i) : MR; \ - REPACK(A + i*lda, lda, rows, k, PA); \ - grid_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); \ + void * PB = aligned_alloc(64, BSZ(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, ASZ(m, k), &fresh); \ + if (PA) { \ + if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ + ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + GEMM(m, n, k, PA, PB, C, ldc, ith, nth); \ + } else { \ + void * PT = aligned_alloc(64, ASZ(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ } \ - free(PA); free(PB); \ + free(PB); \ +} +GRID_ONESHOT_C(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 10) +GRID_ONESHOT_C(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 11) +GRID_ONESHOT_C(gemm_iq2_xxs_q8_K_ppc, block_iq2_xxs_ppc, grid_repack_iq2_xxs, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 12) +GRID_ONESHOT_C(gemm_iq3_xxs_q8_K_ppc, block_iq3_xxs_ppc, grid_repack_iq3_xxs, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 13) +GRID_ONESHOT_C(gemm_iq3_s_q8_K_ppc, block_iq3_s_ppc, grid_repack_iq3_s, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 14) +GRID_ONESHOT_C(gemm_iq1_s_q8_K_ppc, block_iq1_s_ppc, grid_repack_iq1_s, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 15) +GRID_ONESHOT_C(gemm_iq2_xs_q8_K_ppc, block_iq2_xs_ppc, grid16_repack_iq2_xs, grid16_gemm_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 16) +GRID_ONESHOT_C(gemm_iq2_s_q8_K_ppc, block_iq2_s_ppc, grid16_repack_iq2_s, grid16_gemm_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 17) +GRID_ONESHOT_C(gemm_iq1_m_q8_K_ppc, block_iq1_m_ppc, grid16_repack_iq1_m, grid16_gemm_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 18) + +extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_nvfp4_ppc * A = (const block_nvfp4_ppc *)Av; + const block_q8_0_ppc * B = (const block_q8_0_ppc *)Bv; + void * PB = aligned_alloc(64, grid16_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B, ldb, n, k, PB); + int fresh = 0; + void * PA = ppc_apack_cache_acquire(Av, m, k, 19, grid16_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { grid16_repack_nvfp4(A, lda, m, k, (void *)PA); + ppc_apack_cache_publish(Av, m, k, 19); } + grid16_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + } else { + void * PT = aligned_alloc(64, grid16_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + grid16_repack_nvfp4(A + i*lda, lda, rows, k, PT); + grid16_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + } + free(PB); } -GRID_ONESHOT(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0) -GRID_ONESHOT(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0) -GRID_ONESHOT(gemm_iq2_xxs_q8_K_ppc, block_iq2_xxs_ppc, grid_repack_iq2_xxs) -GRID_ONESHOT(gemm_iq3_xxs_q8_K_ppc, block_iq3_xxs_ppc, grid_repack_iq3_xxs) -GRID_ONESHOT(gemm_iq3_s_q8_K_ppc, block_iq3_s_ppc, grid_repack_iq3_s) -GRID_ONESHOT(gemm_iq1_s_q8_K_ppc, block_iq1_s_ppc, grid_repack_iq1_s) #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h b/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h index 4797197a9111..917f4ed0f487 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h +++ b/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h @@ -699,3 +699,396 @@ static const uint64_t iq1s_grid[2048] = { 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000, 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101, }; +static const uint64_t iq2xs_grid[512] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +}; + +static const uint64_t iq2s_grid[1024] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +}; + +static const int8_t kvalues_mxfp4[16] = { + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +}; diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index d5a624c4359a..5bc127e5e150 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -54,6 +54,26 @@ void gemm_iq3_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, void gemm_iq1_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, const void * A, int64_t lda, const void * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth); +void gemm_mxfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq2_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq2_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq1_m_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +// tensor-keyed cache for repacked weights (ppc_pack_cache.cpp) +#include +void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int * fresh); +void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant); #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp index bde49f21df34..abb40715f585 100644 --- a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -292,29 +292,40 @@ extern "C" void leg_gemm_affine(int64_t m, int64_t n, int64_t k, // one-shot drivers (pack per call; see K-quant note in INTEGRATION.md) -#define LEG_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, GEMM) \ +#define LEG_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, GEMM, VARIANT) \ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ float * C, int64_t ldc, int ith, int nth) { \ const BLKA * A = (const BLKA *)Av; \ const YBLK * B = (const YBLK *)Bv; \ - void * PA = aligned_alloc(64, leg_apack_size(MR, k)); \ void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ PACKB(B, ldb, n, k, PB); \ - const int64_t mt = (m + MR - 1) / MR; \ - const int64_t tpt = (mt + nth - 1) / nth; \ - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ - const int64_t i = it*MR; \ - const int64_t rows = (m - i) < MR ? (m - i) : MR; \ - REPACK(A + i*lda, lda, rows, k, PA); \ - GEMM(rows, n, k, PA, PB, C + i, ldc, 0, 1); \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ + leg_apack_size(m, k), &fresh); \ + if (PA) { \ + if (fresh) { REPACK(A, lda, m, k, PA); \ + ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + GEMM(m, n, k, PA, PB, C, ldc, ith, nth); \ + } else { \ + void * PT = aligned_alloc(64, leg_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ } \ - free(PA); free(PB); \ + free(PB); \ } -LEG_ONESHOT(gemm_q4_1_q8_1_ppc, block_q4_1, leg_repack_q4_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine) -LEG_ONESHOT(gemm_q5_0_q8_0_ppc, block_q5_0, leg_repack_q5_0, block_q8_0, leg_pack_b_q8_0, leg_gemm_offset) -LEG_ONESHOT(gemm_q5_1_q8_1_ppc, block_q5_1, leg_repack_q5_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine) +LEG_ONESHOT(gemm_q4_1_q8_1_ppc, block_q4_1, leg_repack_q4_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 25) +LEG_ONESHOT(gemm_q5_0_q8_0_ppc, block_q5_0, leg_repack_q5_0, block_q8_0, leg_pack_b_q8_0, leg_gemm_offset, 26) +LEG_ONESHOT(gemm_q5_1_q8_1_ppc, block_q5_1, leg_repack_q5_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 27) #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp new file mode 100644 index 000000000000..42eb50d2951b --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp @@ -0,0 +1,111 @@ +// ppc_pack_cache.cpp - tensor-keyed cache for MMA-repacked weights. +// +// Repacking weights into MMA layout is deterministic and depends only on +// the (immutable during inference) weight tensor, so it belongs at model +// load. Landing it inside ggml's repack.cpp buffer machinery is the +// eventual home; this cache delivers the same behavior at the dispatch +// layer today: the first llamafile_sgemm call for a tensor packs the +// whole matrix once (other threads block on a condvar until it is +// published), and every subsequent call across the model's lifetime +// reuses the packed form. +// +// Keying and safety assumptions (stated for review): +// * Key = (data pointer, m, k, variant id). Weight tensors in +// llama.cpp inference are immutable and their storage is not +// reallocated, so pointer identity is a sound key for this use. +// Training / repeated model reloads at the same address would need +// explicit invalidation -- out of scope and documented. +// * Capacity-bounded (default 2048 MiB, PPC_MMA_PACK_CACHE_MB to +// override, 0 disables). On miss-with-full-cache or allocation +// failure, acquire returns NULL and callers fall back to the +// per-call packing path -- behavior is never wrong, only slower. +// * Round-robin eviction of ready entries; entries being packed are +// never evicted. + +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include + +#define PPC_PACK_CACHE_SLOTS 128 + +typedef struct { + const void * key; + int64_t m, k; + int variant; + void * buf; + size_t bytes; + int ready; // 0 = packing in progress, 1 = usable + int used; +} slot_t; + +static slot_t g_slots[PPC_PACK_CACHE_SLOTS]; +static size_t g_total = 0; +static size_t g_cap = (size_t)2048 * 1024 * 1024; +static int g_cap_init = 0; +static unsigned g_evict = 0; +static pthread_mutex_t g_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_cv = PTHREAD_COND_INITIALIZER; + +static void cap_init_locked(void) { + if (g_cap_init) return; + g_cap_init = 1; + const char * e = getenv("PPC_MMA_PACK_CACHE_MB"); + if (e) g_cap = (size_t)strtoull(e, NULL, 10) * 1024 * 1024; +} + +extern "C" void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int * fresh) { + *fresh = 0; + pthread_mutex_lock(&g_mu); + cap_init_locked(); + if (g_cap == 0 || bytes > g_cap) { pthread_mutex_unlock(&g_mu); return NULL; } + for (;;) { + slot_t * hit = NULL; + for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { hit = &g_slots[i]; break; } + if (hit) { + while (!hit->ready) pthread_cond_wait(&g_cv, &g_mu); + void * b = hit->buf; + pthread_mutex_unlock(&g_mu); + return b; + } + // insert: find a free or evictable slot, respect capacity + slot_t * dst = NULL; + for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + if (!g_slots[i].used) { dst = &g_slots[i]; break; } + while (!dst || g_total + bytes > g_cap) { + // evict a ready entry round-robin + slot_t * victim = NULL; + for (int t = 0; t < PPC_PACK_CACHE_SLOTS; t++) { + slot_t * s = &g_slots[(g_evict + t) % PPC_PACK_CACHE_SLOTS]; + if (s->used && s->ready) { victim = s; g_evict += t + 1; break; } + } + if (!victim) { pthread_mutex_unlock(&g_mu); return NULL; } + free(victim->buf); + g_total -= victim->bytes; + memset(victim, 0, sizeof(*victim)); + if (!dst) dst = victim; + } + void * buf = aligned_alloc(64, bytes); + if (!buf) { pthread_mutex_unlock(&g_mu); return NULL; } + dst->key = key; dst->m = m; dst->k = k; dst->variant = variant; + dst->buf = buf; dst->bytes = bytes; dst->ready = 0; dst->used = 1; + g_total += bytes; + *fresh = 1; + pthread_mutex_unlock(&g_mu); + return buf; + } +} + +extern "C" void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant) { + pthread_mutex_lock(&g_mu); + for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { g_slots[i].ready = 1; break; } + pthread_cond_broadcast(&g_cv); + pthread_mutex_unlock(&g_mu); +} diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp index 2bf8a5d6de42..853b8dcba95b 100644 --- a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -266,19 +266,29 @@ extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q2_K * A = (const block_q2_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PA = aligned_alloc(64, q2k_apack_size(MR, k)); void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } q2k_pack_b(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - q2k_repack_a(A + i*lda, lda, rows, k, PA); - q2k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + int fresh = 0; + void * PA = ppc_apack_cache_acquire(Av, m, k, 23, q2k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q2k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 23); } + q2k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + } else { + void * PT = aligned_alloc(64, q2k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q2k_repack_a(A + i*lda, lda, rows, k, PT); + q2k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); } - free(PA); free(PB); + free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp index f840e06d0c71..eefbc79ff488 100644 --- a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -262,19 +262,29 @@ extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q3_K * A = (const block_q3_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PA = aligned_alloc(64, q3k_apack_size(MR, k)); void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } q3k_pack_b(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - q3k_repack_a(A + i*lda, lda, rows, k, PA); - q3k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + int fresh = 0; + void * PA = ppc_apack_cache_acquire(Av, m, k, 24, q3k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q3k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 24); } + q3k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + } else { + void * PT = aligned_alloc(64, q3k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q3k_repack_a(A + i*lda, lda, rows, k, PT); + q3k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); } - free(PA); free(PB); + free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp index d0cf08e8d7a7..3eea92d81e3f 100644 --- a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -299,19 +299,29 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q4_K * A = (const block_q4_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PA = aligned_alloc(64, q4k_apack_size(MR, k)); void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } q4k_pack_b(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - q4k_repack_a(A + i*lda, lda, rows, k, PA); - q4k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + int fresh = 0; + void * PA = ppc_apack_cache_acquire(Av, m, k, 20, q4k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q4k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 20); } + q4k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + } else { + void * PT = aligned_alloc(64, q4k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q4k_repack_a(A + i*lda, lda, rows, k, PT); + q4k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); } - free(PA); free(PB); + free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp index bcbb04c92c8a..a263686258bd 100644 --- a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -307,19 +307,29 @@ extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q5_K * A = (const block_q5_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PA = aligned_alloc(64, q5k_apack_size(MR, k)); void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } q5k_pack_b(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - q5k_repack_a(A + i*lda, lda, rows, k, PA); - q5k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + int fresh = 0; + void * PA = ppc_apack_cache_acquire(Av, m, k, 21, q5k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q5k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 21); } + q5k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + } else { + void * PT = aligned_alloc(64, q5k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q5k_repack_a(A + i*lda, lda, rows, k, PT); + q5k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); } - free(PA); free(PB); + free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp index a68b53a36f70..dc41500db7ee 100644 --- a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -257,19 +257,29 @@ extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q6_K * A = (const block_q6_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PA = aligned_alloc(64, q6k_apack_size(MR, k)); void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); - if (!PA || !PB) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } q6k_pack_b(B, ldb, n, k, PB); - const int64_t mt = (m + MR - 1) / MR; - const int64_t tpt = (mt + nth - 1) / nth; - for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { - const int64_t i = it*MR; - const int64_t rows = (m - i) < MR ? (m - i) : MR; - q6k_repack_a(A + i*lda, lda, rows, k, PA); - q6k_gemm_packed(rows, n, k, PA, PB, C + i, ldc, 0, 1); + int fresh = 0; + void * PA = ppc_apack_cache_acquire(Av, m, k, 22, q6k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q6k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 22); } + q6k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + } else { + void * PT = aligned_alloc(64, q6k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q6k_repack_a(A + i*lda, lda, rows, k, PT); + q6k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); } - free(PA); free(PB); + free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp index 43e11db8d1e1..f89abbd11236 100644 --- a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp @@ -1,50 +1,54 @@ -// qbit_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for Q1_0/Q2_0 x Q8_0. -// Imported from https://github.com/mavin2009/ppc-mma-kernels (v3 kernel, -// correctness-verified against double references under qemu -cpu power10). -// See that repo's docs/DESIGN.md for the derivation. +// qbit_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (v4 production API, standalone-verified under qemu -cpu power10). +// Q1_0 / Q2_0 x Q8_0 on POWER10/POWER11 MMA, with the tensor-keyed +// pack cache and a no-packing GEMV fast path for n == 1. #include "ggml-impl.h" #include "ggml-cpu-impl.h" #include "ggml-quants.h" -#include "qbit_ppc_mma.h" +#include "kquants_ppc_mma.h" +#include #include +#include #include #include -#if defined(__MMA__) && defined(__powerpc64__) -#include + + + + +#if defined(__MMA__) && defined(__powerpc64__) + typedef vector unsigned char vuc; typedef vector signed char vsc; typedef vector unsigned int vui; typedef vector signed int vsi; typedef vector float vfl; -// Unaligned 16-byte load via memcpy: single lxv, well-defined for the -// odd struct offsets several block formats use. +// Unaligned 16-byte load via memcpy: compiles to a single lxv (which is +// alignment-agnostic on POWER) while staying well-defined C++ for any +// source alignment -- several block structs place qs at odd offsets. static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } -#define KC_BLKS 16 // 2048-element K slab +#define KC_BLKS 16 #define KC_CHUNKS (KC_BLKS * 4) -#define MR 16 // weight rows per tile -#define NR 8 // activation cols per tile +#define MR 16 +#define NR 8 +#define QBIT_GEMV_NMAX 2 // n <= this uses the raw-weight GEMV path -// Packed weights: unsigned codes in MMA layout. -// v[ch][4*x + b]: depth step x (0..7), row group b (0..3). typedef struct { vuc v[KC_CHUNKS][32]; - vfl dA[KC_BLKS][4]; // per-block scales, 4 rows per vfl + vfl dA[KC_BLKS][4]; } apack_t; -// Packed activations: signed int8, untouched. -// v[ch][2*x + a]: depth step x, col group a (0..1). typedef struct { vuc v[KC_CHUNKS][16]; - vfl dB[KC_CHUNKS][2]; // per-chunk scales, cols 0-3 / 4-7 - vfl E[KC_BLKS][2]; // sum_c dB(j,c)*sum(y_c) per block + vfl dB[KC_CHUNKS][2]; + vfl E[KC_BLKS][2]; } bpack_t; static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { @@ -58,9 +62,6 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); } -// ---- weight unpack: raw UNSIGNED codes (no sign mapping needed) ---- - -// Q1_0: chunk c from the block's 16 qs bytes -> bits 0/1 as bytes. static inline void q1_codes32(vuc raw, int c, vuc v[2]) { const vuc rep0 = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; const vuc rep1 = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; @@ -71,7 +72,6 @@ static inline void q1_codes32(vuc raw, int c, vuc v[2]) { v[1] = vec_and(vec_sr(vec_perm(raw, raw, vec_add(rep1, off)), sh), m1); } -// Q2_0: chunk c from the block's 32 qs bytes (lo/hi) -> codes 0..3. static inline void q2_codes32(vuc lo, vuc hi, int c, vuc v[2]) { const vuc rep0 = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 }; const vuc rep1 = { 4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7 }; @@ -82,106 +82,121 @@ static inline void q2_codes32(vuc lo, vuc hi, int c, vuc v[2]) { v[1] = vec_and(vec_sr(vec_perm(lo, hi, vec_add(rep1, off)), sh), m3); } -// ---- packers ---- +// ---------------- one-time packing API ---------------- -static void pack_A16_q1(const block_q1_0 * A, int64_t lda, int64_t rows, - int64_t blk0, int64_t nblk, apack_t * P) { - for (int64_t b = 0; b < nblk; b++) { - vuc raw[MR]; float d[MR]; - for (int r = 0; r < MR; r++) { - int64_t rr = r < rows ? r : rows - 1; - const block_q1_0 * bp = &A[rr*lda + blk0 + b]; - d[r] = GGML_FP16_TO_FP32(bp->d); - raw[r] = load16u((const uint8_t *)(bp->qs) + (0)); - } - for (int g = 0; g < 4; g++) - P->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; - for (int c = 0; c < 4; c++) { - const int ch = 4*b + c; - vuc t[MR][2]; - for (int r = 0; r < MR; r++) q1_codes32(raw[r], c, t[r]); - vui rows4[4]; - for (int g = 0; g < 4; g++) - for (int h = 0; h < 2; h++) { - for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; - mma_transpose4(rows4, &P->v[ch][16*h + g], 4); - } - } - } +static inline int64_t n_row_tiles(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t n_col_tiles(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t n_slabs(int64_t k) { return (k/128 + KC_BLKS - 1) / KC_BLKS; } + +extern "C" size_t qbit_apack_size(int64_t m, int64_t k) { + return (((size_t)(n_row_tiles(m) * n_slabs(k)) * sizeof(apack_t)) + 63) & ~(size_t)63; +} +extern "C" size_t qbit_bpack_size(int64_t n, int64_t k) { + return (((size_t)(n_col_tiles(n) * n_slabs(k)) * sizeof(bpack_t)) + 63) & ~(size_t)63; } -static void pack_A16_q2(const block_q2_0 * A, int64_t lda, int64_t rows, - int64_t blk0, int64_t nblk, apack_t * P) { - for (int64_t b = 0; b < nblk; b++) { - vuc lo[MR], hi[MR]; float d[MR]; - for (int r = 0; r < MR; r++) { - int64_t rr = r < rows ? r : rows - 1; - const block_q2_0 * bp = &A[rr*lda + blk0 + b]; - d[r] = GGML_FP16_TO_FP32(bp->d); - lo[r] = load16u((const uint8_t *)(bp->qs) + (0)); - hi[r] = load16u((const uint8_t *)(bp->qs) + (16)); - } - for (int g = 0; g < 4; g++) - P->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; - for (int c = 0; c < 4; c++) { - const int ch = 4*b + c; - vuc t[MR][2]; - for (int r = 0; r < MR; r++) q2_codes32(lo[r], hi[r], c, t[r]); - vui rows4[4]; - for (int g = 0; g < 4; g++) - for (int h = 0; h < 2; h++) { - for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; - mma_transpose4(rows4, &P->v[ch][16*h + g], 4); +// packed layout: tile-major, slab-minor: P[tile * n_slabs + slab] + +template +static void repack_rows(const BLK * A, int64_t lda, int64_t m, int64_t k, apack_t * P) { + const int64_t kb = k / 128, ns = n_slabs(k); + for (int64_t it = 0; it < n_row_tiles(m); it++) { + for (int64_t s = 0; s < ns; s++) { + apack_t * T = &P[it*ns + s]; + const int64_t blk0 = s*KC_BLKS; + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + for (int64_t b = 0; b < nblk; b++) { + const BLK * bp[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + blk0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + } + for (int g = 0; g < 4; g++) + T->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; + for (int c = 0; c < 4; c++) { + vuc t[MR][2]; + for (int r = 0; r < MR; r++) CODES(bp[r], c, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[4*b + c][16*h + g], 4); + } } + } } } } -// Activations: no flip; also builds per-chunk scales and the separable -// correction accumulators E. -static void pack_B8(const block_q8_0 * B, int64_t ldb, int64_t cols, - int64_t q8blk0, int64_t nblk, bpack_t * P) { - for (int64_t b = 0; b < nblk; b++) { - vfl E0 = vec_splats(0.0f), E1 = vec_splats(0.0f); - for (int c = 0; c < 4; c++) { - const int64_t ch = 4*b + c; - const block_q8_0 * yb[NR]; - float dB[NR], S[NR]; - for (int j = 0; j < NR; j++) { - int64_t jj = j < cols ? j : cols - 1; - yb[j] = &B[jj*ldb + q8blk0 + ch]; - dB[j] = GGML_FP16_TO_FP32(yb[j]->d); - } - vui rows4[4]; - for (int a = 0; a < 2; a++) { - vuc q[4][2]; - for (int j = 0; j < 4; j++) { - q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (0)); - q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (16)); - vsi z = vec_splats(0); - vsi s = vec_sum4s((vsc)q[j][0], z); - s = vec_sum4s((vsc)q[j][1], s); - S[4*a + j] = (float)(s[0] + s[1] + s[2] + s[3]); - } - for (int h = 0; h < 2; h++) { - for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; - mma_transpose4(rows4, &P->v[ch][8*h + a], 2); +static void q1_blk_codes(const block_q1_0 * bp, int c, vuc v[2]) { + q1_codes32(load16u((const uint8_t *)(bp->qs) + (0)), c, v); +} +static void q2_blk_codes(const block_q2_0 * bp, int c, vuc v[2]) { + q2_codes32(load16u((const uint8_t *)(bp->qs) + (0)), + load16u((const uint8_t *)(bp->qs) + (16)), c, v); +} + +extern "C" void qbit_repack_q1(const block_q1_0 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + repack_rows(A, lda, m, k, (apack_t *)packed); +} +extern "C" void qbit_repack_q2(const block_q2_0 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + repack_rows(A, lda, m, k, (apack_t *)packed); +} + +// Pack all of B once; call from one thread (or split by col tile). +extern "C" void qbit_pack_b(const block_q8_0 * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bpack_t * P = (bpack_t *)packed; + const int64_t kb = k / 128, ns = n_slabs(k); + for (int64_t jt = 0; jt < n_col_tiles(n); jt++) { + for (int64_t s = 0; s < ns; s++) { + bpack_t * T = &P[jt*ns + s]; + const int64_t blk0 = s*KC_BLKS; + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + for (int64_t b = 0; b < nblk; b++) { + vfl E0 = vec_splats(0.0f), E1 = vec_splats(0.0f); + for (int c = 0; c < 4; c++) { + const int64_t ch = 4*b + c; + const block_q8_0 * yb[NR]; + float dB[NR], S[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + 4*(blk0 + b) + c]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + } + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (0)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (16)); + vsi z = vec_splats(0); + vsi sm = vec_sum4s((vsc)q[j][0], z); + sm = vec_sum4s((vsc)q[j][1], sm); + S[4*a + j] = (float)(sm[0] + sm[1] + sm[2] + sm[3]); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + vfl dB0 = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + vfl dB1 = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + T->dB[ch][0] = dB0; T->dB[ch][1] = dB1; + E0 = vec_madd(dB0, (vfl){ S[0],S[1],S[2],S[3] }, E0); + E1 = vec_madd(dB1, (vfl){ S[4],S[5],S[6],S[7] }, E1); } + T->E[b][0] = E0; T->E[b][1] = E1; } - vfl dB0 = (vfl){ dB[0], dB[1], dB[2], dB[3] }; - vfl dB1 = (vfl){ dB[4], dB[5], dB[6], dB[7] }; - P->dB[ch][0] = dB0; - P->dB[ch][1] = dB1; - E0 = vec_madd(dB0, (vfl){ S[0], S[1], S[2], S[3] }, E0); - E1 = vec_madd(dB1, (vfl){ S[4], S[5], S[6], S[7] }, E1); } - P->E[b][0] = E0; - P->E[b][1] = E1; } } -// ---- 16x8 microkernel on all 8 accumulators ---- -// fin[j][g]: activation col j (0..7), weight row group g (0..3). +// ---------------- GEMM from packed operands ---------------- + static void kernel_16x8(const apack_t * PA, const bpack_t * PB, int64_t nblk, float alpha, vfl fin[NR][4]) { const vfl valpha = vec_splats(alpha); @@ -211,7 +226,6 @@ static void kernel_16x8(const apack_t * PA, const bpack_t * PB, __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); } - // fixup: fin += alpha*dB_j * dA_g * P const vfl dBa0 = vec_mul(PB->dB[ch][0], valpha); const vfl dBa1 = vec_mul(PB->dB[ch][1], valpha); for (int i = 0; i < 2; i++) { @@ -231,8 +245,7 @@ static void kernel_16x8(const apack_t * PA, const bpack_t * PB, } } } - // separable correction: fin[j][g] -= sum_b dA[b][g] * E(j,b) - for (int64_t b = 0; b < nblk; b++) { + for (int64_t b = 0; b < nblk; b++) for (int g = 0; g < 4; g++) { const vfl dA = PA->dA[b][g]; fin[0][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 0), fin[0][g]); @@ -244,79 +257,190 @@ static void kernel_16x8(const apack_t * PA, const bpack_t * PB, fin[6][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 2), fin[6][g]); fin[7][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 3), fin[7][g]); } - } } -// ---- driver, templated over weight type ---- -template -static void gemm_qbit(int64_t m, int64_t n, int64_t k, - const BLK * A, int64_t lda, - const block_q8_0 * B, int64_t ldb, - float * C, int64_t ldc, int ith, int nth) { - assert(k % 128 == 0); - const int64_t kb = k / 128; - const int64_t mt = (m + MR - 1) / MR; +// GEMM over pre-packed operands. packedA from qbit_repack_*, +// packedB from qbit_pack_b. Threads split row tiles. +extern "C" void qbit_gemm_packed(int64_t m, int64_t n, int64_t k, float alpha, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const apack_t * PA = (const apack_t *)packedA; + const bpack_t * PB = (const bpack_t *)packedB; + const int64_t kb = k/128, ns = n_slabs(k), mt = n_row_tiles(m), njt = n_col_tiles(n); const int64_t tpt = (mt + nth - 1) / nth; - const int64_t i0 = MR * (ith * tpt); - const int64_t i1 = m < MR * ((ith + 1) * tpt) ? m : MR * ((ith + 1) * tpt); - if (i0 >= m) return; - - const int64_t njt = (n + NR - 1) / NR; - apack_t * PA = (apack_t*)aligned_alloc(64, sizeof(apack_t)); - bpack_t * PB = (bpack_t*)aligned_alloc(64, njt * sizeof(bpack_t)); - vfl (*fin)[NR][4] = (vfl(*)[NR][4])aligned_alloc(64, njt * sizeof(vfl[NR][4])); - if (!PA || !PB || !fin) { GGML_ABORT("ppc-mma: pack buffer allocation failed"); } - - for (int64_t i = i0; i < i1; i += MR) { - const int64_t rows = (i1 - i) < MR ? (i1 - i) : MR; - for (int64_t jt = 0; jt < njt; jt++) - for (int j = 0; j < NR; j++) - for (int g = 0; g < 4; g++) - fin[jt][j][g] = vec_splats(0.0f); - - for (int64_t blk0 = 0; blk0 < kb; blk0 += KC_BLKS) { - const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; - PACK_A(A + i*lda, lda, rows, blk0, nblk, PA); - for (int64_t jt = 0; jt < njt; jt++) { - const int64_t j = NR*jt; - const int64_t cols = (n - j) < NR ? (n - j) : NR; - if (i == i0 || kb > KC_BLKS) - pack_B8(B + j*ldb, ldb, cols, 4*blk0, nblk, &PB[jt]); - kernel_16x8(PA, &PB[jt], nblk, (float)ALPHA, fin[jt]); - } - } + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; for (int64_t jt = 0; jt < njt; jt++) { - const int64_t j = NR*jt; - const int64_t cols = (n - j) < NR ? (n - j) : NR; + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t blk0 = s*KC_BLKS; + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + kernel_16x8(&PA[it*ns + s], &PB[jt*ns + s], nblk, alpha, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; for (int64_t cj = 0; cj < cols; cj++) { - float * dst = C + i + (j + cj)*ldc; + float * dst = C + i + (j0 + cj)*ldc; if (rows == MR) { - for (int g = 0; g < 4; g++) - vec_xst(fin[jt][cj][g], 16*g, dst); + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); } else { for (int64_t r = 0; r < rows; r++) - dst[r] = fin[jt][cj][r >> 2][r & 3]; + dst[r] = fin[cj][r >> 2][r & 3]; } } } } - free(PA); free(PB); free(fin); } +// ---------------- GEMV path (n <= QBIT_GEMV_NMAX), raw weights ---------------- + +typedef struct { float dB; float S; } gemv_bmeta_t; // per chunk + +static void gemv_prep_b(const block_q8_0 * y, int64_t nch, gemv_bmeta_t * M) { + for (int64_t c = 0; c < nch; c++) { + M[c].dB = GGML_FP16_TO_FP32(y[c].d); + vsi z = vec_splats(0); + vsi s = vec_sum4s((vsc)load16u((const uint8_t *)(y[c].qs) + (0)), z); + s = vec_sum4s((vsc)load16u((const uint8_t *)(y[c].qs) + (16)), s); + M[c].S = (float)(s[0] + s[1] + s[2] + s[3]); + } +} + +static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } + +static float gemv_row_q1(const block_q1_0 * a, const block_q8_0 * y, + const gemv_bmeta_t * M, int64_t kb) { + const vuc rep0 = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; + const vuc rep1 = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; + const vuc bitsel = { 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128 }; + float sumf = 0.0f; + for (int64_t b = 0; b < kb; b++) { + const float dA = GGML_FP16_TO_FP32(a[b].d); + vuc raw = load16u((const uint8_t *)(a[b].qs) + (0)); + for (int c = 0; c < 4; c++) { + const vuc off = vec_splats((unsigned char)(4*c)); + vuc e0 = vec_perm(raw, raw, vec_add(rep0, off)); + vuc e1 = vec_perm(raw, raw, vec_add(rep1, off)); + vuc m0 = (vuc)vec_cmpeq(vec_and(e0, bitsel), bitsel); + vuc m1 = (vuc)vec_cmpeq(vec_and(e1, bitsel), bitsel); + const int8_t * q = y[4*b + c].qs; + vuc y0 = load16u((const uint8_t *)(q) + (0)); + vuc y1 = load16u((const uint8_t *)(q) + (16)); + vsi z = vec_splats(0); + vsi p = vec_sum4s((vsc)vec_and(y0, m0), z); + p = vec_sum4s((vsc)vec_and(y1, m1), p); + const gemv_bmeta_t * mm = &M[4*b + c]; + sumf += dA * mm->dB * (2.0f*(float)hsum(p) - mm->S); + } + } + return sumf; +} + +static float gemv_row_q2(const block_q2_0 * a, const block_q8_0 * y, + const gemv_bmeta_t * M, int64_t kb) { + const vuc rep0 = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 }; + const vuc rep1 = { 4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7 }; + const vuc sel0 = { 1,4,16,64, 1,4,16,64, 1,4,16,64, 1,4,16,64 }; + const vuc sel1 = { 2,8,32,128, 2,8,32,128, 2,8,32,128, 2,8,32,128 }; + float sumf = 0.0f; + for (int64_t b = 0; b < kb; b++) { + const float dA = GGML_FP16_TO_FP32(a[b].d); + vuc lo = load16u((const uint8_t *)(a[b].qs) + (0)); + vuc hi = load16u((const uint8_t *)(a[b].qs) + (16)); + for (int c = 0; c < 4; c++) { + const vuc off = vec_splats((unsigned char)(8*c)); + vuc e0 = vec_perm(lo, hi, vec_add(rep0, off)); + vuc e1 = vec_perm(lo, hi, vec_add(rep1, off)); + vuc ma0 = (vuc)vec_cmpeq(vec_and(e0, sel0), sel0); + vuc mb0 = (vuc)vec_cmpeq(vec_and(e0, sel1), sel1); + vuc ma1 = (vuc)vec_cmpeq(vec_and(e1, sel0), sel0); + vuc mb1 = (vuc)vec_cmpeq(vec_and(e1, sel1), sel1); + const int8_t * q = y[4*b + c].qs; + vuc y0 = load16u((const uint8_t *)(q) + (0)); + vuc y1 = load16u((const uint8_t *)(q) + (16)); + vsi z = vec_splats(0); + vsi p0 = vec_sum4s((vsc)vec_and(y0, ma0), z); + p0 = vec_sum4s((vsc)vec_and(y1, ma1), p0); + vsi p1 = vec_sum4s((vsc)vec_and(y0, mb0), z); + p1 = vec_sum4s((vsc)vec_and(y1, mb1), p1); + const gemv_bmeta_t * mm = &M[4*b + c]; + sumf += dA * mm->dB * ((float)hsum(p0) + 2.0f*(float)hsum(p1) - mm->S); + } + } + return sumf; +} -extern "C" void gemm_q1_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, - const void * A, int64_t lda, const void * B, int64_t ldb, +extern "C" void qbit_gemv_q1(int64_t m, int64_t n, int64_t k, + const block_q1_0 * A, int64_t lda, const block_q8_0 * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth) { - gemm_qbit(m, n, k, - (const block_q1_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth); + const int64_t kb = k/128; + gemv_bmeta_t * M = (gemv_bmeta_t *)malloc(sizeof(gemv_bmeta_t)*4*kb); + const int64_t rpt = (m + nth - 1)/nth, i0 = ith*rpt, i1 = (ith+1)*rpt < m ? (ith+1)*rpt : m; + for (int64_t j = 0; j < n; j++) { + gemv_prep_b(B + j*ldb, 4*kb, M); + for (int64_t i = i0; i < i1; i++) + C[i + j*ldc] = gemv_row_q1(A + i*lda, B + j*ldb, M, kb); + } + free(M); } -extern "C" void gemm_q2_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, - const void * A, int64_t lda, const void * B, int64_t ldb, +extern "C" void qbit_gemv_q2(int64_t m, int64_t n, int64_t k, + const block_q2_0 * A, int64_t lda, const block_q8_0 * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth) { - gemm_qbit(m, n, k, - (const block_q2_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth); + const int64_t kb = k/128; + gemv_bmeta_t * M = (gemv_bmeta_t *)malloc(sizeof(gemv_bmeta_t)*4*kb); + const int64_t rpt = (m + nth - 1)/nth, i0 = ith*rpt, i1 = (ith+1)*rpt < m ? (ith+1)*rpt : m; + for (int64_t j = 0; j < n; j++) { + gemv_prep_b(B + j*ldb, 4*kb, M); + for (int64_t i = i0; i < i1; i++) + C[i + j*ldc] = gemv_row_q2(A + i*lda, B + j*ldb, M, kb); + } + free(M); } -#endif // __MMA__ && __powerpc64__ + +// dispatch entries (names preserved from the original v3 integration): +// n == 1 takes the GEMV path -- no packing, no cache, mask-select and +// vsum4s only; larger n uses the packed GEMM through the pack cache. +#define QBIT_DRIVER(NAME, BLKA, REPACK, GEMV, ALPHA, VARIANT) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const block_q8_0 * B = (const block_q8_0 *)Bv; \ + if (n == 1) { GEMV(m, n, k, A, lda, B, ldb, C, ldc, ith, nth); return; } \ + void * PB = aligned_alloc(64, qbit_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B, ldb, n, k, PB); \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ + qbit_apack_size(m, k), &fresh); \ + if (PA) { \ + if (fresh) { REPACK(A, lda, m, k, PA); \ + ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + qbit_gemm_packed(m, n, k, ALPHA, PA, PB, C, ldc, ith, nth); \ + } else { \ + void * PT = aligned_alloc(64, qbit_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + qbit_gemm_packed(rows, n, k, ALPHA, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ + } \ + free(PB); \ +} +QBIT_DRIVER(gemm_q1_0_q8_0_ppc_v3, block_q1_0, qbit_repack_q1, qbit_gemv_q1, 2.0f, 30) +QBIT_DRIVER(gemm_q2_0_q8_0_ppc_v3, block_q2_0, qbit_repack_q2, qbit_gemv_q2, 1.0f, 31) + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 57cb23a75e8e..0b0ff6dd54b7 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -4132,6 +4132,66 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_MXFP4: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + gemm_mxfp4_q8_0_ppc(m, n, k * QK_MXFP4, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ2_XS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq2_xs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ2_S: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq2_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ1_M: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + gemm_iq1_m_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_NVFP4: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + gemm_nvfp4_q8_0_ppc(m, n, k * QK_NVFP4, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + case GGML_TYPE_Q1_0: { if (Btype != GGML_TYPE_Q8_0) return false; From e3ceea08b5ada7fd17cd3d16f2fd59c947b8f8a5 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:05 -0400 Subject: [PATCH 08/21] power-mma: rollout hardening -- cache fingerprint, admission policy, shared B-pack FNV-1a content fingerprint detects model reloads at the same address; admission-without-eviction makes the cache monotonic; column-partitioned activation packing removes per-thread duplicate pack work on cache hits. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 35 ++++++--- .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 55 ++++++++++---- ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 1 + .../src/ggml-cpu/llamafile/legacy_ppc_mma.cpp | 31 +++++--- .../src/ggml-cpu/llamafile/ppc_pack_cache.cpp | 73 ++++++++++++++----- ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp | 24 ++++-- ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp | 24 ++++-- ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 24 ++++-- ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp | 24 ++++-- ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp | 24 ++++-- ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp | 33 ++++++--- 11 files changed, 254 insertions(+), 94 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index edd7eb4063a9..7f2a60c8f6fc 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -345,36 +345,49 @@ extern "C" void iq4_gemm_packed(int64_t m, int64_t n, int64_t k, } -#define IQ4_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, VARIANT) \ +#define IQ4_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, VARIANT) \ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ float * C, int64_t ldc, int ith, int nth) { \ const BLKA * A = (const BLKA *)Av; \ const YBLK * B = (const YBLK *)Bv; \ - void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); \ - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ - PACKB(B, ldb, n, k, PB); \ int fresh = 0; \ void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ - iq4_apack_size(m, k), &fresh); \ + iq4_apack_size(m, k), &fresh); \ if (PA) { \ - if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ + if (fresh) { REPACK(A, lda, m, k, PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ - iq4_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); \ + const int64_t njt = (n + NR - 1) / NR; \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, iq4_bpack_size(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B + j0*ldb, ldb, nc, k, PBl); \ + iq4_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ } else { \ - void * PT = aligned_alloc(64, iq4_apack_size(MR, k)); \ - if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, iq4_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ const int64_t mt = (m + MR - 1) / MR; \ const int64_t tpt = (mt + nth - 1) / nth; \ for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ const int64_t i = it*MR; \ const int64_t rows = (m - i) < MR ? (m - i) : MR; \ REPACK(A + i*lda, lda, rows, k, PT); \ - iq4_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + iq4_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ } \ free(PT); \ + free(PB); \ } \ - free(PB); \ } IQ4_ONESHOT(gemm_iq4_nl_q8_0_ppc, block_iq4_nl, iq4nl_repack_a, block_q8_0, iq4_pack_b_q8_0, 1) IQ4_ONESHOT(gemm_iq4_xs_q8_K_ppc, block_iq4_xs, iq4xs_repack_a, block_q8_K, iq4_pack_b_q8_K, 2) diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 57e52517adb3..1f8d7ddcba1b 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -740,29 +740,42 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ float * C, int64_t ldc, int ith, int nth) { \ const BLKA * A = (const BLKA *)Av; \ const block_q8_K_ppc * B = (const block_q8_K_ppc *)Bv; \ - void * PB = aligned_alloc(64, BSZ(n, k)); \ - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ - PACKB(B, ldb, n, k, PB); \ int fresh = 0; \ void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, ASZ(m, k), &fresh); \ if (PA) { \ - if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ + if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ - GEMM(m, n, k, PA, PB, C, ldc, ith, nth); \ + const int64_t njt = (n + NR - 1) / NR; \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, BSZ(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B + j0*ldb, ldb, nc, k, PBl); \ + GEMM(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ } else { \ - void * PT = aligned_alloc(64, ASZ(MR, k)); \ - if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + void * PB = aligned_alloc(64, BSZ(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, ASZ(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ const int64_t mt = (m + MR - 1) / MR; \ const int64_t tpt = (mt + nth - 1) / nth; \ for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ const int64_t i = it*MR; \ const int64_t rows = (m - i) < MR ? (m - i) : MR; \ - REPACK(A + i*lda, lda, rows, k, PT); \ - GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + REPACK(A + i*lda, lda, rows, k, PT); \ + GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ } \ free(PT); \ + free(PB); \ } \ - free(PB); \ } GRID_ONESHOT_C(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 10) GRID_ONESHOT_C(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 11) @@ -779,16 +792,28 @@ extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_nvfp4_ppc * A = (const block_nvfp4_ppc *)Av; const block_q8_0_ppc * B = (const block_q8_0_ppc *)Bv; - void * PB = aligned_alloc(64, grid16_bpack_size(n, k)); - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } - grid16_pack_b_q8_0(B, ldb, n, k, PB); int fresh = 0; void * PA = ppc_apack_cache_acquire(Av, m, k, 19, grid16_apack_size(m, k), &fresh); if (PA) { if (fresh) { grid16_repack_nvfp4(A, lda, m, k, (void *)PA); ppc_apack_cache_publish(Av, m, k, 19); } - grid16_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + const int64_t njt = (n + NR - 1) / NR; + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, grid16_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B + j0*ldb, ldb, nc, k, PBl); + grid16_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } } else { + void * PB = aligned_alloc(64, grid16_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B, ldb, n, k, PB); void * PT = aligned_alloc(64, grid16_apack_size(MR, k)); if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } const int64_t mt = (m + MR - 1) / MR; @@ -800,8 +825,8 @@ extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, grid16_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); } free(PT); + free(PB); } - free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index 5bc127e5e150..7d08d00c5e68 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -74,6 +74,7 @@ void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, int variant, size_t bytes, int * fresh); void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant); +void ppc_apack_cache_clear(void); #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp index abb40715f585..690871368059 100644 --- a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -298,30 +298,43 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ float * C, int64_t ldc, int ith, int nth) { \ const BLKA * A = (const BLKA *)Av; \ const YBLK * B = (const YBLK *)Bv; \ - void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ - PACKB(B, ldb, n, k, PB); \ int fresh = 0; \ void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ - leg_apack_size(m, k), &fresh); \ + leg_apack_size(m, k), &fresh); \ if (PA) { \ if (fresh) { REPACK(A, lda, m, k, PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ - GEMM(m, n, k, PA, PB, C, ldc, ith, nth); \ + const int64_t njt = (n + NR - 1) / NR; \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, leg_bpack_size(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B + j0*ldb, ldb, nc, k, PBl); \ + GEMM(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ } else { \ - void * PT = aligned_alloc(64, leg_apack_size(MR, k)); \ - if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, leg_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ const int64_t mt = (m + MR - 1) / MR; \ const int64_t tpt = (mt + nth - 1) / nth; \ for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ const int64_t i = it*MR; \ const int64_t rows = (m - i) < MR ? (m - i) : MR; \ REPACK(A + i*lda, lda, rows, k, PT); \ - GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ } \ free(PT); \ + free(PB); \ } \ - free(PB); \ } LEG_ONESHOT(gemm_q4_1_q8_1_ppc, block_q4_1, leg_repack_q4_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 25) LEG_ONESHOT(gemm_q5_0_q8_0_ppc, block_q5_0, leg_repack_q5_0, block_q8_0, leg_pack_b_q8_0, leg_gemm_offset, 26) diff --git a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp index 42eb50d2951b..5a4ed00e7718 100644 --- a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp +++ b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp @@ -19,8 +19,17 @@ // override, 0 disables). On miss-with-full-cache or allocation // failure, acquire returns NULL and callers fall back to the // per-call packing path -- behavior is never wrong, only slower. -// * Round-robin eviction of ready entries; entries being packed are -// never evicted. +// * NO eviction: once full (slots or bytes), new tensors are simply +// not admitted and their callers use the per-call path. Eviction +// was rejected deliberately: an inference working set that exceeds +// the cap cycles through every tensor once per token, and any +// evicting policy then re-packs evicted tensors every token -- +// strictly worse than the per-call baseline. Admission-only makes +// the cache's effect monotonic: cached tensors win, the rest are +// exactly at baseline. +// * A cheap content fingerprint (first+last 32 bytes of the tensor) +// is folded into the key, so a model unload/reload that lands new +// weights at the same address does not serve stale packs. #include "kquants_ppc_mma.h" @@ -33,6 +42,7 @@ typedef struct { const void * key; + uint64_t fp; int64_t m, k; int variant; void * buf; @@ -43,9 +53,20 @@ typedef struct { static slot_t g_slots[PPC_PACK_CACHE_SLOTS]; static size_t g_total = 0; + +static uint64_t fingerprint(const void * p, int64_t m, int64_t k) { + // first and last 32 bytes of the raw tensor data; the row extent is + // conservatively the smallest any format uses (m rows x k/8 bytes). + const uint8_t * b = (const uint8_t *)p; + const size_t approx = (size_t)m * (size_t)(k/8); + const size_t tail = approx >= 32 ? approx - 32 : 0; + uint64_t h = 1469598103934665603ull; + for (int i = 0; i < 32; i++) { h ^= b[i]; h *= 1099511628211ull; } + for (int i = 0; i < 32; i++) { h ^= b[tail + i]; h *= 1099511628211ull; } + return h; +} static size_t g_cap = (size_t)2048 * 1024 * 1024; static int g_cap_init = 0; -static unsigned g_evict = 0; static pthread_mutex_t g_mu = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t g_cv = PTHREAD_COND_INITIALIZER; @@ -62,37 +83,39 @@ extern "C" void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k pthread_mutex_lock(&g_mu); cap_init_locked(); if (g_cap == 0 || bytes > g_cap) { pthread_mutex_unlock(&g_mu); return NULL; } + const uint64_t fp = fingerprint(key, m, k); for (;;) { slot_t * hit = NULL; + slot_t * stale = NULL; for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && - g_slots[i].k == k && g_slots[i].variant == variant) { hit = &g_slots[i]; break; } + g_slots[i].k == k && g_slots[i].variant == variant) { + if (g_slots[i].fp == fp) { hit = &g_slots[i]; } + else { stale = &g_slots[i]; } // same address, new contents + break; + } + if (stale && stale->ready) { // reload detected: retire it + free(stale->buf); + g_total -= stale->bytes; + memset(stale, 0, sizeof(*stale)); + } else if (stale) { // being packed by another thread + while (!stale->ready) pthread_cond_wait(&g_cv, &g_mu); + continue; + } if (hit) { while (!hit->ready) pthread_cond_wait(&g_cv, &g_mu); void * b = hit->buf; pthread_mutex_unlock(&g_mu); return b; } - // insert: find a free or evictable slot, respect capacity + // admission only -- no eviction (see header) slot_t * dst = NULL; for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) if (!g_slots[i].used) { dst = &g_slots[i]; break; } - while (!dst || g_total + bytes > g_cap) { - // evict a ready entry round-robin - slot_t * victim = NULL; - for (int t = 0; t < PPC_PACK_CACHE_SLOTS; t++) { - slot_t * s = &g_slots[(g_evict + t) % PPC_PACK_CACHE_SLOTS]; - if (s->used && s->ready) { victim = s; g_evict += t + 1; break; } - } - if (!victim) { pthread_mutex_unlock(&g_mu); return NULL; } - free(victim->buf); - g_total -= victim->bytes; - memset(victim, 0, sizeof(*victim)); - if (!dst) dst = victim; - } + if (!dst || g_total + bytes > g_cap) { pthread_mutex_unlock(&g_mu); return NULL; } void * buf = aligned_alloc(64, bytes); if (!buf) { pthread_mutex_unlock(&g_mu); return NULL; } - dst->key = key; dst->m = m; dst->k = k; dst->variant = variant; + dst->key = key; dst->fp = fp; dst->m = m; dst->k = k; dst->variant = variant; dst->buf = buf; dst->bytes = bytes; dst->ready = 0; dst->used = 1; g_total += bytes; *fresh = 1; @@ -101,6 +124,18 @@ extern "C" void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k } } +// explicit invalidation for embedders that unload models +extern "C" void ppc_apack_cache_clear(void) { + pthread_mutex_lock(&g_mu); + for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + if (g_slots[i].used && g_slots[i].ready) { + free(g_slots[i].buf); + g_total -= g_slots[i].bytes; + memset(&g_slots[i], 0, sizeof(g_slots[i])); + } + pthread_mutex_unlock(&g_mu); +} + extern "C" void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant) { pthread_mutex_lock(&g_mu); for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp index 853b8dcba95b..3ccc269eadc1 100644 --- a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -266,16 +266,28 @@ extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q2_K * A = (const block_q2_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } - q2k_pack_b(B, ldb, n, k, PB); - int fresh = 0; + int fresh = 0; /* B is packed per-branch below */ void * PA = ppc_apack_cache_acquire(Av, m, k, 23, q2k_apack_size(m, k), &fresh); if (PA) { if (fresh) { q2k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 23); } - q2k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + const int64_t njt = (n + NR - 1) / NR; + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q2k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q2k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q2k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } } else { + void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q2k_pack_b(B, ldb, n, k, PB); void * PT = aligned_alloc(64, q2k_apack_size(MR, k)); if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } const int64_t mt = (m + MR - 1) / MR; @@ -287,8 +299,8 @@ extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q2k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); } free(PT); + free(PB); } - free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp index eefbc79ff488..d21d297b0d0d 100644 --- a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -262,16 +262,28 @@ extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q3_K * A = (const block_q3_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } - q3k_pack_b(B, ldb, n, k, PB); - int fresh = 0; + int fresh = 0; /* B is packed per-branch below */ void * PA = ppc_apack_cache_acquire(Av, m, k, 24, q3k_apack_size(m, k), &fresh); if (PA) { if (fresh) { q3k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 24); } - q3k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + const int64_t njt = (n + NR - 1) / NR; + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q3k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q3k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q3k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } } else { + void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q3k_pack_b(B, ldb, n, k, PB); void * PT = aligned_alloc(64, q3k_apack_size(MR, k)); if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } const int64_t mt = (m + MR - 1) / MR; @@ -283,8 +295,8 @@ extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q3k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); } free(PT); + free(PB); } - free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp index 3eea92d81e3f..5ff07b7373cd 100644 --- a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -299,16 +299,28 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q4_K * A = (const block_q4_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } - q4k_pack_b(B, ldb, n, k, PB); - int fresh = 0; + int fresh = 0; /* B is packed per-branch below */ void * PA = ppc_apack_cache_acquire(Av, m, k, 20, q4k_apack_size(m, k), &fresh); if (PA) { if (fresh) { q4k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 20); } - q4k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + const int64_t njt = (n + NR - 1) / NR; + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q4k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q4k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q4k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } } else { + void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q4k_pack_b(B, ldb, n, k, PB); void * PT = aligned_alloc(64, q4k_apack_size(MR, k)); if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } const int64_t mt = (m + MR - 1) / MR; @@ -320,8 +332,8 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q4k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); } free(PT); + free(PB); } - free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp index a263686258bd..6e7f4ee3ea4a 100644 --- a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -307,16 +307,28 @@ extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q5_K * A = (const block_q5_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } - q5k_pack_b(B, ldb, n, k, PB); - int fresh = 0; + int fresh = 0; /* B is packed per-branch below */ void * PA = ppc_apack_cache_acquire(Av, m, k, 21, q5k_apack_size(m, k), &fresh); if (PA) { if (fresh) { q5k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 21); } - q5k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + const int64_t njt = (n + NR - 1) / NR; + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q5k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q5k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q5k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } } else { + void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q5k_pack_b(B, ldb, n, k, PB); void * PT = aligned_alloc(64, q5k_apack_size(MR, k)); if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } const int64_t mt = (m + MR - 1) / MR; @@ -328,8 +340,8 @@ extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q5k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); } free(PT); + free(PB); } - free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp index dc41500db7ee..21ffb913d9b0 100644 --- a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -257,16 +257,28 @@ extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, float * C, int64_t ldc, int ith, int nth) { const block_q6_K * A = (const block_q6_K *)Av; const block_q8_K * B = (const block_q8_K *)Bv; - void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } - q6k_pack_b(B, ldb, n, k, PB); - int fresh = 0; + int fresh = 0; /* B is packed per-branch below */ void * PA = ppc_apack_cache_acquire(Av, m, k, 22, q6k_apack_size(m, k), &fresh); if (PA) { if (fresh) { q6k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 22); } - q6k_gemm_packed(m, n, k, PA, PB, C, ldc, ith, nth); + const int64_t njt = (n + NR - 1) / NR; + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q6k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q6k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q6k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } } else { + void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q6k_pack_b(B, ldb, n, k, PB); void * PT = aligned_alloc(64, q6k_apack_size(MR, k)); if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } const int64_t mt = (m + MR - 1) / MR; @@ -278,8 +290,8 @@ extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q6k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); } free(PT); + free(PB); } - free(PB); } #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp index f89abbd11236..4c24906f851d 100644 --- a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp @@ -407,37 +407,50 @@ extern "C" void qbit_gemv_q2(int64_t m, int64_t n, int64_t k, // dispatch entries (names preserved from the original v3 integration): // n == 1 takes the GEMV path -- no packing, no cache, mask-select and // vsum4s only; larger n uses the packed GEMM through the pack cache. -#define QBIT_DRIVER(NAME, BLKA, REPACK, GEMV, ALPHA, VARIANT) \ +#define QBIT_DRIVER(NAME, BLKA, REPACK, GEMV, ALPHA, VARIANT) \ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ float * C, int64_t ldc, int ith, int nth) { \ const BLKA * A = (const BLKA *)Av; \ const block_q8_0 * B = (const block_q8_0 *)Bv; \ if (n == 1) { GEMV(m, n, k, A, lda, B, ldb, C, ldc, ith, nth); return; } \ - void * PB = aligned_alloc(64, qbit_bpack_size(n, k)); \ - if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ - qbit_pack_b(B, ldb, n, k, PB); \ int fresh = 0; \ void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ - qbit_apack_size(m, k), &fresh); \ + qbit_apack_size(m, k), &fresh); \ if (PA) { \ if (fresh) { REPACK(A, lda, m, k, PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ - qbit_gemm_packed(m, n, k, ALPHA, PA, PB, C, ldc, ith, nth); \ + const int64_t njt = (n + NR - 1) / NR; \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, qbit_bpack_size(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B + j0*ldb, ldb, nc, k, PBl); \ + qbit_gemm_packed(m, nc, k, ALPHA, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ } else { \ - void * PT = aligned_alloc(64, qbit_apack_size(MR, k)); \ - if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + void * PB = aligned_alloc(64, qbit_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, qbit_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ const int64_t mt = (m + MR - 1) / MR; \ const int64_t tpt = (mt + nth - 1) / nth; \ for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ const int64_t i = it*MR; \ const int64_t rows = (m - i) < MR ? (m - i) : MR; \ REPACK(A + i*lda, lda, rows, k, PT); \ - qbit_gemm_packed(rows, n, k, ALPHA, PT, PB, C + i, ldc, 0, 1); \ + qbit_gemm_packed(rows, n, k, ALPHA, PT, PB, C + i, ldc, 0, 1); \ } \ free(PT); \ + free(PB); \ } \ - free(PB); \ } QBIT_DRIVER(gemm_q1_0_q8_0_ppc_v3, block_q1_0, qbit_repack_q1, qbit_gemv_q1, 2.0f, 30) QBIT_DRIVER(gemm_q2_0_q8_0_ppc_v3, block_q2_0, qbit_repack_q2, qbit_gemv_q2, 1.0f, 31) From 81e7a26dca5b2c899112a79ddac2609204cc60e7 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 09/21] power-mma: prefetch the next packed chunk in the GEMM kernels --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 3 +++ ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index 7f2a60c8f6fc..538862020dad 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -278,8 +278,11 @@ static void kernel_iq4_8x8(const aiq4_t * PA, const biq4_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < nch) { + // each chunk's packed panel spans two 128B lines; touch both __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); } __vector_quad acc[2][2]; for (int g = 0; g < 2; g++) diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 1f8d7ddcba1b..41fefa624f58 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -438,8 +438,11 @@ static void kernel_grid_8x8(const agrid_t * PA, const bgrid_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < nch) { + // each chunk's packed panel spans two 128B lines; touch both __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); } __vector_quad acc[2][2]; for (int g = 0; g < 2; g++) @@ -673,8 +676,11 @@ static void kernel_grid16_8x8(const agrid16_t * PA, const bgrid16_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < nch) { + // each chunk's packed panel spans two 128B lines; touch both __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); } __vector_quad acc[2][2]; for (int g = 0; g < 2; g++) From 273e14e15d98733a330ab10904e041401cc07b1a Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 10/21] power-mma: IQGRID_PINGPONG accumulator-alternation variant (flag) Two accumulator sets alternate so the next chunk's GERs issue before the previous chunk drains. On POWER10 silicon: pp -1..-5%, tg +3-4% pre-0015; standard remains the default. --- .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 41fefa624f58..23e3ffe1e3f5 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -432,7 +432,7 @@ extern "C" void grid_pack_b_q8_K(const block_q8_K_ppc * B, int64_t ldb, } } -static void kernel_grid_8x8(const agrid_t * PA, const bgrid_t * PB, +__attribute__((unused)) static void kernel_grid_8x8(const agrid_t * PA, const bgrid_t * PB, int64_t nch, vfl fin[MR][2]) { for (int64_t ch = 0; ch < nch; ch++) { const vuc * a = PA->v[ch]; @@ -473,6 +473,72 @@ static void kernel_grid_8x8(const agrid_t * PA, const bgrid_t * PB, } } +// --------------------------------------------------------------------------- +// Accumulator ping-pong variant (-DIQGRID_PINGPONG), 32-deep kernel. +// +// Microarchitectural rationale (Power10 MMA): the eight accumulators +// are physically resident in the MMA engine; xxmfacc drains engine +// state to the VRF and serializes against outstanding GERs on that +// accumulator. The default kernel therefore idles the engine through +// every per-chunk fixup. This variant runs two 4-accumulator sets in +// alternation: chunk c+1's GER stream is issued on set B before set A +// is drained, so the drain + VSU fixup of chunk c overlap live GER +// execution. Cost: all 8 accumulators alias VSRs 0-31, raising spill +// pressure (visible in the static count); benefit: engine-idle removal +// (invisible to any static count). Only silicon can arbitrate -- +// this is hardware experiment #1 in docs/BENCHMARKS-QEMU.md. + +static inline void grid_pp_compute(const vuc * a, const vuc * y, + __vector_quad acc[2][2]) { + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 8; x++) { + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } +} + +static inline void grid_pp_fixup(const agrid_t * PA, const bgrid_t * PB, + int64_t ch, __vector_quad acc[2][2], + vfl fin[MR][2]) { + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + for (int r = 0; r < 4; r++) { + vfl t = vec_msub(vec_ctf(rowsP[r],0), + vec_splats(sA[r]), vec_splats(C128[r])); + fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + } + } + } +} + +__attribute__((unused)) static void kernel_grid_8x8_pp(const agrid_t * PA, const bgrid_t * PB, + int64_t nch, vfl fin[MR][2]) { + if (nch <= 0) return; + __vector_quad accA[2][2], accB[2][2]; + grid_pp_compute(PA->v[0], PB->v[0], accA); + int64_t ch = 1; + int aLive = 1; // which set holds chunk ch-1 + for (; ch < nch; ch++) { + if (aLive) { grid_pp_compute(PA->v[ch], PB->v[ch], accB); + grid_pp_fixup(PA, PB, ch - 1, accA, fin); } + else { grid_pp_compute(PA->v[ch], PB->v[ch], accA); + grid_pp_fixup(PA, PB, ch - 1, accB, fin); } + aLive ^= 1; + } + grid_pp_fixup(PA, PB, nch - 1, aLive ? accA : accB, fin); +} + extern "C" void grid_gemm_packed(int64_t m, int64_t n, int64_t k, const void * packedA, const void * packedB, float * C, int64_t ldc, int ith, int nth) { @@ -490,7 +556,11 @@ extern "C" void grid_gemm_packed(int64_t m, int64_t n, int64_t k, for (int64_t s = 0; s < ns; s++) { const int64_t b0 = s*KC_CH; const int64_t nch = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; +#ifdef IQGRID_PINGPONG + kernel_grid_8x8_pp(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); +#else kernel_grid_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); +#endif } const int64_t j0 = jt*NR; const int64_t cols = (n - j0) < NR ? (n - j0) : NR; From bf4cd1251fd32741d7cb8150ece261494e9c56fa Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 11/21] power-mma: route Q8_0/Q4_0 through the pack-cached MMA kernels Replaces tinyBLAS_Q0_PPC for these types; small-n shapes stay on vec_dot pending GEMV forwarding. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 25 +++++++++++++++++++ ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 6 +++++ .../src/ggml-cpu/llamafile/legacy_ppc_mma.cpp | 9 +++++++ ggml/src/ggml-cpu/llamafile/sgemm.cpp | 22 ++++++++-------- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index 538862020dad..b76eb2b7a019 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -150,6 +150,30 @@ extern "C" void iq4nl_repack_a(const block_iq4_nl * A, int64_t lda, } } +// ---- weight repack: Q8_0 (signed int8 weights, identity decode) ---- +extern "C" void q8_0_repack_a(const block_q8_0 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const block_q8_0 * bp = &A[rr*lda + b0 + b]; + sc[r] = GGML_FP16_TO_FP32(bp->d); + w[r][0] = (vsc)load16u(bp->qs); + w[r][1] = (vsc)load16u(bp->qs + 16); + } + iq4_place_chunk(T, b, w, sc); + } + } +} + // ---- weight repack: MXFP4 (same shape as IQ4_NL, different table+scale) ---- extern "C" void mxfp4_repack_a(const block_mxfp4 * A, int64_t lda, int64_t m, int64_t k, void * packed) { @@ -395,6 +419,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ IQ4_ONESHOT(gemm_iq4_nl_q8_0_ppc, block_iq4_nl, iq4nl_repack_a, block_q8_0, iq4_pack_b_q8_0, 1) IQ4_ONESHOT(gemm_iq4_xs_q8_K_ppc, block_iq4_xs, iq4xs_repack_a, block_q8_K, iq4_pack_b_q8_K, 2) IQ4_ONESHOT(gemm_mxfp4_q8_0_ppc, block_mxfp4, mxfp4_repack_a, block_q8_0, iq4_pack_b_q8_0, 3) +IQ4_ONESHOT(gemm_q8_0_q8_0_ppc, block_q8_0, q8_0_repack_a, block_q8_0, iq4_pack_b_q8_0, 4) #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index 7d08d00c5e68..0024f5dff69b 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -69,6 +69,12 @@ void gemm_iq1_m_q8_K_ppc(int64_t m, int64_t n, int64_t k, void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, const void * A, int64_t lda, const void * B, int64_t ldb, float * C, int64_t ldc, int ith, int nth); +void gemm_q8_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q4_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); // tensor-keyed cache for repacked weights (ppc_pack_cache.cpp) #include void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp index 690871368059..012e902ff254 100644 --- a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -132,12 +132,16 @@ static void repack_generic(const BLK * A, int64_t lda, int64_t m, int64_t k, ale } static void codes_q4_1(const block_q4_1 * bp, vuc v[2]) { nibbles32(bp->qs, v); } +static void codes_q4_0(const block_q4_0 * bp, vuc v[2]) { nibbles32(bp->qs, v); } static void codes_q5_0(const block_q5_0 * bp, vuc v[2]) { nibbles32(bp->qs, v); qh_merge(bp->qh, v); } static void codes_q5_1(const block_q5_1 * bp, vuc v[2]) { nibbles32(bp->qs, v); qh_merge(bp->qh, v); } extern "C" void leg_repack_q4_1(const block_q4_1 * A, int64_t lda, int64_t m, int64_t k, void * p) { repack_generic(A, lda, m, k, (aleg_t *)p); } +extern "C" void leg_repack_q4_0(const block_q4_0 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} extern "C" void leg_repack_q5_0(const block_q5_0 * A, int64_t lda, int64_t m, int64_t k, void * p) { repack_generic(A, lda, m, k, (aleg_t *)p); } @@ -188,6 +192,10 @@ static void pack_b_generic(const YBLK * B, int64_t ldb, int64_t n, int64_t k, bl extern "C" void leg_pack_b_q8_0(const block_q8_0 * B, int64_t ldb, int64_t n, int64_t k, void * p) { pack_b_generic(B, ldb, n, k, (bleg_t *)p); } +// SFACT 8 variant for Q4_0's offset (t = q - 8) +extern "C" void leg_pack_b_q8_0_o8(const block_q8_0 * B, int64_t ldb, int64_t n, int64_t k, void * p) { + pack_b_generic(B, ldb, n, k, (bleg_t *)p); +} extern "C" void leg_pack_b_q8_1(const block_q8_1 * B, int64_t ldb, int64_t n, int64_t k, void * p) { pack_b_generic(B, ldb, n, k, (bleg_t *)p); } @@ -339,6 +347,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ LEG_ONESHOT(gemm_q4_1_q8_1_ppc, block_q4_1, leg_repack_q4_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 25) LEG_ONESHOT(gemm_q5_0_q8_0_ppc, block_q5_0, leg_repack_q5_0, block_q8_0, leg_pack_b_q8_0, leg_gemm_offset, 26) LEG_ONESHOT(gemm_q5_1_q8_1_ppc, block_q5_1, leg_repack_q5_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 27) +LEG_ONESHOT(gemm_q4_0_q8_0_ppc, block_q4_0, leg_repack_q4_0, block_q8_0, leg_pack_b_q8_0_o8, leg_gemm_offset, 28) #endif // __MMA__ diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 0b0ff6dd54b7..0add505f7947 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -3952,12 +3952,11 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 return false; if (m < 8 && m != 4) return false; - tinyBLAS_Q0_PPC tb{ - k, (const block_q8_0 *)A, lda, - (const block_q8_0 *)B, ldb, - (float *)C, ldc, - params->ith, params->nth}; - tb.matmul(m, n); + // routed through the pack-cached MMA kernels (patch 0011); replaces + // tinyBLAS_Q0_PPC, whose per-call weight repack, scalar comparray + // fixup and absent GEMV path made it the slow producer here. + gemm_q8_0_q8_0_ppc(m, n, k * QK8_0, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); return true; #else return false; @@ -4243,12 +4242,11 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 return false; if (m < 8 && m != 4) return false; - tinyBLAS_Q0_PPC tb{ - k, (const block_q4_0 *)A, lda, - (const block_q8_0 *)B, ldb, - (float *)C, ldc, - params->ith, params->nth}; - tb.matmul(m, n); + // routed through the pack-cached MMA kernels (patch 0011); replaces + // tinyBLAS_Q0_PPC, whose per-call weight repack, scalar comparray + // fixup and absent GEMV path made it the slow producer here. + gemm_q4_0_q8_0_ppc(m, n, k * QK4_0, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); return true; #else return false; From 4d2118756b424a6e0f2107e74b13789625b6a09b Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 12/21] power-mma: GCC 11 compatibility --- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 0add505f7947..d363f3a63625 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -3627,7 +3627,11 @@ class tinyBLAS_PPC { } else if constexpr(RM == 8 && RN == 8) { KERNEL_8x8(ii, jj); } else { - static_assert(false, "RN/RM values not supported"); + // dependent form: GCC < 13 evaluates a non-dependent + // static_assert(false) eagerly even in a discarded + // constexpr branch (pre-P2593); RM != RM defers it to + // instantiation, which never occurs for supported tiles. + static_assert(RM != RM, "RN/RM values not supported"); } } From 147e9d4e1c7305420234570c06bcfb6c3d946b8e Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 13/21] power-mma: exact-integer 128*codesum corrections The 128*W correction is exactly representable and subtracted from the exact integer accumulator before scaling, matching ggml's scalar rounding order. Field fix from POWER10 validation. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 10 +++---- .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 26 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index b76eb2b7a019..9fd7a37c28f7 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -111,8 +111,8 @@ static void iq4_place_chunk(aiq4_t * T, int64_t ch, W[r] = (float)hsum(s); } T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; - T->C128[ch][g] = (vfl){ 128.0f*W[0]*scale[4*g], 128.0f*W[1]*scale[4*g+1], - 128.0f*W[2]*scale[4*g+2], 128.0f*W[3]*scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0], 128.0f*W[1], + 128.0f*W[2], 128.0f*W[3] }; // exact int; see field FAIL 2026-07-21 } } @@ -329,9 +329,9 @@ static void kernel_iq4_8x8(const aiq4_t * PA, const biq4_t * PB, const vfl dB = PB->dB[ch][cgi]; // fin += dB ⊙ (P*sA_r - C128_r) per weight row r for (int r = 0; r < 4; r++) { - vfl t = vec_msub(vec_ctf(rowsP[r],0), - vec_splats(sA[r]), vec_splats(C128[r])); - fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); } } } diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 23e3ffe1e3f5..1d47f09a6528 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -341,8 +341,8 @@ static void grid_place_chunk(agrid_t * T, int64_t ch, W[r] = (float)hsum(s); } T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; - T->C128[ch][g] = (vfl){ 128.0f*W[0]*scale[4*g], 128.0f*W[1]*scale[4*g+1], - 128.0f*W[2]*scale[4*g+2], 128.0f*W[3]*scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0], 128.0f*W[1], + 128.0f*W[2], 128.0f*W[3] }; // exact int; see field FAIL 2026-07-21 } } @@ -464,9 +464,9 @@ __attribute__((unused)) static void kernel_grid_8x8(const agrid_t * PA, const bg __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); const vfl dB = PB->dB[ch][cgi]; for (int r = 0; r < 4; r++) { - vfl t = vec_msub(vec_ctf(rowsP[r],0), - vec_splats(sA[r]), vec_splats(C128[r])); - fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); } } } @@ -514,9 +514,9 @@ static inline void grid_pp_fixup(const agrid_t * PA, const bgrid_t * PB, __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); const vfl dB = PB->dB[ch][cgi]; for (int r = 0; r < 4; r++) { - vfl t = vec_msub(vec_ctf(rowsP[r],0), - vec_splats(sA[r]), vec_splats(C128[r])); - fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); } } } @@ -607,8 +607,8 @@ static void grid16_place_chunk(agrid16_t * T, int64_t ch, W[r] = (float)hsum(sm); } T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; - T->C128[ch][g] = (vfl){ 128.0f*W[0]*scale[4*g], 128.0f*W[1]*scale[4*g+1], - 128.0f*W[2]*scale[4*g+2], 128.0f*W[3]*scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0], 128.0f*W[1], + 128.0f*W[2], 128.0f*W[3] }; // exact int; see field FAIL 2026-07-21 } } @@ -772,9 +772,9 @@ static void kernel_grid16_8x8(const agrid16_t * PA, const bgrid16_t * PB, __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); const vfl dB = PB->dB[ch][cgi]; for (int r = 0; r < 4; r++) { - vfl t = vec_msub(vec_ctf(rowsP[r],0), - vec_splats(sA[r]), vec_splats(C128[r])); - fin[4*g + r][cgi] = vec_madd(t, dB, fin[4*g + r][cgi]); + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); } } } From b13f8da667fa9503cd06853dcf202ad5ea5d9226 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 14/21] power-mma: row-partition fallback for small n on cache-hit paths n too small to feed every thread by columns (worst case n=1) now row-partitions with the cached pack. Field fix, Q4_K tg32. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 13 ++++++++++ .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 26 +++++++++++++++++++ .../src/ggml-cpu/llamafile/legacy_ppc_mma.cpp | 13 ++++++++++ ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp | 13 ++++++++++ ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp | 13 ++++++++++ ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 13 ++++++++++ ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp | 13 ++++++++++ ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp | 13 ++++++++++ ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp | 13 ++++++++++ 9 files changed, 130 insertions(+) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index 9fd7a37c28f7..758ae0aa02ec 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -385,6 +385,18 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ if (fresh) { REPACK(A, lda, m, k, PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, iq4_bpack_size(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PBs); \ + iq4_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ const int64_t jpt = (njt + nth - 1) / nth; \ const int64_t jt0 = (int64_t)ith*jpt; \ const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ @@ -398,6 +410,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ iq4_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ free(PBl); \ } \ + } \ } else { \ void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); \ if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 1d47f09a6528..83d9a88468a5 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -822,6 +822,18 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, BSZ(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PBs); \ + GEMM(m, n, k, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ const int64_t jpt = (njt + nth - 1) / nth; \ const int64_t jt0 = (int64_t)ith*jpt; \ const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ @@ -835,6 +847,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ GEMM(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ free(PBl); \ } \ + } \ } else { \ void * PB = aligned_alloc(64, BSZ(n, k)); \ if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ @@ -874,6 +887,18 @@ extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, if (fresh) { grid16_repack_nvfp4(A, lda, m, k, (void *)PA); ppc_apack_cache_publish(Av, m, k, 19); } const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, grid16_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B, ldb, n, k, PBs); + grid16_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { const int64_t jpt = (njt + nth - 1) / nth; const int64_t jt0 = (int64_t)ith*jpt; const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; @@ -886,6 +911,7 @@ extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, grid16_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); free(PBl); } + } } else { void * PB = aligned_alloc(64, grid16_bpack_size(n, k)); if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp index 012e902ff254..29db954d621e 100644 --- a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -313,6 +313,18 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ if (fresh) { REPACK(A, lda, m, k, PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, leg_bpack_size(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PBs); \ + GEMM(m, n, k, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ const int64_t jpt = (njt + nth - 1) / nth; \ const int64_t jt0 = (int64_t)ith*jpt; \ const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ @@ -326,6 +338,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ GEMM(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ free(PBl); \ } \ + } \ } else { \ void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp index 3ccc269eadc1..412a97f3d453 100644 --- a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -272,6 +272,18 @@ extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, if (fresh) { q2k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 23); } const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q2k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q2k_pack_b(B, ldb, n, k, PBs); + q2k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { const int64_t jpt = (njt + nth - 1) / nth; const int64_t jt0 = (int64_t)ith*jpt; const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; @@ -284,6 +296,7 @@ extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q2k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); free(PBl); } + } } else { void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp index d21d297b0d0d..fd7960313770 100644 --- a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -268,6 +268,18 @@ extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, if (fresh) { q3k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 24); } const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q3k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q3k_pack_b(B, ldb, n, k, PBs); + q3k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { const int64_t jpt = (njt + nth - 1) / nth; const int64_t jt0 = (int64_t)ith*jpt; const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; @@ -280,6 +292,7 @@ extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q3k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); free(PBl); } + } } else { void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp index 5ff07b7373cd..061991e02254 100644 --- a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -305,6 +305,18 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, if (fresh) { q4k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 20); } const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q4k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q4k_pack_b(B, ldb, n, k, PBs); + q4k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { const int64_t jpt = (njt + nth - 1) / nth; const int64_t jt0 = (int64_t)ith*jpt; const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; @@ -317,6 +329,7 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q4k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); free(PBl); } + } } else { void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp index 6e7f4ee3ea4a..d9af5bb0183e 100644 --- a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -313,6 +313,18 @@ extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, if (fresh) { q5k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 21); } const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q5k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q5k_pack_b(B, ldb, n, k, PBs); + q5k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { const int64_t jpt = (njt + nth - 1) / nth; const int64_t jt0 = (int64_t)ith*jpt; const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; @@ -325,6 +337,7 @@ extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q5k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); free(PBl); } + } } else { void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp index 21ffb913d9b0..aeafb3a0a6f6 100644 --- a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -263,6 +263,18 @@ extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, if (fresh) { q6k_repack_a(A, lda, m, k, PA); ppc_apack_cache_publish(Av, m, k, 22); } const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q6k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q6k_pack_b(B, ldb, n, k, PBs); + q6k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { const int64_t jpt = (njt + nth - 1) / nth; const int64_t jt0 = (int64_t)ith*jpt; const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; @@ -275,6 +287,7 @@ extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, q6k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); free(PBl); } + } } else { void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp index 4c24906f851d..4b5a198bc969 100644 --- a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp @@ -421,6 +421,18 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ if (fresh) { REPACK(A, lda, m, k, PA); \ ppc_apack_cache_publish(Av, m, k, VARIANT); } \ const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, qbit_bpack_size(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B, ldb, n, k, PBs); \ + qbit_gemm_packed(m, n, k, ALPHA, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ const int64_t jpt = (njt + nth - 1) / nth; \ const int64_t jt0 = (int64_t)ith*jpt; \ const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ @@ -434,6 +446,7 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ qbit_gemm_packed(m, nc, k, ALPHA, PA, PBl, C + j0*ldc, ldc, 0, 1); \ free(PBl); \ } \ + } \ } else { \ void * PB = aligned_alloc(64, qbit_bpack_size(n, k)); \ if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ From e6e506afb8369cdaa1adda1123d3273090d40edb Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 15/21] power-mma: n<8 hands generation back to vec_dot; cache slots raised The packed path reads int8-expanded weights (1.8-3x native bytes) and loses generation to vec_dot below one column tile; measured on POWER10 (tg 4.18 -> 35.7 t/s on a 1.5B IQ2_M after this change plus the cache fix). --- .../src/ggml-cpu/llamafile/ppc_pack_cache.cpp | 2 +- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp index 5a4ed00e7718..100f2e4f0dc0 100644 --- a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp +++ b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp @@ -38,7 +38,7 @@ #include #include -#define PPC_PACK_CACHE_SLOTS 128 +#define PPC_PACK_CACHE_SLOTS 1024 typedef struct { const void * key; diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index d363f3a63625..ccb97fdb4eed 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -3971,6 +3971,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q4_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -3983,6 +3987,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q5_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -3995,6 +4003,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q6_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4007,6 +4019,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q2_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4019,6 +4035,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q3_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4031,6 +4051,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq4_xs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4043,6 +4067,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_1) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q4_1_q8_1_ppc(m, n, k * QK4_1, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4055,6 +4083,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_1) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q5_1_q8_1_ppc(m, n, k * QK5_1, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4067,6 +4099,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_tq2_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4079,6 +4115,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_tq1_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4091,6 +4131,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq2_xxs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4103,6 +4147,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq3_xxs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4115,6 +4163,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq3_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4127,6 +4179,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq1_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4139,6 +4195,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_0) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_mxfp4_q8_0_ppc(m, n, k * QK_MXFP4, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4151,6 +4211,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq2_xs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4163,6 +4227,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq2_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4175,6 +4243,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq1_m_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4187,6 +4259,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_0) return false; #if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_nvfp4_q8_0_ppc(m, n, k * QK_NVFP4, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4269,6 +4345,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 tb.matmul(m, n); return true; #elif defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_q5_0_q8_0_ppc(m, n, k * QK5_0, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4289,6 +4369,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 tb.matmul(m, n); return true; #elif defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) gemm_iq4_nl_q8_0_ppc(m, n, k * QK4_NL, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; From efa9b506e7be28072ad3acf75c5095ba17bfe084 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 16/21] power-mma: pack-cache slot table grows; refusals are counted and loud 128 fixed slots lost to the ~197 tensors of a 28-layer model -- a third of the weights re-packed every call. Slot count can no longer bind; the byte cap is the only bound and crossing it prints what it costs. Stats API added. --- .../src/ggml-cpu/llamafile/ppc_pack_cache.cpp | 140 +++++++++++------- 1 file changed, 88 insertions(+), 52 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp index 100f2e4f0dc0..706712da5bda 100644 --- a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp +++ b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp @@ -1,45 +1,39 @@ -// ppc_pack_cache.cpp - tensor-keyed cache for MMA-repacked weights. +// ppc_pack_cache.cpp — tensor-keyed weight-pack cache. // -// Repacking weights into MMA layout is deterministic and depends only on -// the (immutable during inference) weight tensor, so it belongs at model -// load. Landing it inside ggml's repack.cpp buffer machinery is the -// eventual home; this cache delivers the same behavior at the dispatch -// layer today: the first llamafile_sgemm call for a tensor packs the -// whole matrix once (other threads block on a condvar until it is -// published), and every subsequent call across the model's lifetime -// reuses the packed form. +// CANONICAL COPY: this file lives in ppc-mma-kernels/src/ and is +// carried into ggml verbatim by patch 0016. Edit here, regenerate the +// patch; the unit test (xcheck_cache.cpp, `make test`) runs against +// this exact code, including the N > capacity regime — the regime both +// prior cache defects lived in (REVIEW.md defect log: round-robin +// thrash, then 128-slot admission exhaustion). // -// Keying and safety assumptions (stated for review): -// * Key = (data pointer, m, k, variant id). Weight tensors in -// llama.cpp inference are immutable and their storage is not -// reallocated, so pointer identity is a sound key for this use. -// Training / repeated model reloads at the same address would need -// explicit invalidation -- out of scope and documented. -// * Capacity-bounded (default 2048 MiB, PPC_MMA_PACK_CACHE_MB to -// override, 0 disables). On miss-with-full-cache or allocation -// failure, acquire returns NULL and callers fall back to the -// per-call packing path -- behavior is never wrong, only slower. -// * NO eviction: once full (slots or bytes), new tensors are simply -// not admitted and their callers use the per-call path. Eviction -// was rejected deliberately: an inference working set that exceeds -// the cap cycles through every tensor once per token, and any -// evicting policy then re-packs evicted tensors every token -- -// strictly worse than the per-call baseline. Admission-only makes -// the cache's effect monotonic: cached tensors win, the rest are -// exactly at baseline. -// * A cheap content fingerprint (first+last 32 bytes of the tensor) -// is folded into the key, so a model unload/reload that lands new -// weights at the same address does not serve stale packs. +// Policy, stated for review: +// * Key = (data pointer, m, k, variant) + a 64-byte content +// fingerprint that detects model reloads at the same address. +// * Admission-only, no eviction: an admitted pack lives until +// ppc_apack_cache_clear(). The cache's effect is monotonic. +// * The slot table GROWS (doubling, starting at 256): slot count is +// never the binding constraint. The byte capacity is the only +// bound (default 2048 MiB, PPC_MMA_PACK_CACHE_MB overrides, 0 +// disables). +// * A refusal is NEVER silent: the first refusal and every 64th +// after it print to stderr with running totals, and the totals are +// queryable via ppc_apack_cache_stats(). A tensor that packs per +// call is a performance defect (measured 2-8x tg loss on POWER10, +// VALIDATION-POWER10.md D3); it must be visible. -#include "kquants_ppc_mma.h" +#if defined(__has_include) +# if __has_include("kquants_ppc_mma.h") +# include "kquants_ppc_mma.h" // prototypes, when compiled inside ggml +# endif +#endif #include +#include #include #include #include -#define PPC_PACK_CACHE_SLOTS 1024 - typedef struct { const void * key; uint64_t fp; @@ -51,12 +45,17 @@ typedef struct { int used; } slot_t; -static slot_t g_slots[PPC_PACK_CACHE_SLOTS]; -static size_t g_total = 0; +static slot_t * g_slots = NULL; +static int g_nslots = 0; +static size_t g_total = 0; +static size_t g_cap = (size_t)2048 * 1024 * 1024; +static int g_cap_init = 0; +static size_t g_admitted = 0; +static size_t g_refused = 0; +static pthread_mutex_t g_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_cv = PTHREAD_COND_INITIALIZER; static uint64_t fingerprint(const void * p, int64_t m, int64_t k) { - // first and last 32 bytes of the raw tensor data; the row extent is - // conservatively the smallest any format uses (m rows x k/8 bytes). const uint8_t * b = (const uint8_t *)p; const size_t approx = (size_t)m * (size_t)(k/8); const size_t tail = approx >= 32 ? approx - 32 : 0; @@ -65,10 +64,6 @@ static uint64_t fingerprint(const void * p, int64_t m, int64_t k) { for (int i = 0; i < 32; i++) { h ^= b[tail + i]; h *= 1099511628211ull; } return h; } -static size_t g_cap = (size_t)2048 * 1024 * 1024; -static int g_cap_init = 0; -static pthread_mutex_t g_mu = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t g_cv = PTHREAD_COND_INITIALIZER; static void cap_init_locked(void) { if (g_cap_init) return; @@ -77,17 +72,42 @@ static void cap_init_locked(void) { if (e) g_cap = (size_t)strtoull(e, NULL, 10) * 1024 * 1024; } +static int grow_locked(void) { + const int n = g_nslots ? g_nslots * 2 : 256; + slot_t * s = (slot_t *)realloc(g_slots, (size_t)n * sizeof(slot_t)); + if (!s) return 0; + memset(s + g_nslots, 0, (size_t)(n - g_nslots) * sizeof(slot_t)); + g_slots = s; + g_nslots = n; + return 1; +} + +static void refuse_locked(size_t bytes) { + g_refused++; + if (g_refused == 1 || g_refused % 64 == 0) { + fprintf(stderr, + "ppc-mma pack cache: capacity refusal #%zu (%zu MiB requested, " + "%zu/%zu MiB resident, %zu packs admitted) -- this tensor will " + "re-pack EVERY call; raise PPC_MMA_PACK_CACHE_MB\n", + g_refused, bytes >> 20, g_total >> 20, g_cap >> 20, g_admitted); + } +} + extern "C" void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, int variant, size_t bytes, int * fresh) { *fresh = 0; pthread_mutex_lock(&g_mu); cap_init_locked(); - if (g_cap == 0 || bytes > g_cap) { pthread_mutex_unlock(&g_mu); return NULL; } + if (g_cap == 0 || bytes > g_cap) { + if (g_cap != 0) refuse_locked(bytes); + pthread_mutex_unlock(&g_mu); + return NULL; + } const uint64_t fp = fingerprint(key, m, k); for (;;) { slot_t * hit = NULL; slot_t * stale = NULL; - for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + for (int i = 0; i < g_nslots; i++) if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && g_slots[i].k == k && g_slots[i].variant == variant) { if (g_slots[i].fp == fp) { hit = &g_slots[i]; } @@ -110,24 +130,39 @@ extern "C" void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k } // admission only -- no eviction (see header) slot_t * dst = NULL; - for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + for (int i = 0; i < g_nslots; i++) if (!g_slots[i].used) { dst = &g_slots[i]; break; } - if (!dst || g_total + bytes > g_cap) { pthread_mutex_unlock(&g_mu); return NULL; } + if (!dst) { + const int before = g_nslots; + if (!grow_locked()) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst = &g_slots[before]; // first slot the growth added + } + if (g_total + bytes > g_cap) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } void * buf = aligned_alloc(64, bytes); - if (!buf) { pthread_mutex_unlock(&g_mu); return NULL; } + if (!buf) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } dst->key = key; dst->fp = fp; dst->m = m; dst->k = k; dst->variant = variant; dst->buf = buf; dst->bytes = bytes; dst->ready = 0; dst->used = 1; g_total += bytes; + g_admitted++; *fresh = 1; pthread_mutex_unlock(&g_mu); return buf; } } +extern "C" void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant) { + pthread_mutex_lock(&g_mu); + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { g_slots[i].ready = 1; break; } + pthread_cond_broadcast(&g_cv); + pthread_mutex_unlock(&g_mu); +} + // explicit invalidation for embedders that unload models extern "C" void ppc_apack_cache_clear(void) { pthread_mutex_lock(&g_mu); - for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) + for (int i = 0; i < g_nslots; i++) if (g_slots[i].used && g_slots[i].ready) { free(g_slots[i].buf); g_total -= g_slots[i].bytes; @@ -136,11 +171,12 @@ extern "C" void ppc_apack_cache_clear(void) { pthread_mutex_unlock(&g_mu); } -extern "C" void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant) { +// observability: totals since process start +extern "C" void ppc_apack_cache_stats(size_t * admitted, size_t * refused, + size_t * resident_bytes) { pthread_mutex_lock(&g_mu); - for (int i = 0; i < PPC_PACK_CACHE_SLOTS; i++) - if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && - g_slots[i].k == k && g_slots[i].variant == variant) { g_slots[i].ready = 1; break; } - pthread_cond_broadcast(&g_cv); + if (admitted) *admitted = g_admitted; + if (refused) *refused = g_refused; + if (resident_bytes) *resident_bytes = g_total; pthread_mutex_unlock(&g_mu); } From 2fe96e002e84beecef7274d15dce3b00054f1665 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 17/21] power-mma: keep the cached packed path at n=1 for IQ1_M and TQ1_0 These formats' vec_dot runs at 2-3% of the memory wall on POWER10; the packed path wins despite the int8 expansion (tg +50%/+98%). --- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index ccb97fdb4eed..da482f6e7018 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -4115,10 +4115,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) - if (n < 8) - return false; // packed path reads int8-expanded weights; - // below one column tile vec_dot wins the - // bandwidth race on silicon (patch 0015) + // 0015 guard deliberately absent for this format: its ggml + // vec_dot is compute-starved on Power (2-3%% of the memory + // wall, VALIDATION-POWER10.md #9), so the cached packed path + // wins generation despite the int8 expansion (patch 0017). gemm_tq1_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; @@ -4243,10 +4243,10 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) - if (n < 8) - return false; // packed path reads int8-expanded weights; - // below one column tile vec_dot wins the - // bandwidth race on silicon (patch 0015) + // 0015 guard deliberately absent for this format: its ggml + // vec_dot is compute-starved on Power (2-3%% of the memory + // wall, VALIDATION-POWER10.md #9), so the cached packed path + // wins generation despite the int8 expansion (patch 0017). gemm_iq1_m_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, (float *)C, ldc, params->ith, params->nth); return true; From b712ef29c9fbd0e73ece804019f00c4d490e28a7 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 18/21] power-mma: ISA 3.1 experiments -- dcbt streams default for iq4, lxvp flag dcbt TH=8 stream hints measured +13% pp128 on IQ4_XS silicon and become that kernel's default (PPC_DCBT_LINES restores line touches); neutral on K-quants. lxvp vector-pair loads measured -1..-22% and stay behind IQGRID_LXVP as documented negative space. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 5 ++++ .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 23 +++++++++++++++++++ ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp | 5 ++++ ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp | 5 ++++ ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 5 ++++ ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp | 5 ++++ ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp | 5 ++++ 7 files changed, 53 insertions(+) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index 758ae0aa02ec..cdc55ba7399c 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -303,10 +303,15 @@ static void kernel_iq4_8x8(const aiq4_t * PA, const biq4_t * PB, const vuc * y = PB->v[ch]; if (ch + 1 < nch) { // each chunk's packed panel spans two 128B lines; touch both +#ifdef PPC_DCBT_LINES __builtin_prefetch(PA->v[ch + 1], 0, 3); __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); +#else + __asm__ volatile("dcbt 0,%0,8" :: "r"(PA->v[ch + 1])); + __asm__ volatile("dcbt 0,%0,8" :: "r"(PB->v[ch + 1])); +#endif } __vector_quad acc[2][2]; for (int g = 0; g < 2; g++) diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 83d9a88468a5..0bf45e39bdca 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -325,6 +325,13 @@ static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } +#ifdef IQGRID_LXVP +static inline void load_pair(const vuc * p, vuc out[2]) { + __vector_pair vp = *(const __vector_pair *)p; + __builtin_vsx_disassemble_pair((void *)out, &vp); +} +#endif + static void grid_place_chunk(agrid_t * T, int64_t ch, const vsc w[MR][2], const float scale[MR]) { for (int g = 0; g < 2; g++) { @@ -449,8 +456,16 @@ __attribute__((unused)) static void kernel_grid_8x8(const agrid_t * PA, const bg for (int cgi = 0; cgi < 2; cgi++) __builtin_mma_xxsetaccz(&acc[g][cgi]); for (int x = 0; x < 8; x++) { +#ifdef IQGRID_LXVP + vuc wv[2], yv[2]; + load_pair(a + 2*x, wv); + load_pair(y + 2*x, yv); + const vuc w0 = wv[0], w1 = wv[1]; + const vuc y0 = yv[0], y1 = yv[1]; +#else const vuc w0 = a[2*x], w1 = a[2*x + 1]; const vuc y0 = y[2*x], y1 = y[2*x + 1]; +#endif __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); @@ -494,8 +509,16 @@ static inline void grid_pp_compute(const vuc * a, const vuc * y, for (int cgi = 0; cgi < 2; cgi++) __builtin_mma_xxsetaccz(&acc[g][cgi]); for (int x = 0; x < 8; x++) { +#ifdef IQGRID_LXVP + vuc wv[2], yv[2]; + load_pair(a + 2*x, wv); + load_pair(y + 2*x, yv); + const vuc w0 = wv[0], w1 = wv[1]; + const vuc y0 = yv[0], y1 = yv[1]; +#else const vuc w0 = a[2*x], w1 = a[2*x + 1]; const vuc y0 = y[2*x], y1 = y[2*x + 1]; +#endif __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp index 412a97f3d453..cac025d2e3ac 100644 --- a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -170,8 +170,13 @@ static void kernel2k_16x8(const a2k_t * PA, const b2k_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < 16*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else __builtin_prefetch(PA->v[ch + 1], 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif } __vector_quad acc[2][4]; for (int i = 0; i < 2; i++) diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp index fd7960313770..b1c774b5d041 100644 --- a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -173,8 +173,13 @@ static void kernel3k_16x8(const a3k_t * PA, const b3k_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < 16*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else __builtin_prefetch(PA->v[ch + 1], 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif } __vector_quad acc[2][4]; for (int i = 0; i < 2; i++) diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp index 061991e02254..5d55810e7e09 100644 --- a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -202,9 +202,14 @@ static void kernel4k_16x8(const a4k_t * PA, const b4k_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < 8*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else __builtin_prefetch(PA->v[ch + 1], 0, 3); __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif } __vector_quad acc[2][4]; for (int i = 0; i < 2; i++) diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp index d9af5bb0183e..774f27a456aa 100644 --- a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -210,9 +210,14 @@ static void kernel4k_16x8(const a4k_t * PA, const b4k_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < 8*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else __builtin_prefetch(PA->v[ch + 1], 0, 3); __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif } __vector_quad acc[2][4]; for (int i = 0; i < 2; i++) diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp index aeafb3a0a6f6..8cee61dc5654 100644 --- a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -166,8 +166,13 @@ static void kernel6k_16x8(const a6k_t * PA, const b6k_t * PB, const vuc * a = PA->v[ch]; const vuc * y = PB->v[ch]; if (ch + 1 < 16*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else __builtin_prefetch(PA->v[ch + 1], 0, 3); __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif } __vector_quad acc[2][4]; for (int i = 0; i < 2; i++) From 0ffb1e39066b414f9b8f086ac73add5a94da27b9 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 19/21] power-mma: slice-parallel first-touch packing All threads of the op pack disjoint row-tile slices; the last one publishes. Cold start 4.4s -> 1.05s on a 1.5B IQ2_M, +6% steady pp from NUMA-friendly first touch. --- ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp | 13 ++- .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 24 +++-- ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 4 + .../src/ggml-cpu/llamafile/ppc_pack_cache.cpp | 97 +++++++++++++++++++ 4 files changed, 128 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp index cdc55ba7399c..bc5d15127811 100644 --- a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -384,11 +384,16 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const BLKA * A = (const BLKA *)Av; \ const YBLK * B = (const YBLK *)Bv; \ int fresh = 0; \ - void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ - iq4_apack_size(m, k), &fresh); \ + void * PA = ppc_apack_cache_acquire_par(Av, m, k, VARIANT, \ + iq4_apack_size(m, k), nth, &fresh); \ if (PA) { \ - if (fresh) { REPACK(A, lda, m, k, PA); \ - ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + if (fresh) { \ + int64_t i0_, rows_; \ + ppc_apack_slice(m, MR, ith, nth, &i0_, &rows_); \ + if (rows_ > 0) REPACK(A + i0_*lda, lda, rows_, k, \ + (void *)((aiq4_t *)PA + (i0_/MR)*sl32(k))); \ + ppc_apack_cache_slice_done(Av, m, k, VARIANT); \ + } \ const int64_t njt = (n + NR - 1) / NR; \ if (njt < nth) { \ /* n too small to feed every thread by columns (worst \ diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 0bf45e39bdca..957917f8c1a4 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -840,10 +840,16 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const BLKA * A = (const BLKA *)Av; \ const block_q8_K_ppc * B = (const block_q8_K_ppc *)Bv; \ int fresh = 0; \ - void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, ASZ(m, k), &fresh); \ + void * PA = ppc_apack_cache_acquire_par(Av, m, k, VARIANT, \ + ASZ(m, k), nth, &fresh); \ if (PA) { \ - if (fresh) { REPACK(A, lda, m, k, (void *)PA); \ - ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + if (fresh) { \ + int64_t i0_, rows_; \ + ppc_apack_slice(m, MR, ith, nth, &i0_, &rows_); \ + if (rows_ > 0) REPACK(A + i0_*lda, lda, rows_, k, \ + (void *)((agrid_t *)PA + (i0_/MR)*sl(k))); \ + ppc_apack_cache_slice_done(Av, m, k, VARIANT); \ + } \ const int64_t njt = (n + NR - 1) / NR; \ if (njt < nth) { \ /* n too small to feed every thread by columns (worst \ @@ -905,10 +911,16 @@ extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, const block_nvfp4_ppc * A = (const block_nvfp4_ppc *)Av; const block_q8_0_ppc * B = (const block_q8_0_ppc *)Bv; int fresh = 0; - void * PA = ppc_apack_cache_acquire(Av, m, k, 19, grid16_apack_size(m, k), &fresh); + void * PA = ppc_apack_cache_acquire_par(Av, m, k, 19, + grid16_apack_size(m, k), nth, &fresh); if (PA) { - if (fresh) { grid16_repack_nvfp4(A, lda, m, k, (void *)PA); - ppc_apack_cache_publish(Av, m, k, 19); } + if (fresh) { + int64_t i0_, rows_; + ppc_apack_slice(m, MR, ith, nth, &i0_, &rows_); + if (rows_ > 0) grid16_repack_nvfp4(A + i0_*lda, lda, rows_, k, + (void *)((agrid16_t *)PA + (i0_/MR)*sl(k))); + ppc_apack_cache_slice_done(Av, m, k, 19); + } const int64_t njt = (n + NR - 1) / NR; if (njt < nth) { /* n too small to feed every thread by columns (worst diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index 0024f5dff69b..2c2b02e524c0 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -80,6 +80,10 @@ void gemm_q4_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, int variant, size_t bytes, int * fresh); void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant); +void * ppc_apack_cache_acquire_par(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int nth, int * fresh); +void ppc_apack_cache_slice_done(const void * key, int64_t m, int64_t k, int variant); +void ppc_apack_slice(int64_t m, int mr, int ith, int nth, int64_t * i0, int64_t * rows); void ppc_apack_cache_clear(void); #ifdef __cplusplus } diff --git a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp index 706712da5bda..2d01e5f1facc 100644 --- a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp +++ b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp @@ -42,6 +42,7 @@ typedef struct { void * buf; size_t bytes; int ready; // 0 = packing in progress, 1 = usable + int pending; // outstanding parallel pack slices int used; } slot_t; @@ -171,6 +172,102 @@ extern "C" void ppc_apack_cache_clear(void) { pthread_mutex_unlock(&g_mu); } +// Parallel first touch. All nth threads of one ggml op call +// acquire_par with the same (key, m, k, variant, nth); the first +// arrival creates the slot with pending = nth, and EVERY caller that +// sees the slot unready gets the buffer with *fresh = 1: each packs +// its disjoint row-tile slice, then calls slice_done, which counts +// down and blocks until the pack is whole. Later ops hit ready +// slots and take the plain path (*fresh = 0). Single-threaded cold +// start was measured at ~0.5 GB/s for grid decode -- 3 s of one +// thread working and seven waiting on a 1.5B model, worse for bigger +// ones (VALIDATION-POWER10.md, remaining-work item 2). +extern "C" void * ppc_apack_cache_acquire_par(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int nth, + int * fresh) { + *fresh = 0; + pthread_mutex_lock(&g_mu); + cap_init_locked(); + if (g_cap == 0 || bytes > g_cap) { + if (g_cap != 0) refuse_locked(bytes); + pthread_mutex_unlock(&g_mu); + return NULL; + } + const uint64_t fp = fingerprint(key, m, k); + for (;;) { + slot_t * hit = NULL; + slot_t * stale = NULL; + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { + if (g_slots[i].fp == fp) { hit = &g_slots[i]; } + else { stale = &g_slots[i]; } + break; + } + if (stale && stale->ready) { + free(stale->buf); + g_total -= stale->bytes; + memset(stale, 0, sizeof(*stale)); + } else if (stale) { + while (!stale->ready) pthread_cond_wait(&g_cv, &g_mu); + continue; + } + if (hit) { + void * b = hit->buf; + if (!hit->ready) *fresh = 1; // join the parallel fill + pthread_mutex_unlock(&g_mu); + return b; + } + slot_t * dst = NULL; + for (int i = 0; i < g_nslots; i++) + if (!g_slots[i].used) { dst = &g_slots[i]; break; } + if (!dst) { + const int before = g_nslots; + if (!grow_locked()) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst = &g_slots[before]; + } + if (g_total + bytes > g_cap) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + void * buf = aligned_alloc(64, bytes); + if (!buf) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst->key = key; dst->fp = fp; dst->m = m; dst->k = k; dst->variant = variant; + dst->buf = buf; dst->bytes = bytes; dst->ready = 0; dst->pending = nth; dst->used = 1; + g_total += bytes; + g_admitted++; + *fresh = 1; + pthread_mutex_unlock(&g_mu); + return buf; + } +} + +extern "C" void ppc_apack_cache_slice_done(const void * key, int64_t m, int64_t k, int variant) { + pthread_mutex_lock(&g_mu); + slot_t * s = NULL; + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { s = &g_slots[i]; break; } + if (s) { + if (s->pending > 0 && --s->pending == 0) { + s->ready = 1; + pthread_cond_broadcast(&g_cv); + } + while (!s->ready) pthread_cond_wait(&g_cv, &g_mu); + } + pthread_mutex_unlock(&g_mu); +} + +// Row-tile slice assignment for the parallel fill: MR-aligned, evenly +// spread, empty for surplus threads. +extern "C" void ppc_apack_slice(int64_t m, int mr, int ith, int nth, + int64_t * i0, int64_t * rows) { + const int64_t tiles = (m + mr - 1) / mr; + const int64_t tpt = (tiles + nth - 1) / nth; + int64_t t0 = (int64_t)ith * tpt, t1 = t0 + tpt; + if (t1 > tiles) t1 = tiles; + if (t0 >= t1) { *i0 = 0; *rows = 0; return; } + *i0 = t0 * mr; + *rows = (t1 * mr < m ? t1 * mr : m) - *i0; +} + // observability: totals since process start extern "C" void ppc_apack_cache_stats(size_t * admitted, size_t * refused, size_t * resident_bytes) { From 826109371ce43a499d06ed170a6388c4d2defe82 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 20/21] power-mma: two-row VSX GEMV experiment, kept unwired Built to test the stream-parallelism hypothesis at n=1; measured 27-44% slower than vec_dot -- n=1 is issue-rate-bound on POWER10. Kernel and float64 tests remain for wider silicon to re-run. --- ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h | 2 + ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp | 237 ++++++++++++++++++ 2 files changed, 239 insertions(+) diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h index 2c2b02e524c0..ee238855524d 100644 --- a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -84,6 +84,8 @@ void * ppc_apack_cache_acquire_par(const void * key, int64_t m, int64_t k, int variant, size_t bytes, int nth, int * fresh); void ppc_apack_cache_slice_done(const void * key, int64_t m, int64_t k, int variant); void ppc_apack_slice(int64_t m, int mr, int ith, int nth, int64_t * i0, int64_t * rows); +void gemv2_q4_K_q8_K_ppc(int64_t m, int64_t k, const void * Av, int64_t lda, + const void * Bv, float * C, int ith, int nth); void ppc_apack_cache_clear(void); #ifdef __cplusplus } diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp index 5d55810e7e09..dc04870c1a34 100644 --- a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -356,3 +356,240 @@ extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, #endif // __MMA__ +// ---- two-row GEMV, n = 1 (the near-tie squeeze) ----------------------- +// +// Generation cannot use the MMA tiles profitably (the packed operand +// rides at 1.8x native bytes; VALIDATION-POWER10.md D3), and ggml's +// vec_dot keeps one weight stream per thread alive -- measured at 49% +// of the machine's bandwidth on a 27B (s9.1). This kernel walks two +// rows per pass: twice the memory-level parallelism on the weight +// side, and every activation load shared between the rows. Per-row +// arithmetic order is copied exactly from ggml's POWER9 vec_dot, so +// each row's result is bit-identical to the single-row path -- the +// speedup must come from the memory system alone, and correctness +// comparisons stay trivial. +extern "C" void gemv2_q4_K_q8_K_ppc(int64_t m, int64_t k, + const void * Av, int64_t lda, + const void * Bv, float * C, + int ith, int nth) { + const block_q4_K * __restrict A = (const block_q4_K *)Av; + const block_q8_K * __restrict y = (const block_q8_K *)Bv; + const int64_t nb = k / QK_K; + + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector signed char lowMask1 = vec_splats((int8_t)0x3f); + const vector signed char lowMask2 = vec_splats((int8_t)0x30); + const vector int v0 = vec_splats((int32_t)0); + const vector unsigned char v2 = vec_splats((uint8_t)2); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + const int64_t rpt = (m + nth - 1) / nth; + const int64_t lo = (int64_t)ith * rpt; + const int64_t hi = (lo + rpt) < m ? (lo + rpt) : m; + + for (int64_t r = lo; r < hi; r += 2) { + const int two = (r + 1 < hi); + const block_q4_K * xa = A + r*lda; + const block_q4_K * xb = A + (two ? (r + 1)*lda : r*lda); + + vector float sfa0 = vec_splats(0.0f), sfa1 = vec_splats(0.0f); + vector float sfa2 = vec_splats(0.0f), sfa3 = vec_splats(0.0f); + vector float sfb0 = vec_splats(0.0f), sfb1 = vec_splats(0.0f); + vector float sfb2 = vec_splats(0.0f), sfb3 = vec_splats(0.0f); + + for (int64_t i = 0; i < nb; ++i) { + const vector float vyd = vec_splats(y[i].d); + const vector signed short q8ysums0 = vec_xl( 0, y[i].bsums); + const vector signed short q8ysums1 = vec_xl(16, y[i].bsums); + + // ---- per-row scale/min setup (identical to ggml's vec_dot) ---- + vector float vda, vdmina, vdb, vdminb; + vector signed short vscalesa, vscalesb; + vector signed short q4xmins0a, q4xmins1a, q4xmins0b, q4xmins1b; + { + const vector float vxd = vec_splats(ggml_fp16_to_fp32(xa[i].d)); + vda = vec_mul(vxd, vyd); + const vector float vxmin = vec_splats(ggml_fp16_to_fp32(xa[i].dmin)); + vdmina = vec_mul(vxmin, vyd); + vector signed char u0 = (vector signed char)vec_xl_len((unsigned char *)xa[i].scales, 8); + vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2); + vector signed char u2 = (vector signed char)vec_xl_len((unsigned char *)xa[i].scales + 8, 4); + vector signed char u3 = vec_sr(u2, v4); + vector signed char u30 = u1; + vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3); + u1 = vec_and(u0, lowMask1); + u2 = vec_or(u30, u31); + vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2); + vscalesa = vec_unpackh(utmps); + vector signed short q4xmins = vec_unpackl(utmps); + q4xmins0a = vec_mergeh(q4xmins, q4xmins); + q4xmins1a = vec_mergel(q4xmins, q4xmins); + } + { + const vector float vxd = vec_splats(ggml_fp16_to_fp32(xb[i].d)); + vdb = vec_mul(vxd, vyd); + const vector float vxmin = vec_splats(ggml_fp16_to_fp32(xb[i].dmin)); + vdminb = vec_mul(vxmin, vyd); + vector signed char u0 = (vector signed char)vec_xl_len((unsigned char *)xb[i].scales, 8); + vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2); + vector signed char u2 = (vector signed char)vec_xl_len((unsigned char *)xb[i].scales + 8, 4); + vector signed char u3 = vec_sr(u2, v4); + vector signed char u30 = u1; + vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3); + u1 = vec_and(u0, lowMask1); + u2 = vec_or(u30, u31); + vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2); + vscalesb = vec_unpackh(utmps); + vector signed short q4xmins = vec_unpackl(utmps); + q4xmins0b = vec_mergeh(q4xmins, q4xmins); + q4xmins1b = vec_mergel(q4xmins, q4xmins); + } + + { + vector signed int prod0 = vec_mule(q4xmins0a, q8ysums0); + vector signed int prod1 = vec_mule(q4xmins1a, q8ysums1); + vector signed int prod2 = vec_mulo(q4xmins0a, q8ysums0); + vector signed int prod3 = vec_mulo(q4xmins1a, q8ysums1); + sfa0 = vec_nmsub(vec_ctf(prod0, 0), vdmina, sfa0); + sfa1 = vec_nmsub(vec_ctf(prod1, 0), vdmina, sfa1); + sfa2 = vec_nmsub(vec_ctf(prod2, 0), vdmina, sfa2); + sfa3 = vec_nmsub(vec_ctf(prod3, 0), vdmina, sfa3); + } + { + vector signed int prod0 = vec_mule(q4xmins0b, q8ysums0); + vector signed int prod1 = vec_mule(q4xmins1b, q8ysums1); + vector signed int prod2 = vec_mulo(q4xmins0b, q8ysums0); + vector signed int prod3 = vec_mulo(q4xmins1b, q8ysums1); + sfb0 = vec_nmsub(vec_ctf(prod0, 0), vdminb, sfb0); + sfb1 = vec_nmsub(vec_ctf(prod1, 0), vdminb, sfb1); + sfb2 = vec_nmsub(vec_ctf(prod2, 0), vdminb, sfb2); + sfb3 = vec_nmsub(vec_ctf(prod3, 0), vdminb, sfb3); + } + + vector signed int sia0 = v0, sia1 = v0, sia2 = v0, sia3 = v0; + vector signed int sib0 = v0, sib1 = v0, sib2 = v0, sib3 = v0; + + const uint8_t * q4a = xa[i].qs; + const uint8_t * q4b = xb[i].qs; + const int8_t * q8 = y[i].qs; + + for (int j = 0; j < QK_K/64; j += 2) { + __builtin_prefetch(q4a, 0, 1); + __builtin_prefetch(q4b, 0, 1); + __builtin_prefetch(q8, 0, 1); + const vector signed char qxa0 = (vector signed char)vec_xl( 0, q4a); + const vector signed char qxa1 = (vector signed char)vec_xl(16, q4a); + const vector signed char qxa2 = (vector signed char)vec_xl(32, q4a); + const vector signed char qxa3 = (vector signed char)vec_xl(48, q4a); + q4a += 64; + const vector signed char qxb0 = (vector signed char)vec_xl( 0, q4b); + const vector signed char qxb1 = (vector signed char)vec_xl(16, q4b); + const vector signed char qxb2 = (vector signed char)vec_xl(32, q4b); + const vector signed char qxb3 = (vector signed char)vec_xl(48, q4b); + q4b += 64; + + const vector signed char q8y00 = vec_xl( 0, q8); + const vector signed char q8y10 = vec_xl( 16, q8); + const vector signed char q8y01 = vec_xl( 32, q8); + const vector signed char q8y11 = vec_xl( 48, q8); + const vector signed char q8y20 = vec_xl( 64, q8); + const vector signed char q8y30 = vec_xl( 80, q8); + const vector signed char q8y21 = vec_xl( 96, q8); + const vector signed char q8y31 = vec_xl(112, q8); + q8 += 128; + + { + vector unsigned char x00 = (vector unsigned char)vec_and(qxa0, lowMask); + vector unsigned char x01 = (vector unsigned char)vec_sr(qxa0, v4); + vector unsigned char x10 = (vector unsigned char)vec_and(qxa1, lowMask); + vector unsigned char x11 = (vector unsigned char)vec_sr(qxa1, v4); + vector unsigned char x20 = (vector unsigned char)vec_and(qxa2, lowMask); + vector unsigned char x21 = (vector unsigned char)vec_sr(qxa2, v4); + vector unsigned char x30 = (vector unsigned char)vec_and(qxa3, lowMask); + vector unsigned char x31 = (vector unsigned char)vec_sr(qxa3, v4); + vector signed int qv00 = vec_msum(q8y00, x00, v0); + vector signed int qv01 = vec_msum(q8y01, x01, v0); + vector signed int qv10 = vec_msum(q8y10, x10, v0); + vector signed int qv11 = vec_msum(q8y11, x11, v0); + vector signed int qv20 = vec_msum(q8y20, x20, v0); + vector signed int qv21 = vec_msum(q8y21, x21, v0); + vector signed int qv30 = vec_msum(q8y30, x30, v0); + vector signed int qv31 = vec_msum(q8y31, x31, v0); + vector signed int vscales_h = vec_unpackh(vscalesa); + vector signed int vs0 = vec_splat(vscales_h, 0); + vector signed int vs1 = vec_splat(vscales_h, 1); + vector signed int vs2 = vec_splat(vscales_h, 2); + vector signed int vs3 = vec_splat(vscales_h, 3); + vscalesa = vec_sld(vscalesa, vscalesa, 8); + sia0 = vec_add(vec_mul(qv00, vs0), sia0); + sia1 = vec_add(vec_mul(qv01, vs1), sia1); + sia2 = vec_add(vec_mul(qv20, vs2), sia2); + sia3 = vec_add(vec_mul(qv21, vs3), sia3); + sia0 = vec_add(vec_mul(qv10, vs0), sia0); + sia1 = vec_add(vec_mul(qv11, vs1), sia1); + sia2 = vec_add(vec_mul(qv30, vs2), sia2); + sia3 = vec_add(vec_mul(qv31, vs3), sia3); + } + { + vector unsigned char x00 = (vector unsigned char)vec_and(qxb0, lowMask); + vector unsigned char x01 = (vector unsigned char)vec_sr(qxb0, v4); + vector unsigned char x10 = (vector unsigned char)vec_and(qxb1, lowMask); + vector unsigned char x11 = (vector unsigned char)vec_sr(qxb1, v4); + vector unsigned char x20 = (vector unsigned char)vec_and(qxb2, lowMask); + vector unsigned char x21 = (vector unsigned char)vec_sr(qxb2, v4); + vector unsigned char x30 = (vector unsigned char)vec_and(qxb3, lowMask); + vector unsigned char x31 = (vector unsigned char)vec_sr(qxb3, v4); + vector signed int qv00 = vec_msum(q8y00, x00, v0); + vector signed int qv01 = vec_msum(q8y01, x01, v0); + vector signed int qv10 = vec_msum(q8y10, x10, v0); + vector signed int qv11 = vec_msum(q8y11, x11, v0); + vector signed int qv20 = vec_msum(q8y20, x20, v0); + vector signed int qv21 = vec_msum(q8y21, x21, v0); + vector signed int qv30 = vec_msum(q8y30, x30, v0); + vector signed int qv31 = vec_msum(q8y31, x31, v0); + vector signed int vscales_h = vec_unpackh(vscalesb); + vector signed int vs0 = vec_splat(vscales_h, 0); + vector signed int vs1 = vec_splat(vscales_h, 1); + vector signed int vs2 = vec_splat(vscales_h, 2); + vector signed int vs3 = vec_splat(vscales_h, 3); + vscalesb = vec_sld(vscalesb, vscalesb, 8); + sib0 = vec_add(vec_mul(qv00, vs0), sib0); + sib1 = vec_add(vec_mul(qv01, vs1), sib1); + sib2 = vec_add(vec_mul(qv20, vs2), sib2); + sib3 = vec_add(vec_mul(qv21, vs3), sib3); + sib0 = vec_add(vec_mul(qv10, vs0), sib0); + sib1 = vec_add(vec_mul(qv11, vs1), sib1); + sib2 = vec_add(vec_mul(qv30, vs2), sib2); + sib3 = vec_add(vec_mul(qv31, vs3), sib3); + } + } + + sfa0 = vec_madd(vec_ctf(sia0, 0), vda, sfa0); + sfa1 = vec_madd(vec_ctf(sia1, 0), vda, sfa1); + sfa2 = vec_madd(vec_ctf(sia2, 0), vda, sfa2); + sfa3 = vec_madd(vec_ctf(sia3, 0), vda, sfa3); + sfb0 = vec_madd(vec_ctf(sib0, 0), vdb, sfb0); + sfb1 = vec_madd(vec_ctf(sib1, 0), vdb, sfb1); + sfb2 = vec_madd(vec_ctf(sib2, 0), vdb, sfb2); + sfb3 = vec_madd(vec_ctf(sib3, 0), vdb, sfb3); + } + + sfa0 = vec_add(sfa0, sfa2); + sfa1 = vec_add(sfa1, sfa3); + sfa0 = vec_add(sfa0, sfa1); + sfa0 = vec_add(sfa0, vec_sld(sfa0, sfa0, 4)); + sfa0 = vec_add(sfa0, vec_sld(sfa0, sfa0, 8)); + C[r] = vec_extract(sfa0, 0); + + if (two) { + sfb0 = vec_add(sfb0, sfb2); + sfb1 = vec_add(sfb1, sfb3); + sfb0 = vec_add(sfb0, sfb1); + sfb0 = vec_add(sfb0, vec_sld(sfb0, sfb0, 4)); + sfb0 = vec_add(sfb0, vec_sld(sfb0, sfb0, 8)); + C[r + 1] = vec_extract(sfb0, 0); + } + } +} + + From 3fa1f067fff5696e1da86466d07a2746eb42f451 Mon Sep 17 00:00:00 2001 From: Michael Avina <116688+mavin2009@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:09:06 -0400 Subject: [PATCH 21/21] power-mma: GER GEMV over cached tiles at n=1, routed per format by measurement Each xvi8ger4pp retires 4 rows x 4 depth over tiles already decoded in the pack cache. On POWER10: TQ1_0 3.1x over vec_dot, IQ2_XXS +42%, IQ1_S +23%, IQ3_XXS/S +38%, IQ1_M +58%; TQ2_0/IQ2_S/IQ2_XS keep vec_dot, which measured better for them. --- .../ggml-cpu/llamafile/iq_grid_ppc_mma.cpp | 209 +++++++++++++++++- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 8 +- 2 files changed, 203 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp index 957917f8c1a4..e9cc8545d4c0 100644 --- a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -833,7 +833,192 @@ extern "C" void grid16_gemm_packed(int64_t m, int64_t n, int64_t k, } -#define GRID_ONESHOT_C(NAME, BLKA, REPACK, GEMM, ASZ, BSZ, PACKB, VARIANT) \ +// ---- packed GEMV, n = 1: GER over the cached tiles -------------------- +// +// The two-row VSX GEMV lost to vec_dot because n = 1 on this machine +// is issue-rate bound (VALIDATION-POWER10.md s9.2). This kernel +// attacks that constraint with the one unit that crushes instructions +// per byte: each xvi8ger4pp retires 4 rows x 4 depth of dot product, +// and the A-side tiles, per-chunk scales and 128-offset corrections +// already sit decoded in the pack cache. The activation group is +// XOR-flipped and replicated across the GER's four columns (one perm +// per depth group), so every accumulator lane holds the same answer; +// the fixup gathers the four rows into lane form with two merges and +// a permdi. Roughly 62 instructions per 8-row x 32-deep chunk versus +// ~190 for the same work through the per-format vec_dot. +extern "C" void grid_gemv_packed(int64_t m, int64_t k, const void * PAv, + const block_q8_K_ppc * y, float * C, + int ith, int nth) { + const agrid_t * PA = (const agrid_t *)PAv; + const int64_t nt = rt(m), ns = sl(k), ncht = k/32; + const vuc flip = vec_splats((unsigned char)0x80); + const vuc zero = vec_splats((unsigned char)0); + static const vuc repl[4] = { + (vuc){ 0,1,2,3, 0,1,2,3, 0,1,2,3, 0,1,2,3 }, + (vuc){ 4,5,6,7, 4,5,6,7, 4,5,6,7, 4,5,6,7 }, + (vuc){ 8,9,10,11, 8,9,10,11, 8,9,10,11, 8,9,10,11 }, + (vuc){ 12,13,14,15, 12,13,14,15, 12,13,14,15, 12,13,14,15 }, + }; + const int64_t tpt = (nt + nth - 1) / nth; + const int64_t t0 = (int64_t)ith * tpt; + const int64_t t1 = (t0 + tpt) < nt ? (t0 + tpt) : nt; + + for (int64_t it = t0; it < t1; it++) { + vfl fin0 = vec_splats(0.0f); // rows it*MR+0..3, lane-per-row + vfl fin1 = vec_splats(0.0f); // rows it*MR+4..7 + for (int64_t s = 0; s < ns; s++) { + const agrid_t * T = &PA[it*ns + s]; + const int64_t c0 = s*KC_CH; + const int64_t nch = (ncht - c0) < KC_CH ? (ncht - c0) : KC_CH; + for (int64_t ch = 0; ch < nch; ch++) { + if (ch + 1 < nch) { + __builtin_prefetch(T->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)T->v[ch + 1] + 128, 0, 3); + } + const int64_t gc = c0 + ch; + const int8_t * q8 = y[gc/8].qs + 32*(gc%8); + const vfl vdB = vec_splats(y[gc/8].d); + const vuc ylo = vec_xor(load16u(q8), flip); + const vuc yhi = vec_xor(load16u(q8 + 16), flip); + __vector_quad acc0, acc1; + __builtin_mma_xxsetaccz(&acc0); + __builtin_mma_xxsetaccz(&acc1); + const vuc * a = T->v[ch]; + for (int x = 0; x < 8; x++) { + const vuc yg = vec_perm(x < 4 ? ylo : yhi, zero, repl[x & 3]); + __builtin_mma_xvi8ger4pp(&acc0, a[2*x], yg); + __builtin_mma_xvi8ger4pp(&acc1, a[2*x + 1], yg); + } + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, &acc0); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][0]); + fin0 = vec_madd(t, vec_mul(T->sA[ch][0], vdB), fin0); + } + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, &acc1); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][1]); + fin1 = vec_madd(t, vec_mul(T->sA[ch][1], vdB), fin1); + } + } + } + const int64_t r0 = it*MR; + for (int r = 0; r < 4; r++) { + if (r0 + r < m) C[r0 + r] = fin0[r]; + if (r0 + 4 + r < m) C[r0 + 4 + r] = fin1[r]; + } + } +} + + + +// ---- packed GEMV for the 16-deep family (see grid_gemv_packed) -------- +// +// The 16-deep chunks give each accumulator only FOUR GERs before the +// fixup wants it drained, and xxmfacc serializes against that +// accumulator's GER stream -- the engine never reaches a deep +// pipeline (first cut measured 10 t/s where vec_dot does 33). So +// this kernel ping-pongs two accumulator sets across chunks: chunk +// c+1's GERs issue on the idle set while chunk c drains and fixes up. +// The same trade the 8x8 kernels measured at n = 8 is decisive here. +static inline void grid16_gemv_issue(const agrid16_t * T, int64_t ch, + const block_q8_K_ppc * y, int64_t gc, + const vuc repl[4], vuc flip, vuc zero, + __vector_quad * a0, __vector_quad * a1) { + const int8_t * q8 = y[gc/16].qs + 16*(gc%16); + const vuc yv = vec_xor(load16u(q8), flip); + __builtin_mma_xxsetaccz(a0); + __builtin_mma_xxsetaccz(a1); + const vuc * a = T->v[ch]; + for (int t = 0; t < 4; t++) { + const vuc yg = vec_perm(yv, zero, repl[t]); + __builtin_mma_xvi8ger4pp(a0, a[2*t], yg); + __builtin_mma_xvi8ger4pp(a1, a[2*t + 1], yg); + } +} + +static inline void grid16_gemv_fixup(const agrid16_t * T, int64_t ch, + const block_q8_K_ppc * y, int64_t gc, + __vector_quad * a0, __vector_quad * a1, + vfl * fin0, vfl * fin1) { + const vfl vdB = vec_splats(y[gc/16].d); + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, a0); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][0]); + *fin0 = vec_madd(t, vec_mul(T->sA[ch][0], vdB), *fin0); + } + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, a1); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][1]); + *fin1 = vec_madd(t, vec_mul(T->sA[ch][1], vdB), *fin1); + } +} + +extern "C" void grid16_gemv_packed(int64_t m, int64_t k, const void * PAv, + const block_q8_K_ppc * y, float * C, + int ith, int nth) { + const agrid16_t * PA = (const agrid16_t *)PAv; + const int64_t nt = rt(m), ns = sl(k), ncht = k/16; + const vuc flip = vec_splats((unsigned char)0x80); + const vuc zero = vec_splats((unsigned char)0); + static const vuc repl[4] = { + (vuc){ 0,1,2,3, 0,1,2,3, 0,1,2,3, 0,1,2,3 }, + (vuc){ 4,5,6,7, 4,5,6,7, 4,5,6,7, 4,5,6,7 }, + (vuc){ 8,9,10,11, 8,9,10,11, 8,9,10,11, 8,9,10,11 }, + (vuc){ 12,13,14,15, 12,13,14,15, 12,13,14,15, 12,13,14,15 }, + }; + const int64_t tpt = (nt + nth - 1) / nth; + const int64_t t0 = (int64_t)ith * tpt; + const int64_t t1 = (t0 + tpt) < nt ? (t0 + tpt) : nt; + + for (int64_t it = t0; it < t1; it++) { + vfl fin0 = vec_splats(0.0f); + vfl fin1 = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const agrid16_t * T = &PA[it*ns + s]; + const int64_t c0 = s*KC_CH16; + const int64_t nch = (ncht - c0) < KC_CH16 ? (ncht - c0) : KC_CH16; + if (nch <= 0) continue; + __vector_quad accP0, accP1, accQ0, accQ1; + grid16_gemv_issue(T, 0, y, c0, repl, flip, zero, &accP0, &accP1); + int p = 1; + for (int64_t ch = 0; ch < nch; ch++) { + if (ch + 2 < nch) __builtin_prefetch(T->v[ch + 2], 0, 3); + if (ch + 1 < nch) { + if (p) grid16_gemv_issue(T, ch + 1, y, c0 + ch + 1, repl, flip, zero, &accQ0, &accQ1); + else grid16_gemv_issue(T, ch + 1, y, c0 + ch + 1, repl, flip, zero, &accP0, &accP1); + } + if (p) grid16_gemv_fixup(T, ch, y, c0 + ch, &accP0, &accP1, &fin0, &fin1); + else grid16_gemv_fixup(T, ch, y, c0 + ch, &accQ0, &accQ1, &fin0, &fin1); + p ^= 1; + } + } + const int64_t r0 = it*MR; + for (int r = 0; r < 4; r++) { + if (r0 + r < m) C[r0 + r] = fin0[r]; + if (r0 + 4 + r < m) C[r0 + 4 + r] = fin1[r]; + } + } +} + + + +#define GRID_ONESHOT_C(NAME, BLKA, REPACK, GEMM, GEMV, ASZ, BSZ, PACKB, VARIANT) \ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ float * C, int64_t ldc, int ith, int nth) { \ @@ -857,11 +1042,15 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ while scalar row-partitions). Row-partition with the \ cached pack instead; the full activation pack is tiny \ at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + if (n == 1) { \ + GEMV(m, k, PA, B, (float *)C, ith, nth); \ + } else { \ void * PBs = aligned_alloc(64, BSZ(n, k)); \ if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ PACKB(B, ldb, n, k, PBs); \ GEMM(m, n, k, PA, PBs, C, ldc, ith, nth); \ free(PBs); \ + } \ } else { \ const int64_t jpt = (njt + nth - 1) / nth; \ const int64_t jt0 = (int64_t)ith*jpt; \ @@ -895,15 +1084,15 @@ extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ free(PB); \ } \ } -GRID_ONESHOT_C(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 10) -GRID_ONESHOT_C(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 11) -GRID_ONESHOT_C(gemm_iq2_xxs_q8_K_ppc, block_iq2_xxs_ppc, grid_repack_iq2_xxs, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 12) -GRID_ONESHOT_C(gemm_iq3_xxs_q8_K_ppc, block_iq3_xxs_ppc, grid_repack_iq3_xxs, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 13) -GRID_ONESHOT_C(gemm_iq3_s_q8_K_ppc, block_iq3_s_ppc, grid_repack_iq3_s, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 14) -GRID_ONESHOT_C(gemm_iq1_s_q8_K_ppc, block_iq1_s_ppc, grid_repack_iq1_s, grid_gemm_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 15) -GRID_ONESHOT_C(gemm_iq2_xs_q8_K_ppc, block_iq2_xs_ppc, grid16_repack_iq2_xs, grid16_gemm_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 16) -GRID_ONESHOT_C(gemm_iq2_s_q8_K_ppc, block_iq2_s_ppc, grid16_repack_iq2_s, grid16_gemm_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 17) -GRID_ONESHOT_C(gemm_iq1_m_q8_K_ppc, block_iq1_m_ppc, grid16_repack_iq1_m, grid16_gemm_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 18) +GRID_ONESHOT_C(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 10) +GRID_ONESHOT_C(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 11) +GRID_ONESHOT_C(gemm_iq2_xxs_q8_K_ppc, block_iq2_xxs_ppc, grid_repack_iq2_xxs, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 12) +GRID_ONESHOT_C(gemm_iq3_xxs_q8_K_ppc, block_iq3_xxs_ppc, grid_repack_iq3_xxs, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 13) +GRID_ONESHOT_C(gemm_iq3_s_q8_K_ppc, block_iq3_s_ppc, grid_repack_iq3_s, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 14) +GRID_ONESHOT_C(gemm_iq1_s_q8_K_ppc, block_iq1_s_ppc, grid_repack_iq1_s, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 15) +GRID_ONESHOT_C(gemm_iq2_xs_q8_K_ppc, block_iq2_xs_ppc, grid16_repack_iq2_xs, grid16_gemm_packed, grid16_gemv_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 16) +GRID_ONESHOT_C(gemm_iq2_s_q8_K_ppc, block_iq2_s_ppc, grid16_repack_iq2_s, grid16_gemm_packed, grid16_gemv_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 17) +GRID_ONESHOT_C(gemm_iq1_m_q8_K_ppc, block_iq1_m_ppc, grid16_repack_iq1_m, grid16_gemm_packed, grid16_gemv_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 18) extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, const void * Av, int64_t lda, const void * Bv, int64_t ldb, diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index da482f6e7018..790cdbf94a4d 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -4131,7 +4131,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) - if (n < 8) + if (n > 1 && n < 8) return false; // packed path reads int8-expanded weights; // below one column tile vec_dot wins the // bandwidth race on silicon (patch 0015) @@ -4147,7 +4147,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) - if (n < 8) + if (n > 1 && n < 8) return false; // packed path reads int8-expanded weights; // below one column tile vec_dot wins the // bandwidth race on silicon (patch 0015) @@ -4163,7 +4163,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) - if (n < 8) + if (n > 1 && n < 8) return false; // packed path reads int8-expanded weights; // below one column tile vec_dot wins the // bandwidth race on silicon (patch 0015) @@ -4179,7 +4179,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 if (Btype != GGML_TYPE_Q8_K) return false; #if defined(__MMA__) - if (n < 8) + if (n > 1 && n < 8) return false; // packed path reads int8-expanded weights; // below one column tile vec_dot wins the // bandwidth race on silicon (patch 0015)