Add RVV qs8/qu8 gavgpool kernels, configs and tests.

cmake/gen/rvv_microkernels.cmake

@@ -54,9 +54,13 @@ SET(PROD_RVV_MICROKERNEL_SRCS
   src/f32-vrnd/gen/f32-vrndu-rvv-u4v.c
   src/f32-vrnd/gen/f32-vrndz-rvv-u4v.c
   src/f32-vrsqrt/gen/f32-vrsqrt-rvv-rsqrt-u4v.c
+  src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u2v.c
+  src/qs8-gavgpool/gen/qs8-gavgpool-7x-minmax-fp32-rvv-u2v.c
   src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u2v.c
   src/qs8-vmul/gen/qs8-vmul-minmax-f32-rvv-u2v.c
   src/qs8-vmulc/gen/qs8-vmulc-minmax-f32-rvv-u2v.c
+  src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-rvv-u2v.c
+  src/qu8-gavgpool/gen/qu8-gavgpool-7x-minmax-fp32-rvv-u2v.c
   src/qu8-f32-vcvt/gen/qu8-f32-vcvt-rvv-u2v.c
   src/qu8-vmul/gen/qu8-vmul-minmax-f32-rvv-u2v.c
   src/qu8-vmulc/gen/qu8-vmulc-minmax-f32-rvv-u2v.c
@@ -180,9 +184,13 @@ SET(NON_PROD_RVV_MICROKERNEL_SRCS
   src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-6x4v-minmax-rvv.c
   src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-7x4v-minmax-rvv.c
   src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-8x4v-minmax-rvv.c
+  src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c
+  src/qs8-gavgpool/gen/qs8-gavgpool-7x-minmax-fp32-rvv-u1v.c
   src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u1v.c
   src/qs8-vmul/gen/qs8-vmul-minmax-f32-rvv-u1v.c
   src/qs8-vmulc/gen/qs8-vmulc-minmax-f32-rvv-u1v.c
+  src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c
+  src/qu8-gavgpool/gen/qu8-gavgpool-7x-minmax-fp32-rvv-u1v.c
   src/qu8-f32-vcvt/gen/qu8-f32-vcvt-rvv-u1v.c
   src/qu8-vmul/gen/qu8-vmul-minmax-f32-rvv-u1v.c
   src/qu8-vmulc/gen/qu8-vmulc-minmax-f32-rvv-u1v.c

gen/rvv_microkernels.bzl

@@ -50,9 +50,13 @@ PROD_RVV_MICROKERNEL_SRCS = [
     "src/f32-vrnd/gen/f32-vrndu-rvv-u4v.c",
     "src/f32-vrnd/gen/f32-vrndz-rvv-u4v.c",
     "src/f32-vrsqrt/gen/f32-vrsqrt-rvv-rsqrt-u4v.c",
+    "src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u2v.c",
+    "src/qs8-gavgpool/gen/qs8-gavgpool-7x-minmax-fp32-rvv-u2v.c",
     "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u2v.c",
     "src/qs8-vmul/gen/qs8-vmul-minmax-f32-rvv-u2v.c",
     "src/qs8-vmulc/gen/qs8-vmulc-minmax-f32-rvv-u2v.c",
+    "src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-rvv-u2v.c",
+    "src/qu8-gavgpool/gen/qu8-gavgpool-7x-minmax-fp32-rvv-u2v.c",
     "src/qu8-f32-vcvt/gen/qu8-f32-vcvt-rvv-u2v.c",
     "src/qu8-vmul/gen/qu8-vmul-minmax-f32-rvv-u2v.c",
     "src/qu8-vmulc/gen/qu8-vmulc-minmax-f32-rvv-u2v.c",
@@ -177,9 +181,13 @@ NON_PROD_RVV_MICROKERNEL_SRCS = [
     "src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-6x4v-minmax-rvv.c",
     "src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-7x4v-minmax-rvv.c",
     "src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-8x4v-minmax-rvv.c",
+    "src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c",
+    "src/qs8-gavgpool/gen/qs8-gavgpool-7x-minmax-fp32-rvv-u1v.c",
     "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u1v.c",
     "src/qs8-vmul/gen/qs8-vmul-minmax-f32-rvv-u1v.c",
     "src/qs8-vmulc/gen/qs8-vmulc-minmax-f32-rvv-u1v.c",
+    "src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c",
+    "src/qu8-gavgpool/gen/qu8-gavgpool-7x-minmax-fp32-rvv-u1v.c",
     "src/qu8-f32-vcvt/gen/qu8-f32-vcvt-rvv-u1v.c",
     "src/qu8-vmul/gen/qu8-vmul-minmax-f32-rvv-u1v.c",
     "src/qu8-vmulc/gen/qu8-vmulc-minmax-f32-rvv-u1v.c",

scripts/generate-qs8-gavgpool.sh

@@ -168,4 +168,18 @@ tools/xngen src/qs8-gavgpool/multipass-sse4.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7
 tools/xngen src/qs8-gavgpool/multipass-sse4.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7 -D CHANNEL_TILE=16 -D REQUANTIZATION=FP32 -D DATATYPE=QU8 -o src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-sse41-c16.c &
 tools/xngen src/qs8-gavgpool/multipass-sse4.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7 -D CHANNEL_TILE=24 -D REQUANTIZATION=FP32 -D DATATYPE=QU8 -o src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-sse41-c24.c &
 
+
+################################ RISC-V Vector ################################
+tools/xngen src/qs8-gavgpool/unipass-rvv.c.in -D ROW_TILE=7 -D REQUANTIZATION=FP32 -D LMUL=1 -D DATATYPE=QS8 -o src/qs8-gavgpool/gen/qs8-gavgpool-7x-minmax-fp32-rvv-u1v.c &
+tools/xngen src/qs8-gavgpool/unipass-rvv.c.in -D ROW_TILE=7 -D REQUANTIZATION=FP32 -D LMUL=2 -D DATATYPE=QS8 -o src/qs8-gavgpool/gen/qs8-gavgpool-7x-minmax-fp32-rvv-u2v.c &
+
+tools/xngen src/qs8-gavgpool/unipass-rvv.c.in -D ROW_TILE=7 -D REQUANTIZATION=FP32 -D LMUL=1 -D DATATYPE=QU8 -o src/qu8-gavgpool/gen/qu8-gavgpool-7x-minmax-fp32-rvv-u1v.c &
+tools/xngen src/qs8-gavgpool/unipass-rvv.c.in -D ROW_TILE=7 -D REQUANTIZATION=FP32 -D LMUL=2 -D DATATYPE=QU8 -o src/qu8-gavgpool/gen/qu8-gavgpool-7x-minmax-fp32-rvv-u2v.c &
+
+tools/xngen src/qs8-gavgpool/multipass-rvv.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7 -D REQUANTIZATION=FP32 -D LMUL=1 -D DATATYPE=QS8 -o src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c &
+tools/xngen src/qs8-gavgpool/multipass-rvv.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7 -D REQUANTIZATION=FP32 -D LMUL=2 -D DATATYPE=QS8 -o src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u2v.c &
+
+tools/xngen src/qs8-gavgpool/multipass-rvv.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7 -D REQUANTIZATION=FP32 -D LMUL=1 -D DATATYPE=QU8 -o src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c &
+tools/xngen src/qs8-gavgpool/multipass-rvv.c.in -D ROW_TILE=7 -D ROW_SUBTILE=7 -D REQUANTIZATION=FP32 -D LMUL=2 -D DATATYPE=QU8 -o src/qu8-gavgpool/gen/qu8-gavgpool-7p7x-minmax-fp32-rvv-u2v.c &
+
 wait

src/configs/gavgpool-config.c

@@ -193,6 +193,13 @@ static void init_qs8_gavgpool_config(void) {
     qs8_gavgpool_config.update.qs8 = xnn_update_qs8_avgpool_minmax_fp32_scalar_imagic_params;
     qs8_gavgpool_config.row_tile = 7;
     qs8_gavgpool_config.channel_tile = 4;
+  #elif XNN_ARCH_RISCV && XNN_ENABLE_RISCV_VECTOR
+    qs8_gavgpool_config.unipass = (xnn_gavgpool_unipass_ukernel_fn) xnn_qs8_gavgpool_minmax_fp32_ukernel_7x__rvv_u2v;
+    qs8_gavgpool_config.multipass = (xnn_gavgpool_multipass_ukernel_fn) xnn_qs8_gavgpool_minmax_fp32_ukernel_7p7x__rvv_u2v;
+    qs8_gavgpool_config.init.qs8 = xnn_init_qs8_avgpool_minmax_fp32_scalar_fmagic_params;
+    qs8_gavgpool_config.update.qs8 = xnn_update_qs8_avgpool_minmax_fp32_scalar_fmagic_params;
+    qs8_gavgpool_config.row_tile = 7;
+    qs8_gavgpool_config.channel_tile = 32;
   #else
     qs8_gavgpool_config.unipass = (xnn_gavgpool_unipass_ukernel_fn) xnn_qs8_gavgpool_minmax_fp32_ukernel_7x__scalar_imagic_c1;
     qs8_gavgpool_config.multipass = (xnn_gavgpool_multipass_ukernel_fn) xnn_qs8_gavgpool_minmax_fp32_ukernel_7p7x__scalar_imagic_c1;
@@ -261,6 +268,13 @@ static void init_qu8_gavgpool_config(void) {
     qu8_gavgpool_config.update.qu8 = xnn_update_qu8_avgpool_minmax_fp32_scalar_imagic_params;
     qu8_gavgpool_config.row_tile = 7;
     qu8_gavgpool_config.channel_tile = 4;
+  #elif XNN_ARCH_RISCV && XNN_ENABLE_RISCV_VECTOR
+    qu8_gavgpool_config.unipass = (xnn_gavgpool_unipass_ukernel_fn) xnn_qu8_gavgpool_minmax_fp32_ukernel_7x__rvv_u2v;
+    qu8_gavgpool_config.multipass = (xnn_gavgpool_multipass_ukernel_fn) xnn_qu8_gavgpool_minmax_fp32_ukernel_7p7x__rvv_u2v;
+    qu8_gavgpool_config.init.qu8 = xnn_init_qu8_avgpool_minmax_fp32_scalar_fmagic_params;
+    qu8_gavgpool_config.update.qu8 = xnn_update_qu8_avgpool_minmax_fp32_scalar_fmagic_params;
+    qu8_gavgpool_config.row_tile = 7;
+    qu8_gavgpool_config.channel_tile = 32;
   #else
     qu8_gavgpool_config.unipass = (xnn_gavgpool_unipass_ukernel_fn) xnn_qu8_gavgpool_minmax_fp32_ukernel_7x__scalar_imagic_c1;
     qu8_gavgpool_config.multipass = (xnn_gavgpool_multipass_ukernel_fn) xnn_qu8_gavgpool_minmax_fp32_ukernel_7p7x__scalar_imagic_c1;

src/qs8-gavgpool/gen/qs8-gavgpool-7p7x-minmax-fp32-rvv-u1v.c

@@ -0,0 +1,175 @@
+// Auto-generated file. Do not edit!
+//   Template: src/qs8-gavgpool/multipass-rvv.c.in
+//   Generator: tools/xngen
+//
+// Copyright 2024 Imagination Technologies, inc.
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <riscv_vector.h>
+
+#include "xnnpack/gavgpool.h"
+#include "xnnpack/math.h"
+
+
+void xnn_qs8_gavgpool_minmax_fp32_ukernel_7p7x__rvv_u1v(
+    size_t rows,
+    size_t channels,
+    const int8_t* input,
+    size_t input_stride,
+    const int8_t* zero,
+    int32_t* buffer,
+    int8_t* output,
+    const union xnn_qs8_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+  assert(rows > 7);
+  assert(channels != 0);
+
+  const int8_t* i0 = input;
+  const int8_t* i1 = (const int8_t*) ((uintptr_t) i0 + input_stride);
+  if XNN_UNPREDICTABLE(rows < 2) {
+    i1 = zero;
+  }
+  const int8_t* i2 = (const int8_t*) ((uintptr_t) i1 + input_stride);
+  if XNN_UNPREDICTABLE(rows <= 2) {
+    i2 = zero;
+  }
+  const int8_t* i3 = (const int8_t*) ((uintptr_t) i2 + input_stride);
+  if XNN_UNPREDICTABLE(rows < 4) {
+    i3 = zero;
+  }
+  const int8_t* i4 = (const int8_t*) ((uintptr_t) i3 + input_stride);
+  if XNN_UNPREDICTABLE(rows <= 4) {
+    i4 = zero;
+  }
+  const int8_t* i5 = (const int8_t*) ((uintptr_t) i4 + input_stride);
+  if XNN_UNPREDICTABLE(rows < 6) {
+    i5 = zero;
+  }
+  const int8_t* i6 = (const int8_t*) ((uintptr_t) i5 + input_stride);
+  if XNN_UNPREDICTABLE(rows <= 6) {
+    i6 = zero;
+  }
+  const size_t input_increment = 7 * input_stride - channels;
+
+  const int32_t init_bias = params->fp32_scalar_fmagic.init_bias;
+  int32_t* b = buffer;
+  int32_t c = (int32_t) channels;
+  do {
+    int32_t n = __riscv_vsetvl_e8m1(c); c -= n;
+    vint8m1_t i0_i8v = __riscv_vle8_v_i8m1(i0, n); i0 += n;
+    vint8m1_t i1_i8v = __riscv_vle8_v_i8m1(i1, n); i1 += n;
+    vint16m2_t acc_i16v = __riscv_vwadd_vv_i16m2(i0_i8v, i1_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i2, n); i2 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i3, n); i3 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i4, n); i4 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i5, n); i5 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i6, n); i6 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+
+    vint32m4_t acc_i32v = __riscv_vwadd_vx_i32m4(acc_i16v, init_bias, n);
+    __riscv_vse32_v_i32m4(b, acc_i32v, n); b += n;
+  } while (c > 0);
+
+  for (rows -= 7; rows > 7; rows -= 7) {
+    i0 = (const int8_t*) ((uintptr_t) i0 + input_increment);
+    i1 = (const int8_t*) ((uintptr_t) i1 + input_increment);
+    i2 = (const int8_t*) ((uintptr_t) i2 + input_increment);
+    i3 = (const int8_t*) ((uintptr_t) i3 + input_increment);
+    i4 = (const int8_t*) ((uintptr_t) i4 + input_increment);
+    i5 = (const int8_t*) ((uintptr_t) i5 + input_increment);
+    i6 = (const int8_t*) ((uintptr_t) i6 + input_increment);
+
+    int32_t* b = buffer;
+    int32_t c = (int32_t) channels;
+    do {
+      int32_t n = __riscv_vsetvl_e8m1(c); c -= n;
+      vint8m1_t i0_i8v = __riscv_vle8_v_i8m1(i0, n); i0 += n;
+      vint8m1_t i1_i8v = __riscv_vle8_v_i8m1(i1, n); i1 += n;
+      vint16m2_t acc_i16v = __riscv_vwadd_vv_i16m2(i0_i8v, i1_i8v, n);
+      i0_i8v = __riscv_vle8_v_i8m1(i2, n); i2 += n;
+      acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+      i0_i8v = __riscv_vle8_v_i8m1(i3, n); i3 += n;
+      acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+      i0_i8v = __riscv_vle8_v_i8m1(i4, n); i4 += n;
+      acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+      i0_i8v = __riscv_vle8_v_i8m1(i5, n); i5 += n;
+      acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+      i0_i8v = __riscv_vle8_v_i8m1(i6, n); i6 += n;
+      acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+
+      vint32m4_t acc_i32v = __riscv_vle32_v_i32m4(b, n);
+      acc_i32v = __riscv_vwadd_wv_i32m4(acc_i32v, acc_i16v, n);
+      __riscv_vse32_v_i32m4(b, acc_i32v, n); b += n;
+    } while (c > 0);
+  }
+
+  i0 = (const int8_t*) ((uintptr_t) i0 + input_increment);
+  i1 = (const int8_t*) ((uintptr_t) i1 + input_increment);
+  if XNN_UNPREDICTABLE(rows < 2) {
+    i1 = zero;
+  }
+  i2 = (const int8_t*) ((uintptr_t) i2 + input_increment);
+  if XNN_UNPREDICTABLE(rows <= 2) {
+    i2 = zero;
+  }
+  i3 = (const int8_t*) ((uintptr_t) i3 + input_increment);
+  if XNN_UNPREDICTABLE(rows < 4) {
+    i3 = zero;
+  }
+  i4 = (const int8_t*) ((uintptr_t) i4 + input_increment);
+  if XNN_UNPREDICTABLE(rows <= 4) {
+    i4 = zero;
+  }
+  i5 = (const int8_t*) ((uintptr_t) i5 + input_increment);
+  if XNN_UNPREDICTABLE(rows < 6) {
+    i5 = zero;
+  }
+  i6 = (const int8_t*) ((uintptr_t) i6 + input_increment);
+  if XNN_UNPREDICTABLE(rows <= 6) {
+    i6 = zero;
+  }
+
+  const float scale = params->fp32_scalar_fmagic.scale;
+  const float output_min_less_zero_point = params->fp32_scalar_fmagic.output_min_less_zero_point;
+  const float output_max_less_zero_point = params->fp32_scalar_fmagic.output_max_less_zero_point;
+  const float magic_bias = params->fp32_scalar_fmagic.magic_bias;
+  const int32_t magic_bias_less_output_zero_point = params->fp32_scalar_fmagic.magic_bias_less_output_zero_point;
+
+  do {
+    int32_t n = __riscv_vsetvl_e8m1(channels); channels -= n;
+    vint8m1_t i0_i8v = __riscv_vle8_v_i8m1(i0, n); i0 += n;
+    vint8m1_t i1_i8v = __riscv_vle8_v_i8m1(i1, n); i1 += n;
+    vint16m2_t acc_i16v = __riscv_vwadd_vv_i16m2(i0_i8v, i1_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i2, n); i2 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i3, n); i3 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i4, n); i4 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i5, n); i5 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+    i0_i8v = __riscv_vle8_v_i8m1(i6, n); i6 += n;
+    acc_i16v = __riscv_vwadd_wv_i16m2(acc_i16v, i0_i8v, n);
+
+    vint32m4_t acc_i32v = __riscv_vle32_v_i32m4(buffer, n); buffer += n;
+    acc_i32v = __riscv_vwadd_wv_i32m4(acc_i32v, acc_i16v, n);
+    vfloat32m4_t acc_f32v = __riscv_vfcvt_f_x_v_f32m4(acc_i32v, n);
+    acc_f32v = __riscv_vfmul_vf_f32m4(acc_f32v, scale, n);
+    acc_f32v = __riscv_vfmin_vf_f32m4(__riscv_vfmax_vf_f32m4(acc_f32v, output_min_less_zero_point, n), output_max_less_zero_point, n);
+    acc_f32v = __riscv_vfadd_vf_f32m4(acc_f32v, magic_bias, n);
+
+    vint32m4_t out_i32v = __riscv_vfcvt_x_f_v_i32m4(acc_f32v, n);
+    vint16m2_t out_i16v = __riscv_vncvt_x_x_w_i16m2(out_i32v, n);
+    out_i16v = __riscv_vsub_vx_i16m2(out_i16v, magic_bias_less_output_zero_point, n);
+    vint8m1_t out_i8v = __riscv_vncvt_x_x_w_i8m1(out_i16v, n);
+    __riscv_vse8_v_i8m1(output, out_i8v, n); output += n;
+  } while (channels != 0);
+}