From 5728ad8673682f0fa95f61ae2a660f6f14534380 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 17 Jul 2026 03:40:33 +0000 Subject: [PATCH 1/2] chore: update llama.cpp to b10054 (submodule ref) --- third_party/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/llama.cpp b/third_party/llama.cpp index 99f3dc322..ac2557cb2 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit 99f3dc32296f825fec94f202da1e9fede1e78cf9 +Subproject commit ac2557cb24def295888ef47f1a35b401d978c510 From f10ab9677dc2cc125fda3f1eca76e9f485f5e503 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 17 Jul 2026 03:43:07 +0000 Subject: [PATCH 2/2] chore(sync): update cpp/ directory after llama.cpp b10054 bootstrap --- cpp/common/build-info.cpp | 4 +- cpp/common/chat-auto-parser-generator.cpp | 7 +- cpp/common/chat-auto-parser.h | 7 +- cpp/common/chat-diff-analyzer.cpp | 34 +- cpp/common/common.h | 15 + cpp/common/jinja/value.cpp | 39 + cpp/common/speculative.cpp | 5 +- cpp/ggml-backend-meta.cpp | 5 + cpp/ggml-cpu.h | 1 + cpp/ggml-cpu/arch/arm/cpu-feats.cpp | 5 + cpp/ggml-cpu/ggml-cpu.c | 32 +- cpp/ggml-cpu/ggml-cpu.cpp | 8 +- cpp/ggml-cpu/ops.cpp | 390 ++++++++- cpp/ggml-cpu/ops.h | 3 + cpp/ggml-hexagon/ggml-hexagon.cpp | 325 +++++--- cpp/ggml-hexagon/htp-drv.cpp | 47 +- cpp/ggml-hexagon/htp-drv.h | 8 +- cpp/ggml-hexagon/htp/CMakeLists.txt | 9 +- cpp/ggml-hexagon/htp/act-ops.c | 11 +- cpp/ggml-hexagon/htp/argsort-ops.c | 4 +- cpp/ggml-hexagon/htp/binary-ops.c | 1 + cpp/ggml-hexagon/htp/cumsum-ops.c | 11 +- cpp/ggml-hexagon/htp/dma-queue.c | 106 +++ cpp/ggml-hexagon/htp/dma-queue.h | 396 +++++++++ cpp/ggml-hexagon/htp/flash-attn-ops.c | 44 +- cpp/ggml-hexagon/htp/hex-bitmap.h | 24 + cpp/ggml-hexagon/htp/hex-dma.h | 377 +-------- cpp/ggml-hexagon/htp/hex-profile.h | 10 +- cpp/ggml-hexagon/htp/hex-utils.h | 31 +- cpp/ggml-hexagon/htp/hmx-mm-kernels-tiled.h | 81 +- cpp/ggml-hexagon/htp/hmx-queue.c | 59 +- cpp/ggml-hexagon/htp/hmx-queue.h | 41 +- cpp/ggml-hexagon/htp/htp-ctx.h | 23 +- cpp/ggml-hexagon/htp/htp-ops.h | 11 +- cpp/ggml-hexagon/htp/htp-tensor.c | 204 +++++ cpp/ggml-hexagon/htp/htp-tensor.h | 36 + cpp/ggml-hexagon/htp/main.c | 765 +++++++++++------- cpp/ggml-hexagon/htp/matmul-ops.c | 467 +++++++++-- cpp/ggml-hexagon/htp/matmul-ops.h | 132 +++ cpp/ggml-hexagon/htp/rope-ops.c | 11 +- cpp/ggml-hexagon/htp/unary-ops.c | 15 +- cpp/ggml-hexagon/htp/work-queue.c | 244 ++++++ cpp/ggml-hexagon/htp/work-queue.h | 38 + cpp/ggml-metal/ggml-metal-device.cpp | 27 + cpp/ggml-metal/ggml-metal-device.h | 1 + cpp/ggml-metal/ggml-metal-device.m | 8 +- cpp/ggml-metal/ggml-metal-impl.h | 8 + cpp/ggml-metal/ggml-metal-ops.cpp | 101 +++ cpp/ggml-metal/ggml-metal-ops.h | 1 + cpp/ggml-metal/ggml-metal.metal | 234 ++++++ cpp/ggml-opencl/ggml-opencl.cpp | 204 +++-- cpp/ggml-opencl/kernels/flash_attn_f32_f16.cl | 4 + .../kernels/flash_attn_f32_q4_0.cl | 4 + .../kernels/flash_attn_f32_q8_0.cl | 4 + .../kernels/gemm_moe_mxfp4_f32_ns.cl | 10 +- .../kernels/gemv_noshuffle_iq4_nl_f32.cl | 7 +- .../kernels/gemv_noshuffle_q1_0_f32.cl | 6 +- .../kernels/gemv_noshuffle_q4_0_f32.cl | 7 +- .../kernels/gemv_noshuffle_q4_0_f32_spec.cl | 6 +- .../kernels/gemv_noshuffle_q4_1_f32.cl | 7 +- .../kernels/gemv_noshuffle_q4_k_f32.cl | 7 +- .../kernels/gemv_noshuffle_q5_0_f32.cl | 7 +- .../kernels/gemv_noshuffle_q5_1_f32.cl | 7 +- .../kernels/gemv_noshuffle_q5_k_f32.cl | 7 +- .../kernels/gemv_noshuffle_q6_k_f32.cl | 7 +- .../kernels/gemv_noshuffle_q8_0_f32.cl | 6 +- cpp/ggml-opencl/kernels/mul_mv_f16_f16.cl | 9 +- cpp/ggml-opencl/kernels/mul_mv_f16_f32.cl | 9 +- .../kernels/mul_mv_f16_f32_1row.cl | 9 +- cpp/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl | 4 + cpp/ggml-opencl/kernels/mul_mv_q4_k_f32.cl | 2 + cpp/ggml.c | 171 +++- cpp/ggml.h | 46 ++ cpp/gguf.cpp | 5 + cpp/gguf.h | 13 +- cpp/llama-arch.cpp | 1 + cpp/llama-arch.h | 1 + cpp/llama-context.cpp | 31 + cpp/llama-cparams.h | 4 + cpp/llama-graph.h | 3 + cpp/llama-kv-cache-dsv4.cpp | 100 ++- cpp/llama-kv-cache-dsv4.h | 25 +- cpp/llama-model.cpp | 26 +- cpp/models/deepseek4.cpp | 83 +- cpp/models/hy-v3.cpp | 390 +++++++++ cpp/models/minimax-m2.cpp | 2 + cpp/models/models.h | 21 +- example/ios/Podfile.lock | 8 +- src/version.ts | 4 +- 89 files changed, 4466 insertions(+), 1267 deletions(-) create mode 100644 cpp/ggml-hexagon/htp/dma-queue.c create mode 100644 cpp/ggml-hexagon/htp/dma-queue.h create mode 100644 cpp/ggml-hexagon/htp/hex-bitmap.h create mode 100644 cpp/ggml-hexagon/htp/htp-tensor.c create mode 100644 cpp/ggml-hexagon/htp/htp-tensor.h create mode 100644 cpp/ggml-hexagon/htp/work-queue.c create mode 100644 cpp/ggml-hexagon/htp/work-queue.h create mode 100644 cpp/models/hy-v3.cpp diff --git a/cpp/common/build-info.cpp b/cpp/common/build-info.cpp index 16be7f7a2..adbb5ce56 100644 --- a/cpp/common/build-info.cpp +++ b/cpp/common/build-info.cpp @@ -3,8 +3,8 @@ #include #include -int LLAMA_BUILD_NUMBER = 9982; -char const * LLAMA_COMMIT = "99f3dc3"; +int LLAMA_BUILD_NUMBER = 10054; +char const * LLAMA_COMMIT = "ac2557c"; char const * LLAMA_COMPILER = "unknown"; char const * LLAMA_BUILD_TARGET = "unknown"; diff --git a/cpp/common/chat-auto-parser-generator.cpp b/cpp/common/chat-auto-parser-generator.cpp index 36aab7ecb..3f91364c5 100644 --- a/cpp/common/chat-auto-parser-generator.cpp +++ b/cpp/common/chat-auto-parser-generator.cpp @@ -147,7 +147,8 @@ common_peg_arena autoparser::build_parser(const generation_params & inputs, cons } else { parser = content.build_parser(ctx); } - return pure_content ? p.prefix(generation_prompt, reasoning.start) + parser : p.prefix(generation_prompt, reasoning.start) << parser; + const std::string reasoning_start = trim_whitespace(reasoning.start); + return pure_content ? p.prefix(generation_prompt, reasoning_start) + parser : p.prefix(generation_prompt, reasoning_start) << parser; }); } @@ -261,6 +262,10 @@ common_peg_parser analyze_tools::build_func_parser(common_chat_peg_builder & p, bool matched_atomic = false; common_peg_parser func_parser = p.eps(); + if (!function.args_separator.empty()) { + open = open + p.space() + p.literal(function.args_separator); + } + if (!function.name_suffix.empty()) { func_parser = open + call_id_section + p.space() + args; matched_atomic = true; diff --git a/cpp/common/chat-auto-parser.h b/cpp/common/chat-auto-parser.h index 9e8113f24..d47b09dcf 100644 --- a/cpp/common/chat-auto-parser.h +++ b/cpp/common/chat-auto-parser.h @@ -192,9 +192,10 @@ struct tool_format_analysis { }; struct tool_function_analysis { - std::string name_prefix; // e.g., "", "\"", ":0" - std::string close; // e.g., "", "" (for tag-based) + std::string name_prefix; // e.g., "", "\"", ":0" + std::string args_separator; // e.g., "" (marker between function name and arguments) + std::string close; // e.g., "", "" (for tag-based) }; struct tool_arguments_analysis { diff --git a/cpp/common/chat-diff-analyzer.cpp b/cpp/common/chat-diff-analyzer.cpp index b166ee5a1..127278dfb 100644 --- a/cpp/common/chat-diff-analyzer.cpp +++ b/cpp/common/chat-diff-analyzer.cpp @@ -124,16 +124,16 @@ static std::vector"); analysis.preserved_tokens.push_back(""); analysis.preserved_tokens.push_back(""); analysis.preserved_tokens.push_back(""); @@ -259,6 +259,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) { LOG_DBG("per_call_end: '%s'\n", tools.format.per_call_end.c_str()); LOG_DBG("func_name_prefix: '%s'\n", tools.function.name_prefix.c_str()); LOG_DBG("func_name_suffix: '%s'\n", tools.function.name_suffix.c_str()); + LOG_DBG("func_args_separator: '%s'\n", tools.function.args_separator.c_str()); LOG_DBG("func_close: '%s'\n", tools.function.close.c_str()); LOG_DBG("call_id_prefix: '%s'\n", tools.call_id.prefix.c_str()); LOG_DBG("call_id_suffix: '%s'\n", tools.call_id.suffix.c_str()); @@ -302,6 +303,7 @@ void autoparser::collect_preserved_tokens() { add_token(tools.format.per_call_end); add_token(tools.function.name_prefix); add_token(tools.function.name_suffix); + add_token(tools.function.args_separator); add_token(tools.function.close); add_token(tools.arguments.start); add_token(tools.arguments.end); @@ -1051,6 +1053,23 @@ void analyze_tools::check_per_call_markers() { format.section_start.clear(); format.section_end.clear(); } + + if (!format.per_call_end.empty()) { + auto count_occurrences = [](const std::string & haystack, const std::string & needle) { + size_t count = 0; + for (size_t pos = haystack.find(needle); pos != std::string::npos; + pos = haystack.find(needle, pos + needle.size())) { + count++; + } + return count; + }; + size_t calls_one = count_occurrences(one_vs_two->output_A, format.per_call_end); + size_t calls_two = count_occurrences(one_vs_two->output_B, format.per_call_end); + if (calls_one > 0 && calls_one == calls_two) { + format.section_end = format.per_call_end; + format.per_call_end.clear(); + } + } } void analyze_tools::extract_function_markers() { @@ -1132,6 +1151,17 @@ void analyze_tools::extract_function_markers() { auto suf_result = suffix_parser.parse_and_extract(diff.suffix); if (suf_result.result.success()) { function.name_suffix += suf_result.tags["ext"]; + + auto arg_start = [&](common_peg_parser_builder &p) { + return p.marker() + p.space() + p.choice({ p.literal(ARG_FIRST), p.literal(ARG_SECOND) }); + }; + auto sep_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) { + return p.tag("sep", p.zero_or_more(p.negate(arg_start(p)) + p.any())) + arg_start(p); + }); + auto sep_result = sep_parser.parse_and_extract(diff.suffix.substr(suf_result.tags["ext"].size())); + if (sep_result.result.success()) { + function.args_separator = trim_whitespace(sep_result.tags["sep"]); + } } } diff --git a/cpp/common/common.h b/cpp/common/common.h index 231e3a0e8..5181ed969 100644 --- a/cpp/common/common.h +++ b/cpp/common/common.h @@ -105,6 +105,7 @@ enum llama_example { LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, LLAMA_EXAMPLE_DOWNLOAD, + LLAMA_EXAMPLE_TOKENIZE, LLAMA_EXAMPLE_COUNT, }; @@ -635,6 +636,14 @@ struct common_params { std::string api_prefix = ""; // NOLINT std::string chat_template = ""; // NOLINT bool use_jinja = true; // NOLINT + + // server CORS params + std::string cors_origins = "*"; + std::string cors_methods = "GET, POST, DELETE, OPTIONS"; + std::string cors_headers = "*"; + bool cors_credentials = true; + bool cors_origins_explicit = false; // for --agent option + bool enable_chat_template = true; bool force_pure_content_parser = false; common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; @@ -721,6 +730,12 @@ struct common_params { // batched-bench params bool batched_bench_output_jsonl = false; + // tokenize params + bool tokenize_ids = false; // if true, only print the token IDs + bool tokenize_stdin = false; // if true, read the prompt from stdin + bool tokenize_no_bos = false; // if true, do not add the BOS token + bool tokenize_show_count = false; // if true, print the total token count + // common params std::string out_file; // output filename for all example programs // optional callback for model loading progress and cancellation: diff --git a/cpp/common/jinja/value.cpp b/cpp/common/jinja/value.cpp index a2852acfa..fd59bd627 100644 --- a/cpp/common/jinja/value.cpp +++ b/cpp/common/jinja/value.cpp @@ -750,11 +750,50 @@ const func_builtins & value_string_t::get_builtins() const { res->val_str.mark_input_based_on(args.get_pos(0)->val_str); return res; }}, + {"format", [](const func_args & args) -> value { + value val_input = args.get_pos(0); + if (!is_val(val_input)) { + throw raised_exception("format() first argument must be a string"); + } + const jinja::string & fmt = val_input->as_string(); + const bool fmt_is_input = fmt.all_parts_are_input(); + + const std::string str = fmt.str(); + jinja::string result; + std::string literal; + auto flush_literal = [&]() { + if (!literal.empty()) { + result.parts.push_back({fmt_is_input, literal}); + literal.clear(); + } + }; + + size_t arg_idx = 1; // positional args follow the format string + for (size_t i = 0; i < str.size(); ++i) { + if (str[i] != '{') { + literal += str[i]; + continue; + } + if (i + 1 >= str.size() || str[i + 1] != '}') { + throw not_implemented_exception("format() only supports simple '{}' placeholders"); + } + ++i; + flush_literal(); + const jinja::string arg_str = args.get_pos(arg_idx++)->as_string(); + result.parts.insert(result.parts.end(), arg_str.parts.begin(), arg_str.parts.end()); + } + flush_literal(); + return mk_val(result); + }}, {"int", [](const func_args & args) -> value { value val_input = args.get_pos(0); value val_default = args.get_kwarg_or_pos("default", 1); value val_base = args.get_kwarg_or_pos("base", 2); const int base = val_base->is_undefined() ? 10 : val_base->as_int(); + if (base != 0 && (base < 2 || base > 36)) { + // an out-of-range base makes std::stoi fail fast on the MSVC CRT instead of throwing + throw raised_exception("int() base must be 0 or between 2 and 36"); + } if (is_val(val_input) == false) { throw raised_exception("int() first argument must be a string"); } diff --git a/cpp/common/speculative.cpp b/cpp/common/speculative.cpp index 2ad72f1f5..9cdae397a 100644 --- a/cpp/common/speculative.cpp +++ b/cpp/common/speculative.cpp @@ -260,7 +260,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { bool process(const llama_batch & batch) override { auto * ctx_dft = params.ctx_dft; - const int ret = llama_decode(ctx_dft, batch); + llama_batch batch_dft = batch; + batch_dft.logits = nullptr; + + const int ret = llama_decode(ctx_dft, batch_dft); if (ret != 0) { SPC_ERR("failed to decode draft batch, ret = %d\n", ret); diff --git a/cpp/ggml-backend-meta.cpp b/cpp/ggml-backend-meta.cpp index 4596e5709..d23fd1db6 100644 --- a/cpp/ggml-backend-meta.cpp +++ b/cpp/ggml-backend-meta.cpp @@ -984,6 +984,11 @@ static struct lm_ggml_backend_meta_split_state lm_ggml_backend_meta_get_split_st case LM_GGML_OP_GATED_DELTA_NET: { split_state = handle_gated_delta_net(src_ss); } break; + case LM_GGML_OP_DSV4_HC_COMB: + case LM_GGML_OP_DSV4_HC_PRE: + case LM_GGML_OP_DSV4_HC_POST: { + split_state = handle_generic(src_ss, /*scalar_only =*/ true); + } break; case LM_GGML_OP_UNARY: { split_state = handle_generic(src_ss, /*scalar_only =*/ false); } break; diff --git a/cpp/ggml-cpu.h b/cpp/ggml-cpu.h index 9d022cb98..1024e5f60 100644 --- a/cpp/ggml-cpu.h +++ b/cpp/ggml-cpu.h @@ -100,6 +100,7 @@ extern "C" { LM_GGML_BACKEND_API int lm_ggml_cpu_has_sve (void); LM_GGML_BACKEND_API int lm_ggml_cpu_get_sve_cnt (void); // sve vector length in bytes LM_GGML_BACKEND_API int lm_ggml_cpu_has_sme (void); + LM_GGML_BACKEND_API int lm_ggml_cpu_has_sme2 (void); // other LM_GGML_BACKEND_API int lm_ggml_cpu_has_riscv_v (void); LM_GGML_BACKEND_API int lm_ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes diff --git a/cpp/ggml-cpu/arch/arm/cpu-feats.cpp b/cpp/ggml-cpu/arch/arm/cpu-feats.cpp index 9afa09d3a..e7287ea1f 100644 --- a/cpp/ggml-cpu/arch/arm/cpu-feats.cpp +++ b/cpp/ggml-cpu/arch/arm/cpu-feats.cpp @@ -28,6 +28,7 @@ struct aarch64_features { bool has_sve2 = false; bool has_i8mm = false; bool has_sme = false; + bool has_sme2 = false; aarch64_features() { #if defined(__linux__) @@ -56,6 +57,10 @@ struct aarch64_features { has_sme = static_cast(oldp); } + if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) { + has_sme2 = static_cast(oldp); + } + // Apple apparently does not implement SVE yet #endif } diff --git a/cpp/ggml-cpu/ggml-cpu.c b/cpp/ggml-cpu/ggml-cpu.c index f83812cf4..50169765c 100644 --- a/cpp/ggml-cpu/ggml-cpu.c +++ b/cpp/ggml-cpu/ggml-cpu.c @@ -2064,6 +2064,18 @@ static void lm_ggml_compute_forward(struct lm_ggml_compute_params * params, stru { lm_ggml_compute_forward_lightning_indexer(params, tensor); } break; + case LM_GGML_OP_DSV4_HC_COMB: + { + lm_ggml_compute_forward_dsv4_hc_comb(params, tensor); + } break; + case LM_GGML_OP_DSV4_HC_PRE: + { + lm_ggml_compute_forward_dsv4_hc_pre(params, tensor); + } break; + case LM_GGML_OP_DSV4_HC_POST: + { + lm_ggml_compute_forward_dsv4_hc_post(params, tensor); + } break; case LM_GGML_OP_MAP_CUSTOM1: { lm_ggml_compute_forward_map_custom1(params, tensor); @@ -2244,6 +2256,9 @@ static int lm_ggml_get_n_tasks(struct lm_ggml_tensor * node, int n_threads) { case LM_GGML_OP_COUNT_EQUAL: case LM_GGML_OP_SOLVE_TRI: case LM_GGML_OP_GATED_DELTA_NET: + case LM_GGML_OP_DSV4_HC_COMB: + case LM_GGML_OP_DSV4_HC_PRE: + case LM_GGML_OP_DSV4_HC_POST: { n_tasks = n_threads; } break; @@ -2859,7 +2874,14 @@ struct lm_ggml_cplan lm_ggml_graph_plan( } break; case LM_GGML_OP_OUT_PROD: { - if (lm_ggml_is_quantized(node->src[0]->type)) { + if (lm_ggml_is_quantized(node->src[0]->type) || + node->src[0]->type == LM_GGML_TYPE_F16) { + cur = lm_ggml_type_size(LM_GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; + } + } break; + case LM_GGML_OP_SET_ROWS: + { + if (node->src[0]->type == LM_GGML_TYPE_F16 && node->type != LM_GGML_TYPE_F16) { cur = lm_ggml_type_size(LM_GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } } break; @@ -3785,6 +3807,14 @@ int lm_ggml_cpu_has_sme(void) { #endif } +int lm_ggml_cpu_has_sme2(void) { +#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME2) + return 1; +#else + return 0; +#endif +} + void lm_ggml_cpu_init(void) { // needed to initialize lm_ggml_time { diff --git a/cpp/ggml-cpu/ggml-cpu.cpp b/cpp/ggml-cpu/ggml-cpu.cpp index f0bfca43e..8ba54b17d 100644 --- a/cpp/ggml-cpu/ggml-cpu.cpp +++ b/cpp/ggml-cpu/ggml-cpu.cpp @@ -462,11 +462,12 @@ static bool lm_ggml_backend_cpu_device_supports_op(lm_ggml_backend_dev_t dev, co return max_bias == 0.0f; } case LM_GGML_OP_IM2COL_BACK: - return src0->type == LM_GGML_TYPE_F32 && src1->type == LM_GGML_TYPE_F32; + return src0->type == LM_GGML_TYPE_F32 && (src1->type == LM_GGML_TYPE_F32 || src1->type == LM_GGML_TYPE_F16); case LM_GGML_OP_GET_ROWS_BACK: return src0->type == LM_GGML_TYPE_F32 || src0->type == LM_GGML_TYPE_F16; case LM_GGML_OP_OUT_PROD: - return (src0->type == LM_GGML_TYPE_F32 || (lm_ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && + return (src0->type == LM_GGML_TYPE_F32 || + ((src0->type == LM_GGML_TYPE_F16 || lm_ggml_is_quantized(src0->type)) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && src1->type == LM_GGML_TYPE_F32 && op->type == LM_GGML_TYPE_F32; default: return true; @@ -594,6 +595,9 @@ static lm_ggml_backend_feature * lm_ggml_backend_cpu_get_features(lm_ggml_backen if (lm_ggml_cpu_has_sme()) { features.push_back({ "SME", "1" }); } + if (lm_ggml_cpu_has_sme2()) { + features.push_back({ "SME2", "1" }); + } if (lm_ggml_cpu_has_riscv_v()) { features.push_back({ "RISCV_V", "1" }); } diff --git a/cpp/ggml-cpu/ops.cpp b/cpp/ggml-cpu/ops.cpp index 02db2b2b0..abcff58c7 100644 --- a/cpp/ggml-cpu/ops.cpp +++ b/cpp/ggml-cpu/ops.cpp @@ -2081,8 +2081,8 @@ void lm_ggml_compute_forward_concat( const lm_ggml_tensor * src1 = dst->src[1]; if (lm_ggml_is_quantized(src0->type)) { - LM_GGML_ASSERT(lm_ggml_is_contiguous(src0)); - LM_GGML_ASSERT(lm_ggml_is_contiguous(src1)); + LM_GGML_ASSERT(lm_ggml_is_contiguous_rows(src0)); + LM_GGML_ASSERT(lm_ggml_is_contiguous_rows(src1)); LM_GGML_ASSERT(src0->ne[0] % lm_ggml_blck_size(src0->type) == 0); LM_GGML_ASSERT(src1->ne[0] % lm_ggml_blck_size(src1->type) == 0); } @@ -4449,6 +4449,70 @@ static void lm_ggml_compute_forward_out_prod_q_f32( } } +static void lm_ggml_compute_forward_out_prod_f16_f32( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + + const lm_ggml_tensor * src0 = dst->src[0]; + const lm_ggml_tensor * src1 = dst->src[1]; + + LM_GGML_TENSOR_BINARY_OP_LOCALS; + + const int ith = params->ith; + const int nth = params->nth; + + LM_GGML_ASSERT(src0->type == LM_GGML_TYPE_F16); + LM_GGML_ASSERT(src1->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(dst->type == LM_GGML_TYPE_F32); + + LM_GGML_ASSERT(ne02 == ne12); + LM_GGML_ASSERT(ne03 == ne13); + LM_GGML_ASSERT(ne2 == ne12); + LM_GGML_ASSERT(ne3 == ne13); + + LM_GGML_ASSERT(nb00 == sizeof(lm_ggml_fp16_t)); + LM_GGML_ASSERT(nb0 == sizeof(float)); + + LM_GGML_ASSERT(ne0 == ne00); + LM_GGML_ASSERT(ne1 == ne10); + LM_GGML_ASSERT(ne2 == ne02); + LM_GGML_ASSERT(ne3 == ne03); + + if (ith == 0) { + lm_ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0); + } + lm_ggml_barrier(params->threadpool); + + const int64_t nr = ne1*ne2*ne3; + const int64_t dr = (nr + nth - 1)/nth; + const int64_t ir0 = dr*ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith; + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i3 = ir/(ne2*ne1); + const int64_t i2 = (ir - i3*ne2*ne1)/ne1; + const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1); + + const int64_t i02 = i2; + const int64_t i03 = i3; + + const int64_t i12 = i2; + const int64_t i13 = i3; + + float * d = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)); + + for (int64_t i01 = 0; i01 < ne01; ++i01) { + const int64_t i11 = i01; + lm_ggml_fp16_t * s0 = (lm_ggml_fp16_t *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03)); + float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13)); + lm_ggml_fp16_to_fp32_row(s0, wdata, ne0); + lm_ggml_vec_mad_f32(ne0, d, wdata, *s1); + } + } +} + void lm_ggml_compute_forward_out_prod( const lm_ggml_compute_params * params, lm_ggml_tensor * dst) { @@ -4486,9 +4550,8 @@ void lm_ggml_compute_forward_out_prod( } break; case LM_GGML_TYPE_F16: { - LM_GGML_ABORT("fatal error"); // todo - // lm_ggml_compute_forward_out_prod_f16_f32(params, dst); - } + lm_ggml_compute_forward_out_prod_f16_f32(params, dst); + } break; case LM_GGML_TYPE_F32: { lm_ggml_compute_forward_out_prod_f32(params, dst); @@ -5041,7 +5104,7 @@ static void lm_ggml_compute_forward_set_rows_impl( assert(ne0 == nc); assert(ne2 == ne02); assert(ne3 == ne03); - LM_GGML_ASSERT(src0->type == LM_GGML_TYPE_F32 || (src0->type == LM_GGML_TYPE_F16 && dst->type == LM_GGML_TYPE_F16)); + LM_GGML_ASSERT(src0->type == LM_GGML_TYPE_F32 || src0->type == LM_GGML_TYPE_F16); assert(ne02 % ne11 == 0); assert(ne03 % ne12 == 0); @@ -5075,10 +5138,19 @@ static void lm_ggml_compute_forward_set_rows_impl( (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); } else if constexpr (std::is_same_v) { - memcpy( + if (dst->type == LM_GGML_TYPE_F16) { + memcpy( ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), rs); + } else { + float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; + lm_ggml_fp16_to_fp32_row( + (const lm_ggml_fp16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + wdata, nc); + from_float(wdata, + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + } } else { LM_GGML_ABORT("src0->type = %d (%s) not supported", src0->type, lm_ggml_type_name(src0->type)); } @@ -5107,16 +5179,12 @@ void lm_ggml_compute_forward_set_rows( } break; case LM_GGML_TYPE_F16: { - if (dst->type == LM_GGML_TYPE_F16) { - if (src1->type == LM_GGML_TYPE_I64) { - lm_ggml_compute_forward_set_rows_impl(params, dst); - } else if (src1->type == LM_GGML_TYPE_I32) { - lm_ggml_compute_forward_set_rows_impl(params, dst); - } else { - LM_GGML_ABORT("src1->type = %d (%s) not supported", src1->type, lm_ggml_type_name(src1->type)); - } + if (src1->type == LM_GGML_TYPE_I64) { + lm_ggml_compute_forward_set_rows_impl(params, dst); + } else if (src1->type == LM_GGML_TYPE_I32) { + lm_ggml_compute_forward_set_rows_impl(params, dst); } else { - LM_GGML_ABORT("dst->type = %d (%s) not supported with src0->type = %d (%s)", dst->type, lm_ggml_type_name(dst->type), src0->type, lm_ggml_type_name(src0->type)); + LM_GGML_ABORT("src1->type = %d (%s) not supported", src1->type, lm_ggml_type_name(src1->type)); } } break; default: @@ -6362,7 +6430,6 @@ static void lm_ggml_compute_forward_im2col_f16( const lm_ggml_tensor * src0 = dst->src[0]; const lm_ggml_tensor * src1 = dst->src[1]; - LM_GGML_ASSERT(src0->type == LM_GGML_TYPE_F16); LM_GGML_ASSERT(src1->type == LM_GGML_TYPE_F16 || src1->type == LM_GGML_TYPE_F32); LM_GGML_ASSERT( dst->type == LM_GGML_TYPE_F16); @@ -6393,7 +6460,6 @@ static void lm_ggml_compute_forward_im2col_f16( int ofs0 = is_2D ? nb13 : nb12; int ofs1 = is_2D ? nb12 : nb11; - LM_GGML_ASSERT(nb00 == sizeof(lm_ggml_fp16_t)); LM_GGML_ASSERT(nb10 == lm_ggml_type_size(src1->type)); // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW] @@ -6466,7 +6532,7 @@ void lm_ggml_compute_forward_im2col_back_f32( const lm_ggml_tensor * src1 = dst->src[1]; // convolution kernel LM_GGML_ASSERT(src0->type == LM_GGML_TYPE_F32); - LM_GGML_ASSERT(src1->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(src1->type == LM_GGML_TYPE_F32 || src1->type == LM_GGML_TYPE_F16); LM_GGML_ASSERT( dst->type == LM_GGML_TYPE_F32); LM_GGML_TENSOR_BINARY_OP_LOCALS; @@ -6563,7 +6629,6 @@ static void lm_ggml_compute_forward_im2col_3d_f16( const lm_ggml_tensor * src0 = dst->src[0]; const lm_ggml_tensor * src1 = dst->src[1]; - LM_GGML_ASSERT(src0->type == LM_GGML_TYPE_F16); LM_GGML_ASSERT(src1->type == LM_GGML_TYPE_F32); LM_GGML_ASSERT( dst->type == LM_GGML_TYPE_F16); @@ -10879,6 +10944,291 @@ void lm_ggml_compute_forward_gated_delta_net( } } + +// lm_ggml_compute_forward_dsv4_hc_comb + +static void lm_ggml_dsv4_hc_comb_norm_cols(float * comb, float eps) { + constexpr int64_t hc = 4; + + for (int64_t idst = 0; idst < hc; ++idst) { + float sum = eps; + for (int64_t isrc = 0; isrc < hc; ++isrc) { + sum += comb[idst + hc*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int64_t isrc = 0; isrc < hc; ++isrc) { + comb[idst + hc*isrc] *= inv_sum; + } + } +} + +static void lm_ggml_dsv4_hc_comb_norm_rows(float * comb, float eps) { + constexpr int64_t hc = 4; + + for (int64_t isrc = 0; isrc < hc; ++isrc) { + float sum = eps; + for (int64_t idst = 0; idst < hc; ++idst) { + sum += comb[idst + hc*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int64_t idst = 0; idst < hc; ++idst) { + comb[idst + hc*isrc] *= inv_sum; + } + } +} + +static void lm_ggml_compute_forward_dsv4_hc_comb_f32( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + const lm_ggml_tensor * mixes = dst->src[0]; + const lm_ggml_tensor * scale = dst->src[1]; + const lm_ggml_tensor * base = dst->src[2]; + + LM_GGML_ASSERT(mixes->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(scale->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(base->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(dst->type == LM_GGML_TYPE_F32); + + constexpr int64_t hc = 4; + constexpr int64_t comb_offset = 2*hc; + constexpr int64_t hc_mix_dim = (2 + hc)*hc; + + const int64_t n_tokens = mixes->ne[1]; + + LM_GGML_ASSERT(mixes->ne[0] == hc_mix_dim); + LM_GGML_ASSERT(dst->ne[0] == hc); + LM_GGML_ASSERT(dst->ne[1] == hc); + LM_GGML_ASSERT(dst->ne[2] == n_tokens); + LM_GGML_ASSERT(scale->ne[0] >= 3); + LM_GGML_ASSERT(base->ne[0] == hc_mix_dim); + + LM_GGML_TENSOR_LOCALS(size_t, nbm, mixes, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbs, scale, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbb, base, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const float eps = lm_ggml_get_op_params_f32(dst, 0); + const int32_t n_iter = lm_ggml_get_op_params_i32(dst, 1); + LM_GGML_ASSERT(n_iter > 0); + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t dr = (n_tokens + nth - 1) / nth; + const int64_t it0 = dr * ith; + const int64_t it1 = MIN(it0 + dr, n_tokens); + + const float scale_comb = *(const float *) ((const char *) scale->data + 2*nbs0); + + for (int64_t it = it0; it < it1; ++it) { + float comb[hc*hc]; + + for (int64_t isrc = 0; isrc < hc; ++isrc) { + float max = -INFINITY; + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + const float xv = *(const float *) ((const char *) mixes->data + (comb_offset + idx)*nbm0 + it*nbm1); + const float bv = *(const float *) ((const char *) base->data + (comb_offset + idx)*nbb0); + const float v = xv * scale_comb + bv; + comb[idx] = v; + max = MAX(max, v); + } + + float sum = 0.0f; + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + const float v = expf(comb[idx] - max); + comb[idx] = v; + sum += v; + } + + const float inv_sum = 1.0f / sum; + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + comb[idx] = comb[idx] * inv_sum + eps; + } + } + + lm_ggml_dsv4_hc_comb_norm_cols(comb, eps); + for (int32_t i = 1; i < n_iter; ++i) { + lm_ggml_dsv4_hc_comb_norm_rows(comb, eps); + lm_ggml_dsv4_hc_comb_norm_cols(comb, eps); + } + + for (int64_t isrc = 0; isrc < hc; ++isrc) { + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + *(float *) ((char *) dst->data + idst*nbd0 + isrc*nbd1 + it*nbd2) = comb[idx]; + } + } + } +} + +void lm_ggml_compute_forward_dsv4_hc_comb( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + const lm_ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case LM_GGML_TYPE_F32: + { + lm_ggml_compute_forward_dsv4_hc_comb_f32(params, dst); + } break; + default: + { + LM_GGML_ABORT("fatal error"); + } + } +} + +// lm_ggml_compute_forward_dsv4_hc_pre + +static void lm_ggml_compute_forward_dsv4_hc_pre_f32( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + const lm_ggml_tensor * x = dst->src[0]; + const lm_ggml_tensor * weights = dst->src[1]; + + LM_GGML_ASSERT(x->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(weights->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(dst->type == LM_GGML_TYPE_F32); + + const int64_t n_embd = x->ne[0]; + const int64_t hc = x->ne[1]; + const int64_t n_tokens = x->ne[2]; + + LM_GGML_ASSERT(dst->ne[0] == n_embd); + LM_GGML_ASSERT(dst->ne[1] == n_tokens); + LM_GGML_ASSERT(weights->ne[0] == hc); + LM_GGML_ASSERT(weights->ne[1] == n_tokens); + + LM_GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbw, weights, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t nr = n_embd * n_tokens; + const int64_t dr = (nr + nth - 1) / nth; + const int64_t ir0 = dr * ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i0 = ir % n_embd; + const int64_t it = ir / n_embd; + + float sum = 0.0f; + for (int64_t ih = 0; ih < hc; ++ih) { + const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + ih*nbx1 + it*nbx2); + const float wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1); + sum += xv * wv; + } + + *(float *) ((char *) dst->data + i0*nbd0 + it*nbd1) = sum; + } +} + +void lm_ggml_compute_forward_dsv4_hc_pre( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + const lm_ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case LM_GGML_TYPE_F32: + { + lm_ggml_compute_forward_dsv4_hc_pre_f32(params, dst); + } break; + default: + { + LM_GGML_ABORT("fatal error"); + } + } +} + +// lm_ggml_compute_forward_dsv4_hc_post + +static void lm_ggml_compute_forward_dsv4_hc_post_f32( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + const lm_ggml_tensor * x = dst->src[0]; + const lm_ggml_tensor * residual = dst->src[1]; + const lm_ggml_tensor * post = dst->src[2]; + const lm_ggml_tensor * comb = dst->src[3]; + + LM_GGML_ASSERT(x->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(residual->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(post->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(comb->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(dst->type == LM_GGML_TYPE_F32); + + const int64_t n_embd = x->ne[0]; + const int64_t n_tokens = x->ne[1]; + const int64_t hc = residual->ne[1]; + + LM_GGML_ASSERT(dst->ne[0] == n_embd); + LM_GGML_ASSERT(dst->ne[1] == hc); + LM_GGML_ASSERT(dst->ne[2] == n_tokens); + LM_GGML_ASSERT(residual->ne[0] == n_embd); + LM_GGML_ASSERT(residual->ne[2] == n_tokens); + LM_GGML_ASSERT(post->ne[0] == hc); + LM_GGML_ASSERT(post->ne[1] == n_tokens); + LM_GGML_ASSERT(comb->ne[0] == hc); + LM_GGML_ASSERT(comb->ne[1] == hc); + LM_GGML_ASSERT(comb->ne[2] == n_tokens); + + LM_GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbp, post, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbc, comb, nb); + LM_GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t nr = n_embd * hc * n_tokens; + const int64_t dr = (nr + nth - 1) / nth; + const int64_t ir0 = dr * ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i0 = ir % n_embd; + const int64_t idst = (ir / n_embd) % hc; + const int64_t it = ir / (n_embd * hc); + + const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + it*nbx1); + const float pv = *(const float *) ((const char *) post->data + idst*nbp0 + it*nbp1); + + float sum = xv * pv; + for (int64_t isrc = 0; isrc < hc; ++isrc) { + const float rv = *(const float *) ((const char *) residual->data + i0*nbr0 + isrc*nbr1 + it*nbr2); + const float cv = *(const float *) ((const char *) comb->data + idst*nbc0 + isrc*nbc1 + it*nbc2); + sum += rv * cv; + } + + *(float *) ((char *) dst->data + i0*nbd0 + idst*nbd1 + it*nbd2) = sum; + } +} + +void lm_ggml_compute_forward_dsv4_hc_post( + const lm_ggml_compute_params * params, + lm_ggml_tensor * dst) { + const lm_ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case LM_GGML_TYPE_F32: + { + lm_ggml_compute_forward_dsv4_hc_post_f32(params, dst); + } break; + default: + { + LM_GGML_ABORT("fatal error"); + } + } +} + // lm_ggml_compute_forward_rwkv_wkv7 static void lm_ggml_compute_forward_rwkv_wkv7_f32( diff --git a/cpp/ggml-cpu/ops.h b/cpp/ggml-cpu/ops.h index 2cb2e191f..c36ff2524 100644 --- a/cpp/ggml-cpu/ops.h +++ b/cpp/ggml-cpu/ops.h @@ -106,6 +106,9 @@ void lm_ggml_compute_forward_solve_tri(const struct lm_ggml_compute_params * par void lm_ggml_compute_forward_gla(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); void lm_ggml_compute_forward_gated_delta_net(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); void lm_ggml_compute_forward_lightning_indexer(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); +void lm_ggml_compute_forward_dsv4_hc_comb(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); +void lm_ggml_compute_forward_dsv4_hc_pre(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); +void lm_ggml_compute_forward_dsv4_hc_post(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); void lm_ggml_compute_forward_map_custom1(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); void lm_ggml_compute_forward_map_custom2(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); void lm_ggml_compute_forward_map_custom3(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst); diff --git a/cpp/ggml-hexagon/ggml-hexagon.cpp b/cpp/ggml-hexagon/ggml-hexagon.cpp index 4d1c55754..7d09e14cb 100644 --- a/cpp/ggml-hexagon/ggml-hexagon.cpp +++ b/cpp/ggml-hexagon/ggml-hexagon.cpp @@ -21,6 +21,11 @@ #include #ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include # include #else # include @@ -28,7 +33,9 @@ #endif #pragma clang diagnostic ignored "-Wnested-anon-types" +#pragma clang diagnostic ignored "-Wlanguage-extension-token" #pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#pragma clang diagnostic ignored "-Wmicrosoft-enum-value" #include #include @@ -134,6 +141,8 @@ static const char * htp_event_name(uint16_t id) { case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP"; case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP"; case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP"; + case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH"; + case HTP_TRACE_EVT_INIT: return "INIT"; default: return "UNKNOWN"; } } @@ -501,6 +510,8 @@ static void repack_q4_0_tiled(lm_ggml_tensor * t, const void * data, size_t size } } } + + LM_GGML_UNUSED(size); } // repack q4_0_tiled tensor into q4_0 data @@ -554,6 +565,8 @@ static void repack_tiled_q4_0(void * data, const lm_ggml_tensor * t, size_t size } } } + + LM_GGML_UNUSED(size); } // repack q4_1 data into q4_1_tiled tensor @@ -611,6 +624,8 @@ static void repack_q4_1_tiled(lm_ggml_tensor * t, const void * data, size_t size } } } + + LM_GGML_UNUSED(size); } // repack q4_1_tiled tensor into q4_1 data @@ -665,6 +680,8 @@ static void repack_tiled_q4_1(void * data, const lm_ggml_tensor * t, size_t size } } } + + LM_GGML_UNUSED(size); } // repack q8_0 data into q8_0_tiled tensor @@ -711,6 +728,8 @@ static void repack_q8_0_tiled(lm_ggml_tensor * t, const void * data, size_t size } } } + + LM_GGML_UNUSED(size); } // repack q8_0_tiled tensor into q8_0 data @@ -761,6 +780,8 @@ static void repack_tiled_q8_0(void * data, const lm_ggml_tensor * t, size_t size } } } + + LM_GGML_UNUSED(size); } // repack mxfp4 data into mxfp4_tiled tensor @@ -812,6 +833,8 @@ static void repack_mxfp4_tiled(lm_ggml_tensor * t, const void * data, size_t siz } } } + + LM_GGML_UNUSED(size); } // repack mxfp4_tiled tensor into mxfp4 data @@ -865,6 +888,8 @@ static void repack_tiled_mxfp4(void * data, const lm_ggml_tensor * t, size_t siz } } } + + LM_GGML_UNUSED(size); } static void lm_ggml_backend_hexagon_buffer_set_tensor(lm_ggml_backend_buffer_t buffer, @@ -965,11 +990,12 @@ static void lm_ggml_backend_hexagon_buffer_get_tensor(lm_ggml_backend_buffer_t b static bool lm_ggml_backend_hexagon_buffer_cpy_tensor(lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst) { + // we might optimize this later, for now take the slow path (ie get/set_tensor) + return false; + LM_GGML_UNUSED(buffer); LM_GGML_UNUSED(src); LM_GGML_UNUSED(dst); - // we might optimize this later, for now take the slow path (ie get/set_tensor) - return false; } static void lm_ggml_backend_hexagon_buffer_clear(lm_ggml_backend_buffer_t buffer, uint8_t value) { @@ -1025,9 +1051,9 @@ static lm_ggml_backend_buffer_t lm_ggml_backend_hexagon_repack_buffer_type_alloc } } -static size_t lm_ggml_backend_hexagon_buffer_type_get_alignment(lm_ggml_backend_buffer_type_t buffer_type) { +static size_t lm_ggml_backend_hexagon_buffer_type_get_alignment(lm_ggml_backend_buffer_type_t buft) { return 128; // HVX alignment - LM_GGML_UNUSED(buffer_type); + LM_GGML_UNUSED(buft); } static size_t lm_ggml_backend_hexagon_buffer_type_get_alloc_size(lm_ggml_backend_buffer_type_t buft, const struct lm_ggml_tensor * t) { @@ -1039,20 +1065,24 @@ static size_t lm_ggml_backend_hexagon_buffer_type_get_alloc_size(lm_ggml_backend return lm_ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3; } return lm_ggml_nbytes(t); + + LM_GGML_UNUSED(buft); } -static size_t lm_ggml_backend_hexagon_buffer_type_get_max_size(lm_ggml_backend_buffer_type_t buffer_type) { - auto * context = static_cast(buffer_type->context); +static size_t lm_ggml_backend_hexagon_buffer_type_get_max_size(lm_ggml_backend_buffer_type_t buft) { + auto * context = static_cast(buft->context); return context->sess->max_bufsize; } static bool lm_ggml_backend_hexagon_buffer_type_is_host(lm_ggml_backend_buffer_type_t buft) { return opt_hostbuf; + LM_GGML_UNUSED(buft); } static bool lm_ggml_backend_hexagon_repack_buffer_type_is_host(lm_ggml_backend_buffer_type_t buft) { return false; + LM_GGML_UNUSED(buft); } @@ -1098,6 +1128,14 @@ struct lm_ggml_hexagon_opbatch { std::unordered_map t_map; // tensor ptr to index std::unordered_multimap d_map; // tensor data to index + struct tensor_range { + uint64_t start; + uint64_t end; + int bi; + std::vector tensors; + }; + std::vector ranges; + unsigned int n_bufs; // num buffers in the batch unsigned int n_tens; // num tensors ... unsigned int n_ops; // num ops ... @@ -1117,6 +1155,7 @@ struct lm_ggml_hexagon_opbatch { b_map.clear(); t_map.clear(); d_map.clear(); + ranges.clear(); } lm_ggml_hexagon_opbatch(lm_ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) { @@ -1124,7 +1163,7 @@ struct lm_ggml_hexagon_opbatch { n_bufs_max = HTP_OP_MAX_BUFS; n_ops_max = batch_size; - n_tens_max = n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS; + n_tens_max = std::min(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS); b_vmem_max = max_vmem; @@ -1170,6 +1209,71 @@ struct lm_ggml_hexagon_opbatch { return bi; } + void add_range(const htp_tensor * h, int ti) { + uint64_t t_start = h->data; + uint64_t t_end = t_start + h->size; + int bi = h->bi; + + int first_match = -1; + int unused_idx = -1; + for (size_t i = 0; i < ranges.size(); i++) { + if (ranges[i].bi == -1) { + unused_idx = i; + continue; + } + if (ranges[i].bi != bi) { + continue; + } + if (ranges[i].start >= t_end || ranges[i].end <= t_start) { + continue; + } + + if (first_match == -1) { + first_match = i; + HEX_VERBOSE("ggml-hex: %s range-grow #%d : bi %d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n", + sess->c_name(), (int) i, ranges[i].bi, + (void *) (h_bufs[ranges[i].bi].base + ranges[i].start), + (void *) (h_bufs[ranges[i].bi].base + ranges[i].end), + ti, + (void *) (h_bufs[bi].base + t_start), + (void *) (h_bufs[bi].base + t_end), + (void *) (h_bufs[ranges[i].bi].base + std::min(ranges[i].start, t_start)), + (void *) (h_bufs[ranges[i].bi].base + std::max(ranges[i].end, t_end))); + + ranges[i].start = std::min(ranges[i].start, t_start); + ranges[i].end = std::max(ranges[i].end, t_end); + ranges[i].tensors.push_back(ti); + } else { + HEX_VERBOSE("ggml-hex: %s range-merge #%d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n", + sess->c_name(), first_match, + (void *) (h_bufs[bi].base + ranges[first_match].start), + (void *) (h_bufs[bi].base + ranges[first_match].end), + (int) i, + (void *) (h_bufs[bi].base + ranges[i].start), + (void *) (h_bufs[bi].base + ranges[i].end), + (void *) (h_bufs[bi].base + std::min(ranges[first_match].start, ranges[i].start)), + (void *) (h_bufs[bi].base + std::max(ranges[first_match].end, ranges[i].end))); + + ranges[first_match].start = std::min(ranges[first_match].start, ranges[i].start); + ranges[first_match].end = std::max(ranges[first_match].end, ranges[i].end); + ranges[first_match].tensors.insert( + ranges[first_match].tensors.end(), + ranges[i].tensors.begin(), + ranges[i].tensors.end() + ); + ranges[i].bi = -1; + } + } + + if (first_match == -1) { + if (unused_idx != -1) { + ranges[unused_idx] = {t_start, t_end, bi, {ti}}; + } else { + ranges.push_back({t_start, t_end, bi, {ti}}); + } + } + } + bool same_shape(const htp_tensor * h, const lm_ggml_tensor * t) const { int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -1213,6 +1317,7 @@ struct lm_ggml_hexagon_opbatch { htp_tensor &h = h_tens[ti]; h.bi = add_buffer(sbuf); + h.ti = ti; h.data = t_offset; h.type = t->type; @@ -1235,8 +1340,11 @@ struct lm_ggml_hexagon_opbatch { h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3]; } + h.alias = ti; + add_range(&h, ti); + h.flags = 0; - if (lm_ggml_backend_buffer_get_usage(t->buffer) == LM_GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + if (lm_ggml_backend_buffer_get_usage(t->buffer) != LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { h.flags |= HTP_TENSOR_COMPUTE; } @@ -1313,6 +1421,17 @@ struct lm_ggml_hexagon_opbatch { o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff; } } + + void finalize_ranges() { + for (const auto & r : ranges) { + if (r.bi == -1) { + continue; + } + for (size_t i = 0; i < r.tensors.size(); i++) { + h_tens[r.tensors[i]].alias = r.tensors[(i + 1) % r.tensors.size()]; + } + } + } }; struct lm_ggml_hexagon_opqueue { @@ -1571,6 +1690,8 @@ void lm_ggml_hexagon_session::flush_pending(bool all) { void lm_ggml_hexagon_session::flush_batch() { if (op_batch->empty()) { return; } + op_batch->finalize_ranges(); + htp_opbatch_req req {}; dspqueue_buffer dbuf{}; @@ -1647,7 +1768,7 @@ void lm_ggml_hexagon_session::allocate(int dev_id) noexcept(false) { LM_GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); - domain * my_domain = get_domain(this->domain_id); + domain * my_domain = htpdrv_get_domain(this->domain_id); if (my_domain == NULL) { LM_GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n"); throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)"); @@ -1793,16 +1914,6 @@ void lm_ggml_hexagon_session::allocate(int dev_id) noexcept(false) { } } - if (opt_profile) { - htp_iface_pmu_conf pmu_conf{}; - std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); - - err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); - if (err != 0) { - LM_GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); - } - } - // Allocate buffers and state for op batching this->op_queue = new lm_ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); @@ -1821,6 +1932,16 @@ void lm_ggml_hexagon_session::allocate(int dev_id) noexcept(false) { throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); } this->valid_iface = true; + + if (opt_profile) { + htp_iface_pmu_conf pmu_conf{}; + std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); + + err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); + if (err != 0) { + LM_GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); + } + } } void lm_ggml_hexagon_session::release() noexcept(true) { @@ -1929,6 +2050,8 @@ static bool lm_ggml_hexagon_flash_attn_is_hmx_eligible( } return true; + + LM_GGML_UNUSED(sinks); } static bool lm_ggml_hexagon_precompute_flash_attn_params( @@ -2149,8 +2272,9 @@ static bool lm_ggml_hexagon_supported_gated_delta_net(const struct lm_ggml_hexag return false; } - LM_GGML_UNUSED(sess); return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_matmul_is_hmx_eligible( @@ -2198,6 +2322,8 @@ static bool lm_ggml_hexagon_matmul_is_hmx_eligible( } return true; + + LM_GGML_UNUSED(dst); } static bool lm_ggml_hexagon_precompute_hmx_mm_params( @@ -2234,109 +2360,15 @@ static bool lm_ggml_hexagon_precompute_hmx_mm_params( if (is_batched_val && wtype == LM_GGML_TYPE_F16 && group_size > 1) { // Try grouped path first const bool use_dma_activation = (src1->nb[1]/sizeof(float) > (size_t)ne00_padded); - size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; - size_t best_m_chunk = 0; - size_t best_n_chunk = 0; - size_t best_vtcm_size = 0; - - int act_threads = n_threads; - while (act_threads >= 1) { - const size_t f32_scratch_size = use_dma_activation ? hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE) : 0; - size_t group_overhead = 256 + f32_scratch_size; - size_t group_size_per_n, group_size_per_m, group_size_per_mn; - htp_mm_hmx_get_batched_chunk_costs(ne00_padded, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); - - size_t m_chunk_candidate = 0; - size_t n_chunk_candidate = 0; - size_t vtcm_size_candidate = 0; - - if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, - (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, - &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads); - if (exact_size <= vtcm_budget) { - size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { - best_mblocks = mblocks; - best_act_threads = act_threads; - best_m_chunk = m_chunk_candidate; - best_n_chunk = n_chunk_candidate; - best_vtcm_size = exact_size; - } - } - } - if (act_threads == 1) { - act_threads = 0; - } else { - act_threads /= 2; - } - } - - if (best_act_threads > 0) { - m_chunk = best_m_chunk; - n_chunk = best_n_chunk; - vtcm_size = best_vtcm_size; - act_threads_selected = best_act_threads; + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, use_dma_activation, n_threads, pipeline, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { use_grouped = true; } } if (!use_grouped) { // Fallback to simple 2D path (group_size = 1) - size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; - size_t best_m_chunk = 0; - size_t best_n_chunk = 0; - size_t best_vtcm_size = 0; - - // For MUL_MAT_ID the kernel runs one 2D matmul per expert, with M equal to the number of rows routed to that expert. - // A single expert can receive up to all routed rows (dst->ne[1]*dst->ne[2] = n_expert_used*n_tokens), so size the chunk - // search for that upper bound rather than ne12 (token positions only). - // We recompute m_chunk per expert against the actual count in the NPU kernel. - const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]); - const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; - const int m_for_cost = is_matmul_id ? m_id_rows : ne11; - - int act_threads = n_threads; - while (act_threads >= 1) { - const size_t act_f32_size = is_matmul_id ? 0 : hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE); - size_t simple_2d_overhead = 256 + act_f32_size; - size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; - htp_mm_hmx_get_2d_chunk_costs(wtype, ne00_padded, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); - - size_t m_chunk_candidate = 0; - size_t n_chunk_candidate = 0; - size_t vtcm_size_candidate = 0; - - if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, - (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, - &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size); - if (exact_size <= vtcm_budget) { - size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { - best_mblocks = mblocks; - best_act_threads = act_threads; - best_m_chunk = m_chunk_candidate; - best_n_chunk = n_chunk_candidate; - best_vtcm_size = exact_size; - } - } - } - if (act_threads == 1) { - act_threads = 0; - } else { - act_threads /= 2; - } - } - - if (best_act_threads > 0) { - m_chunk = best_m_chunk; - n_chunk = best_n_chunk; - vtcm_size = best_vtcm_size; - act_threads_selected = best_act_threads; - } else { + const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]); + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { return false; } } @@ -2352,6 +2384,8 @@ static bool lm_ggml_hexagon_precompute_hmx_mm_params( kparams->src1_row_size = (wtype == LM_GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); kparams->vtcm_size = vtcm_size; kparams->vtcm_src0_size = 0; + kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); + kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); kparams->vtcm_src1_size = 0; kparams->vtcm_dst_size = 0; @@ -2361,6 +2395,8 @@ static bool lm_ggml_hexagon_precompute_hmx_mm_params( kparams->kernel_type = HTP_MM_KERNEL_HMX_2D; } return true; + + LM_GGML_UNUSED(src0); } static void lm_ggml_hexagon_precompute_hvx_mm_params( @@ -2955,6 +2991,8 @@ static bool lm_ggml_hexagon_supported_binary(const struct lm_ggml_hexagon_sessio } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_add_id(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -2981,6 +3019,8 @@ static bool lm_ggml_hexagon_supported_add_id(const struct lm_ggml_hexagon_sessio } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_unary(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3006,6 +3046,8 @@ static bool lm_ggml_hexagon_supported_unary(const struct lm_ggml_hexagon_session } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_sum_rows(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3025,10 +3067,11 @@ static bool lm_ggml_hexagon_supported_sum_rows(const struct lm_ggml_hexagon_sess } return true; + + LM_GGML_UNUSED(sess); } -static bool lm_ggml_hexagon_supported_activations(const struct lm_ggml_hexagon_session * sess, - const struct lm_ggml_tensor * op) { +static bool lm_ggml_hexagon_supported_activations(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { const struct lm_ggml_tensor * src0 = op->src[0]; const struct lm_ggml_tensor * src1 = op->src[1]; const struct lm_ggml_tensor * dst = op; @@ -3057,6 +3100,8 @@ static bool lm_ggml_hexagon_supported_activations(const struct lm_ggml_hexagon_s } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_softmax(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3122,6 +3167,8 @@ static bool lm_ggml_hexagon_supported_softmax(const struct lm_ggml_hexagon_sessi } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_set_rows(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3142,6 +3189,8 @@ static bool lm_ggml_hexagon_supported_set_rows(const struct lm_ggml_hexagon_sess } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_get_rows(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3162,6 +3211,8 @@ static bool lm_ggml_hexagon_supported_get_rows(const struct lm_ggml_hexagon_sess } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_argsort(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3182,6 +3233,8 @@ static bool lm_ggml_hexagon_supported_argsort(const struct lm_ggml_hexagon_sessi } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_rope(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3243,6 +3296,8 @@ static bool lm_ggml_hexagon_supported_rope(const struct lm_ggml_hexagon_session return false; } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_ssm_conv(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3282,6 +3337,8 @@ static bool lm_ggml_hexagon_supported_ssm_conv(const struct lm_ggml_hexagon_sess } return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_pad(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3292,8 +3349,9 @@ static bool lm_ggml_hexagon_supported_pad(const struct lm_ggml_hexagon_session * return false; } - LM_GGML_UNUSED(sess); return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_cumsum(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3308,8 +3366,9 @@ static bool lm_ggml_hexagon_supported_cumsum(const struct lm_ggml_hexagon_sessio return false; } - LM_GGML_UNUSED(sess); return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_diag(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3331,8 +3390,9 @@ static bool lm_ggml_hexagon_supported_diag(const struct lm_ggml_hexagon_session return false; } - LM_GGML_UNUSED(sess); return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_solve_tri(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3364,8 +3424,9 @@ static bool lm_ggml_hexagon_supported_solve_tri(const struct lm_ggml_hexagon_ses return false; } - LM_GGML_UNUSED(sess); return true; + + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_tri(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -3812,6 +3873,8 @@ static void lm_ggml_backend_hexagon_graph_optimize(lm_ggml_backend_t backend, lm } } } + + LM_GGML_UNUSED(backend); } static struct lm_ggml_backend_i hexagon_backend_i = { @@ -3930,6 +3993,8 @@ static bool lm_ggml_hexagon_supported_buffers(lm_ggml_hexagon_session *sess, con } static bool lm_ggml_hexagon_supported_cpy(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { + LM_GGML_UNUSED(sess); + const struct lm_ggml_tensor * src0 = op->src[0]; const struct lm_ggml_tensor * dst = op; @@ -4000,6 +4065,7 @@ static bool lm_ggml_hexagon_supported_concat(const struct lm_ggml_hexagon_sessio } return true; + LM_GGML_UNUSED(sess); } static bool lm_ggml_hexagon_supported_fill(const struct lm_ggml_hexagon_session * sess, const struct lm_ggml_tensor * op) { @@ -4009,8 +4075,8 @@ static bool lm_ggml_hexagon_supported_fill(const struct lm_ggml_hexagon_session return false; } - LM_GGML_UNUSED(sess); return true; + LM_GGML_UNUSED(sess); } static bool lm_ggml_backend_hexagon_device_supports_op(lm_ggml_backend_dev_t dev, const struct lm_ggml_tensor * op) { @@ -4295,6 +4361,7 @@ static void * lm_ggml_backend_hexagon_get_proc_address(lm_ggml_backend_reg_t reg } return NULL; + LM_GGML_UNUSED(reg); } template std::vector str_to_vec(const char* str) { @@ -4353,10 +4420,18 @@ static void lm_ggml_hexagon_init(lm_ggml_backend_reg * reg) { // Init Arch first since it affects other defaults if (!str_arch) { - int err = get_hex_arch_ver(CDSP_DOMAIN_ID, &opt_arch); + int err = htpdrv_get_arch(CDSP_DOMAIN_ID, &opt_arch); if (err != 0) { LM_GGML_LOG_ERROR("ggml-hex: failed to query HTP version (err %d) defaulting to v73\n", err); opt_arch = 73; + } else { + if (opt_arch < 73) { + LM_GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch); + opt_arch = 73; + } else if (opt_arch > 81) { + LM_GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v81\n", opt_arch); + opt_arch = 81; + } } } else { if (str_arch[0] == 'v' || str_arch[0] == 'V') { diff --git a/cpp/ggml-hexagon/htp-drv.cpp b/cpp/ggml-hexagon/htp-drv.cpp index 75294d34e..41019b0ac 100644 --- a/cpp/ggml-hexagon/htp-drv.cpp +++ b/cpp/ggml-hexagon/htp-drv.cpp @@ -1,13 +1,8 @@ -// sample drv interface - -#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" -#pragma clang diagnostic ignored "-Wmissing-prototypes" -#pragma clang diagnostic ignored "-Wsign-compare" - #include #include #include #include + #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX @@ -16,9 +11,17 @@ # include # include #else -# include -# include +# include +# include #endif + +#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#pragma clang diagnostic ignored "-Wmissing-prototypes" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wlanguage-extension-token" +#pragma clang diagnostic ignored "-Wmicrosoft-enum-value" +#pragma clang diagnostic ignored "-Wnested-anon-types" + #include "ggml-impl.h" #include "htp-drv.h" #include "libdl.h" @@ -359,7 +362,7 @@ int htpdrv_init() { return AEE_SUCCESS; } -domain * get_domain(int domain_id) { +domain * htpdrv_get_domain(int domain_id) { int i = 0; int size = sizeof(supported_domains) / sizeof(domain); @@ -372,7 +375,7 @@ domain * get_domain(int domain_id) { return NULL; } -int get_hex_arch_ver(int domain, int * arch) { +int htpdrv_get_arch(int domain, int * arch) { if (!remote_handle_control_pfn) { LM_GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n"); return AEE_EUNSUPPORTEDAPI; @@ -394,25 +397,7 @@ int get_hex_arch_ver(int domain, int * arch) { return err; } - switch (arch_ver.capability & 0xff) { - case 0x68: - *arch = 68; - return 0; - case 0x69: - *arch = 69; - return 0; - case 0x73: - *arch = 73; - return 0; - case 0x75: - *arch = 75; - return 0; - case 0x79: - *arch = 79; - return 0; - case 0x81: - *arch = 81; - return 0; - } - return -1; + uint32_t val = arch_ver.capability & 0xff; + *arch = (int) ((val >> 4) * 10 + (val & 0x0f)); + return 0; } diff --git a/cpp/ggml-hexagon/htp-drv.h b/cpp/ggml-hexagon/htp-drv.h index 2757680ec..e59633788 100644 --- a/cpp/ggml-hexagon/htp-drv.h +++ b/cpp/ggml-hexagon/htp-drv.h @@ -96,17 +96,17 @@ extern "C" { HTPDRV_API int htpdrv_init(void); /** - * get_domain API: get domain struct from domain value. + * htpdrv_get_domain API: get domain struct from domain value. * * @param[in] domain value of a domain * @return Returns domain struct of the domain if it is supported or else * returns NULL. * */ -HTPDRV_API domain * get_domain(int domain_id); +HTPDRV_API domain * htpdrv_get_domain(int domain_id); /** - * get_hex_arch_ver API: query the Hexagon processor architecture version information + * htpdrv_get_arch API: query the Hexagon processor architecture version information * * @param[in] domain_id value of a domain * @param[out] Arch version (73, 75, ...) @@ -114,7 +114,7 @@ HTPDRV_API domain * get_domain(int domain_id); * non-zero if error, return value points to the error. * */ -HTPDRV_API int get_hex_arch_ver(int domain, int * arch); +HTPDRV_API int htpdrv_get_arch(int domain, int * arch); #ifdef __cplusplus } diff --git a/cpp/ggml-hexagon/htp/CMakeLists.txt b/cpp/ggml-hexagon/htp/CMakeLists.txt index cf9e726c0..4fb526f0c 100644 --- a/cpp/ggml-hexagon/htp/CMakeLists.txt +++ b/cpp/ggml-hexagon/htp/CMakeLists.txt @@ -17,9 +17,12 @@ set(HTP_LIB ggml-htp-${DSP_VERSION}) add_library(${HTP_LIB} SHARED main.c htp_iface_skel.c - worker-pool.c - hex-dma.c + work-queue.c + dma-queue.c hmx-queue.c + htp-tensor.c + matmul-ops.c + flash-attn-ops.c gated-delta-net-ops.c binary-ops.c unary-ops.c @@ -38,8 +41,6 @@ add_library(${HTP_LIB} SHARED diag-ops.c solve-tri-ops.c pad-ops.c - flash-attn-ops.c - matmul-ops.c argsort-ops.c ) diff --git a/cpp/ggml-hexagon/htp/act-ops.c b/cpp/ggml-hexagon/htp/act-ops.c index 5cff846d2..df157af34 100644 --- a/cpp/ggml-hexagon/htp/act-ops.c +++ b/cpp/ggml-hexagon/htp/act-ops.c @@ -16,6 +16,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-ops.h" +#include "htp-tensor.h" #define htp_act_preamble \ const struct htp_tensor * src0 = actx->octx->src[0]; \ @@ -766,17 +767,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { } int op_activations(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_activations_f32(octx); - break; + return execute_op_activations_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/cpp/ggml-hexagon/htp/argsort-ops.c b/cpp/ggml-hexagon/htp/argsort-ops.c index 4589362f1..11f1284aa 100644 --- a/cpp/ggml-hexagon/htp/argsort-ops.c +++ b/cpp/ggml-hexagon/htp/argsort-ops.c @@ -345,7 +345,7 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i int32_t * indices_buf = (int32_t *) (spad + values_size); \ uint32_t nb01 = src0->nb[1]; \ uint32_t nb1 = dst->nb[1]; \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[i]; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); \ for (uint32_t r = start_row; r < end_row; r++) { \ uint32_t src_offset = r * nb01; \ @@ -411,7 +411,7 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data const HVX_Vector ind_init_vec = *(HVX_Vector *)argosrt_ramp_lut; const HVX_Vector ind_diff_vec = Q6_V_vsplat_R(32); - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); for (uint32_t r = start_row; r < end_row; r++) { diff --git a/cpp/ggml-hexagon/htp/binary-ops.c b/cpp/ggml-hexagon/htp/binary-ops.c index 8cc9d86b4..6117716b3 100644 --- a/cpp/ggml-hexagon/htp/binary-ops.c +++ b/cpp/ggml-hexagon/htp/binary-ops.c @@ -16,6 +16,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-ops.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) diff --git a/cpp/ggml-hexagon/htp/cumsum-ops.c b/cpp/ggml-hexagon/htp/cumsum-ops.c index 5f65c27b4..7b0249df4 100644 --- a/cpp/ggml-hexagon/htp/cumsum-ops.c +++ b/cpp/ggml-hexagon/htp/cumsum-ops.c @@ -9,6 +9,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" #include "hex-dma.h" @@ -255,16 +256,10 @@ int op_cumsum_f32(struct htp_ops_context * octx) { int op_cumsum(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; - int err = HTP_STATUS_OK; - switch (dst->type) { case HTP_TYPE_F32: - err = op_cumsum_f32(octx); - break; + return op_cumsum_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/cpp/ggml-hexagon/htp/dma-queue.c b/cpp/ggml-hexagon/htp/dma-queue.c new file mode 100644 index 000000000..8f60a2c9d --- /dev/null +++ b/cpp/ggml-hexagon/htp/dma-queue.c @@ -0,0 +1,106 @@ +#include "dma-queue.h" + +#include +#include +#include + +#pragma clang diagnostic ignored "-Wunused-function" + +static inline uint32_t pow2_ceil(uint32_t x) { + if (x <= 1) { + return 1; + } + int p = 2; + x--; + while (x >>= 1) { + p <<= 1; + } + return p; +} + +static inline uintptr_t align_up(uintptr_t addr, size_t align) { + return (addr + align - 1) & ~(align - 1); +} + +size_t dma_queue_sizeof(size_t capacity) { + capacity = pow2_ceil(capacity); + + size_t size_q = sizeof(dma_queue); + size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r = sizeof(dma_ring); + size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE); + size_t size_desc = capacity * sizeof(dma_descriptor_2d); + size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + size_t size_dptr = capacity * sizeof(dma_ptr); + + return offset_dptr + size_dptr; +} + +size_t dma_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) { + capacity = pow2_ceil(capacity); + + size_t size_q = sizeof(dma_queue); + size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r = sizeof(dma_ring); + size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE); + size_t size_desc = capacity * sizeof(dma_descriptor_2d); + size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + size_t size_dptr = capacity * sizeof(dma_ptr); + + size_t total_size = offset_dptr + size_dptr; + memset(ptr, 0, total_size); + + dma_queue * q = (dma_queue *) ptr; + dma_ring * r = (dma_ring *) ((uintptr_t) ptr + offset_r); + + q->ring = r; + q->nocache = 0; + q->alias = false; + + r->trace = trace; + r->vtcm_base = vtcm_base; + r->vtcm_end = vtcm_base + vtcm_size; + r->capacity = capacity; + r->idx_mask = capacity - 1; + r->push_idx = 0; + r->pop_idx = 0; + + r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc); + r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr); + r->tail = &r->desc[capacity - 1]; + + FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size); + + return q; +} + +void dma_queue_free(dma_queue_t q) { + (void) q; +} + +size_t dma_queue_alias_sizeof(void) { + return sizeof(dma_queue); +} + +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) { + dma_queue * q = (dma_queue *) ptr; + memset(q, 0, sizeof(dma_queue)); + + q->ring = main_q->ring; + q->nocache = nocache; + q->alias = true; + + return q; +} + +void dma_queue_alias_free(dma_queue_t q) { + (void) q; +} + +void dma_queue_flush(dma_queue_t q) { + while (dma_queue_pop(q).dst != NULL) ; +} diff --git a/cpp/ggml-hexagon/htp/dma-queue.h b/cpp/ggml-hexagon/htp/dma-queue.h new file mode 100644 index 000000000..dba452cfb --- /dev/null +++ b/cpp/ggml-hexagon/htp/dma-queue.h @@ -0,0 +1,396 @@ +#ifndef HTP_DMA_H +#define HTP_DMA_H + +#include +#include +#include +#include +#include "hex-utils.h" + +#include "hex-profile.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date +typedef struct dma_descriptor_1d_s { + void * next; + uint32_t size:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + void * src; + void * dst; +} dma_descriptor_1d; + +#if __HVX_ARCH__ < 75 + +typedef struct dma_descriptor_2d_s { + void * next; + uint32_t reserved0:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + void * src; + void * dst; + uint32_t desc_type:8; + uint32_t reserved1:24; + uint32_t row_size:16; + uint32_t nrows:16; + uint32_t src_stride:16; + uint32_t dst_stride:16; + uint32_t src_offset:16; + uint32_t dst_offset:16; +} dma_descriptor_2d; + +#else + +typedef struct dma_descriptor_2d_s { + void * next; + uint32_t dst_stride:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + void * src; + void * dst; + uint32_t desc_type:8; + uint32_t reserved0:24; + uint32_t row_size:24; + uint32_t nrows_lo:8; + uint32_t nrows_hi:8; + uint32_t src_stride:24; + uint32_t offset:24; + uint32_t reserved1:8; +} dma_descriptor_2d; + +#endif + +typedef struct { + void *dst; + const void *src; +} dma_ptr; + +typedef struct dma_ring_s dma_ring; +struct dma_ring_s { + dma_descriptor_2d * desc; // descriptor pointers + dma_descriptor_2d * tail; // tail pointer + dma_ptr * dptr; // dst/src pointers + uint32_t push_idx; + uint32_t pop_idx; + uint32_t capacity; + uint32_t idx_mask; + struct htp_thread_trace * trace; + uintptr_t vtcm_base; + uintptr_t vtcm_end; +}; + +typedef struct dma_queue_s dma_queue; +typedef dma_queue * dma_queue_t; + +struct dma_queue_s { + dma_ring * ring; // Points to the descriptor ring state + uint8_t nocache; // Queue-specific bypass flag + bool alias; // When set, dma_queue_delete will not free the ring +}; + +void dma_queue_flush(dma_queue_t q); + +size_t dma_queue_sizeof(size_t capacity); +size_t dma_queue_alignof(void); +dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace); +void dma_queue_free(dma_queue_t q); + +size_t dma_queue_alias_sizeof(void); +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache); +void dma_queue_alias_free(dma_queue_t q); + +// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead +// but those do not seem to always compiler properly. +static inline void dmstart(void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmstart(%0)" : : "r"(next)); +} + +static inline void dmlink(void * cur, void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); +} + +static inline unsigned int dmpoll(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); + return ret; +} + +static inline unsigned int dmwait(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); + return ret; +} + +static inline dma_ptr dma_make_ptr(void *dst, const void *src) +{ + dma_ptr p = { dst, src }; + return p; +} + +static inline bool dma_is_vtcm(const dma_queue * q, const void * ptr) { + return (uintptr_t) ptr >= q->ring->vtcm_base && (uintptr_t) ptr < q->ring->vtcm_end; +} + +static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) { + dma_ring * r = q->ring; + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { + FARF(HIGH, "dma-push: queue full\n"); + return false; + } + + dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx]; + desc->src = (void *) dptr.src; + desc->dst = (void *) dptr.dst; + desc->size = size; + + r->dptr[r->push_idx] = dptr; + + if (size) { + desc->next = NULL; + desc->desc_size = 0; // 1D mode + desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache; + desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; + desc->order = 0; + desc->done = 0; + + htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); + dmlink(r->tail, desc); + r->tail = (dma_descriptor_2d *) desc; + } else { + desc->desc_size = 0; + desc->done = 1; + } + + r->push_idx = (r->push_idx + 1) & r->idx_mask; + return true; +} + +static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + dma_ring * r = q->ring; + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { + FARF(HIGH, "dma-push: queue full\n"); + return false; + } + + dma_descriptor_2d * desc = &r->desc[r->push_idx]; + + desc->next = NULL; + desc->reserved0 = 0; + desc->reserved1 = 0; + desc->desc_size = 1; // 2d mode + desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache; + desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; + desc->src_comp = 0; + desc->dst_comp = 0; + desc->order = 0; + desc->done = 0; + desc->src_stride = src_stride; + desc->dst_stride = dst_stride; + desc->src = (void *) dptr.src; + desc->dst = (void *) dptr.dst; + desc->row_size = row_size; + +#if __HVX_ARCH__ < 75 + desc->desc_type = 0; // 2d (16-bit) mode + desc->nrows = nrows; + desc->src_offset = 0; + desc->dst_offset = 0; +#else + desc->desc_type = 9; // 2d (24-bit) mode + desc->nrows_lo = (nrows & 0xff); + desc->nrows_hi = (nrows >> 8); + desc->offset = 0; +#endif + + r->dptr[r->push_idx] = dptr; + + if (nrows) { + htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); + dmlink(r->tail, desc); + r->tail = desc; + } else { + desc->done = 1; + } + + r->push_idx = (r->push_idx + 1) & r->idx_mask; + return true; +} + +static inline dma_ptr dma_queue_pop(dma_queue * q) { + dma_ring * r = q->ring; + dma_ptr dptr = { NULL }; + + if (r->push_idx == r->pop_idx) { + return dptr; + } + + dma_descriptor_2d * desc = &r->desc[r->pop_idx]; + + // Wait for desc to complete + if (!desc->done) { + while (!desc->done) { + dmpoll(); + } + } + htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); + + dptr = r->dptr[r->pop_idx]; + + r->pop_idx = (r->pop_idx + 1) & r->idx_mask; + return dptr; +} + +static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) { + dma_ring * r = q->ring; + dma_ptr dptr = { NULL }; + + if (r->push_idx == r->pop_idx) { + return dptr; + } + + dptr = r->dptr[r->pop_idx]; + + r->pop_idx = (r->pop_idx + 1) & r->idx_mask; + return dptr; +} + +static inline bool dma_queue_empty(dma_queue * q) { + return q->ring->push_idx == q->ring->pop_idx; +} + +static inline uint32_t dma_queue_depth(dma_queue * q) { + return (q->ring->push_idx - q->ring->pop_idx) & q->ring->idx_mask; +} + +static inline uint32_t dma_queue_capacity(dma_queue * q) { + return q->ring->capacity; +} + +#if __HVX_ARCH__ < 75 + +// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535. +// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions. + +#define DMA_MAX_FIELD_VAL 65535u + +static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + // Fast path: everything fits in 16 bits + if (nrows == 0 || __builtin_expect( + row_size <= DMA_MAX_FIELD_VAL && + nrows <= DMA_MAX_FIELD_VAL && + src_stride <= DMA_MAX_FIELD_VAL && + dst_stride <= DMA_MAX_FIELD_VAL, 1)) { + return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); + } + + // Contiguous block + // Use 1d DMA mode which supports sizes up to 24-bits (16MB) + if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { + size_t total = row_size * nrows; + return dma_queue_push_single_1d(q, dptr, total); + } + + // Stride overflow - fall back to row-by-row. + { + const uint8_t *src = (const uint8_t *) dptr.src; + uint8_t *dst = (uint8_t *) dptr.dst; + for (size_t r = 0; r < nrows; ++r) { + dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); + if (!dma_queue_push_single_1d(q, p, row_size)) + return false; + if (r + 1 < nrows) + dma_queue_pop(q); + } + return true; + } +} + +#else // HVX_ARCH >= 75 + +static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + // On v75 and up we always use 2d 24-bit mode + return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); +} + +#endif + +static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { + return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows); +} + +static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { + return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); +} + +#define DMA_CACHE_MAX_SIZE 256U + +typedef struct { + uint8_t *base; + uint32_t line_size; + uint32_t capacity; + uint32_t src[DMA_CACHE_MAX_SIZE]; + uint16_t age[DMA_CACHE_MAX_SIZE]; +} dma_cache; + +static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity) +{ + c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; + c->base = base; + c->line_size = line_size; + + for (unsigned i=0; i < c->capacity; i++) { + c->src[i] = 0; + c->age[i] = 0; + } +} + +static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) +{ + uint32_t o_idx = 0; + uint16_t o_age = 0; + uint8_t * dst = 0; + + for (unsigned i=0; i < c->capacity; i++) { + if (c->src[i] == (uint32_t) src) { + c->age[i] = 0; + dst = c->base + (i * c->line_size); nrows = 0; // dummy dma + } else { + c->age[i]++; + if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } + } + } + if (!dst) { + c->age[o_idx] = 0; + c->src[o_idx] = (uint32_t) src; + dst = c->base + o_idx * c->line_size; // normal nrows dma + return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); + } + + return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* HTP_DMA_H */ diff --git a/cpp/ggml-hexagon/htp/flash-attn-ops.c b/cpp/ggml-hexagon/htp/flash-attn-ops.c index cb04ee925..8a48814fc 100644 --- a/cpp/ggml-hexagon/htp/flash-attn-ops.c +++ b/cpp/ggml-hexagon/htp/flash-attn-ops.c @@ -24,7 +24,7 @@ #include "hvx-reduce.h" #include "hvx-flash-attn.h" #include "htp-vtcm.h" -#include "worker-pool.h" +#include "work-queue.h" #define LM_GGML_COMMON_DECL_C #include "ggml-common.h" @@ -204,7 +204,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * if (ir0 >= ir1) return; - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; dma_queue * dma = octx->ctx->dma[ith]; @@ -486,7 +486,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, (const __fp16 *) args->curr_k, total_rows, factx->DK, args->src_stride, start, end); @@ -494,7 +494,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) } static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { n = factx->n_threads; @@ -502,7 +502,7 @@ static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_row uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t }; if (n > 1) { - worker_pool_run_func(wp, fa_k_interleave_thread, &args, n); + work_queue_run(wp, fa_k_interleave_thread, &args, n); } else { fa_k_interleave_thread(1, 0, &args); } @@ -534,7 +534,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV, args->src_stride, (uint32_t) args->n_col_tiles, start, end); @@ -548,7 +548,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, void * v_tiles_dst, size_t n_col_tiles, uint32_t kv_start) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { n = factx->n_threads; @@ -556,7 +556,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; if (n > 1) { - worker_pool_run_func(wp, fa_v_interleave_thread, &args, n); + work_queue_run(wp, fa_v_interleave_thread, &args, n); } else { fa_v_interleave_thread(1, 0, &args); } @@ -589,7 +589,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t start = (size_t) i * rows_per_t; const size_t end = hex_smin(start + rows_per_t, factx->g_br); - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); // Parallel initialization of per-block state @@ -720,7 +720,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { n = factx->n_threads; @@ -739,7 +739,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, args.q_transposed = q->nb[1] < q->nb[2]; atomic_init(&args.barrier, n); if (n > 1) { - worker_pool_run_func(wp, fa_q_load_thread, &args, n); + work_queue_run(wp, fa_q_load_thread, &args, n); } else { fa_q_load_thread(1, 0, &args); } @@ -772,7 +772,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -820,7 +820,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -862,7 +862,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { n = factx->n_threads; @@ -871,7 +871,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; if (n > 1) { - worker_pool_run_func(wp, store_fn, &args, n); + work_queue_run(wp, store_fn, &args, n); } else { store_fn(1, 0, &args); } @@ -930,7 +930,7 @@ static inline void fa_softmax_impl( return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride @@ -1290,7 +1290,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, fa_softmax_args_t * sargs, size_t n_row_tiles, size_t n_row_tiles_g_br) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); worker_callback_t softmax_fn = fa_softmax_thread; @@ -1307,7 +1307,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); sargs->thread_div = init_fastdiv_values(n_use); - worker_pool_run_func(wp, softmax_fn, sargs, n_use); + work_queue_run(wp, softmax_fn, sargs, n_use); } else { softmax_fn(1, 0, sargs); } @@ -1519,8 +1519,8 @@ static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) { // ============================================================================ int hmx_flash_attn_ext(struct htp_ops_context * octx) { - struct htp_thread_trace * tr_hvx = octx->ctx ? &octx->ctx->trace[0] : NULL; - struct htp_thread_trace * tr_hmx = octx->ctx ? &octx->ctx->trace[HTP_MAX_NTHREADS] : NULL; + struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0]; + struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS]; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; const struct htp_tensor * v = octx->src[2]; @@ -1735,7 +1735,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const size_t k_src_stride = size_k_row_padded / sizeof(__fp16); const size_t v_src_stride = size_v_row_padded / sizeof(__fp16); - struct hmx_queue * hmx_q = ctx->hmx_queue; + hmx_queue_t hmx_q = ctx->hmx_queue; if (factx.pipeline) { // Pipeline path @@ -2084,7 +2084,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { } if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads); } return HTP_STATUS_OK; diff --git a/cpp/ggml-hexagon/htp/hex-bitmap.h b/cpp/ggml-hexagon/htp/hex-bitmap.h new file mode 100644 index 000000000..140898852 --- /dev/null +++ b/cpp/ggml-hexagon/htp/hex-bitmap.h @@ -0,0 +1,24 @@ +#ifndef HEX_BITMAP_H +#define HEX_BITMAP_H + +#include +#include +#include + +static inline void bitmap_set(uint32_t * bitmap, uint32_t idx) { + bitmap[idx / 32] |= (1U << (idx % 32)); +} + +static inline void bitmap_clear(uint32_t * bitmap, uint32_t idx) { + bitmap[idx / 32] &= ~(1U << (idx % 32)); +} + +static inline bool bitmap_test(const uint32_t * bitmap, uint32_t idx) { + return (bitmap[idx / 32] & (1U << (idx % 32))) != 0; +} + +static inline void bitmap_reset(uint32_t * bitmap, size_t size_in_bits) { + memset(bitmap, 0, ((size_in_bits + 31) / 32) * sizeof(uint32_t)); +} + +#endif // HEX_BITMAP_H diff --git a/cpp/ggml-hexagon/htp/hex-dma.h b/cpp/ggml-hexagon/htp/hex-dma.h index 98fcc9fda..9e9a5f950 100644 --- a/cpp/ggml-hexagon/htp/hex-dma.h +++ b/cpp/ggml-hexagon/htp/hex-dma.h @@ -1,375 +1,2 @@ -#ifndef HTP_DMA_H -#define HTP_DMA_H - -#include -#include -#include -#include -#include "hex-utils.h" - -#include "hex-profile.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date -typedef struct dma_descriptor_1d_s { - void * next; - uint32_t size:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; -} dma_descriptor_1d; - -#if __HVX_ARCH__ < 75 - -typedef struct dma_descriptor_2d_s { - void * next; - uint32_t reserved0:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; - uint32_t desc_type:8; - uint32_t reserved1:24; - uint32_t row_size:16; - uint32_t nrows:16; - uint32_t src_stride:16; - uint32_t dst_stride:16; - uint32_t src_offset:16; - uint32_t dst_offset:16; -} dma_descriptor_2d; - -#else - -typedef struct dma_descriptor_2d_s { - void * next; - uint32_t dst_stride:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; - uint32_t desc_type:8; - uint32_t reserved0:24; - uint32_t row_size:24; - uint32_t nrows_lo:8; - uint32_t nrows_hi:8; - uint32_t src_stride:24; - uint32_t offset:24; - uint32_t reserved1:8; -} dma_descriptor_2d; - -#endif - -typedef struct { - void *dst; - const void *src; -} dma_ptr; - -typedef struct { - dma_descriptor_2d * desc; // descriptor pointers - dma_descriptor_2d * tail; // tail pointer - dma_ptr * dptr; // dst/src pointers - uint32_t push_idx; - uint32_t pop_idx; - uint32_t capacity; - uint32_t idx_mask; - struct htp_thread_trace * trace; -} dma_queue; - -dma_queue * dma_queue_create(size_t capacity); -void dma_queue_delete(dma_queue * q); -void dma_queue_flush(dma_queue * q); - -// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead -// but those do not seem to always compiler properly. -static inline void dmstart(void * next) { - asm volatile(" release(%0):at" : : "r"(next)); - asm volatile(" dmstart(%0)" : : "r"(next)); -} - -static inline void dmlink(void * cur, void * next) { - asm volatile(" release(%0):at" : : "r"(next)); - asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); -} - -static inline unsigned int dmpoll(void) { - unsigned int ret = 0; - asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); - return ret; -} - -static inline unsigned int dmwait(void) { - unsigned int ret = 0; - asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); - return ret; -} - -static inline dma_ptr dma_make_ptr(void *dst, const void *src) -{ - dma_ptr p = { dst, src }; - return p; -} - -static const uint32_t dma_src_l2_bypass_on = 1; -static const uint32_t dma_dst_l2_bypass_on = 1; - -static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) { - if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) { - FARF(HIGH, "dma-push: queue full\n"); - return false; - } - - dma_descriptor_1d * desc = (dma_descriptor_1d *) &q->desc[q->push_idx]; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->size = size; - - q->dptr[q->push_idx] = dptr; - - if (size) { - desc->next = NULL; - desc->desc_size = 0; // 1D mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->order = 0; - desc->done = 0; - - htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx); - dmlink(q->tail, desc); - q->tail = (dma_descriptor_2d *) desc; - } else { - desc->desc_size = 0; - desc->done = 1; - } - - q->push_idx = (q->push_idx + 1) & q->idx_mask; - return true; -} - -static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) { - FARF(HIGH, "dma-push: queue full\n"); - return false; - } - - dma_descriptor_2d * desc = &q->desc[q->push_idx]; - - desc->next = NULL; - desc->reserved0 = 0; - desc->reserved1 = 0; - desc->desc_size = 1; // 2d mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->src_comp = 0; - desc->dst_comp = 0; - desc->order = 0; - desc->done = 0; - desc->src_stride = src_stride; - desc->dst_stride = dst_stride; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->row_size = row_size; - -#if __HVX_ARCH__ < 75 - desc->desc_type = 0; // 2d (16-bit) mode - desc->nrows = nrows; - desc->src_offset = 0; - desc->dst_offset = 0; -#else - desc->desc_type = 9; // 2d (24-bit) mode - desc->nrows_lo = (nrows & 0xff); - desc->nrows_hi = (nrows >> 8); - desc->offset = 0; -#endif - - q->dptr[q->push_idx] = dptr; - - if (nrows) { - htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx); - dmlink(q->tail, desc); - q->tail = desc; - } else { - desc->done = 1; - } - - // FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src); - q->push_idx = (q->push_idx + 1) & q->idx_mask; - return true; -} - -static inline dma_ptr dma_queue_pop(dma_queue * q) { - dma_ptr dptr = { NULL }; - - if (q->push_idx == q->pop_idx) { - return dptr; - } - - dma_descriptor_2d * desc = &q->desc[q->pop_idx]; - - // Wait for desc to complete - if (!desc->done) { - while (!desc->done) { - dmpoll(); - } - } - htp_trace_event_stop(q->trace, HTP_TRACE_EVT_DMA, q->pop_idx); - - dptr = q->dptr[q->pop_idx]; - - // FARF(ERROR, "dma-pop: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src); - q->pop_idx = (q->pop_idx + 1) & q->idx_mask; - return dptr; -} - -static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) { - dma_ptr dptr = { NULL }; - - if (q->push_idx == q->pop_idx) { - return dptr; - } - - dptr = q->dptr[q->pop_idx]; - - // FARF(ERROR, "dma-pop-nowait: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src); - q->pop_idx = (q->pop_idx + 1) & q->idx_mask; - return dptr; -} - -static inline bool dma_queue_empty(dma_queue * q) { - return q->push_idx == q->pop_idx; -} - -static inline uint32_t dma_queue_depth(dma_queue * q) { - return (q->push_idx - q->pop_idx) & q->idx_mask; -} - -static inline uint32_t dma_queue_capacity(dma_queue * q) { - return q->capacity; -} - -#if __HVX_ARCH__ < 75 - -// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535. -// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions. - -#define DMA_MAX_FIELD_VAL 65535u - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // Fast path: everything fits in 16 bits - if (nrows == 0 || __builtin_expect( - row_size <= DMA_MAX_FIELD_VAL && - nrows <= DMA_MAX_FIELD_VAL && - src_stride <= DMA_MAX_FIELD_VAL && - dst_stride <= DMA_MAX_FIELD_VAL, 1)) { - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); - } - - // Contiguous block - // Use 1d DMA mode which supports sizes up to 24-bits (16MB) - if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { - size_t total = row_size * nrows; - return dma_queue_push_single_1d(q, dptr, total); - } - - // Stride overflow — fall back to row-by-row. - { - const uint8_t *src = (const uint8_t *) dptr.src; - uint8_t *dst = (uint8_t *) dptr.dst; - for (size_t r = 0; r < nrows; ++r) { - dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); - if (!dma_queue_push_single_1d(q, p, row_size)) - return false; - if (r + 1 < nrows) - dma_queue_pop(q); - } - return true; - } -} - -#else // HVX_ARCH >= 75 - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // On v75 and up we always use 2d 24-bit mode - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); -} - -#endif - -static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows); -} - -static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); -} - -#define DMA_CACHE_MAX_SIZE 256U - -typedef struct { - uint8_t *base; - uint32_t line_size; - uint32_t capacity; - uint32_t src[DMA_CACHE_MAX_SIZE]; - uint16_t age[DMA_CACHE_MAX_SIZE]; -} dma_cache; - -static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity) -{ - c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; - c->base = base; - c->line_size = line_size; - - for (unsigned i=0; i < c->capacity; i++) { - c->src[i] = 0; - c->age[i] = 0; - } -} - -static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) -{ - uint32_t o_idx = 0; - uint16_t o_age = 0; - uint8_t * dst = 0; - - for (unsigned i=0; i < c->capacity; i++) { - if (c->src[i] == (uint32_t) src) { - c->age[i] = 0; - dst = c->base + (i * c->line_size); nrows = 0; // dummy dma - } else { - c->age[i]++; - if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } - } - } - if (!dst) { - c->age[o_idx] = 0; - c->src[o_idx] = (uint32_t) src; - dst = c->base + o_idx * c->line_size; // normal nrows dma - return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); - } - - return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0); -} - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* HTP_DMA_H */ +#pragma once +#include "dma-queue.h" diff --git a/cpp/ggml-hexagon/htp/hex-profile.h b/cpp/ggml-hexagon/htp/hex-profile.h index 8a37a4a06..a26961fc9 100644 --- a/cpp/ggml-hexagon/htp/hex-profile.h +++ b/cpp/ggml-hexagon/htp/hex-profile.h @@ -44,11 +44,11 @@ struct htp_thread_trace { }; static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { - if (tr && tr->events && tr->count < tr->max_events) { - uint32_t idx = tr->count; - tr->events[idx].id = id; - tr->events[idx].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); - tr->events[idx].cycles = (uint32_t) hex_get_cycles(); + if (tr->count < tr->max_events) { + uint32_t i = tr->count; + tr->events[i].id = id; + tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); + tr->events[i].cycles = (uint32_t) hex_get_cycles(); tr->count++; } } diff --git a/cpp/ggml-hexagon/htp/hex-utils.h b/cpp/ggml-hexagon/htp/hex-utils.h index 07930bef6..93e87efcb 100644 --- a/cpp/ggml-hexagon/htp/hex-utils.h +++ b/cpp/ggml-hexagon/htp/hex-utils.h @@ -30,21 +30,26 @@ static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, Q6_l2fetch_AP((void *) p, control); } -#define HEX_L2_LINE_SIZE 64 -#define HEX_L2_FLUSH_SIZE (128 * 1024) +static inline void hex_l2fetch_block(const void * addr, size_t size) { + if (size == 0) return; + const uint32_t width = 16384; // 16KB rows + const uint32_t height = (size + width - 1) / width; + hex_l2fetch(addr, width, width, height); +} + +#define HEX_L2_LINE_SIZE 128 +#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) +#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) +#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) static inline void hex_l2flush(void * addr, size_t size) { - if (size > HEX_L2_FLUSH_SIZE) { - qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); - } else { - const uint32_t s = (uint32_t) addr; - const uint32_t e = s + size; - for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE * 4) { - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3); - } + const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); + const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1); + for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) { + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0); + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1); + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2); + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3); } } diff --git a/cpp/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/cpp/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 740a8f87d..a4efb1b00 100644 --- a/cpp/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/cpp/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -1005,10 +1005,62 @@ static void transfer_activation_row_pair_fp32_to_fp16( } } +static void transfer_activation_row_pair_fp32_to_fp16_col_chunk( + __fp16 *restrict vtcm_dst, + const float *restrict row0, // offset by c_first + const float *restrict row1, // offset by c_first + uint32_t r, + uint32_t k_block, + uint32_t c_first, + uint32_t c_len, + uint32_t k_chunk_valid, + bool row0_valid, + bool row1_valid) { + + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + uint32_t c = 0; + for (; c + 32 <= k_chunk_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < c_len) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + uint32_t rem = (k_chunk_valid > c) ? (k_chunk_valid - c) : 0; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } +} + static void transfer_activation_chunk_fp32_to_fp16_gathered( __fp16 *restrict vtcm_dst, const float *restrict src, uint32_t start_row, + uint32_t vtcm_start_row, uint32_t n_rows, uint32_t k_block, const struct mmid_row_mapping *matrix_rows, @@ -1029,8 +1081,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered( for (r = 0; r < n_rows_tiled; r += 2) { uint32_t r_idx0 = start_row + r + 0; uint32_t r_idx1 = start_row + r + 1; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; @@ -1073,9 +1126,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered( } for (; r < n_rows_padded; r += 2) { - uint32_t r_idx0 = start_row + r; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx const bool row0_valid = (start_row + r + 0) < cne1; const bool row1_valid = (start_row + r + 1) < cne1; @@ -1135,6 +1188,7 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( __fp16 *restrict vtcm_dst, const float *restrict src, uint32_t start_row, + uint32_t vtcm_start_row, uint32_t n_rows, uint32_t k_block, const struct mmid_row_mapping *matrix_rows, @@ -1152,8 +1206,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( for (r = 0; r < n_rows_tiled; r += 2) { uint32_t r_idx0 = start_row + r + 0; uint32_t r_idx1 = start_row + r + 1; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; @@ -1193,9 +1248,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( } for (; r < n_rows_padded; r += 2) { - uint32_t r_idx0 = start_row + r; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx const bool row0_valid = (start_row + r + 0) < cne1; const bool row1_valid = (start_row + r + 1) < cne1; @@ -1253,6 +1308,7 @@ static void transfer_output_chunk_fp16_to_fp32_scattered( float *restrict dst, const __fp16 *restrict vtcm_src, uint32_t start_row, + uint32_t vtcm_start_row, uint32_t n_rows, uint32_t n_cols, const struct mmid_row_mapping *matrix_rows, @@ -1269,8 +1325,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered( for (size_t r = 0; r < n_rows; r += 2) { uint32_t r_idx0 = start_row + r + 0; uint32_t r_idx1 = start_row + r + 1; - const size_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; - const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile + uint32_t lr = vtcm_start_row + r; // vtcm-local row + const size_t r0 = (lr / HTP_MM_HMX_TILE_N_ROWS); + const size_t r1 = (lr % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; if (r_idx0 >= cne1) break; diff --git a/cpp/ggml-hexagon/htp/hmx-queue.c b/cpp/ggml-hexagon/htp/hmx-queue.c index 5f6a5e206..c369d3dd2 100644 --- a/cpp/ggml-hexagon/htp/hmx-queue.c +++ b/cpp/ggml-hexagon/htp/hmx-queue.c @@ -14,7 +14,7 @@ #define QURT_LOWEST_PRIO (254) -static inline void hmx_lock(struct hmx_queue *q) +static inline void hmx_lock(hmx_queue_t q) { if (!q->hmx_locked) { HAP_compute_res_hmx_lock(q->hap_rctx); @@ -22,7 +22,7 @@ static inline void hmx_lock(struct hmx_queue *q) } } -static inline void hmx_unlock(struct hmx_queue *q) +static inline void hmx_unlock(hmx_queue_t q) { if (q->hmx_locked) { HAP_compute_res_hmx_unlock(q->hap_rctx); @@ -30,7 +30,7 @@ static inline void hmx_unlock(struct hmx_queue *q) } } -static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { +static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { unsigned int ir = atomic_load(&q->idx_read); while (ir != atomic_load(&q->idx_write)) { @@ -38,7 +38,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { if (!d->done) { FARF(HIGH, "hmx-queue-process: ir %u func %p data %p", ir, d->func, d->data); - enum hmx_queue_signal sig = (enum hmx_queue_signal) (unsigned int) d->func; + uintptr_t sig = (uintptr_t) d->func; switch (sig) { case HMX_QUEUE_NOOP: /* noop */; break; case HMX_QUEUE_KILL: *killed = true; break; @@ -61,7 +61,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { } static void hmx_queue_thread(void * arg) { - struct hmx_queue * q = (struct hmx_queue *) arg; + hmx_queue_t q = (hmx_queue_t) arg; FARF(HIGH, "hmx-queue-thread: started"); @@ -93,34 +93,41 @@ static void hmx_queue_thread(void * arg) { FARF(HIGH, "hmx-queue-thread: stopped"); } -struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) { +size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) { capacity = hex_ceil_pow2(capacity); + size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); + size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); + size_t size_stack = stack_size; + return size_q + size_desc + size_stack; +} + +size_t hmx_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) { + capacity = hex_ceil_pow2(capacity); + size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); + size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); + + uint8_t * block = (uint8_t *) ptr; + + hmx_queue_t q = (hmx_queue_t) block; block += size_q; + memset(q, 0, sizeof(struct hmx_queue_s)); - struct hmx_queue * q = (struct hmx_queue *) memalign(32, sizeof(struct hmx_queue)); - if (q == NULL) { - FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__); - return NULL; - } - memset(q, 0, sizeof(struct hmx_queue)); q->capacity = capacity; q->idx_mask = capacity - 1; q->hap_rctx = hap_rctx; + q->external_mem = true; - q->desc = (struct hmx_queue_desc *) memalign(64, capacity * sizeof(struct hmx_queue_desc)); - if (!q->desc) { - FARF(ERROR, "hmx-queue: failed to allocate HMX queue descriptors\n"); - return NULL; - } + q->desc = (struct hmx_queue_desc *) block; block += size_desc; memset(q->desc, 0, capacity * sizeof(struct hmx_queue_desc)); - const size_t stack_size = HMX_QUEUE_THREAD_STACK_SIZE; - q->stack = (unsigned char *) memalign(64, stack_size); - if (!q->stack) { - FARF(ERROR, "hmx-queue: thread stack allocation failed (%zu bytes)", stack_size); - return NULL; - } + q->stack = block; memset(q->stack, 0, stack_size); + q->trace = trace; + // Match caller thread priority (same pattern as worker-pool.c). int prio = qurt_thread_get_priority(qurt_thread_get_id()); if (prio < 1) { @@ -148,7 +155,7 @@ struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) { return q; } -void hmx_queue_delete(struct hmx_queue * q) { +void hmx_queue_free(hmx_queue_t q) { if (!q) { return; } @@ -160,8 +167,4 @@ void hmx_queue_delete(struct hmx_queue * q) { int status; qurt_thread_join(q->thread, &status); - - free(q->desc); - free(q->stack); - free(q); } diff --git a/cpp/ggml-hexagon/htp/hmx-queue.h b/cpp/ggml-hexagon/htp/hmx-queue.h index b176fa179..c2b1859a2 100644 --- a/cpp/ggml-hexagon/htp/hmx-queue.h +++ b/cpp/ggml-hexagon/htp/hmx-queue.h @@ -17,8 +17,6 @@ extern "C" { #endif -#define HMX_QUEUE_THREAD_STACK_SIZE (16 * 1024) - #if __HVX_ARCH__ > 79 #define HMX_QUEUE_POLL_COUNT 2000 #else @@ -41,7 +39,7 @@ struct hmx_queue_desc { atomic_uint done; }; -struct hmx_queue { +struct hmx_queue_s { struct hmx_queue_desc * desc; atomic_uint idx_write; // updated by producer (push) atomic_uint idx_read; // updated by consumer (process) @@ -55,19 +53,24 @@ struct hmx_queue { uint32_t hap_rctx; bool hmx_locked; struct htp_thread_trace * trace; + bool external_mem; // memory owned externally }; -struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx); -void hmx_queue_delete(struct hmx_queue * q); +typedef struct hmx_queue_s * hmx_queue_t; + +size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size); +size_t hmx_queue_alignof(void); +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace); +void hmx_queue_free(hmx_queue_t q); static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) { struct hmx_queue_desc d = { func, data }; return d; } -static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) { +static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) { unsigned int ir = atomic_load(&q->idx_read); - unsigned int iw = q->idx_write; + unsigned int iw = atomic_load(&q->idx_write); if (((iw + 1) & q->idx_mask) == ir) { FARF(HIGH, "hmx-queue-push: queue is full\n"); @@ -87,25 +90,25 @@ static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) return true; } -static inline bool hmx_queue_signal(struct hmx_queue *q, enum hmx_queue_signal sig) { +static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) { return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL)); } -static inline bool hmx_queue_empty(struct hmx_queue * q) { - return q->idx_pop == q->idx_write; +static inline bool hmx_queue_empty(hmx_queue_t q) { + return q->idx_pop == atomic_load(&q->idx_write); } -static inline uint32_t hmx_queue_depth(struct hmx_queue * q) { - return (q->idx_read - q->idx_read) & q->idx_mask; +static inline uint32_t hmx_queue_depth(hmx_queue_t q) { + return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask; } -static inline uint32_t hmx_queue_capacity(struct hmx_queue * q) { +static inline uint32_t hmx_queue_capacity(hmx_queue_t q) { return q->capacity; } -static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) { +static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) { unsigned int ip = q->idx_pop; - unsigned int iw = q->idx_write; + unsigned int iw = atomic_load(&q->idx_write); struct hmx_queue_desc rd = { NULL, NULL }; if (ip == iw) { @@ -126,7 +129,7 @@ static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) { return rd; } -static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { +static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) { while (1) { struct hmx_queue_desc d = hmx_queue_pop_one(q); @@ -138,15 +141,15 @@ static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { } } -static inline void hmx_queue_flush(struct hmx_queue * q) { +static inline void hmx_queue_flush(hmx_queue_t q) { while (hmx_queue_pop_one(q).func != NULL) ; } -static inline void hmx_queue_wakeup(struct hmx_queue * q) { +static inline void hmx_queue_wakeup(hmx_queue_t q) { hmx_queue_signal(q, HMX_QUEUE_WAKEUP); } -static inline void hmx_queue_suspend(struct hmx_queue *q) { +static inline void hmx_queue_suspend(hmx_queue_t q) { hmx_queue_signal(q, HMX_QUEUE_SUSPEND); } diff --git a/cpp/ggml-hexagon/htp/htp-ctx.h b/cpp/ggml-hexagon/htp/htp-ctx.h index e13103fb1..c4b60eb26 100644 --- a/cpp/ggml-hexagon/htp/htp-ctx.h +++ b/cpp/ggml-hexagon/htp/htp-ctx.h @@ -5,7 +5,8 @@ #include "hmx-queue.h" #include "htp-ops.h" #include "hex-profile.h" -#include "worker-pool.h" +#include "work-queue.h" +#include "hex-fastdiv.h" #include #include @@ -52,6 +53,9 @@ struct htp_ops_context { const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS]; }; + dma_queue ** src_dma[HTP_OP_MAX_INPUTS]; + dma_queue ** dst_dma[HTP_OP_MAX_OUTPUTS]; + // TODO convert these to an array struct htp_spad src0_spad; struct htp_spad src1_spad; @@ -65,11 +69,16 @@ struct htp_ops_context { // Main context for htp DSP backend struct htp_context { - dspqueue_t queue; - dma_queue * dma[HTP_MAX_NTHREADS]; + dspqueue_t dsp_queue; + struct htp_mmap mmap[HTP_MAX_MMAPS]; - worker_pool_context_t worker_pool; + dma_queue_t dma[HTP_MAX_NTHREADS]; + dma_queue_t dma_cached[HTP_MAX_NTHREADS]; + work_queue_t work_queue; + hmx_queue_t hmx_queue; + uint32_t n_threads; + struct fastdiv_values n_threads_div; int thread_id; int thread_prio; @@ -86,6 +95,7 @@ struct htp_context { atomic_bool vtcm_needs_release; uint64_t max_vmem; + uint32_t dirty_map[HTP_OP_MAX_TENSORS / 32]; // Persistent DDR scratchpad for MUL_MAT_ID mappings void * ddr_spad_base; @@ -93,7 +103,10 @@ struct htp_context { struct htp_ops_context octx; - struct hmx_queue * hmx_queue; // Async HMX queue for pipeline overlap + qurt_thread_t main_thread; + void * main_stack; + atomic_bool killed; + size_t footprint; }; int op_matmul(struct htp_ops_context * octx); diff --git a/cpp/ggml-hexagon/htp/htp-ops.h b/cpp/ggml-hexagon/htp/htp-ops.h index 49ae4382d..797d605ef 100644 --- a/cpp/ggml-hexagon/htp/htp-ops.h +++ b/cpp/ggml-hexagon/htp/htp-ops.h @@ -108,8 +108,7 @@ enum htp_op_code { #define HTP_OP_MAX_KERN_PARAMS 32 #define HTP_OP_MAX_BUFS 16 -#define HTP_OP_MAX_REQS 256 -#define HTP_OP_MAX_TENSORS (HTP_OP_MAX_REQS * HTP_OP_MAX_INPUTS + HTP_OP_MAX_REQS) +#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) #define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) @@ -117,16 +116,18 @@ enum htp_op_code { enum htp_tensor_flags { HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights) - HTP_TENSOR_FLUSHED = (1U << 1) // Tensor buffer has been flushed (set by the NPU) + HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed }; // Tensor descriptor struct htp_tensor { uint32_t data; // Buffer offset in the messages, and data pointer on the NPU + uint32_t alias; // Index of the canonical tensor for this memory buffer uint32_t size; // Data size in bytes uint32_t flags; // Buffer / tensor flags - uint16_t type; // Data type + uint32_t type; // Data type uint16_t bi; // Buffer index + uint16_t ti; // Tensor index uint32_t ne[HTP_OP_MAX_DIMS]; // Number of elements uint32_t nb[HTP_OP_MAX_DIMS]; // Stride in bytes (see ggml.h lm_ggml_tensor) }; @@ -169,6 +170,8 @@ enum htp_profiler_mode { enum htp_trace_event_id { HTP_TRACE_EVT_DMA = 0, + HTP_TRACE_EVT_L2FLUSH = 1, + HTP_TRACE_EVT_INIT = 2, HTP_TRACE_EVT_HVX_COMP = 20, HTP_TRACE_EVT_HVX_A_QUANT = 21, diff --git a/cpp/ggml-hexagon/htp/htp-tensor.c b/cpp/ggml-hexagon/htp/htp-tensor.c new file mode 100644 index 000000000..3f028a7e4 --- /dev/null +++ b/cpp/ggml-hexagon/htp/htp-tensor.c @@ -0,0 +1,204 @@ +#include "htp-tensor.h" + +#include +#include + +#include "hex-common.h" +#include "hex-utils.h" +#include "hex-fastdiv.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "work-queue.h" + +struct l2flush_task { + struct htp_thread_trace * trace; + uint32_t start; + uint32_t end; + uint32_t chunk_size; + uint32_t ti; +}; + +static void l2flush_thread_worker(unsigned int n, unsigned int i, void * data) { + struct l2flush_task * task = (struct l2flush_task *) data; + const uint32_t start = task->start; + const uint32_t end = task->end; + const uint32_t ti = task->ti; + const uint32_t chunk_size = task->chunk_size; + + const uint32_t thread_s = start + i * chunk_size; + if (thread_s >= end) { + return; + } + uint32_t thread_e = thread_s + chunk_size; + if (thread_e > end) { + thread_e = end; + } + + struct htp_thread_trace * tr = &task->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, ti); + hex_l2flush((void *) (uintptr_t) thread_s, thread_e - thread_s); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, ti); +} + +static void flush_all_dcache(struct htp_context * ctx) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + hex_l2fetch_block(ctx, ctx->footprint); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS); +} + +static void flush_tensor_range(struct htp_context * ctx, const struct htp_tensor * t) { + struct htp_thread_trace * tr = &ctx->trace[0]; + + if (t->size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_task task; + task.start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); + task.end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); + task.ti = t->ti; + task.trace = ctx->trace; + + const uint32_t total_size = task.end - task.start; + const uint32_t n_blocks = (total_size + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + const uint32_t blocks_per_thread = fastdiv(n_blocks + ctx->n_threads - 1, &ctx->n_threads_div); + task.chunk_size = blocks_per_thread * HEX_L2_BLOCK_SIZE; + + work_queue_run(ctx->work_queue, l2flush_thread_worker, &task, ctx->n_threads); + } else { + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + hex_l2flush((void *) t->data, t->size); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + } + + htp_tensor_make_clean(t, ctx->dirty_map); +} + +void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t) { + if (!bitmap_test(ctx->dirty_map, t->ti)) { + return; + } + + if (t->size > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + flush_tensor_range(ctx, t); +} + +// One dirty tensor's line-aligned range, placed in the flattened global block space. +struct l2flush_range { + uint32_t start; // line-aligned start address + uint32_t end; // line-aligned end address + uint32_t block_first; // global block index of this range's first block + uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial) +}; + +struct l2flush_multi_task { + struct htp_thread_trace * trace; + struct l2flush_range ranges[HTP_OP_MAX_INPUTS]; + uint32_t n_ranges; + uint32_t total_blocks; + uint32_t blocks_per_thread; +}; + +static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct l2flush_multi_task * task = (struct l2flush_multi_task *) data; + + const uint32_t gb_first = i * task->blocks_per_thread; + uint32_t gb_last = gb_first + task->blocks_per_thread; + if (gb_last > task->total_blocks) { + gb_last = task->total_blocks; + } + if (gb_first >= gb_last) { + return; + } + + struct htp_thread_trace * tr = &task->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); + + for (uint32_t r = 0; r < task->n_ranges; r++) { + const struct l2flush_range * rg = &task->ranges[r]; + const uint32_t rb_first = rg->block_first; + const uint32_t rb_last = rg->block_first + rg->n_blocks; + + const uint32_t lo = gb_first > rb_first ? gb_first : rb_first; + const uint32_t hi = gb_last < rb_last ? gb_last : rb_last; + if (lo >= hi) { + continue; + } + + const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE; + uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE; + if (e > rg->end) { + e = rg->end; + } + hex_l2flush((void *) (uintptr_t) s, e - s); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); +} + +void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { + uint64_t total_dirty = 0; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (t && bitmap_test(ctx->dirty_map, t->ti)) { + total_dirty += t->size; + } + } + + if (total_dirty == 0) { + return; + } + + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + // Aggregate is small enough to walk. Thread it across all dirty ranges at once + // when it is worth the dispatch, otherwise flush sequentially. + if (total_dirty > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = 0; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (!t || !bitmap_test(ctx->dirty_map, t->ti)) { + continue; + } + // Clear as we go: dedups a tensor passed as multiple srcs (e.g. mul(x,x)). + htp_tensor_make_clean(t, ctx->dirty_map); + + struct l2flush_range * rg = &task.ranges[task.n_ranges++]; + rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + return; + } + + struct htp_thread_trace * tr = &ctx->trace[0]; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (!t || !bitmap_test(ctx->dirty_map, t->ti)) { + continue; + } + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + hex_l2flush((void *) t->data, t->size); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + htp_tensor_make_clean(t, ctx->dirty_map); + } +} diff --git a/cpp/ggml-hexagon/htp/htp-tensor.h b/cpp/ggml-hexagon/htp/htp-tensor.h new file mode 100644 index 000000000..e503bb90f --- /dev/null +++ b/cpp/ggml-hexagon/htp/htp-tensor.h @@ -0,0 +1,36 @@ +#ifndef HTP_TENSOR_H +#define HTP_TENSOR_H + +#include +#include "htp-ops.h" +#include "hex-bitmap.h" + +static inline struct htp_tensor * htp_tensor_alias(const struct htp_tensor * t) { + return (struct htp_tensor *) (uintptr_t) t->alias; +} + +static inline void * htp_tensor_data(const struct htp_tensor * t) { + return (void *) (uintptr_t) t->data; +} + +static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { + return (uint32_t *) &t->flags; +} + +static inline void htp_tensor_make_dirty(const struct htp_tensor * t, uint32_t * dirty_map) { + struct htp_tensor * curr = (struct htp_tensor *) t; + do { + bitmap_set(dirty_map, curr->ti); + curr = htp_tensor_alias(curr); + } while (curr != t); +} + +static inline void htp_tensor_make_clean(const struct htp_tensor * t, uint32_t * dirty_map) { + bitmap_clear(dirty_map, t->ti); +} + +struct htp_context; +void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t); +void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); + +#endif // HTP_TENSOR_H diff --git a/cpp/ggml-hexagon/htp/main.c b/cpp/ggml-hexagon/htp/main.c index 3bc24846c..eaafe5426 100644 --- a/cpp/ggml-hexagon/htp/main.c +++ b/cpp/ggml-hexagon/htp/main.c @@ -25,112 +25,44 @@ #define LM_GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-bitmap.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "htp_iface.h" -#include "worker-pool.h" - -AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) { - struct htp_context * ctx; - int err = 0; - - ctx = calloc(1, sizeof(*ctx)); - if (ctx == NULL) { - return AEE_ENOMEMORY; - } - - // Use the context structure as the handle - *handle = (remote_handle64) ctx; - - // Enable FARF logs - HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0); - - // Set client class - { - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_apptype; - request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; - - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; - } - } - - { - HAP_power_request_t request; - memset(&request, 0, sizeof(request)); +#include "work-queue.h" +#include "hex-profile.h" - request.type = HAP_power_set_DCVS_v3; - request.dcvs_v3.set_dcvs_enable = TRUE; - request.dcvs_v3.dcvs_enable = FALSE; - request.dcvs_v3.set_bus_params = TRUE; - request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.set_core_params = TRUE; - request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.set_sleep_disable = TRUE; - request.dcvs_v3.sleep_disable = TRUE; +#define HMX_QUEUE_CAPACITY 16 +#define HMX_QUEUE_STACK_SIZE 16384 +#define WORK_QUEUE_CAPACITY 16 +#define WORK_QUEUE_STACK_SIZE 16384 +#define MAIN_THREAD_STACK_SIZE 32768 -#if (__HEXAGON_ARCH__ >= 79) - HAP_set_dcvs_v3_protected_bus_corners(&request, 1); -#endif - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; - } +_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS, + "work-queue thread cap must be >= HTP_MAX_NTHREADS"); - memset(&request, 0, sizeof(request)); - request.type = HAP_power_set_HVX; - request.hvx.power_up = TRUE; - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; - } - } +struct htp_handle { + struct htp_context * ctx; +}; -#if __HVX_ARCH__ >= 75 - { - // Power on HMX and set HMX clock - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_HMX_v2; - request.hmx_v2.set_power = TRUE; - request.hmx_v2.power_up = TRUE; - request.hmx_v2.set_clock = TRUE; - request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; - FARF(ALWAYS, "Setting HMX clock\n"); - err = HAP_power_set((void *) ctx, &request); - if (err != AEE_SUCCESS) { - FARF(ERROR, "ggml-hex: error setting HMX clock."); - return err; - } - } -#else - { - // Power on HMX - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_HMX; - request.hmx.power_up = TRUE; - FARF(ALWAYS, "Powering HMX on\n"); - err = HAP_power_set((void *) ctx, &request); - if (err != AEE_SUCCESS) { - FARF(ERROR, "ggml-hex: error powering on HMX."); - return err; - } +AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) { + (void) uri; + struct htp_handle * h = calloc(1, sizeof(*h)); + if (h == NULL) { + return AEE_ENOMEMORY; } -#endif + *handle = (remote_handle64) h; return AEE_SUCCESS; } AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { + return AEE_EBADPARM; + } + int err = enable ? HAP_user_etm_enable() : HAP_user_etm_disable(); if (err) { if (err == AEE_EVERSIONNOTSUPPORT) { @@ -143,10 +75,11 @@ AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) { } AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_iface_pmu_conf* pmu_conf) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; if (mode == HTP_PROF_PMU) { const uint32_t* events = pmu_conf->events; @@ -179,48 +112,55 @@ AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_if } AEEResult htp_iface_close(remote_handle64 handle) { - struct htp_context * ctx = (struct htp_context *) handle; - - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { return AEE_EBADPARM; } - if (ctx->queue) { - FARF(ERROR, "Closing handle with queue still open"); - return AEE_EITEMBUSY; - } + struct htp_context * ctx = h->ctx; + if (ctx) { + if (ctx->dsp_queue) { + FARF(ERROR, "Closing handle with queue still open"); + return AEE_EITEMBUSY; + } - // release the mmaps (if any) - for (uint32_t i=0; immap[i].size) { + // release the mmaps (if any) + for (uint32_t i=0; immap[i].size) { #if __HVX_ARCH__ > 73 - HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); + HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); #else - HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); + HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); #endif - ctx->mmap[i].size = 0; - ctx->mmap[i].base = NULL; - ctx->mmap[i].fd = -1; + ctx->mmap[i].size = 0; + ctx->mmap[i].base = NULL; + ctx->mmap[i].fd = -1; + } } - } - if (ctx->profiler) { - qurt_pmu_enable(1); - } + if (ctx->profiler) { + qurt_pmu_enable(1); + } + + if (ctx->etm) { + HAP_user_etm_disable(); + } - if (ctx->etm) { - HAP_user_etm_disable(); + // Free the unified block (ctx is the base address of the block) + free(ctx); + h->ctx = NULL; } - free(ctx); + free(h); return AEE_SUCCESS; } AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; // See if we already have this mapping for (uint32_t i=0; ictx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; for (uint32_t i=0; immap[i]; @@ -358,91 +299,268 @@ static void vtcm_free(struct htp_context * ctx) { } } +static void htp_main_thread(void * context); static void htp_packet_callback(dspqueue_t queue, int error, void * context); static void htp_error_callback(dspqueue_t queue, int error, void * context); AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) { - struct htp_context * ctx = (struct htp_context *) handle; - - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { return AEE_EBADPARM; } - if (ctx->queue) { + if (h->ctx) { FARF(ERROR, "Queue already open"); return AEE_EITEMBUSY; } - // Import queue created on the CPU - int err = dspqueue_import(dsp_queue_id, // Queue ID from dspqueue_export - htp_packet_callback, // Packet callback - htp_error_callback, // Error callback; no errors expected on the DSP - (void *) ctx, // Callback context - &ctx->queue); + // Cache the original FastRPC thread priority, then calculate compute priority + int fastrpc_tid = qurt_thread_get_id(); + int fastrpc_prio = qurt_thread_get_priority(fastrpc_tid); + int main_prio = fastrpc_prio - 10; + if (main_prio < 1) main_prio = 1; + + dspqueue_t dsp_queue = NULL; + bool use_callbacks = false; + + // Import queue with NULL callbacks to avoid starting dspueue internal threads + int err = dspqueue_import(dsp_queue_id, NULL, NULL, (void *) h, &dsp_queue); + if (err == AEE_EBADPARM) { + // Fallback for devices that don't support NULL callbacks + FARF(HIGH, "dspqueue import with NULL callbacks failed, trying with callbacks"); + use_callbacks = true; + err = dspqueue_import(dsp_queue_id, htp_packet_callback, htp_error_callback, (void *) h, &dsp_queue); + } + if (err) { FARF(ERROR, "Queue import failed with 0x%08x", (unsigned) err); return err; } + qurt_sysenv_max_hthreads_t hw_threads; + qurt_sysenv_get_max_hw_threads(&hw_threads); + uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + + if (n_hvx == 0) { + n_hvx = hw_nhvx; + } + if (n_hvx > hw_threads.max_hthreads) { + n_hvx = hw_threads.max_hthreads; + } + if (n_hvx > HTP_MAX_NTHREADS) { + n_hvx = HTP_MAX_NTHREADS; + } + + // layout segments of our contiguous block + + // 1. htp_context : sits at the base (block is 4K-aligned via memalign below) + size_t offset = sizeof(struct htp_context); + + // 2. main_stack + size_t offset_main_stack = 0; + size_t size_main_stack = 0; + if (!use_callbacks) { + offset_main_stack = hex_align_up(offset, 4096); + size_main_stack = MAIN_THREAD_STACK_SIZE; + offset = offset_main_stack + size_main_stack; + } + + // 3. work_queue + size_t wq_align = work_queue_alignof(); + size_t offset_wq = hex_align_up(offset, wq_align); + size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + offset = offset_wq + size_wq; + + // 4. dma_queue + size_t dma_align = dma_queue_alignof(); + size_t offset_dma = hex_align_up(offset, dma_align); + size_t size_dma = 0; + for (uint32_t i = 0; i < n_hvx; i++) { + size_dma = hex_align_up(size_dma, dma_queue_alignof()); + size_dma += dma_queue_sizeof(256); + size_dma = hex_align_up(size_dma, dma_queue_alignof()); + size_dma += dma_queue_alias_sizeof(); + } + offset = offset_dma + size_dma; + + // 5. hmx_queue + size_t offset_hmx = 0; + size_t size_hmx = 0; + if (n_hmx) { + size_t hmx_align = hmx_queue_alignof(); + offset_hmx = hex_align_up(offset, hmx_align); + size_hmx = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE); + offset = offset_hmx + size_hmx; + } + + size_t footprint = hex_align_up(offset, 128); + + void * block = memalign(4096, footprint); + if (!block) { + FARF(ERROR, "Unable to allocate unified block of size %zu\n", footprint); + dspqueue_close(dsp_queue); + return AEE_ENOMEMORY; + } + memset(block, 0, footprint); + + h->ctx = (struct htp_context *) block; + struct htp_context * ctx = h->ctx; + ctx->footprint = footprint; + + ctx->thread_id = fastrpc_tid; + ctx->thread_prio = main_prio; ctx->max_vmem = max_vmem; - ctx->thread_id = qurt_thread_get_id(); - ctx->thread_prio = qurt_thread_get_priority(ctx->thread_id); + ctx->dsp_queue = dsp_queue; - // allocate VTCM err = vtcm_alloc(ctx); if (err != AEE_SUCCESS) { FARF(ERROR, "Unable to allocate VTCM"); + htp_iface_stop(handle); return AEE_ENOMEMORY; } - ctx->hmx_enabled = n_hmx; - ctx->hmx_queue = NULL; - if (n_hmx) { - ctx->hmx_queue = hmx_queue_create(16, ctx->vtcm_rctx); - if (ctx->hmx_queue) { - ctx->hmx_queue->trace = &ctx->trace[HTP_MAX_NTHREADS]; - } else { - FARF(ERROR, "hmx-queue-create failed"); - ctx->hmx_enabled = false; + HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0); + + // Set client class + { + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_apptype; + request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; + + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; } } - FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); - qurt_sysenv_max_hthreads_t hw_threads; - qurt_sysenv_get_max_hw_threads(&hw_threads); - uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + // DCVS setup + { + HAP_power_request_t request; + memset(&request, 0, sizeof(request)); - if (n_hvx == 0) { - n_hvx = hw_nhvx; + request.type = HAP_power_set_DCVS_v3; + request.dcvs_v3.set_dcvs_enable = TRUE; + request.dcvs_v3.dcvs_enable = FALSE; + request.dcvs_v3.set_bus_params = TRUE; + request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.set_core_params = TRUE; + request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.set_sleep_disable = TRUE; + request.dcvs_v3.sleep_disable = TRUE; + +#if (__HEXAGON_ARCH__ >= 79) + HAP_set_dcvs_v3_protected_bus_corners(&request, 1); +#endif + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; + } + + memset(&request, 0, sizeof(request)); + request.type = HAP_power_set_HVX; + request.hvx.power_up = TRUE; + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; + } } - if (n_hvx > hw_threads.max_hthreads) { - n_hvx = hw_threads.max_hthreads; + +#if __HVX_ARCH__ >= 75 + { + // Power on HMX and set HMX clock + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HMX_v2; + request.hmx_v2.set_power = TRUE; + request.hmx_v2.power_up = TRUE; + request.hmx_v2.set_clock = TRUE; + request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; + request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX; + request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX; + request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; + FARF(ALWAYS, "Setting HMX clock\n"); + err = HAP_power_set((void *) ctx, &request); + if (err != AEE_SUCCESS) { + FARF(ERROR, "ggml-hex: error setting HMX clock."); + htp_iface_stop(handle); + return err; + } } - if (n_hvx > HTP_MAX_NTHREADS) { - n_hvx = HTP_MAX_NTHREADS; +#else + { + // Power on HMX + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HMX; + request.hmx.power_up = TRUE; + FARF(ALWAYS, "Powering HMX on\n"); + err = HAP_power_set((void *) ctx, &request); + if (err != AEE_SUCCESS) { + FARF(ERROR, "ggml-hex: error powering on HMX."); + htp_iface_stop(handle); + return err; + } } +#endif + + ctx->hmx_enabled = n_hmx; + ctx->hmx_queue = NULL; + if (n_hmx) { + void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); + ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]); + } + FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); ctx->n_threads = n_hvx; + ctx->n_threads_div = init_fastdiv_values(ctx->n_threads); + + // Initialize DMA queues + uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); + size_t size_dma_q = dma_queue_sizeof(256); + size_t size_dma_alias = dma_queue_alias_sizeof(); + for (int i = 0; i < ctx->n_threads; i++) { - ctx->dma[i] = dma_queue_create(256); // queue depth - if (ctx->dma[i]) { - ctx->dma[i]->trace = &ctx->trace[i]; - } + dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); + ctx->dma_cached[i] = dma_queue_init(dma_ptr_curr, 256, (uintptr_t) ctx->vtcm_base, ctx->vtcm_size, &ctx->trace[i]); + dma_ptr_curr += size_dma_q; + + dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); + ctx->dma[i] = dma_queue_alias_init(dma_ptr_curr, ctx->dma_cached[i], 1); + dma_ptr_curr += size_dma_alias; } ctx->ddr_spad_size = 512 * 1024; // 512 KB ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size); - // init worker pool - err = worker_pool_init(&ctx->worker_pool, n_hvx); - if (err != AEE_SUCCESS) { - FARF(ERROR, "Unable to create worker pool"); - if (ctx->ddr_spad_base) { - free(ctx->ddr_spad_base); - ctx->ddr_spad_base = NULL; - ctx->ddr_spad_size = 0; + void * wq_ptr = (void *) ((uintptr_t) block + offset_wq); + ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + + ctx->main_stack = NULL; + ctx->main_thread = 0; + atomic_store(&ctx->killed, false); + + if (!use_callbacks) { + // Start main compute thread + ctx->main_stack = (void *) ((uintptr_t) block + offset_main_stack); + + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + qurt_thread_attr_set_stack_addr(&attr, ctx->main_stack); + qurt_thread_attr_set_stack_size(&attr, size_main_stack); + qurt_thread_attr_set_priority(&attr, main_prio); + qurt_thread_attr_set_name(&attr, "htp-main"); + + int err_thread = qurt_thread_create(&ctx->main_thread, &attr, htp_main_thread, ctx); + if (err_thread) { + FARF(ERROR, "Unable to create htp main thread: %d", err_thread); + htp_iface_stop(handle); + return AEE_ENOMEMORY; } - return err; } FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n", @@ -452,35 +570,34 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } AEEResult htp_iface_stop(remote_handle64 handle) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; - if (!ctx->queue) { - FARF(ERROR, "Queue not open"); - return AEE_EBADSTATE; + if (ctx->main_thread) { + atomic_store(&ctx->killed, true); + int status; + (void) qurt_thread_join(ctx->main_thread, &status); + ctx->main_thread = 0; } - // Close queue. dspqueue_close() will also wait for callbacks to finish. - int err = dspqueue_close(ctx->queue); - ctx->queue = NULL; + int err = dspqueue_close(ctx->dsp_queue); ctx->dsp_queue = NULL; if (err != 0) { FARF(ERROR, "Queue close failed with 0x%08x", (unsigned) err); return err; } - if (ctx->worker_pool) { - // Release worker pool - worker_pool_release(&ctx->worker_pool); - } + work_queue_free(ctx->work_queue); for (int i = 0; i < ctx->n_threads; i++) { - dma_queue_delete(ctx->dma[i]); + dma_queue_alias_free(ctx->dma[i]); + dma_queue_free(ctx->dma_cached[i]); } if (ctx->hmx_queue) { - hmx_queue_delete(ctx->hmx_queue); + hmx_queue_free(ctx->hmx_queue); ctx->hmx_queue = NULL; } ctx->hmx_enabled = false; @@ -493,6 +610,9 @@ AEEResult htp_iface_stop(remote_handle64 handle) { ctx->ddr_spad_size = 0; } + free(ctx); + h->ctx = NULL; + return AEE_SUCCESS; } @@ -671,8 +791,6 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_INVALID: break; - - // No default to catch missing cases } FARF(ERROR, "Unknown Op %u", octx->op); @@ -778,12 +896,14 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin } } -static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t idx, struct htp_tensor *t) { +static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) { uint32_t offset = t->data; uint32_t size = t->size; uint32_t bi = t->bi; + uint32_t alias = t->alias; - t->data = bufs[bi].base + offset; // update data to the actual pointer + t->data = (uint32_t) (bufs[bi].base + offset); // update data to the actual pointer + t->alias = (uint32_t) (tens + alias); // update alias to the actual pointer FARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data, t->ne[0], t->ne[1], t->ne[3], t->ne[3]); @@ -791,7 +911,7 @@ static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) { for (uint32_t i=0; i < n_tens; i++) { - prep_tensor(ctx, bufs, i, tens + i); + prep_tensor(ctx, bufs, tens, i, tens + i); } } @@ -805,29 +925,36 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u // Prep input tensors for (uint32_t i=0; isrc[i] == 0xffff ? NULL : tens + op->src[i]; - - octx->src[i] = src; - if (!src) continue; - - if (!(src->flags & HTP_TENSOR_FLUSHED) && (src->flags & HTP_TENSOR_COMPUTE)) { - // flush compute buffers on input - hex_l2flush((void *) src->data, src->size); + uint16_t src_idx = op->src[i]; + if (src_idx == 0xffff) { + octx->src[i] = NULL; + octx->src_dma[i] = NULL; + continue; } + struct htp_tensor *src = tens + src_idx; + octx->src[i] = src; + octx->src_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma; + FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size, src->ne[0], src->ne[1], src->ne[3], src->ne[3]); } + htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS); + // Prep output tensors for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { uint16_t dst_idx = op->dst[i]; if (dst_idx == 0xffff) { - octx->dsts[i] = NULL; + octx->dsts[i] = NULL; + octx->dst_dma[i] = NULL; continue; } struct htp_tensor *dst = tens + dst_idx; - octx->dsts[i] = dst; + octx->dsts[i] = dst; + octx->dst_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma; + + htp_tensor_make_dirty(dst, octx->ctx->dirty_map); FARF(HIGH, "prep-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, dst_idx, (void*) dst->data, dst->size, dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); @@ -841,34 +968,134 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; - // flush buffers on output - for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { - if (octx->dsts[i]) { - struct htp_tensor *dst = (struct htp_tensor *)octx->dsts[i]; - hex_l2flush((void *) dst->data, dst->size); - dst->flags |= HTP_TENSOR_FLUSHED; + return status; +} + +static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) { + dspqueue_t queue = ctx->dsp_queue; + int err; - FARF(HIGH, "post-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, op->dst[i], (void*) dst->data, dst->size, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + const uint32_t n_bufs = req->n_bufs; + const uint32_t n_tens = req->n_tensors; + const uint32_t n_ops = req->n_ops; + + const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs; + const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; + const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; + const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; + const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc); + + if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) { + FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size); + return; + } + + FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id, + n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); + + // Clean cache at the start of the batch + // We cant trace this part because the trace buffer is setup later + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + hex_l2fetch_block(ctx, ctx->footprint); + bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS); + + // Setup descriptor pointers + uint8_t * m_ptr = dbuf->ptr; + struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; + struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size; + struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size; + struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr; + + prep_op_bufs(ctx, bufs, n_bufs); + prep_tensors(ctx, bufs, tens, n_tens); + + struct htp_ops_context *octx = &ctx->octx; + memset(octx, 0, sizeof(*octx)); + octx->n_threads = ctx->n_threads; + octx->ctx = ctx; + + memset(ctx->trace, 0, sizeof(ctx->trace)); + if (ctx->profiler == HTP_PROF_TRACE) { + struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); + for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + ctx->trace[t].events = &trace_events[t * req->n_traces]; + ctx->trace[t].max_events = req->n_traces; } } - return status; + work_queue_wakeup(ctx->work_queue); + if (ctx->hmx_queue) { + hmx_queue_wakeup(ctx->hmx_queue); + } + + int op_status = HTP_STATUS_OK; + for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) { + struct profile_data prof; + + profile_start(ctx->profiler, &prof); + + op_status = proc_op_req(octx, tens, i, &ops[i]); + + profile_stop(ctx->profiler, &prof); + + if (ctx->profiler) { + pds[i].opcode = ops[i].opcode; + pds[i].usecs = prof.usecs; + pds[i].cycles_start = prof.cycles_start; + pds[i].cycles_stop = prof.cycles_stop; + for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) { + pds[i].pmu[j] = prof.pmu_counters[j]; + } + } + } + + if (ctx->hmx_queue) { + hmx_queue_suspend(ctx->hmx_queue); + hmx_queue_flush(ctx->hmx_queue); + } + work_queue_suspend(ctx->work_queue); + + struct htp_opbatch_rsp rsp; + memset(&rsp, 0, sizeof(rsp)); + rsp.id = req->id; + rsp.status = op_status; + rsp.n_bufs = n_bufs; + rsp.n_tensors = n_tens; + rsp.n_ops = n_ops; + + if (ctx->profiler == HTP_PROF_TRACE) { + for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + rsp.n_traces[t] = ctx->trace[t].count; + } + } + + struct dspqueue_buffer write_dbuf = *dbuf; + write_dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; + + // Flush remaining dirty tensors at the end of the batch + htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + + err = dspqueue_write(queue, 0, 1, &write_dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE); + if (err != 0) { + FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err); + } } +#define DSPQUEUE_READ_TIMEOUT_USEC 5000 #define DSPQUEUE_POLL_TIMEOUT_USEC 100 #define DSPQUEUE_POLL_COUNT 100 -static void htp_packet_callback(dspqueue_t queue, int error, void * context) { - struct htp_context * ctx = (struct htp_context *) context; - +static void process_ops(struct htp_context * ctx) { + dspqueue_t queue = ctx->dsp_queue; int err; uint32_t poll_count = DSPQUEUE_POLL_COUNT; vtcm_acquire(ctx); - while (!ctx->vtcm_needs_release) { + while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) { struct htp_opbatch_req req; uint32_t r_size = sizeof(req); @@ -898,111 +1125,41 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) { // Reset poll count for valid requests poll_count = DSPQUEUE_POLL_COUNT; - const uint32_t n_bufs = req.n_bufs; - const uint32_t n_tens = req.n_tensors; - const uint32_t n_ops = req.n_ops; - - const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs; - const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; - const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; - const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; - const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(struct htp_trace_desc); - - if (dbuf.size < b_size + t_size + o_size + p_size + tr_size) { - FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf.size, b_size + t_size + o_size + p_size + tr_size); - break; - } - - FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req.id, - n_bufs, n_tens, n_ops, req.n_traces, dbuf.size, b_size, t_size, o_size); - - // Setup descriptor pointers - uint8_t * m_ptr = dbuf.ptr; - struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; - struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size; - struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size; - struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr; - - prep_op_bufs(ctx, bufs, n_bufs); - prep_tensors(ctx, bufs, tens, n_tens); - - struct htp_ops_context *octx = &ctx->octx; - memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->ctx = ctx; - - if (ctx->profiler == HTP_PROF_TRACE) { - memset(ctx->trace, 0, sizeof(ctx->trace)); - struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { - ctx->trace[t].events = &trace_events[t * req.n_traces]; - ctx->trace[t].max_events = req.n_traces; - } - } else { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { - ctx->trace[t].events = NULL; - ctx->trace[t].max_events = 0; - } - } - - int op_status = HTP_STATUS_OK; - uint32_t op_wakeup = n_ops / 2; // half-way throgh the batch - - hmx_queue_wakeup(ctx->hmx_queue); - - for (uint32_t i=0; i < n_ops; i++) { - struct profile_data prof; - - if (i == op_wakeup) { - dspqueue_write_early_wakeup_noblock(queue, 0, 0); - } + process_opbatch(ctx, &req, &dbuf); + } - profile_start(ctx->profiler, &prof); + vtcm_release(ctx); +} - op_status = proc_op_req(octx, tens, i, &ops[i]); +static void htp_packet_callback(dspqueue_t queue, int error, void * context) { + (void) queue; + (void) error; + struct htp_handle * h = (struct htp_handle *) context; + if (h && h->ctx) { + process_ops(h->ctx); + } +} - profile_stop(ctx->profiler, &prof); +static void htp_main_thread(void * context) { + struct htp_context * ctx = (struct htp_context *) context; - if (op_status != HTP_STATUS_OK) { - break; - } + FARF(HIGH, "htp-main-thread: started"); - if (ctx->profiler) { - pds[i].opcode = ops[i].opcode; - pds[i].usecs = prof.usecs; - pds[i].cycles_start = prof.cycles_start; - pds[i].cycles_stop = prof.cycles_stop; - for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) { - pds[i].pmu[j] = prof.pmu_counters[j]; - } - } - } + while (!atomic_load(&ctx->killed)) { + uint32_t flags = 0; + uint32_t num_buffers = 0; + uint32_t message_length = 0; - hmx_queue_suspend(ctx->hmx_queue); - - struct htp_opbatch_rsp rsp; - rsp.id = req.id; - rsp.status = op_status; - rsp.n_bufs = n_bufs; - rsp.n_tensors = n_tens; - rsp.n_ops = n_ops; - memset(rsp.pad, 0, sizeof(rsp.pad)); - if (ctx->profiler == HTP_PROF_TRACE) { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { - rsp.n_traces[t] = ctx->trace[t].count; - } + int err = dspqueue_peek(ctx->dsp_queue, &flags, &num_buffers, &message_length, 50000); + if (err == 0) { + process_ops(ctx); + } else if (err == AEE_EWOULDBLOCK || err == AEE_EEXPIRED) { + continue; } else { - memset(rsp.n_traces, 0, sizeof(rsp.n_traces)); - } - - dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - - err = dspqueue_write(queue, 0, 1, &dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE); - if (err != 0) { - FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err); + FARF(ERROR, "dspqueue_peek failed: 0x%08x", (unsigned) err); break; } } - vtcm_release(ctx); + FARF(HIGH, "htp-main-thread: stopped"); } diff --git a/cpp/ggml-hexagon/htp/matmul-ops.c b/cpp/ggml-hexagon/htp/matmul-ops.c index 04fc101d9..af481c377 100644 --- a/cpp/ggml-hexagon/htp/matmul-ops.c +++ b/cpp/ggml-hexagon/htp/matmul-ops.c @@ -92,10 +92,10 @@ struct htp_mm_context { // Per thread quant tasks // Precomputed block-parallel quantization values worker_callback_t quant_task_func; - uint32_t quant_ib_first[MAX_NUM_WORKERS]; - uint32_t quant_ib_last[MAX_NUM_WORKERS]; - uint32_t quant_r[MAX_NUM_WORKERS]; - uint32_t quant_c[MAX_NUM_WORKERS]; + uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS]; uint32_t n_quant_tasks; uint32_t n_quant_rows_per_thread; atomic_uint quant_barrier; @@ -254,7 +254,7 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start); const uint32_t blck_0 = 64; @@ -309,7 +309,7 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ @@ -410,7 +410,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ @@ -523,7 +523,7 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; \ uint8_t * restrict src1_data = mmctx->vtcm_src1; \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ @@ -699,7 +699,7 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \ uint8_t * restrict src1_data = mmctx->vtcm_src1; \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \ @@ -820,7 +820,7 @@ static void name(unsigned int nth, unsigned int ith, void * data) { return; \ } \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ \ uint8_t * restrict dst = mmctx->vtcm_src1; \ @@ -846,7 +846,7 @@ QUANTIZE_IMPL(quantize_f16_f16_flat, "quantize-f16-f16", quantize_f16_f static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = octx->src[1]; @@ -870,7 +870,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = octx->src[1]; @@ -944,7 +944,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -1040,7 +1040,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_start_row = src0_nrows_per_thread * ith; const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -1155,7 +1155,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1244,7 +1244,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1338,6 +1338,9 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type static int hvx_mm_matmul(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; @@ -1557,9 +1560,6 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; - if (need_quant) { mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -1570,8 +1570,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->n_quant_tasks = 0; } - const uint32_t n_matmul_jobs = octx->n_threads; - worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads); return HTP_STATUS_OK; } @@ -1874,7 +1875,7 @@ static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) { #define DEQUANTIZE_WORKER_LOOP_IMPL(SUFFIX) \ static void dequantize_tiled_worker_loop_##SUFFIX(unsigned int n, unsigned int i, void *data) { \ tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; \ - struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; \ + struct htp_thread_trace * tr = &state->traces[i]; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); \ for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { \ int start = task_id * state->n_tiles_per_task; \ @@ -1892,7 +1893,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q8_0) static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) { tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; - struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; + struct htp_thread_trace * tr = &state->traces[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { int start = task_id * state->n_tiles_per_task; @@ -1905,7 +1906,7 @@ static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) { tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; - struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; + struct htp_thread_trace * tr = &state->traces[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, i); for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { @@ -1920,7 +1921,7 @@ static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int start_chunk_idx = i * st->n_chunks_per_task; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); @@ -1955,6 +1956,170 @@ typedef struct { uint32_t dma_step_rows_shift; } activation_transfer_task_state_t; +typedef struct { + __fp16 *dst; + const float *src; + uint32_t n_rows; + uint32_t k_block; + uint32_t k_stride; + uint32_t k_valid; + uint32_t n_col_chunks; + struct fastdiv_values n_threads_div; + float *vtcm_f32_act; + size_t vtcm_f32_act_bytes; + struct htp_thread_trace *traces; + struct htp_context *ctx; + uint32_t dma_step_rows; + uint32_t dma_step_rows_shift; +} activation_transfer_col_chunk_state_t; + +static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( + dma_queue *dma_q, + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t k_chunk_valid, + uint32_t c_first, + uint32_t c_len, + float *thread_f32_act, + struct htp_thread_trace *tr, + uint32_t dma_step_rows, + uint32_t dma_step_rows_shift) { + + const uint32_t R = dma_step_rows; + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + + const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift; + + // Push step 0 + if (n_steps > 0 && n_rows > 0) { + uint32_t nrows_to_fetch = hex_smin(n_rows, R); + dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src + c_first), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + // Push step 1 + if (n_steps > 1) { + uint32_t next_r = R * 1; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + const float *next_src = src + next_r * k_stride + c_first; + float *next_buf = thread_f32_act + 1 * R * c_len; + dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + } + for (uint32_t s = 0; s < n_steps; ++s) { + uint32_t r = s << dma_step_rows_shift; + float *curr_buf = thread_f32_act; + + if (r < n_rows) { + curr_buf = (float *) dma_queue_pop(dma_q).dst; + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + float *pair_buf = curr_buf + (p << 1) * c_len; + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); + + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, pair_buf, pair_buf + c_len, row_idx, k_block, c_first, c_len, k_chunk_valid, r0_valid, r1_valid + ); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + + // Push step s + 2 + uint32_t next_s = s + 2; + uint32_t next_r = next_s << dma_step_rows_shift; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + const float *next_src = src + next_r * k_stride + c_first; + dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + } +} + +static void transfer_activation_chunk_fp32_to_fp16_col_chunk( + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t c_first, + uint32_t c_len, + uint32_t k_chunk_valid) { + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + const uint32_t n_rows_tiled = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS; + + uint32_t r = 0; + + #pragma unroll(2) + for (r = 0; r < n_rows_tiled; r += 2) { + const float *ptr_in0 = src + (r + 0) * k_stride + c_first; + const float *ptr_in1 = src + (r + 1) * k_stride + c_first; + + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, true, true + ); + } + + for (; r < n_rows_padded; r += 2) { + const bool row0_valid = r < n_rows; + const bool row1_valid = (r + 1) < n_rows; + + const float *ptr_in0 = row0_valid ? (src + (r + 0) * k_stride + c_first) : NULL; + const float *ptr_in1 = row1_valid ? (src + (r + 1) * k_stride + c_first) : NULL; + + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, row0_valid, row1_valid + ); + } +} + +static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { + activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data; + struct htp_thread_trace * tr = &st->traces[i]; + + uint32_t n_blocks = st->k_block / 32; + uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t c_first = b_first * 32; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; + + if (c_len == 0) { + return; + } + + uint32_t k_chunk_valid = 0; + if (st->k_valid > c_first) { + k_chunk_valid = hex_smin(st->k_valid, c_last) - c_first; + } + + __fp16 *dst = st->dst; + const float *src = st->src; + + if (st->vtcm_f32_act) { + size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128); + float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes); + + transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( + st->ctx->dma[i], dst, src, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, + c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift + ); + } else { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first); + transfer_activation_chunk_fp32_to_fp16_col_chunk( + dst, src, st->n_rows, st->k_block, st->k_stride, c_first, c_len, k_chunk_valid + ); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first); + } +} + static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( dma_queue *dma_q, __fp16 *restrict vtcm_dst, @@ -2024,7 +2189,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { int chunk_idx = task_id * st->n_chunks_per_task; @@ -2085,15 +2250,16 @@ typedef struct { static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; - int start_row = st->start_row + chunk_idx * chunk_size; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; int n_rows = hex_smin(st->cne1 - start_row, chunk_size); if (n_rows > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); transfer_activation_chunk_fp32_to_fp16_gathered( - st->dst, st->src, start_row, n_rows, st->k_block, + st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block, st->matrix_rows, st->cur_a, st->mapping_stride, st->ne11, &st->ne11_div, st->nb11, st->nb12, st->cne1, st->k_valid); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); @@ -2102,15 +2268,16 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; - int start_row = st->start_row + chunk_idx * chunk_size; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; int n_rows = hex_smin(st->cne1 - start_row, chunk_size); if (n_rows > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); transfer_activation_chunk_fp32_to_fp16_gathered_flat( - st->dst, st->src, start_row, n_rows, st->k_block, + st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block, st->matrix_rows, st->cur_a, st->mapping_stride, st->nb12, st->cne1, st->k_valid); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); @@ -2119,15 +2286,16 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) { output_transfer_scattered_task_state_t *st = data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; - int start_row = st->start_row + chunk_idx * chunk_size; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; int n_rows = hex_smin(st->cne1 - start_row, chunk_size); if (n_rows > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx); transfer_output_chunk_fp16_to_fp32_scattered( - st->dst, st->vtcm_src, start_row, n_rows, st->n_cols, + st->dst, st->vtcm_src, start_row, vtcm_start_row, n_rows, st->n_cols, st->matrix_rows, st->cur_a, st->mapping_stride, st->dst_nb1, st->dst_nb2, st->cne1); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx); @@ -2210,42 +2378,81 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, } } -static void transfer_activation_chunk_threaded( - struct htp_context *ctx, - __fp16 *dst, - const float *src, - int n_rows, - int k_block, - int k_stride, - int n_threads, - int k_valid, - float *vtcm_f32_act, - size_t vtcm_f32_act_bytes) { +struct activation_transfer_params { + struct htp_context * ctx; + __fp16 * dst; + const float * src; + int n_rows; + int k_block; + int k_stride; + int n_threads; + const struct fastdiv_values * act_threads_div; + const struct fastdiv_values * k_div; + int k_valid; + float * vtcm_f32_act; + size_t vtcm_f32_act_bytes; +}; + +static void transfer_activation_chunk_threaded(const struct activation_transfer_params * params) { + struct htp_context * ctx = params->ctx; + __fp16 * dst = params->dst; + const float * src = params->src; + int n_rows = params->n_rows; + int k_block = params->k_block; + int k_stride = params->k_stride; + int n_threads = params->n_threads; + const struct fastdiv_values * act_threads_div = params->act_threads_div; + const struct fastdiv_values * k_div = params->k_div; + int k_valid = params->k_valid; + float * vtcm_f32_act = params->vtcm_f32_act; + size_t vtcm_f32_act_bytes = params->vtcm_f32_act_bytes; + if (n_rows <= 0) { return; } + const size_t n_tasks = (n_rows + 31) >> 5; + if (n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) { + // Calculate step rows parameters for column-chunked dma pipelining + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) { + size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float); + size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } + } + + activation_transfer_col_chunk_state_t col_state; + col_state.dst = dst; + col_state.src = src; + col_state.n_rows = n_rows; + col_state.k_block = k_block; + col_state.k_stride = k_stride; + col_state.k_valid = k_valid; + col_state.n_col_chunks = n_threads; + col_state.n_threads_div = *act_threads_div; + col_state.vtcm_f32_act = vtcm_f32_act; + col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes; + col_state.traces = ctx->trace; + col_state.ctx = ctx; + col_state.dma_step_rows = dma_step_rows; + col_state.dma_step_rows_shift = dma_step_rows_shift; + + worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads); + return; + } + assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address - uint32_t dma_step_rows = 2; - uint32_t dma_step_rows_shift = 1; - if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) { - size_t thread_scratch_elements = vtcm_f32_act_bytes / (n_threads * sizeof(float)); - size_t dma_step_rows_max = (thread_scratch_elements / 2) / k_block; - if (dma_step_rows_max >= 4) { - dma_step_rows = 4; - dma_step_rows_shift = 2; - } else { - dma_step_rows = 2; - dma_step_rows_shift = 1; - } - } - activation_transfer_task_state_t state; - state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; + state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2258,7 +2465,18 @@ static void transfer_activation_chunk_threaded( state.vtcm_f32_act = vtcm_f32_act; int active_threads = hex_smin(n_threads, (int)state.n_tasks); - state.vtcm_f32_act_bytes_per_thread = (vtcm_f32_act_bytes / active_threads) & ~127u; + state.vtcm_f32_act_bytes_per_thread = hex_align_down(vtcm_f32_act_bytes / active_threads, 128); + + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (vtcm_f32_act && state.vtcm_f32_act_bytes_per_thread > 0 && k_block > 0) { + size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float); + size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } + } state.dma_step_rows = dma_step_rows; state.dma_step_rows_shift = dma_step_rows_shift; @@ -2321,9 +2539,14 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int pipeline, int n_threads, int act_threads, + const struct fastdiv_values * act_threads_div, + const struct fastdiv_values * k_div, int tile_size, int aligned_tile_size, int vtcm_size) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + if (k % 32 != 0 || n % 32 != 0) { return -1; } if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN)) { return -1; } @@ -2393,6 +2616,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + if (pipeline) { // --- Asynchronous Pipelined Loop --- hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors @@ -2403,7 +2628,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; - transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes); + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .src = activation + mr * act_stride, + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + }; + transfer_activation_chunk_threaded(&act_params); // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); @@ -2480,7 +2719,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes); + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .src = activation + mr * act_stride, + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + }; + transfer_activation_chunk_threaded(&act_params); // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { @@ -2570,7 +2823,8 @@ static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_pa static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) { + int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, + const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) { int ret = 0; for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { @@ -2582,14 +2836,17 @@ static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, params->act_stride, params->weight_stride * (int)sizeof(__fp16), HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n, m_chunk, n_chunk, pipeline, n_threads, act_threads, - 0, 0, vtcm_size); + act_threads_div, k_div, 0, 0, vtcm_size); } } return ret; } static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) { + int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, + const struct fastdiv_values * act_threads_div, + const struct fastdiv_values * k_div, + int vtcm_size) { if (params->act_stride < params->k || params->weight_stride < params->k || params->dst_stride < params->n) { return -1; } if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; } if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; } @@ -2604,9 +2861,12 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ // Grouped path is only valid if group_size > 1 and it fits within VTCM budget. bool run_grouped = (group_size > 1 && (size_t)vtcm_size <= vtcm_budget); if (!run_grouped) { - return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size); + return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); } + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const size_t vec_dot_size = params->k * sizeof(__fp16); const bool use_dma_activation = (params->act_stride > params->k); @@ -2622,7 +2882,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); - return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); } uint8_t * const base = (uint8_t *) ctx->vtcm_base; @@ -2644,6 +2905,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + hmx_matmul_job_t job; for (int b3 = 0; b3 < params->ne13; ++b3) { @@ -2662,9 +2925,21 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ for (int g = 0; g < group_size; ++g) { const float *activation_chunk = hmx_mm_activation_batch_ptr(params, b2_base + g, b3) + mr * params->act_stride; __fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; - transfer_activation_chunk_threaded(ctx, vtcm_act_g, - activation_chunk, (int) n_rows, - params->k, params->act_stride, act_threads, params->k, vtcm_f32_act, L.act_f32_bytes); + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_act_g, + .src = activation_chunk, + .n_rows = (int) n_rows, + .k_block = params->k, + .k_stride = params->act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = params->k, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + }; + transfer_activation_chunk_threaded(&act_params); } // Prologue: Push A0 and A1 (if exists) @@ -2835,6 +3110,9 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const struct mmid_row_mapping *matrix_rows, int cur_a, int mapping_stride) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const int cne1 = m; const int m_padded = hex_align_up(m, 32); @@ -2913,6 +3191,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + hmx_matmul_job_t job; for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) { @@ -2980,10 +3260,6 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const int act_stride = (int)(src1->nb[1] / sizeof(float)); const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - const float * src2_ptr = NULL; uint32_t src2_stride = 0; size_t src2_nb2 = 0; @@ -3027,6 +3303,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, + &kparams->div_n_act_threads, + &kparams->div_ne00_padded, kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( @@ -3035,6 +3313,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k (int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0], kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, + &kparams->div_n_act_threads, + &kparams->div_ne00_padded, kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size ); } @@ -3103,6 +3383,10 @@ static int hvx_mm_matmul_id( bool must_free_mapping ) { htp_matmul_tensors_preamble; + + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const struct htp_tensor * restrict ids = octx->src[2]; const size_t src0_row_size = nb01; @@ -3175,8 +3459,9 @@ static int hvx_mm_matmul_id( mmctx->n_quant_tasks = n_quant_tasks; atomic_init(&mmctx->quant_barrier, n_quant_tasks); - const uint32_t n_matmul_jobs = octx->n_threads; - worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, octx->n_threads); if (must_free_mapping) free(mapping_buf); return HTP_STATUS_OK; @@ -3185,6 +3470,9 @@ static int hvx_mm_matmul_id( int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; @@ -3262,10 +3550,7 @@ int op_matmul_id(struct htp_ops_context * octx) { } } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - if (must_free_mapping) free(mapping_buf); - return HTP_STATUS_OK; - } + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); if (kparams->n_hmx) { return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping); @@ -3275,6 +3560,9 @@ int op_matmul_id(struct htp_ops_context * octx) { } int op_matmul_qkv(struct htp_ops_context * octx) { + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const struct htp_tensor * restrict src0 = octx->src[0]; // Wk const struct htp_tensor * restrict src1 = octx->src[1]; // x const struct htp_tensor * restrict src2 = octx->src[2]; // Wv @@ -3379,9 +3667,6 @@ int op_matmul_qkv(struct htp_ops_context * octx) { mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads; mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; mmctx->n_quant_tasks = n_quant_tasks; @@ -3413,12 +3698,18 @@ int op_matmul_qkv(struct htp_ops_context * octx) { } else { matmul_job_func = hvx_mm_qkv_2d; } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); return HTP_STATUS_OK; } int op_matmul_ffn(struct htp_ops_context * octx) { + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate const struct htp_tensor * restrict src1 = octx->src[1]; // y const struct htp_tensor * restrict src2 = octx->src[2]; // Wup @@ -3516,9 +3807,6 @@ int op_matmul_ffn(struct htp_ops_context * octx) { mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads; mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; mmctx->n_quant_tasks = n_quant_tasks; @@ -3550,6 +3838,9 @@ int op_matmul_ffn(struct htp_ops_context * octx) { } else { matmul_job_func = hvx_mm_ffn_2d; } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); return HTP_STATUS_OK; diff --git a/cpp/ggml-hexagon/htp/matmul-ops.h b/cpp/ggml-hexagon/htp/matmul-ops.h index 2e131bc3d..445044660 100644 --- a/cpp/ggml-hexagon/htp/matmul-ops.h +++ b/cpp/ggml-hexagon/htp/matmul-ops.h @@ -95,6 +95,8 @@ struct htp_mm_kernel_params { struct fastdiv_values div_r2; struct fastdiv_values div_r3; struct fastdiv_values div_ne11; + struct fastdiv_values div_n_act_threads; + struct fastdiv_values div_ne00_padded; }; #if defined(__cplusplus) @@ -643,6 +645,136 @@ static inline size_t htp_mm_hmx_get_batched_vtcm_size( return L.total_bytes; } +static inline bool htp_mm_hmx_solve_batched_params( + int wtype, + uint32_t k, + uint32_t ne01_padded, + uint32_t ne11, + uint32_t group_size, + bool use_dma_activation, + int n_threads, + bool pipeline, + size_t vtcm_budget, + size_t * m_chunk_out, + size_t * n_chunk_out, + int * act_threads_out, + size_t * vtcm_size_out +) { + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + int act_threads = n_threads; + while (act_threads >= 1) { + size_t group_overhead = 256; + size_t group_size_per_n, group_size_per_m, group_size_per_mn; + htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); + + size_t m_chunk_candidate = 0; + size_t n_chunk_candidate = 0; + size_t vtcm_size_candidate = 0; + + if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_candidate; + best_n_chunk = n_chunk_candidate; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) { + act_threads = 0; + } else { + act_threads /= 2; + } + } + + if (best_act_threads > 0) { + *m_chunk_out = best_m_chunk; + *n_chunk_out = best_n_chunk; + *vtcm_size_out = best_vtcm_size; + *act_threads_out = best_act_threads; + return true; + } + return false; +} + +static inline bool htp_mm_hmx_solve_2d_params( + int wtype, + uint32_t k, + uint32_t m_id_rows, + uint32_t ne01_padded, + uint32_t ne11_padded, + uint32_t m_for_cost, + int n_threads, + bool pipeline, + bool is_matmul_id, + uint32_t aligned_tile_size, + size_t vtcm_budget, + size_t * m_chunk_out, + size_t * n_chunk_out, + int * act_threads_out, + size_t * vtcm_size_out +) { + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; + + int act_threads = n_threads; + while (act_threads >= 1) { + size_t simple_2d_overhead = 256; + size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; + htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); + + size_t m_chunk_candidate = 0; + size_t n_chunk_candidate = 0; + size_t vtcm_size_candidate = 0; + + if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_candidate; + best_n_chunk = n_chunk_candidate; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) { + act_threads = 0; + } else { + act_threads /= 2; + } + } + + if (best_act_threads > 0) { + *m_chunk_out = best_m_chunk; + *n_chunk_out = best_n_chunk; + *vtcm_size_out = best_vtcm_size; + *act_threads_out = best_act_threads; + return true; + } + return false; +} + #ifdef __cplusplus } #endif diff --git a/cpp/ggml-hexagon/htp/rope-ops.c b/cpp/ggml-hexagon/htp/rope-ops.c index fa4c68f0c..56736406d 100644 --- a/cpp/ggml-hexagon/htp/rope-ops.c +++ b/cpp/ggml-hexagon/htp/rope-ops.c @@ -18,6 +18,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-ops.h" +#include "htp-tensor.h" // Redefined the rope type constants as we can't include ggml.h #define HTP_ROPE_TYPE_NORMAL 0 @@ -712,17 +713,11 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { } int op_rope(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_rope_f32(octx); - break; + return execute_op_rope_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/cpp/ggml-hexagon/htp/unary-ops.c b/cpp/ggml-hexagon/htp/unary-ops.c index 42ac43994..3c2b8e301 100644 --- a/cpp/ggml-hexagon/htp/unary-ops.c +++ b/cpp/ggml-hexagon/htp/unary-ops.c @@ -19,6 +19,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "htp-vtcm.h" #include "hex-profile.h" @@ -397,7 +398,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat struct htp_ops_context * octx = uctx->octx; \ const struct htp_tensor * src = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ htp_unary_preamble; \ \ @@ -558,7 +559,7 @@ static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void struct htp_ops_context * octx = uctx->octx; \ const struct htp_tensor * src = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ htp_unary_preamble; \ \ @@ -922,17 +923,11 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { } int op_unary(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_unary_f32(octx); - break; + return execute_op_unary_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/cpp/ggml-hexagon/htp/work-queue.c b/cpp/ggml-hexagon/htp/work-queue.c new file mode 100644 index 000000000..bb73e205a --- /dev/null +++ b/cpp/ggml-hexagon/htp/work-queue.c @@ -0,0 +1,244 @@ +#include "work-queue.h" +#include "hex-utils.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "HAP_farf.h" + +#define LOWEST_USABLE_QURT_PRIO (254) + +// internal structure kept in thread-local storage per instance of work queue +typedef struct { + work_queue_t queue; + unsigned int id; +} worker_context_t; + +struct work_queue_task_s { + work_queue_func_t func; + void * data; + unsigned int n_threads; + atomic_uint barrier; +}; + +// internal structure kept in thread-local storage per instance of work queue +struct work_queue_s { + atomic_uint seqn; // seqno used to detect new jobs + atomic_uint idx_read; // Updated by producer (pop/reclaim) + unsigned int idx_write; // Updated by producer (push) + uint32_t idx_mask; + uint32_t capacity; + + qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers + worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts + void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers + unsigned int n_threads; // total threads (workers + main) + unsigned int n_workers; // number of active threads (just workers) + + atomic_bool active; // workers are polling/active + atomic_bool killed; // threads need to exit + bool external_mem; // memory owned externally + + struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE))); +}; + +static void work_queue_thread(void * context) { + worker_context_t * me = (worker_context_t *) context; + work_queue_t q = me->queue; + + FARF(HIGH, "work-queue: thread %u started", me->id); + + unsigned int prev_seqn = 0; + + while (!atomic_load_explicit(&q->killed, memory_order_relaxed)) { + unsigned int seqn = atomic_load_explicit(&q->seqn, memory_order_acquire); + if (seqn == prev_seqn) { + if (atomic_load_explicit(&q->active, memory_order_relaxed)) { + hex_pause(); + } else { + qurt_futex_wait(&q->seqn, prev_seqn); + } + continue; + } + + prev_seqn = seqn; + + // Process all active tasks in the queue + unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed); + unsigned int iw = q->idx_write; + + while (ir != iw) { + struct work_queue_task_s * task = &q->queue[ir]; + + unsigned int n = task->n_threads; + unsigned int i = me->id; + if (i < n) { + task->func(n, i, task->data); + + atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); + } else { + while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) { + hex_pause(); + } + } + + ir = (ir + 1) & q->idx_mask; + } + } + + FARF(HIGH, "work-queue: thread %u stopped", me->id); +} + +bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n > q->n_threads) { + FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads); + return false; + } + + unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed); + unsigned int iw = q->idx_write; + + if (((iw + 1) & q->idx_mask) == ir) { + FARF(ERROR, "work-queue-push: queue is full\n"); + return false; + } + + struct work_queue_task_s * task = &q->queue[iw]; + task->func = func; + task->data = data; + task->n_threads = n; + atomic_store_explicit(&task->barrier, n, memory_order_relaxed); + + q->idx_write = (iw + 1) & q->idx_mask; + + // publish job to workers (already awake and polling) + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + + // main thread runs job #0 + func(n, 0, data); + + atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); + + while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) { + hex_pause(); + } + + atomic_thread_fence(memory_order_acquire); + + atomic_store_explicit(&q->idx_read, (ir + 1) & q->idx_mask, memory_order_relaxed); + + return true; +} + +size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { + capacity = hex_ceil_pow2(capacity); + uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; + size_t size_stacks = stack_size * n_workers; + size_t size_q = hex_align_up(sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s), HEX_L2_LINE_SIZE); + return size_stacks + size_q; +} + +size_t work_queue_alignof(void) { + return 4096; +} + +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { + capacity = hex_ceil_pow2(capacity); + uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; + unsigned char * mem_blob = (unsigned char *) ptr; + + work_queue_t q = (work_queue_t) (mem_blob + stack_size * n_workers); + memset(q, 0, sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s)); + + q->n_threads = n_threads; + q->n_workers = n_workers; + q->external_mem = true; + q->capacity = capacity; + + for (unsigned int i = 0; i < n_workers; i++) { + q->stack[i] = mem_blob; mem_blob += stack_size; + q->thread[i] = 0; + q->context[i].id = i + 1; + q->context[i].queue = q; + } + + atomic_init(&q->idx_read, 0); + atomic_init(&q->seqn, 0); + atomic_init(&q->active, false); + q->idx_write = 0; + q->idx_mask = capacity - 1; + q->killed = 0; + for (int i = 0; i < (int) capacity; i++) { + atomic_init(&q->queue[i].barrier, 0); + q->queue[i].func = NULL; + q->queue[i].data = NULL; + q->queue[i].n_threads = 0; + } + + // launch the workers + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + + for (unsigned int i = 0; i < n_workers; i++) { + qurt_thread_attr_set_stack_addr(&attr, q->stack[i]); + qurt_thread_attr_set_stack_size(&attr, stack_size); + + char thread_name[32]; + snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i); + qurt_thread_attr_set_name(&attr, thread_name); + + // set up priority - by default, match the creating thread's prio + int prio = qurt_thread_get_priority(qurt_thread_get_id()); + if (prio < 1) { + prio = 1; + } + if (prio > LOWEST_USABLE_QURT_PRIO) { + prio = LOWEST_USABLE_QURT_PRIO; + } + + qurt_thread_attr_set_priority(&attr, prio); + + int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]); + if (err) { + FARF(ERROR, "Could not launch worker threads!"); + work_queue_free(q); + return NULL; + } + } + + return q; +} + +void work_queue_free(work_queue_t q) { + if (!q) { return; } + + atomic_store_explicit(&q->killed, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + qurt_futex_wake(&q->seqn, q->n_workers); + + for (unsigned int i = 0; i < q->n_workers; i++) { + if (q->thread[i]) { + int status; + (void) qurt_thread_join(q->thread[i], &status); + } + } +} + +void work_queue_wakeup(work_queue_t q) { + if (!atomic_load_explicit(&q->active, memory_order_relaxed)) { + atomic_store_explicit(&q->active, true, memory_order_release); + // Increment seqn and wake workers to transition them out of sleep + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + qurt_futex_wake(&q->seqn, q->n_workers); + } +} + +void work_queue_suspend(work_queue_t q) { + atomic_store_explicit(&q->active, false, memory_order_release); +} diff --git a/cpp/ggml-hexagon/htp/work-queue.h b/cpp/ggml-hexagon/htp/work-queue.h new file mode 100644 index 000000000..09ca4b1f4 --- /dev/null +++ b/cpp/ggml-hexagon/htp/work-queue.h @@ -0,0 +1,38 @@ +#ifndef HTP_WORK_QUEUE_H +#define HTP_WORK_QUEUE_H + +#include +#include +#include + +typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *); + +struct work_queue_s; +typedef struct work_queue_s * work_queue_t; + +#define WORK_QUEUE_MAX_N_THREADS 10 + +size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +size_t work_queue_alignof(void); +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +void work_queue_free(work_queue_t q); + +void work_queue_wakeup(work_queue_t q); +void work_queue_suspend(work_queue_t q); + +bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); + +static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n <= 1) { + func(n, 0, data); + return true; + } + return work_queue_run_async(q, func, data, n); +} + +// Legacy compatibility +typedef work_queue_func_t worker_callback_t; +#define worker_pool_run_func work_queue_run +#define worker_pool work_queue + +#endif // #ifndef HTP_WORK_QUEUE_H diff --git a/cpp/ggml-metal/ggml-metal-device.cpp b/cpp/ggml-metal/ggml-metal-device.cpp index ec43dc351..5c66b4568 100644 --- a/cpp/ggml-metal/ggml-metal-device.cpp +++ b/cpp/ggml-metal/ggml-metal-device.cpp @@ -805,6 +805,11 @@ lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_mul_mv(lm_ nsg = N_SG_Q1_0; nr0 = N_R0_Q1_0; } break; + case LM_GGML_TYPE_Q2_0: + { + nsg = N_SG_Q2_0; + nr0 = N_R0_Q2_0; + } break; case LM_GGML_TYPE_Q4_0: { nsg = N_SG_Q4_0; @@ -1029,6 +1034,11 @@ lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_mul_mv_id( nsg = N_SG_Q1_0; nr0 = N_R0_Q1_0; } break; + case LM_GGML_TYPE_Q2_0: + { + nsg = N_SG_Q2_0; + nr0 = N_R0_Q2_0; + } break; case LM_GGML_TYPE_Q4_0: { nsg = N_SG_Q4_0; @@ -1824,6 +1834,23 @@ lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_col2im_1d( return res; } +lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_snake(lm_ggml_metal_library_t lib, enum lm_ggml_type type) { + LM_GGML_ASSERT(type == LM_GGML_TYPE_F32 || type == LM_GGML_TYPE_F16 || type == LM_GGML_TYPE_BF16); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_snake_%s", lm_ggml_type_name(type)); + snprintf(name, 256, "%s", base); + + lm_ggml_metal_pipeline_with_params res = lm_ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = lm_ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_conv_transpose_2d(lm_ggml_metal_library_t lib, const lm_ggml_tensor * op) { assert(op->op == LM_GGML_OP_CONV_TRANSPOSE_2D); diff --git a/cpp/ggml-metal/ggml-metal-device.h b/cpp/ggml-metal/ggml-metal-device.h index 060752cf6..50a9bd9d7 100644 --- a/cpp/ggml-metal/ggml-metal-device.h +++ b/cpp/ggml-metal/ggml-metal-device.h @@ -151,6 +151,7 @@ struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_im2 struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_conv_transpose_1d (lm_ggml_metal_library_t lib, const struct lm_ggml_tensor * op); struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_conv_transpose_2d (lm_ggml_metal_library_t lib, const struct lm_ggml_tensor * op); struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_col2im_1d (lm_ggml_metal_library_t lib, const struct lm_ggml_tensor * op); +struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_snake (lm_ggml_metal_library_t lib, enum lm_ggml_type type); struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_conv_2d (lm_ggml_metal_library_t lib, const struct lm_ggml_tensor * op); struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_conv_2d_dw (lm_ggml_metal_library_t lib, const struct lm_ggml_tensor * op, bool tiled); struct lm_ggml_metal_pipeline_with_params lm_ggml_metal_library_get_pipeline_conv_3d (lm_ggml_metal_library_t lib, const struct lm_ggml_tensor * op); diff --git a/cpp/ggml-metal/ggml-metal-device.m b/cpp/ggml-metal/ggml-metal-device.m index 97920ad56..d82ad7d71 100644 --- a/cpp/ggml-metal/ggml-metal-device.m +++ b/cpp/ggml-metal/ggml-metal-device.m @@ -1289,6 +1289,7 @@ bool lm_ggml_metal_device_supports_op(lm_ggml_metal_device_t dev, const struct l case LM_GGML_TYPE_BF16: case LM_GGML_TYPE_Q8_0: case LM_GGML_TYPE_Q1_0: + case LM_GGML_TYPE_Q2_0: case LM_GGML_TYPE_Q4_0: case LM_GGML_TYPE_Q4_1: case LM_GGML_TYPE_Q5_0: @@ -1316,6 +1317,7 @@ bool lm_ggml_metal_device_supports_op(lm_ggml_metal_device_t dev, const struct l return false; } case LM_GGML_TYPE_Q1_0: + case LM_GGML_TYPE_Q2_0: case LM_GGML_TYPE_Q4_0: case LM_GGML_TYPE_Q4_1: case LM_GGML_TYPE_Q5_0: @@ -1338,7 +1340,11 @@ bool lm_ggml_metal_device_supports_op(lm_ggml_metal_device_t dev, const struct l return op->src[0]->type != LM_GGML_TYPE_NVFP4; case LM_GGML_OP_SET_ROWS: { - if (op->src[0]->type != LM_GGML_TYPE_F32 && op->src[0]->type != LM_GGML_TYPE_F16) { + if (op->src[0]->type == LM_GGML_TYPE_F16) { + return op->type == LM_GGML_TYPE_F16; + } + + if (op->src[0]->type != LM_GGML_TYPE_F32) { return false; } diff --git a/cpp/ggml-metal/ggml-metal-impl.h b/cpp/ggml-metal/ggml-metal-impl.h index 2fab5f41e..75e6b6b96 100644 --- a/cpp/ggml-metal/ggml-metal-impl.h +++ b/cpp/ggml-metal/ggml-metal-impl.h @@ -24,6 +24,9 @@ #define N_R0_Q1_0 8 #define N_SG_Q1_0 2 +#define N_R0_Q2_0 8 +#define N_SG_Q2_0 2 + #define N_R0_Q4_0 4 #define N_SG_Q4_0 2 @@ -613,6 +616,11 @@ typedef struct { int32_t p0; } lm_ggml_metal_kargs_col2im_1d; +typedef struct { + int32_t T; + int32_t C; +} lm_ggml_metal_kargs_snake; + typedef struct { int32_t IC; int32_t IH; diff --git a/cpp/ggml-metal/ggml-metal-ops.cpp b/cpp/ggml-metal/ggml-metal-ops.cpp index 2a88e2007..a1dcddcc3 100644 --- a/cpp/ggml-metal/ggml-metal-ops.cpp +++ b/cpp/ggml-metal/ggml-metal-ops.cpp @@ -2077,6 +2077,7 @@ int lm_ggml_metal_op_mul_mat(lm_ggml_metal_op_t ctx, int idx) { op->src[0]->type == LM_GGML_TYPE_F16 || op->src[0]->type == LM_GGML_TYPE_BF16 || op->src[0]->type == LM_GGML_TYPE_Q1_0 || + op->src[0]->type == LM_GGML_TYPE_Q2_0 || op->src[0]->type == LM_GGML_TYPE_Q4_0 || op->src[0]->type == LM_GGML_TYPE_Q4_1 || op->src[0]->type == LM_GGML_TYPE_Q5_0 || @@ -3083,7 +3084,58 @@ int lm_ggml_metal_op_flash_attn_ext(lm_ggml_metal_op_t ctx, int idx) { return 1; } +// Snake activation autofuse: mul -> sin -> sqr -> mul -> add +static bool lm_ggml_metal_op_can_fuse_snake(lm_ggml_metal_op_t ctx, int idx) { + static constexpr lm_ggml_op snake_ops[5] = { LM_GGML_OP_MUL, LM_GGML_OP_SIN, LM_GGML_OP_SQR, LM_GGML_OP_MUL, LM_GGML_OP_ADD }; + + if (ctx->node(idx)->op != LM_GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) { + return false; + } + + const lm_ggml_tensor * mul0 = ctx->node(idx + 0); + const lm_ggml_tensor * sin_node = ctx->node(idx + 1); + const lm_ggml_tensor * sqr = ctx->node(idx + 2); + const lm_ggml_tensor * mul1 = ctx->node(idx + 3); + const lm_ggml_tensor * add = ctx->node(idx + 4); + + // x carries the full activation shape, a is the broadcast operand + const lm_ggml_tensor * x = lm_ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const lm_ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const lm_ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add reads the same x as the leading mul + const lm_ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // x is in the supported whitelist and every chain intermediate shares x's type. + // a and inv_b bind as device const float * in the kernel, so they stay F32. + const bool types_ok = + (x->type == LM_GGML_TYPE_F32 || x->type == LM_GGML_TYPE_F16 || x->type == LM_GGML_TYPE_BF16) && + (a->type == LM_GGML_TYPE_F32) && (inv_b->type == LM_GGML_TYPE_F32) && + (mul0->type == x->type) && (sin_node->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D + const bool shape_ok = lm_ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + const bool dim_ok = + (x->ne[2] == 1) && (x->ne[3] == 1) && + (add->ne[2] == 1) && (add->ne[3] == 1) && + (a->ne[2] == 1) && (a->ne[3] == 1) && + (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = + lm_ggml_is_contiguous(x) && lm_ggml_is_contiguous(add) && + lm_ggml_is_contiguous(a) && lm_ggml_is_contiguous(inv_b); + + return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; +} + int lm_ggml_metal_op_bin(lm_ggml_metal_op_t ctx, int idx) { + if (ctx->use_fusion && lm_ggml_metal_op_can_fuse_snake(ctx, idx)) { + return lm_ggml_metal_op_snake_fused(ctx, idx); + } + lm_ggml_tensor * op = ctx->node(idx); lm_ggml_metal_library_t lib = ctx->lib; @@ -3990,6 +4042,55 @@ int lm_ggml_metal_op_col2im_1d(lm_ggml_metal_op_t ctx, int idx) { return 1; } +// Dispatch the fused snake kernel from the matched mul -> sin -> sqr -> mul -> add chain. +// idx points at the leading mul. The caller has validated the chain. +int lm_ggml_metal_op_snake_fused(lm_ggml_metal_op_t ctx, int idx) { + lm_ggml_metal_library_t lib = ctx->lib; + lm_ggml_metal_encoder_t enc = ctx->enc; + + const lm_ggml_tensor * mul0 = ctx->node(idx + 0); + const lm_ggml_tensor * sqr = ctx->node(idx + 2); + const lm_ggml_tensor * mul1 = ctx->node(idx + 3); + lm_ggml_tensor * add = ctx->node(idx + 4); + + const lm_ggml_tensor * x = lm_ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const lm_ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + const lm_ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + const int T = (int) x->ne[0]; + const int C = (int) x->ne[1]; + const int total = T * C; + + // the encode loop pre-checked the leading mul only, check the rest of the chain + for (int i = 1; i < 5; ++i) { + if (!lm_ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) { + lm_ggml_metal_op_concurrency_reset(ctx); + + break; + } + } + + auto pipeline = lm_ggml_metal_library_get_pipeline_snake(lib, x->type); + + lm_ggml_metal_kargs_snake args = { + /*.T =*/ T, + /*.C =*/ C, + }; + + lm_ggml_metal_encoder_set_pipeline(enc, pipeline); + lm_ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + lm_ggml_metal_encoder_set_buffer (enc, lm_ggml_metal_get_buffer_id(x), 1); + lm_ggml_metal_encoder_set_buffer (enc, lm_ggml_metal_get_buffer_id(a), 2); + lm_ggml_metal_encoder_set_buffer (enc, lm_ggml_metal_get_buffer_id(inv_b), 3); + lm_ggml_metal_encoder_set_buffer (enc, lm_ggml_metal_get_buffer_id(add), 4); + + const int nth = 256; + const int ntg = (total + nth - 1) / nth; + lm_ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + + return 5; +} + int lm_ggml_metal_op_conv_transpose_2d(lm_ggml_metal_op_t ctx, int idx) { lm_ggml_tensor * op = ctx->node(idx); diff --git a/cpp/ggml-metal/ggml-metal-ops.h b/cpp/ggml-metal/ggml-metal-ops.h index dc581a743..d1b2311f2 100644 --- a/cpp/ggml-metal/ggml-metal-ops.h +++ b/cpp/ggml-metal/ggml-metal-ops.h @@ -80,6 +80,7 @@ int lm_ggml_metal_op_conv_3d (lm_ggml_metal_op_t ctx, int idx); int lm_ggml_metal_op_conv_transpose_1d (lm_ggml_metal_op_t ctx, int idx); int lm_ggml_metal_op_conv_transpose_2d (lm_ggml_metal_op_t ctx, int idx); int lm_ggml_metal_op_col2im_1d (lm_ggml_metal_op_t ctx, int idx); +int lm_ggml_metal_op_snake_fused (lm_ggml_metal_op_t ctx, int idx); int lm_ggml_metal_op_upscale (lm_ggml_metal_op_t ctx, int idx); int lm_ggml_metal_op_pad (lm_ggml_metal_op_t ctx, int idx); int lm_ggml_metal_op_pad_reflect_1d (lm_ggml_metal_op_t ctx, int idx); diff --git a/cpp/ggml-metal/ggml-metal.metal b/cpp/ggml-metal/ggml-metal.metal index a12080403..9464c4fe7 100644 --- a/cpp/ggml-metal/ggml-metal.metal +++ b/cpp/ggml-metal/ggml-metal.metal @@ -1937,6 +1937,9 @@ LM_GGML_TABLE_END() #define N_R0_Q1_0 8 #define N_SG_Q1_0 2 +#define N_R0_Q2_0 8 +#define N_SG_Q2_0 2 + #define N_R0_Q4_0 4 #define N_SG_Q4_0 2 @@ -2526,6 +2529,11 @@ typedef struct { int32_t p0; } lm_ggml_metal_kargs_col2im_1d; +typedef struct { + int32_t T; + int32_t C; +} lm_ggml_metal_kargs_snake; + typedef struct { int32_t IC; int32_t IH; @@ -3288,6 +3296,39 @@ void dequantize_q1_0_t4(device const block_q1_0 * xb, short il, thread type4 & r reg = (type4) reg_f; } +template +void dequantize_q2_0(device const block_q2_0 * xb, short il, thread type4x4 & reg) { + device const uint8_t * qs = xb->qs; + const float d = xb->d; + + const int byte_offset = il * 4; // il*16 elements = il*4 bytes (4 elements per byte) + float4x4 reg_f; + + for (int i = 0; i < 4; i++) { + const uint8_t b = qs[byte_offset + i]; + reg_f[i][0] = ((float)((b >> 0) & 3) - 1.0f) * d; + reg_f[i][1] = ((float)((b >> 2) & 3) - 1.0f) * d; + reg_f[i][2] = ((float)((b >> 4) & 3) - 1.0f) * d; + reg_f[i][3] = ((float)((b >> 6) & 3) - 1.0f) * d; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q2_0_t4(device const block_q2_0 * xb, short il, thread type4 & reg) { + const float d = xb->d; + const uint8_t b = xb->qs[il]; + + float4 reg_f; + reg_f[0] = ((float)((b >> 0) & 3) - 1.0f) * d; + reg_f[1] = ((float)((b >> 2) & 3) - 1.0f) * d; + reg_f[2] = ((float)((b >> 4) & 3) - 1.0f) * d; + reg_f[3] = ((float)((b >> 6) & 3) - 1.0f) * d; + + reg = (type4) reg_f; +} + template void dequantize_q4_0(device const block_q4_0 * xb, short il, thread type4x4 & reg) { device const uint16_t * qs = ((device const uint16_t *)xb + 1); @@ -3339,6 +3380,27 @@ void quantize_q1_0(device const float * src, device block_q1_0 & dst) { } } +void quantize_q2_0(device const float * src, device block_q2_0 & dst) { + float amax = 0.0f; + for (int j = 0; j < QK2_0; j++) { + float a = fabs(src[j]); + if (a > amax) amax = a; + } + const float d = amax; + dst.d = d; + + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + for (int j = 0; j < QK2_0 / 4; j++) { + dst.qs[j] = 0; + } + for (int j = 0; j < QK2_0; j++) { + int q = (int)round(src[j] * id) + 1; + q = max(0, min(3, q)); + dst.qs[j / 4] |= (q << (2 * (j % 4))); + } +} + void quantize_q4_0(device const float * src, device block_q4_0 & dst) { #pragma METAL fp math_mode(safe) float amax = 0.0f; // absolute max @@ -6407,6 +6469,55 @@ inline float block_q_n_dot_y(device const block_q1_0 * qb_curr, float sumy, thre return qb_curr->d * (2.0f * acc - sumy); } +// Q2_0 dot: d * (sum_lo(y) + 2*sum_hi(y) - sumy) via per-bit conditional adds +inline float block_q_n_dot_y(device const block_q2_0 * qb_curr, float sumy, thread float * yl, int il) { + device const uint8_t * qs = qb_curr->qs + (il / 4); + const uint8_t b0 = qs[0]; + const uint8_t b1 = qs[1]; + const uint8_t b2 = qs[2]; + const uint8_t b3 = qs[3]; + + // Accumulate where low bit is set (bits 0,2,4,6 of each byte) + float acc_lo = 0.0f; + acc_lo += select(0.0f, yl[ 0], bool(b0 & 0x01)); + acc_lo += select(0.0f, yl[ 1], bool(b0 & 0x04)); + acc_lo += select(0.0f, yl[ 2], bool(b0 & 0x10)); + acc_lo += select(0.0f, yl[ 3], bool(b0 & 0x40)); + acc_lo += select(0.0f, yl[ 4], bool(b1 & 0x01)); + acc_lo += select(0.0f, yl[ 5], bool(b1 & 0x04)); + acc_lo += select(0.0f, yl[ 6], bool(b1 & 0x10)); + acc_lo += select(0.0f, yl[ 7], bool(b1 & 0x40)); + acc_lo += select(0.0f, yl[ 8], bool(b2 & 0x01)); + acc_lo += select(0.0f, yl[ 9], bool(b2 & 0x04)); + acc_lo += select(0.0f, yl[10], bool(b2 & 0x10)); + acc_lo += select(0.0f, yl[11], bool(b2 & 0x40)); + acc_lo += select(0.0f, yl[12], bool(b3 & 0x01)); + acc_lo += select(0.0f, yl[13], bool(b3 & 0x04)); + acc_lo += select(0.0f, yl[14], bool(b3 & 0x10)); + acc_lo += select(0.0f, yl[15], bool(b3 & 0x40)); + + // Accumulate where high bit is set (bits 1,3,5,7 of each byte) + float acc_hi = 0.0f; + acc_hi += select(0.0f, yl[ 0], bool(b0 & 0x02)); + acc_hi += select(0.0f, yl[ 1], bool(b0 & 0x08)); + acc_hi += select(0.0f, yl[ 2], bool(b0 & 0x20)); + acc_hi += select(0.0f, yl[ 3], bool(b0 & 0x80)); + acc_hi += select(0.0f, yl[ 4], bool(b1 & 0x02)); + acc_hi += select(0.0f, yl[ 5], bool(b1 & 0x08)); + acc_hi += select(0.0f, yl[ 6], bool(b1 & 0x20)); + acc_hi += select(0.0f, yl[ 7], bool(b1 & 0x80)); + acc_hi += select(0.0f, yl[ 8], bool(b2 & 0x02)); + acc_hi += select(0.0f, yl[ 9], bool(b2 & 0x08)); + acc_hi += select(0.0f, yl[10], bool(b2 & 0x20)); + acc_hi += select(0.0f, yl[11], bool(b2 & 0x80)); + acc_hi += select(0.0f, yl[12], bool(b3 & 0x02)); + acc_hi += select(0.0f, yl[13], bool(b3 & 0x08)); + acc_hi += select(0.0f, yl[14], bool(b3 & 0x20)); + acc_hi += select(0.0f, yl[15], bool(b3 & 0x80)); + + return qb_curr->d * (acc_lo + 2.0f * acc_hi - sumy); +} + // function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i]) // il indicates where the q4 quants begin (0 or QK4_0/4) // we assume that the yl's have been multiplied with the appropriate scale factor @@ -6710,6 +6821,86 @@ kernel void kernel_mul_mv_q1_0_f32( kernel_mul_mv_q1_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } +template +void kernel_mul_mv_q2_0_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK2_0; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13; + + device const float * y = (device const float *) (src1 + offset1); + + device const block_q2_0 * ax[nr0]; + for (int row = 0; row < nr0; ++row) { + const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + ax[row] = (device const block_q2_0 *) ((device char *) src0 + offset0); + } + + float yl[16]; + float sumf[nr0] = {0.f}; + + // group 64: 4 sub-blocks of 16 weights per Q2_0 block + const short ix = (tiisg/4); + const short il = (tiisg%4)*16; + + device const float * yb = y + ix*QK2_0 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) { + float sumy = 0.f; + + FOR_UNROLL (short i = 0; i < 16; i++) { + yl[i] = yb[i]; + sumy += yb[i]; + } + + FOR_UNROLL (short row = 0; row < nr0; row++) { + sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il); + } + + yb += QK2_0 * (N_SIMDWIDTH/4); + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0; ++row) { + const float tot = simd_sum(sumf[row]); + + if (tiisg == 0 && first_row + row < args.ne01) { + dst_f32[first_row + row] = tot; + } + } +} + +[[host_name("kernel_mul_mv_q2_0_f32")]] +kernel void kernel_mul_mv_q2_0_f32( + constant lm_ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_q2_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + kernel void kernel_mul_mv_q4_0_f32( constant lm_ggml_metal_kargs_mul_mv & args, device const char * src0, @@ -7107,6 +7298,11 @@ template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_3")]] kernel mul_mv_ext_q4 template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q1_0, 128, dequantize_q1_0_t4>; template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q1_0, 128, dequantize_q1_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q2_0, 64, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q2_0, 64, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q2_0, 64, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q2_0, 64, dequantize_q2_0_t4>; + template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0, 32, dequantize_q4_0_t4>; template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0, 32, dequantize_q4_0_t4>; template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0, 32, dequantize_q4_0_t4>; @@ -8336,6 +8532,35 @@ template [[host_name("kernel_col2im_1d_bf16")]] kernel void kernel_col2im_1d +kernel void kernel_snake( + constant lm_ggml_metal_kargs_snake & args, + device const T * x, + device const float * a, + device const float * inv_b, + device T * dst, + uint tgpig [[threadgroup_position_in_grid]], + uint tpitg [[thread_position_in_threadgroup]], + uint ntg [[threads_per_threadgroup]]) { + + const int idx = tgpig * ntg + tpitg; + if (idx >= args.T * args.C) { + return; + } + + const int c = idx / args.T; // x is [T, C], a / inv_b collapse to [1, C] + const float xi = float(x[idx]); + const float si = sin(a[c] * xi); + dst[idx] = T(xi + si * si * inv_b[c]); +} + +template [[host_name("kernel_snake_f32")]] kernel void kernel_snake(constant lm_ggml_metal_kargs_snake &, device const float *, device const float *, device const float *, device float *, uint, uint, uint); +template [[host_name("kernel_snake_f16")]] kernel void kernel_snake(constant lm_ggml_metal_kargs_snake &, device const half *, device const float *, device const float *, device half *, uint, uint, uint); +#if defined(LM_GGML_METAL_HAS_BF16) +template [[host_name("kernel_snake_bf16")]] kernel void kernel_snake(constant lm_ggml_metal_kargs_snake &, device const bfloat *, device const float *, device const float *, device bfloat *, uint, uint, uint); +#endif + + typedef void (conv_transpose_2d_t)( constant lm_ggml_metal_kargs_conv_transpose_2d & args, device const float * src0, @@ -10818,6 +11043,7 @@ typedef decltype(kernel_cpy_f32_q) cpy_f_q_ template [[host_name("kernel_cpy_f32_q8_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q1_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q2_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q4_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q4_1")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q5_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; @@ -10863,6 +11089,7 @@ kernel void kernel_cpy_q_f32( typedef decltype(kernel_cpy_q_f32) cpy_q_f_t; template [[host_name("kernel_cpy_q1_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; @@ -10870,6 +11097,7 @@ template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32< template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; @@ -12714,6 +12942,7 @@ template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_ro typedef decltype(kernel_get_rows_q) get_rows_q_t; template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q2_0")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q; @@ -13584,6 +13813,7 @@ template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; #endif template [[host_name("kernel_mul_mm_q1_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mul_mm_t kernel_mul_mm; @@ -13608,6 +13838,7 @@ template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_f32_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_f16_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q1_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_0_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_1_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q5_0_f16")]] kernel mul_mm_t kernel_mul_mm; @@ -13641,6 +13872,7 @@ template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mul_mm_id kernel_m template [[host_name("kernel_mul_mm_id_bf16_f32")]] kernel mul_mm_id kernel_mul_mm_id; #endif template [[host_name("kernel_mul_mm_id_q1_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; @@ -13665,6 +13897,7 @@ template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mul_mm_id kernel_m template [[host_name("kernel_mul_mm_id_f32_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_f16_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q1_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_1_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q5_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; @@ -13820,6 +14053,7 @@ template [[host_name("kernel_mul_mv_id_bf16_f32_4")]] kernel kernel_mul_mv_id_4 template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q1_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; diff --git a/cpp/ggml-opencl/ggml-opencl.cpp b/cpp/ggml-opencl/ggml-opencl.cpp index 277cf5253..e9c4c2c42 100644 --- a/cpp/ggml-opencl/ggml-opencl.cpp +++ b/cpp/ggml-opencl/ggml-opencl.cpp @@ -114,6 +114,7 @@ enum GPU_FAMILY { enum ADRENO_GPU_GEN { ADRENO_UNKNOWN, + A6X, A7X, A8X, X1E, @@ -122,6 +123,7 @@ enum ADRENO_GPU_GEN { enum ADRENO_CL_COMPILER_TYPE { E031, + E17, DX, }; @@ -243,14 +245,29 @@ static lm_ggml_cl_version get_opencl_c_version(lm_ggml_cl_version platform_versi } static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { + if (strstr(device_name, "610") || strstr(device_name, "612") || + strstr(device_name, "613") || strstr(device_name, "615") || + strstr(device_name, "616") || strstr(device_name, "618") || + strstr(device_name, "619") || strstr(device_name, "620") || + strstr(device_name, "630") || strstr(device_name, "640") || + strstr(device_name, "642") || strstr(device_name, "643") || + strstr(device_name, "644") || strstr(device_name, "650") || + strstr(device_name, "660") || strstr(device_name, "663") || + strstr(device_name, "680") || strstr(device_name, "685") || + strstr(device_name, "690")) { + return ADRENO_GPU_GEN::A6X; + } + if (strstr(device_name, "730") || strstr(device_name, "740") || strstr(device_name, "750")) { return ADRENO_GPU_GEN::A7X; } - if (strstr(device_name, "830") || - strstr(device_name, "840")) { + if (strstr(device_name, "810") || + strstr(device_name, "830") || + strstr(device_name, "840") || + strstr(device_name, "850")) { return ADRENO_GPU_GEN::A8X; } @@ -274,6 +291,17 @@ static lm_ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *dr size_t compiler_minor_offset = 8; size_t compiler_patch_offset = 11; + if (compiler_ver_pos == std::string::npos) { + compiler_ver_pos = driver_ver_str.find("E17"); + if (compiler_ver_pos != std::string::npos) { + type = ADRENO_CL_COMPILER_TYPE::E17; + compiler_ver_len = 12; + compiler_major_offset = 4; + compiler_minor_offset = 7; + compiler_patch_offset = 10; + } + } + if (compiler_ver_pos == std::string::npos) { compiler_ver_pos = driver_ver_str.find("DX"); if (compiler_ver_pos == std::string::npos) { @@ -282,6 +310,8 @@ static lm_ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *dr type = ADRENO_CL_COMPILER_TYPE::DX; compiler_ver_len = 11; compiler_major_offset = 3; + compiler_minor_offset = 6; + compiler_patch_offset = 9; } std::string compiler_ver_str = driver_ver_str.substr(compiler_ver_pos, compiler_ver_len); @@ -532,6 +562,7 @@ struct lm_ggml_backend_opencl_context { bool fp16_support; bool has_vector_subgroup_broadcast; bool has_subgroup_shuffle = false; // cl_khr_subgroup_shuffle or cl_qcom_subgroup_shuffle + bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8 bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle bool disable_fusion; @@ -834,7 +865,7 @@ struct lm_ggml_backend_opencl_context { cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns; cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin; cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in) - cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a; // dp4a (int8) prefill GEMM variant + cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) prefill GEMM variant cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16] @@ -844,12 +875,12 @@ struct lm_ggml_backend_opencl_context { cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8] cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns; cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns; - cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a; // dp4a (int8) q6_K MoE prefill GEMM + cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) q6_K MoE prefill GEMM cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32; cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin; cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV - cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a; // dp4a (int8) mxfp4 MoE prefill GEMM - cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a; // dp4a (int8) q4_0 MoE prefill GEMM + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMM + cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_moe_reorder_b; cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum @@ -1037,10 +1068,10 @@ struct lm_ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32; cl_kernel kernel_gemm_noshuffle_q4_k_f32; - cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a; // dp4a (int8) dense prefill GEMM - cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) - cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a; // dp4a (int8) dense q5_K prefill GEMM - cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a; // dp4a (int8) dense q6_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass cl_kernel kernel_gemv_noshuffle_q6_K_f32; cl_kernel kernel_gemm_noshuffle_q6_K_f32; @@ -1640,6 +1671,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { // those compiler versions since it is anyway not used for Adreno. if (backend_ctx->gpu_family != ADRENO || backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) || + backend_ctx->adreno_cl_compiler_version.type == E17 || backend_ctx->adreno_cl_compiler_version.type == DX) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { @@ -3490,7 +3522,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h" @@ -3580,7 +3612,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h" @@ -3595,7 +3627,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h" @@ -3708,7 +3740,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h" @@ -3730,7 +3762,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h" @@ -3746,7 +3778,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h" @@ -3761,7 +3793,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h" @@ -4091,7 +4123,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_k_q8_1_dp4a.cl.h" @@ -4108,7 +4140,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h" @@ -4125,7 +4157,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_0_q8_1_dp4a.cl.h" @@ -4142,7 +4174,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q8_1_dp4a.cl.h" @@ -4256,7 +4288,7 @@ static void load_cl_kernels(lm_ggml_backend_opencl_context *backend_ctx) { } // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM) - { + if (backend_ctx->has_integer_dot) { #ifdef LM_GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q6_k_q8_1_dp4a.cl.h" @@ -5602,6 +5634,8 @@ static void lm_ggml_opencl_print_backend_info(lm_ggml_backend_opencl_device_cont backend_ctx->has_subgroup_shuffle ? "true" : "false"); LM_GGML_LOG_INFO("lm_ggml_opencl: device FP16 support: %s\n", backend_ctx->fp16_support ? "true" : "false"); + LM_GGML_LOG_INFO("lm_ggml_opencl: khr dot product support: %s\n", + backend_ctx->has_integer_dot ? "true" : "false"); LM_GGML_LOG_INFO("lm_ggml_opencl: mem base addr align: %u\n", backend_ctx->alignment); LM_GGML_LOG_INFO("lm_ggml_opencl: global mem size: %zu MB\n", @@ -5810,6 +5844,12 @@ static lm_ggml_backend_opencl_context * lm_ggml_cl_init(lm_ggml_backend_dev_t de strstr(ext_buffer, "cl_khr_subgroup_shuffle") != NULL || backend_ctx->has_qcom_subgroup_shuffle; + // check for cl_khr_integer_dot_product + // cl_qcom_dot_product8 uses signed * unsigned + // while cl_khr_integer_dot_product uses signed * signed -- we stick with khr for now + backend_ctx->has_integer_dot = + strstr(ext_buffer, "cl_khr_integer_dot_product") != NULL; + cl_uint base_align_in_bits; CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); LM_GGML_ASSERT(base_align_in_bits % 8u == 0); @@ -6923,8 +6963,31 @@ inline bool use_adreno_kernels(const lm_ggml_backend_opencl_context *backend_ctx return threashold_ok; } +static bool adreno_e17_compiler_quirks(const lm_ggml_backend_opencl_context *backend_ctx) { + if (!backend_ctx || backend_ctx->gpu_family != GPU_FAMILY::ADRENO || + backend_ctx->adreno_cl_compiler_version.type != ADRENO_CL_COMPILER_TYPE::E17) { + return false; + } + const char * env = getenv("LM_GGML_OPENCL_ADRENO_E17_QUIRKS"); + return !(env && env[0] == '0'); +} + inline bool use_adreno_moe_kernels(const lm_ggml_backend_opencl_context *backend_ctx, const lm_ggml_tensor *tensor) { - LM_GGML_UNUSED(backend_ctx); + // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. + // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. + // So, exclude A6x and A7x from using Adreno MoE kernels for now. + // The quants that have a general mul_mat_id kernel fallback to the general version; the + // rest fallback to CPU. + if (backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN)) { + return false; + } + + if (adreno_e17_compiler_quirks(backend_ctx)) { + return false; + } + int ne01 = tensor->ne[1]; return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0); } @@ -7257,6 +7320,10 @@ static bool lm_ggml_opencl_supports_op(lm_ggml_backend_dev_t dev, const struct l case LM_GGML_OP_MEAN: return op->src[0]->type == LM_GGML_TYPE_F32; case LM_GGML_OP_FLASH_ATTN_EXT: { + // The E17 compilers segfault while building FA kernels, skip E17 for now + if (adreno_e17_compiler_quirks(backend_ctx)) { + return false; + } const lm_ggml_tensor * q = op->src[0]; const lm_ggml_tensor * k = op->src[1]; const lm_ggml_tensor * v = op->src[2]; @@ -7310,6 +7377,14 @@ static bool lm_ggml_opencl_supports_op(lm_ggml_backend_dev_t dev, const struct l return false; } + // Some compilers for A7x (Adreno 740, compiler E031.41) crashes when + // building FA kernels with mixed or quant types (f32_f16, f32_q8_0, f32_q4_0) + // Here we skip all A7x for these kernels to avoid crash + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } + if (dk == 512) { if (backend_ctx->gpu_family == INTEL) { return false; @@ -10516,10 +10591,16 @@ static lm_ggml_backend_buffer_t lm_ggml_backend_opencl_buffer_type_alloc_buffer( cl_int err; cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); +#if LM_GGML_OPENCL_TARGET_VERSION >= 300 + // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than + // that do not export the symbol, so a build targeting them fails to link. The large + // buffer extension is only ever enabled on drivers that are well past 3.0, so this path + // is dead there anyway. if (err != CL_SUCCESS && backend_ctx->adreno_use_large_buffer) { cl_mem_properties props[] = { 0x41A6 /* CL_LARGE_BUFFER_QCOM */, 1, 0 }; mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); } +#endif if (err != CL_SUCCESS) { LM_GGML_LOG_INFO("%s: failed to allocate %.2f MiB\n", __func__, size / 1024.0 / 1024.0); @@ -15819,18 +15900,14 @@ static void lm_ggml_cl_mul_mat_q4_0_f32_adreno(lm_ggml_backend_t backend, const CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { - // dp4a (int8) dense prefill GEMM: quant activations to q8_1, then the int8 - // dp4a inner-loop GEMM, in place of the transpose + f16 half-dot kernel. - // q4_0 = d*(q-8); mirrors the IQ4_NL/q8_0 dense dp4a paths (+ the sum term). - // OPT-IN / DEFAULT OFF: correct, but neutral on X2E. q4_0's dequant - // ((q-8)*scale) is already trivial so the f16 GEMM is weight-BW-bound and the - // int8 ALU win has nothing to beat -- same as q5_0 dense (unlike IQ4_NL, whose - // codebook dequant is expensive enough for dp4a to help). Kept for A/B; force - // on with LM_GGML_OPENCL_Q4_0_DENSE_DP4A=1. Needs N>8, K%32==0, M%64==0. + // dp4a (int8) dense prefill GEMM, default off static const char * q4_0_dense_dp4a_env = getenv("LM_GGML_OPENCL_Q4_0_DENSE_DP4A"); - const bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env + bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env ? (atoi(q4_0_dense_dp4a_env) != 0) : false; + // dot prod has to be available + q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on; + if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; @@ -16253,27 +16330,16 @@ static void lm_ggml_cl_mul_mat_q5_0_f32_adreno(lm_ggml_backend_t backend, const CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { - // dp4a (int8) dense q5_0 prefill GEMM. Quantizes the [N,K] activations to - // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch - // (ne1>8) only. q5_0 weight = (x-16)*d (x = nibble | hi<<4); x packed as a - // 0..31 byte (dp4a), the -16 centering folded into a single min term - // (d*16) via the q8_1 block sum. Reads the qs/qh/d buffers byte-identically - // to the f16 kernel (greedy byte-identical, MUL_MAT NMSE-OK). - // - // OPT-IN / DEFAULT OFF. Unlike q8_0/q4_K dense, dp4a is not a win for q5_0 on - // X2E: the q5_0 model is bottlenecked elsewhere, so the dense-GEMM int8 win - // has nothing to surface and the q8_1 prepass slightly hurts. Kept correct + - // opt-in for the X1 A/B (different texture-cache dynamic) and the - // weight-texture variant. Env: LM_GGML_OPENCL_Q5_DENSE_DP4A=1. - // Weight-as-texture variant (X1 lever): routes the dominant qs nibble plane - // through an image1d_buffer (qh stays a buffer). Opt-in - // LM_GGML_OPENCL_Q5_DENSE_DP4A_WIMG; when set it also forces the dp4a path on. + // dp4a (int8) dense q5_0 prefill GEMM, default off static const char * q5_dense_dp4a_env = getenv("LM_GGML_OPENCL_Q5_DENSE_DP4A"); static const char * q5_dense_wimg_env = getenv("LM_GGML_OPENCL_Q5_DENSE_DP4A_WIMG"); const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0); - const bool q5_dense_dp4a_on = q5_dense_wimg_on + bool q5_dense_dp4a_on = q5_dense_wimg_on ? true : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0)); + // dot prod has to be available + q5_dense_dp4a_on = backend_ctx->has_integer_dot && q5_dense_dp4a_on; + if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; @@ -16708,15 +16774,14 @@ static void lm_ggml_cl_mul_mat_iq4_nl_f32_adreno(lm_ggml_backend_t backend, cons } else { // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch - // (ne1>8) only. IQ4_NL weight = kvalues[nibble]*d; the codebook value IS the - // int8 (no min term), so this is the q8_0 dense case plus a nibble->int8 LUT - // unpack. Reads the q/d buffers byte-identically to the f16 kernel. No bin - // kernel for IQ4_NL -> baseline is f16, default ON for X2E (like q4_K/q6_K - // dense dp4a). X1 stays on f16. Env: LM_GGML_OPENCL_IQ4NL_DENSE_DP4A. + // (ne1>8) only static const char * iq4nl_dense_dp4a_env = getenv("LM_GGML_OPENCL_IQ4NL_DENSE_DP4A"); - const bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env + bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env ? (atoi(iq4nl_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + iq4nl_dense_dp4a_on = backend_ctx->has_integer_dot && iq4nl_dense_dp4a_on; + if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; @@ -16966,13 +17031,15 @@ static void lm_ggml_cl_mul_mat_q8_0_f32_adreno(lm_ggml_backend_t backend, const static const char * q8_dense_wimg_env = getenv("LM_GGML_OPENCL_Q8_DENSE_DP4A_WIMG"); const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0); - const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); + const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); // bin kernel takes precedence - const bool q8_dense_dp4a_on = q8_dense_wimg_on + bool q8_dense_dp4a_on = q8_dense_wimg_on ? true : q8_dense_dp4a_env ? (atoi(q8_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded); + // dot prod has to be available + q8_dense_dp4a_on = backend_ctx->has_integer_dot && q8_dense_dp4a_on; if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { @@ -17379,13 +17446,16 @@ static void lm_ggml_cl_mul_mat_q4_k_f32_adreno(lm_ggml_backend_t backend, const static const char * q4k_dense_dp4a_env = getenv("LM_GGML_OPENCL_Q4K_DENSE_DP4A"); static const char * q4k_dense_wimg_env = getenv("LM_GGML_OPENCL_Q4K_DENSE_DP4A_WIMG"); - const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); - const bool q4k_dense_dp4a_on = q4k_dense_wimg_on + const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); + bool q4k_dense_dp4a_on = q4k_dense_wimg_on ? true : q4k_dense_dp4a_env ? (atoi(q4k_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dp4 has to be available + q4k_dense_dp4a_on = backend_ctx->has_integer_dot && q4k_dense_dp4a_on; + // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8 static const char * q4k_dp4a_minn_env = getenv("LM_GGML_OPENCL_Q4K_DP4A_MINN"); const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9; @@ -17608,9 +17678,11 @@ static void lm_ggml_cl_mul_mat_q6_K_f32_adreno(lm_ggml_backend_t backend, const // dp4a (int8) dense q6_K prefill GEMM static const char * q6k_dense_dp4a_env = getenv("LM_GGML_OPENCL_Q6K_DENSE_DP4A"); - static const bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) + bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) ? (atoi(q6k_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E); + // dot prod has to be available + q6k_dense_dp4a_on = backend_ctx->has_integer_dot && q6k_dense_dp4a_on; const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 || strncmp(src0->name, "token_embd", 10) == 0; @@ -17901,9 +17973,11 @@ static void lm_ggml_cl_mul_mat_q5_K_f32_adreno(lm_ggml_backend_t backend, const // dp4a (int8) dense q5_K prefill GEMM static const char * q5k_dense_dp4a_env = getenv("LM_GGML_OPENCL_Q5K_DENSE_DP4A"); - const bool q5k_dense_dp4a_on = q5k_dense_dp4a_env + bool q5k_dense_dp4a_on = q5k_dense_dp4a_env ? (atoi(q5k_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + q5k_dense_dp4a_on = backend_ctx->has_integer_dot && q5k_dense_dp4a_on; if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { const int Mm = ne01, Nn = ne1, Kk = ne00; @@ -20640,6 +20714,8 @@ static void lm_ggml_cl_mul_mat_id(lm_ggml_backend_t backend, const lm_ggml_tenso bool use_moe_dp4a = q4_0_moe_dp4a_env ? (atoi(q4_0_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr; @@ -21815,6 +21891,8 @@ static void lm_ggml_cl_mul_mat_id(lm_ggml_backend_t backend, const lm_ggml_tenso bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr) ? (atoi(q4k_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; @@ -22316,10 +22394,12 @@ static void lm_ggml_cl_mul_mat_id(lm_ggml_backend_t backend, const lm_ggml_tenso // dp4a (int8) q6_K MoE prefill GEMM static const char * q6k_moe_dp4a_env = getenv("LM_GGML_OPENCL_Q6K_MOE_DP4A"); - static const bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr) + bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr) ? (atoi(q6k_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; cl_buffer_region region; region.origin = 0; @@ -22569,6 +22649,8 @@ static void lm_ggml_cl_mul_mat_id(lm_ggml_backend_t backend, const lm_ggml_tenso bool use_moe_dp4a = mxfp4_moe_dp4a_env ? (atoi(mxfp4_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr; diff --git a/cpp/ggml-opencl/kernels/flash_attn_f32_f16.cl b/cpp/ggml-opencl/kernels/flash_attn_f32_f16.cl index 1cc0cc8c3..6e43ee81e 100644 --- a/cpp/ggml-opencl/kernels/flash_attn_f32_f16.cl +++ b/cpp/ggml-opencl/kernels/flash_attn_f32_f16.cl @@ -30,6 +30,10 @@ #elif defined(cl_qcom_subgroup_shuffle) #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable #define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) #endif #define ACC_TYPE float diff --git a/cpp/ggml-opencl/kernels/flash_attn_f32_q4_0.cl b/cpp/ggml-opencl/kernels/flash_attn_f32_q4_0.cl index de09a1eaa..95d215971 100644 --- a/cpp/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +++ b/cpp/ggml-opencl/kernels/flash_attn_f32_q4_0.cl @@ -10,6 +10,10 @@ #elif defined(cl_qcom_subgroup_shuffle) #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable #define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) #endif // Flash attention: Q=f32, K=q4_0, V=q4_0. diff --git a/cpp/ggml-opencl/kernels/flash_attn_f32_q8_0.cl b/cpp/ggml-opencl/kernels/flash_attn_f32_q8_0.cl index 46bc4bc9d..7e89ed0bd 100644 --- a/cpp/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +++ b/cpp/ggml-opencl/kernels/flash_attn_f32_q8_0.cl @@ -10,6 +10,10 @@ #elif defined(cl_qcom_subgroup_shuffle) #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable #define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) #endif // Flash attention: Q=f32, K=q8_0, V=q8_0. diff --git a/cpp/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl b/cpp/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl index 834050a4f..10c8855c1 100644 --- a/cpp/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +++ b/cpp/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl @@ -274,8 +274,9 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns( shared_b[b_local_offset.y] = bx8_f16.hi; // Dequantization - reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * s; - reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * s; + // Cast the e8m0 scale to half to satisfy E17 compilers + reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * (half)s; + reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * (half)s; sub_group_barrier(CLK_LOCAL_MEM_FENCE); @@ -304,8 +305,9 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns( shared_b[b_local_offset.y] = bx8_f16.hi; // Dequantization - reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * s; - reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * s; + // Cast the e8m0 scale to half to satisfy E17 compilers + reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * (half)s; + reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * (half)s; sub_group_barrier(CLK_LOCAL_MEM_FENCE); diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl index 9386bf25a..1f832cb25 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl @@ -296,7 +296,12 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl index e83c5d068..9efede294 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl @@ -116,6 +116,10 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32( if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - dst[gid] = totalSum; + // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize, + // so when ne01 is not a multiple of the wave size the tail work-items run past + // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical + // when ne01 is wave-aligned (no padding). + if (gid < M) dst[gid] = totalSum; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl index 106832069..8de0de1cc 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl @@ -268,7 +268,12 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl index 571a375da..0dca20f71 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl @@ -262,7 +262,11 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows against the padded x-grid tail overrunning dst. + // The current shape specializations are all ne01 % 128 == 0 (no padding), so + // this is a no-op / byte-identical today; keep it in lockstep with the base kernel. + if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl index fdc147245..5fa312780 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl @@ -277,7 +277,12 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl index dd1e2b55c..2eb20e2f7 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl @@ -312,7 +312,12 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl index c228f717a..7dbf5a3bb 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl @@ -285,7 +285,12 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl index daf1308ea..ba0e2a711 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl @@ -288,7 +288,12 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl index c40db1666..446f46533 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl @@ -321,6 +321,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl index 6f89cf968..51682eceb 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl @@ -288,6 +288,11 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( if (grp == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(total_sum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor (garbage downstream). + // No-op / byte-identical when ne01 % 128 == 0 (no padding). + if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = total_sum.s0; + if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1; } } diff --git a/cpp/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/cpp/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl index f5c6fb3e8..09bae2d55 100644 --- a/cpp/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +++ b/cpp/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl @@ -190,6 +190,10 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( // 1 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - dst[gid] = totalSum; + // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize, + // so when ne01 is not a multiple of the wave size the tail work-items run past + // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical + // when ne01 is wave-aligned (no padding). + if (gid < M) dst[gid] = totalSum; } } diff --git a/cpp/ggml-opencl/kernels/mul_mv_f16_f16.cl b/cpp/ggml-opencl/kernels/mul_mv_f16_f16.cl index 9393b5494..b4b03eb11 100644 --- a/cpp/ggml-opencl/kernels/mul_mv_f16_f16.cl +++ b/cpp/ggml-opencl/kernels/mul_mv_f16_f16.cl @@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f16( global half * x = (global half *) (src0 + offset_src0); - if (ne00 < 128) { + // The vector path below casts the row pointers to half4, which must be 8-byte aligned. + // A row address is r0*nb01 + ..., and a permuted or strided src leaves nb01/nb11 + // unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. Every + // src1 row this work-item walks is src1_base + r1*nb11, so require both. + const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13); + const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 7) == 0 && (nb11 & 7) == 0; + + if (ne00 < 128 || !row_aligned) { for (int row = 0; row < N_F16_F16; ++row) { int r1 = rb + row; if (r1 >= ne11) { diff --git a/cpp/ggml-opencl/kernels/mul_mv_f16_f32.cl b/cpp/ggml-opencl/kernels/mul_mv_f16_f32.cl index e52d3c6d4..8f3ed9c7b 100644 --- a/cpp/ggml-opencl/kernels/mul_mv_f16_f32.cl +++ b/cpp/ggml-opencl/kernels/mul_mv_f16_f32.cl @@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f32( global half * x = (global half *) (src0 + offset_src0); - if (ne00 < 128) { + // The vector path below casts the row pointers to half4/float4, which must be 8- and + // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves + // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. + // Every src1 row this work-item walks is src1_base + r1*nb11, so require both. + const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13); + const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 15) == 0 && (nb11 & 15) == 0; + + if (ne00 < 128 || !row_aligned) { for (int row = 0; row < N_F16_F32; ++row) { int r1 = rb + row; if (r1 >= ne11) { diff --git a/cpp/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl b/cpp/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl index 28d30212c..eca45615e 100644 --- a/cpp/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +++ b/cpp/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl @@ -64,8 +64,15 @@ kernel void kernel_mul_mat_f16_f32_1row( global half * x = (global half *) (src0 + offset_src0); global float * y = (global float *) (src1 + offset_src1); + // The vector path below casts the row pointers to half4/float4, which must be 8- and + // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves + // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. + // Take the vector path only when the rows this work-item touches are actually aligned; + // the scalar loop has no such requirement. + const bool row_aligned = (((ulong) x) & 7) == 0 && (((ulong) y) & 15) == 0; + float sumf = 0; - if (ne00 < 128) { + if (ne00 < 128 || !row_aligned) { for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) { sumf += (float) x[i] * (float) y[i]; } diff --git a/cpp/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl b/cpp/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl index da2e14ae9..97148d370 100644 --- a/cpp/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +++ b/cpp/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl @@ -24,6 +24,10 @@ #elif defined(cl_qcom_subgroup_shuffle) #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable #define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) #endif // Assumes row size (ne00) is a multiple of 4 diff --git a/cpp/ggml-opencl/kernels/mul_mv_q4_k_f32.cl b/cpp/ggml-opencl/kernels/mul_mv_q4_k_f32.cl index 71ab98982..4c3d5f00c 100644 --- a/cpp/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +++ b/cpp/ggml-opencl/kernels/mul_mv_q4_k_f32.cl @@ -1,3 +1,5 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + #ifdef cl_intel_required_subgroup_size #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable #define INTEL_GPU 1 diff --git a/cpp/ggml.c b/cpp/ggml.c index fade32bd2..a14ac872f 100644 --- a/cpp/ggml.c +++ b/cpp/ggml.c @@ -1088,6 +1088,9 @@ static const char * LM_GGML_OP_NAME[LM_GGML_OP_COUNT] = { "SOLVE_TRI", "GATED_DELTA_NET", "LIGHTNING_INDEXER", + "DSV4_HC_COMB", + "DSV4_HC_PRE", + "DSV4_HC_POST", "UNARY", @@ -1105,7 +1108,7 @@ static const char * LM_GGML_OP_NAME[LM_GGML_OP_COUNT] = { "GLU", }; -static_assert(LM_GGML_OP_COUNT == 98, "LM_GGML_OP_COUNT != 98"); +static_assert(LM_GGML_OP_COUNT == 101, "LM_GGML_OP_COUNT != 101"); static const char * LM_GGML_OP_SYMBOL[LM_GGML_OP_COUNT] = { "none", @@ -1200,6 +1203,9 @@ static const char * LM_GGML_OP_SYMBOL[LM_GGML_OP_COUNT] = { "A X = B, A triangular, solve X", "gated_delta_net(q, k, v, g, beta, s)", "lightning_indexer(q, k, weights, mask)", + "dsv4_hc_comb(mixes, scale, base)", + "dsv4_hc_pre(x, weights)", + "dsv4_hc_post(x, residual, post, comb)", "unary(x)", @@ -1217,7 +1223,7 @@ static const char * LM_GGML_OP_SYMBOL[LM_GGML_OP_COUNT] = { "glu(x)", }; -static_assert(LM_GGML_OP_COUNT == 98, "LM_GGML_OP_COUNT != 98"); +static_assert(LM_GGML_OP_COUNT == 101, "LM_GGML_OP_COUNT != 101"); static_assert(LM_GGML_OP_POOL_COUNT == 2, "LM_GGML_OP_POOL_COUNT != 2"); @@ -1472,14 +1478,14 @@ bool lm_ggml_is_transposed(const struct lm_ggml_tensor * tensor) { return tensor->nb[0] > tensor->nb[1]; } -static bool lm_ggml_is_contiguous_n(const struct lm_ggml_tensor * tensor, int n) { +static bool lm_ggml_is_contiguous_m_n(const struct lm_ggml_tensor * tensor, int m, int n) { size_t next_nb = lm_ggml_type_size(tensor->type); if (tensor->ne[0] != lm_ggml_blck_size(tensor->type) && tensor->nb[0] != next_nb) { return false; } next_nb *= tensor->ne[0]/lm_ggml_blck_size(tensor->type); - for (int i = 1; i < LM_GGML_MAX_DIMS; i++) { - if (i > n) { + for (int i = 1; i < n; i++) { + if (i > m) { if (tensor->ne[i] != 1 && tensor->nb[i] != next_nb) { return false; } @@ -1497,15 +1503,27 @@ bool lm_ggml_is_contiguous(const struct lm_ggml_tensor * tensor) { } bool lm_ggml_is_contiguous_0(const struct lm_ggml_tensor * tensor) { - return lm_ggml_is_contiguous_n(tensor, 0); + return lm_ggml_is_contiguous_m_n(tensor, 0, LM_GGML_MAX_DIMS); } bool lm_ggml_is_contiguous_1(const struct lm_ggml_tensor * tensor) { - return lm_ggml_is_contiguous_n(tensor, 1); + return lm_ggml_is_contiguous_m_n(tensor, 1, LM_GGML_MAX_DIMS); } bool lm_ggml_is_contiguous_2(const struct lm_ggml_tensor * tensor) { - return lm_ggml_is_contiguous_n(tensor, 2); + return lm_ggml_is_contiguous_m_n(tensor, 2, LM_GGML_MAX_DIMS); +} + +bool lm_ggml_is_contiguous_to_1(const struct lm_ggml_tensor * tensor) { + return lm_ggml_is_contiguous_m_n(tensor, 0, 1); +} + +bool lm_ggml_is_contiguous_to_2(const struct lm_ggml_tensor * tensor) { + return lm_ggml_is_contiguous_m_n(tensor, 0, 2); +} + +bool lm_ggml_is_contiguous_to_3(const struct lm_ggml_tensor * tensor) { + return lm_ggml_is_contiguous_m_n(tensor, 0, 3); } bool lm_ggml_is_contiguously_allocated(const struct lm_ggml_tensor * tensor) { @@ -4515,7 +4533,7 @@ struct lm_ggml_tensor * lm_ggml_conv_1d( int s0, int p0, int d0) { - struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, LM_GGML_TYPE_F16); // [N, OL, IC * K] + struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, a->type == LM_GGML_TYPE_BF16 ? LM_GGML_TYPE_F32 : LM_GGML_TYPE_F16); // [N, OL, IC * K] struct lm_ggml_tensor * result = lm_ggml_mul_mat(ctx, @@ -4549,7 +4567,7 @@ struct lm_ggml_tensor * lm_ggml_conv_1d_dw( int d0) { struct lm_ggml_tensor * new_b = lm_ggml_reshape_4d(ctx, b, b->ne[0], 1, b->ne[1], b->ne[2]); - struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, a, new_b, s0, 0, p0, 0, d0, 0, false, LM_GGML_TYPE_F16); + struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, a, new_b, s0, 0, p0, 0, d0, 0, false, a->type == LM_GGML_TYPE_BF16 ? LM_GGML_TYPE_F32 : LM_GGML_TYPE_F16); struct lm_ggml_tensor * result = lm_ggml_mul_mat(ctx, im2col, a); @@ -4655,7 +4673,7 @@ struct lm_ggml_tensor * lm_ggml_conv_2d( int p1, int d0, int d1) { - struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, a->type); // [N, OH, OW, IC * KH * KW] + struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, a->type == LM_GGML_TYPE_BF16 ? LM_GGML_TYPE_F32 : LM_GGML_TYPE_F16); // [N, OH, OW, IC * KH * KW] struct lm_ggml_tensor * result = lm_ggml_mul_mat(ctx, @@ -4737,7 +4755,7 @@ struct lm_ggml_tensor * lm_ggml_conv_3d( int d1, // dilation height int d2 // dilation depth ) { - struct lm_ggml_tensor * im2col = lm_ggml_im2col_3d(ctx, a, b, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, a->type); // [N*OD, OH, OW, IC * KD * KH * KW] + struct lm_ggml_tensor * im2col = lm_ggml_im2col_3d(ctx, a, b, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, a->type == LM_GGML_TYPE_BF16 ? LM_GGML_TYPE_F32 : LM_GGML_TYPE_F16); // [N*OD, OH, OW, IC * KD * KH * KW] int64_t OC = a->ne[3] / IC; int64_t N = b->ne[3] / IC; @@ -4787,7 +4805,7 @@ struct lm_ggml_tensor * lm_ggml_conv_2d_dw( struct lm_ggml_tensor * new_a = lm_ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]); struct lm_ggml_tensor * im2col = lm_ggml_im2col(ctx, new_a, lm_ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]), - s0, s1, p0, p1, d0, d1, true, LM_GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW] + s0, s1, p0, p1, d0, d1, true, a->type == LM_GGML_TYPE_BF16 ? LM_GGML_TYPE_F32 : LM_GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW] struct lm_ggml_tensor * new_b = lm_ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3]); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW] new_a = lm_ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1); // [OC，1, KH, KW] => [1, OC, 1, KH * KW] @@ -5433,6 +5451,7 @@ struct lm_ggml_tensor * lm_ggml_flash_attn_ext( return result; } + void lm_ggml_flash_attn_ext_set_prec( struct lm_ggml_tensor * a, enum lm_ggml_prec prec) { @@ -6333,6 +6352,132 @@ struct lm_ggml_tensor * lm_ggml_lightning_indexer( return result; } +// lm_ggml_dsv4_hc_comb + +struct lm_ggml_tensor * lm_ggml_dsv4_hc_comb( + struct lm_ggml_context * ctx, + struct lm_ggml_tensor * mixes, + struct lm_ggml_tensor * scale, + struct lm_ggml_tensor * base, + float eps, + int32_t n_iter) { + LM_GGML_ASSERT(mixes->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(scale->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(base->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(n_iter > 0); + + const int64_t hc_mix_dim = mixes->ne[0]; + const int64_t n_tokens = mixes->ne[1]; + + int64_t hc = 0; + for (int64_t i = 1; i*i + 2*i <= hc_mix_dim; ++i) { + if ((2 + i)*i == hc_mix_dim) { + hc = i; + break; + } + } + + LM_GGML_ASSERT(hc > 0); + LM_GGML_ASSERT(hc == 4); + LM_GGML_ASSERT(mixes->ne[2] == 1); + LM_GGML_ASSERT(mixes->ne[3] == 1); + LM_GGML_ASSERT(scale->ne[0] >= 3); + LM_GGML_ASSERT(scale->ne[1] == 1); + LM_GGML_ASSERT(scale->ne[2] == 1); + LM_GGML_ASSERT(scale->ne[3] == 1); + LM_GGML_ASSERT(base->ne[0] == hc_mix_dim); + LM_GGML_ASSERT(base->ne[1] == 1); + LM_GGML_ASSERT(base->ne[2] == 1); + LM_GGML_ASSERT(base->ne[3] == 1); + + struct lm_ggml_tensor * result = lm_ggml_new_tensor_3d(ctx, LM_GGML_TYPE_F32, hc, hc, n_tokens); + + lm_ggml_set_op_params_f32(result, 0, eps); + lm_ggml_set_op_params_i32(result, 1, n_iter); + + result->op = LM_GGML_OP_DSV4_HC_COMB; + result->src[0] = mixes; + result->src[1] = scale; + result->src[2] = base; + + return result; +} + +// lm_ggml_dsv4_hc_pre + +struct lm_ggml_tensor * lm_ggml_dsv4_hc_pre( + struct lm_ggml_context * ctx, + struct lm_ggml_tensor * x, + struct lm_ggml_tensor * weights) { + LM_GGML_ASSERT(x->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(weights->type == LM_GGML_TYPE_F32); + + const int64_t n_embd = x->ne[0]; + const int64_t hc = x->ne[1]; + const int64_t n_tokens = x->ne[2]; + + LM_GGML_ASSERT(hc > 0); + LM_GGML_ASSERT(x->ne[3] == 1); + LM_GGML_ASSERT(weights->ne[0] == hc); + LM_GGML_ASSERT(weights->ne[1] == n_tokens); + LM_GGML_ASSERT(weights->ne[2] == 1); + LM_GGML_ASSERT(weights->ne[3] == 1); + + struct lm_ggml_tensor * result = lm_ggml_new_tensor_2d(ctx, LM_GGML_TYPE_F32, n_embd, n_tokens); + + result->op = LM_GGML_OP_DSV4_HC_PRE; + result->src[0] = x; + result->src[1] = weights; + + return result; +} + +// lm_ggml_dsv4_hc_post + +struct lm_ggml_tensor * lm_ggml_dsv4_hc_post( + struct lm_ggml_context * ctx, + struct lm_ggml_tensor * x, + struct lm_ggml_tensor * residual, + struct lm_ggml_tensor * post, + struct lm_ggml_tensor * comb) { + LM_GGML_ASSERT(x->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(residual->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(post->type == LM_GGML_TYPE_F32); + LM_GGML_ASSERT(comb->type == LM_GGML_TYPE_F32); + + const int64_t n_embd = x->ne[0]; + const int64_t n_tokens = x->ne[1]; + const int64_t hc = residual->ne[1]; + + LM_GGML_ASSERT(hc > 0); + LM_GGML_ASSERT(x->ne[2] == 1); + LM_GGML_ASSERT(x->ne[3] == 1); + + LM_GGML_ASSERT(residual->ne[0] == n_embd); + LM_GGML_ASSERT(residual->ne[2] == n_tokens); + LM_GGML_ASSERT(residual->ne[3] == 1); + + LM_GGML_ASSERT(post->ne[0] == hc); + LM_GGML_ASSERT(post->ne[1] == n_tokens); + LM_GGML_ASSERT(post->ne[2] == 1); + LM_GGML_ASSERT(post->ne[3] == 1); + + LM_GGML_ASSERT(comb->ne[0] == hc); + LM_GGML_ASSERT(comb->ne[1] == hc); + LM_GGML_ASSERT(comb->ne[2] == n_tokens); + LM_GGML_ASSERT(comb->ne[3] == 1); + + struct lm_ggml_tensor * result = lm_ggml_new_tensor_3d(ctx, LM_GGML_TYPE_F32, n_embd, hc, n_tokens); + + result->op = LM_GGML_OP_DSV4_HC_POST; + result->src[0] = x; + result->src[1] = residual; + result->src[2] = post; + result->src[3] = comb; + + return result; +} + //////////////////////////////////////////////////////////////////////////////// struct lm_ggml_hash_set lm_ggml_hash_set_new(size_t size) { diff --git a/cpp/ggml.h b/cpp/ggml.h index e87ad55fd..0758833d8 100644 --- a/cpp/ggml.h +++ b/cpp/ggml.h @@ -571,6 +571,9 @@ extern "C" { LM_GGML_OP_SOLVE_TRI, LM_GGML_OP_GATED_DELTA_NET, LM_GGML_OP_LIGHTNING_INDEXER, + LM_GGML_OP_DSV4_HC_COMB, + LM_GGML_OP_DSV4_HC_PRE, + LM_GGML_OP_DSV4_HC_POST, LM_GGML_OP_UNARY, @@ -780,6 +783,10 @@ extern "C" { LM_GGML_API bool lm_ggml_is_contiguous_1(const struct lm_ggml_tensor * tensor); // contiguous for dims >= 1 LM_GGML_API bool lm_ggml_is_contiguous_2(const struct lm_ggml_tensor * tensor); // contiguous for dims >= 2 + LM_GGML_API bool lm_ggml_is_contiguous_to_1(const struct lm_ggml_tensor * tensor); // contiguous for dims < 1 + LM_GGML_API bool lm_ggml_is_contiguous_to_2(const struct lm_ggml_tensor * tensor); // contiguous for dims < 2 + LM_GGML_API bool lm_ggml_is_contiguous_to_3(const struct lm_ggml_tensor * tensor); // contiguous for dims < 3 + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) LM_GGML_API bool lm_ggml_is_contiguously_allocated(const struct lm_ggml_tensor * tensor); @@ -2594,6 +2601,45 @@ extern "C" { struct lm_ggml_tensor * weights, struct lm_ggml_tensor * mask); + // DeepSeek V4 hyper-connections (ref. https://arxiv.org/pdf/2512.24880) + // In short these operations are replacements for the original residual connection (x = transformer(x) + x) + // using a richer representation through streams. + // + // hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc] + // -> [dst_hc, src_hc, n_tokens] + // logits[dst, src, t] = mixes[2*hc + dst + hc*src, t]*scale[2] + // + base[2*hc + dst + hc*src] + // Softmax over dst, add eps, normalize over src, then repeat normalization + // over dst followed by src for iterations 1 through n_iter - 1. + LM_GGML_API struct lm_ggml_tensor * lm_ggml_dsv4_hc_comb( + struct lm_ggml_context * ctx, + struct lm_ggml_tensor * mixes, + struct lm_ggml_tensor * scale, + struct lm_ggml_tensor * base, + float eps, + int32_t n_iter); + + // hc_pre: x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens] + // result[i, t] = sum_h x[i, h, t]*weights[h, t] + // + LM_GGML_API struct lm_ggml_tensor * lm_ggml_dsv4_hc_pre( + struct lm_ggml_context * ctx, + struct lm_ggml_tensor * x, + struct lm_ggml_tensor * weights); + + // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens], + // post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens] + // -> [n_embd, hc, n_tokens] + // result[i, dst, t] = x[i, t]*post[dst, t] + // + sum_src residual[i, src, t]*comb[dst, src, t] + // + LM_GGML_API struct lm_ggml_tensor * lm_ggml_dsv4_hc_post( + struct lm_ggml_context * ctx, + struct lm_ggml_tensor * x, + struct lm_ggml_tensor * residual, + struct lm_ggml_tensor * post, + struct lm_ggml_tensor * comb); + // custom operators typedef void (*lm_ggml_custom1_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, int ith, int nth, void * userdata); diff --git a/cpp/gguf.cpp b/cpp/gguf.cpp index c0ec9fea5..8b71a175e 100644 --- a/cpp/gguf.cpp +++ b/cpp/gguf.cpp @@ -1186,6 +1186,11 @@ const char * lm_gguf_get_tensor_name(const struct lm_gguf_context * ctx, int64_t return ctx->info[tensor_id].t.name; } +const int64_t * lm_gguf_get_tensor_ne(const struct lm_gguf_context * ctx, int64_t tensor_id) { + LM_GGML_ASSERT(tensor_id >= 0 && tensor_id < lm_gguf_get_n_tensors(ctx)); + return ctx->info[tensor_id].t.ne; +} + enum lm_ggml_type lm_gguf_get_tensor_type(const struct lm_gguf_context * ctx, int64_t tensor_id) { LM_GGML_ASSERT(tensor_id >= 0 && tensor_id < lm_gguf_get_n_tensors(ctx)); return ctx->info[tensor_id].t.type; diff --git a/cpp/gguf.h b/cpp/gguf.h index 1ae410b03..df6ba5f10 100644 --- a/cpp/gguf.h +++ b/cpp/gguf.h @@ -125,12 +125,13 @@ extern "C" { // get ith C string from array with given key_id LM_GGML_API const char * lm_gguf_get_arr_str (const struct lm_gguf_context * ctx, int64_t key_id, size_t i); - LM_GGML_API int64_t lm_gguf_get_n_tensors (const struct lm_gguf_context * ctx); - LM_GGML_API int64_t lm_gguf_find_tensor (const struct lm_gguf_context * ctx, const char * name); // returns -1 if the tensor is not found - LM_GGML_API size_t lm_gguf_get_tensor_offset(const struct lm_gguf_context * ctx, int64_t tensor_id); - LM_GGML_API const char * lm_gguf_get_tensor_name (const struct lm_gguf_context * ctx, int64_t tensor_id); - LM_GGML_API enum lm_ggml_type lm_gguf_get_tensor_type (const struct lm_gguf_context * ctx, int64_t tensor_id); - LM_GGML_API size_t lm_gguf_get_tensor_size (const struct lm_gguf_context * ctx, int64_t tensor_id); + LM_GGML_API int64_t lm_gguf_get_n_tensors (const struct lm_gguf_context * ctx); + LM_GGML_API int64_t lm_gguf_find_tensor (const struct lm_gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + LM_GGML_API size_t lm_gguf_get_tensor_offset(const struct lm_gguf_context * ctx, int64_t tensor_id); + LM_GGML_API const char * lm_gguf_get_tensor_name (const struct lm_gguf_context * ctx, int64_t tensor_id); + LM_GGML_API const int64_t * lm_gguf_get_tensor_ne (const struct lm_gguf_context * ctx, int64_t tensor_id); // returns ne, an array of LM_GGML_MAX_DIMS elements; ne[dim] is 1 for dim >= n_dims + LM_GGML_API enum lm_ggml_type lm_gguf_get_tensor_type (const struct lm_gguf_context * ctx, int64_t tensor_id); + LM_GGML_API size_t lm_gguf_get_tensor_size (const struct lm_gguf_context * ctx, int64_t tensor_id); // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) LM_GGML_API int64_t lm_gguf_remove_key(struct lm_gguf_context * ctx, const char * key); diff --git a/cpp/llama-arch.cpp b/cpp/llama-arch.cpp index e9f1175a5..3e764f7a2 100644 --- a/cpp/llama-arch.cpp +++ b/cpp/llama-arch.cpp @@ -113,6 +113,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_HUNYUAN_MOE, "hunyuan-moe" }, { LLM_ARCH_HUNYUAN_DENSE, "hunyuan-dense" }, { LLM_ARCH_HUNYUAN_VL, "hunyuan_vl" }, + { LLM_ARCH_HY_V3, "hy_v3" }, { LLM_ARCH_SMOLLM3, "smollm3" }, { LLM_ARCH_OPENAI_MOE, "gpt-oss" }, { LLM_ARCH_LFM2, "lfm2" }, diff --git a/cpp/llama-arch.h b/cpp/llama-arch.h index 50e76c907..af588a0e4 100644 --- a/cpp/llama-arch.h +++ b/cpp/llama-arch.h @@ -118,6 +118,7 @@ enum llm_arch { LLM_ARCH_HUNYUAN_MOE, LLM_ARCH_HUNYUAN_DENSE, LLM_ARCH_HUNYUAN_VL, + LLM_ARCH_HY_V3, LLM_ARCH_SMOLLM3, LLM_ARCH_OPENAI_MOE, LLM_ARCH_LFM2, diff --git a/cpp/llama-context.cpp b/cpp/llama-context.cpp index 7751cba88..470588c64 100644 --- a/cpp/llama-context.cpp +++ b/cpp/llama-context.cpp @@ -61,6 +61,24 @@ static const llm_fused_op_probe llm_fused_op_lid_probe = { /*.n_tokens_per_seq =*/ 1, }; +static const llm_fused_op_probe llm_fused_op_dsv4_hc_pre_probe = { + /*.op =*/ LLM_FUSED_OP_DSV4_HC_PRE, + /*.name =*/ "fused DeepSeek V4 HC pre", + /*.n_tokens_per_seq =*/ 1, +}; + +static const llm_fused_op_probe llm_fused_op_dsv4_hc_comb_probe = { + /*.op =*/ LLM_FUSED_OP_DSV4_HC_COMB, + /*.name =*/ "fused DeepSeek V4 HC comb", + /*.n_tokens_per_seq =*/ 1, +}; + +static const llm_fused_op_probe llm_fused_op_dsv4_hc_post_probe = { + /*.op =*/ LLM_FUSED_OP_DSV4_HC_POST, + /*.name =*/ "fused DeepSeek V4 HC post", + /*.n_tokens_per_seq =*/ 1, +}; + llama_context::llama_context( const llama_model & model, llama_context_params params) : @@ -235,6 +253,11 @@ llama_context::llama_context( cparams.fused_lid = true; cparams.auto_flid = true; + cparams.fused_dsv4_hc_pre = true; + cparams.fused_dsv4_hc_comb = true; + cparams.fused_dsv4_hc_post = true; + cparams.auto_fhc = true; + // with causal attention, the batch size is limited by the context size cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; @@ -537,6 +560,14 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3 resolve(llm_fused_op_lid_probe, cparams.fused_lid); cparams.auto_flid = false; } + + if (cparams.auto_fhc) { + LLAMA_LOG_INFO("%s: resolving fused DeepSeek V4 HC support:\n", func); + resolve(llm_fused_op_dsv4_hc_pre_probe, cparams.fused_dsv4_hc_pre); + resolve(llm_fused_op_dsv4_hc_comb_probe, cparams.fused_dsv4_hc_comb); + resolve(llm_fused_op_dsv4_hc_post_probe, cparams.fused_dsv4_hc_post); + cparams.auto_fhc = false; + } } void llama_context::sched_reserve() { diff --git a/cpp/llama-cparams.h b/cpp/llama-cparams.h index 02de8ec72..34fdddb55 100644 --- a/cpp/llama-cparams.h +++ b/cpp/llama-cparams.h @@ -43,6 +43,10 @@ struct llama_cparams { bool auto_fgdn; bool fused_lid; // use fused lightning indexer bool auto_flid; + bool fused_dsv4_hc_pre; + bool fused_dsv4_hc_comb; + bool fused_dsv4_hc_post; + bool auto_fhc; bool no_perf; bool warmup; // TODO: remove [TAG_LLAMA_GRAPH_NO_WARMUP] bool op_offload; diff --git a/cpp/llama-graph.h b/cpp/llama-graph.h index 25ce5544f..92c2d2e76 100644 --- a/cpp/llama-graph.h +++ b/cpp/llama-graph.h @@ -43,6 +43,9 @@ enum llm_fused_op { LLM_FUSED_OP_GDN_AR, LLM_FUSED_OP_GDN_CH, LLM_FUSED_OP_LIGHTNING_INDEXER, + LLM_FUSED_OP_DSV4_HC_PRE, + LLM_FUSED_OP_DSV4_HC_COMB, + LLM_FUSED_OP_DSV4_HC_POST, }; enum llm_ffn_op_type : int { diff --git a/cpp/llama-kv-cache-dsv4.cpp b/cpp/llama-kv-cache-dsv4.cpp index 7bcb8b6d5..ac5d67708 100644 --- a/cpp/llama-kv-cache-dsv4.cpp +++ b/cpp/llama-kv-cache-dsv4.cpp @@ -720,7 +720,7 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( auto it = ctx_map.find(buft); if (it == ctx_map.end()) { lm_ggml_init_params params = { - /*.mem_size =*/ size_t(2u*hparams.n_layer()*lm_ggml_tensor_overhead()), + /*.mem_size =*/ size_t(2u*(1 + n_stream)*hparams.n_layer()*lm_ggml_tensor_overhead()), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; @@ -767,9 +767,17 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( lm_ggml_format_name(kv, "dsv4_%s_state_kv_l%d", name, il); lm_ggml_format_name(score, "dsv4_%s_state_score_l%d", name, il); + std::vector kv_stream; + std::vector score_stream; + + for (uint32_t s = 0; s < n_stream; ++s) { + kv_stream.push_back(lm_ggml_view_2d(ctx, kv, n_embd_state, state_size, kv->nb[1], s*kv->nb[2])); + score_stream.push_back(lm_ggml_view_2d(ctx, score, n_embd_state, state_size, score->nb[1], s*score->nb[2])); + } + map_layer_ids[il] = layers.size(); - layers.push_back({ il, kv, score }); + layers.push_back({ il, kv, score, std::move(kv_stream), std::move(score_stream) }); } for (auto & [buft, ctx] : ctx_map) { @@ -809,6 +817,30 @@ void llama_dsv4_comp_state::clear(llama_seq_id seq_id, bool data) { } } +void llama_dsv4_comp_state::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst) { + LM_GGML_ASSERT(seq_id_src >= 0 && (uint32_t) seq_id_src < n_stream); + LM_GGML_ASSERT(seq_id_dst >= 0 && (uint32_t) seq_id_dst < n_stream); + + if (seq_id_src == seq_id_dst) { + return; + } + + sc_info.ssrc.push_back((uint32_t) seq_id_src); + sc_info.sdst.push_back((uint32_t) seq_id_dst); +} + +void llama_dsv4_comp_state::apply_copies(const stream_copy_info & sc_info) const { + for (size_t i = 0; i < sc_info.ssrc.size(); ++i) { + const uint32_t ssrc = sc_info.ssrc[i]; + const uint32_t sdst = sc_info.sdst[i]; + + for (const auto & layer : layers) { + lm_ggml_backend_tensor_copy(layer.kv_stream[ssrc], layer.kv_stream[sdst]); + lm_ggml_backend_tensor_copy(layer.score_stream[ssrc], layer.score_stream[sdst]); + } + } +} + uint32_t llama_dsv4_comp_state::get_ratio() const { return ratio; } @@ -1154,7 +1186,13 @@ llama_memory_context_ptr llama_kv_cache_dsv4::init_full() { } llama_memory_context_ptr llama_kv_cache_dsv4::init_update(llama_context * lctx, bool optimize) { - return std::make_unique(this, lctx, optimize); + return std::make_unique( + this, + lctx, + optimize, + std::move(csa_state->sc_info), + std::move(hca_state->sc_info), + std::move(lid_state->sc_info)); } bool llama_kv_cache_dsv4::get_can_shift() const { @@ -1174,14 +1212,19 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1 } if (p0 > 0) { - // DSV4 compressed cache rows are derived from running compressor state, - // so arbitrary rollback is not reconstructible from the raw cache alone. - // Allow the common prompt-cache cleanup no-op: remove [end, infinity). - if (seq_id >= 0 && p0 > kv_raw->seq_pos_max(seq_id)) { - return true; + if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max || + p0 <= kv_raw->seq_pos_max(seq_id)) { + return false; } - return false; + bool res = true; + + res = res & kv_raw->seq_rm(seq_id, p0, -1); + res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); + res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1); + res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); + + return res; } const bool res = kv_raw->seq_rm(seq_id, p0, p1); @@ -1194,7 +1237,16 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1 } void llama_kv_cache_dsv4::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + LM_GGML_ASSERT(p0 <= 0 && p1 < 0 && "DSV4 only supports full sequence copies"); + kv_raw->seq_cp(seq_id_src, seq_id_dst, p0, p1); + kv_csa->seq_cp(seq_id_src, seq_id_dst, -1, -1); + kv_hca->seq_cp(seq_id_src, seq_id_dst, -1, -1); + kv_lid->seq_cp(seq_id_src, seq_id_dst, -1, -1); + + csa_state->seq_cp(seq_id_src, seq_id_dst); + hca_state->seq_cp(seq_id_src, seq_id_dst); + lid_state->seq_cp(seq_id_src, seq_id_dst); } void llama_kv_cache_dsv4::seq_keep(llama_seq_id seq_id) { @@ -1639,20 +1691,26 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( llama_kv_cache_dsv4 * kv, llama_context * lctx, - bool optimize) : + bool optimize, + stream_copy_info sc_info_csa, + stream_copy_info sc_info_hca, + stream_copy_info sc_info_lid) : ctx_raw(std::make_unique(kv->get_raw(), lctx, optimize)), ctx_csa_mem(kv->get_csa()->init_update(lctx, optimize)), ctx_hca_mem(kv->get_hca()->init_update(lctx, optimize)), ctx_lid_mem(kv->get_lid()->init_update(lctx, optimize)), - ctx_csa(std::make_unique(kv->get_csa())), - ctx_hca(std::make_unique(kv->get_hca())), - ctx_lid(std::make_unique(kv->get_lid())), csa_state(kv->get_csa_state()), hca_state(kv->get_hca_state()), lid_state(kv->get_lid_state()), + sc_info_csa(std::move(sc_info_csa)), + sc_info_hca(std::move(sc_info_hca)), + sc_info_lid(std::move(sc_info_lid)), status(llama_memory_status_combine( - llama_memory_status_combine(ctx_raw->get_status(), ctx_csa_mem->get_status()), - llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status()))) { + llama_memory_status_combine( + llama_memory_status_combine(ctx_raw->get_status(), ctx_csa_mem->get_status()), + llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status())), + this->sc_info_csa.empty() && this->sc_info_hca.empty() && this->sc_info_lid.empty() ? + LLAMA_MEMORY_STATUS_NO_UPDATE : LLAMA_MEMORY_STATUS_SUCCESS)) { } llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( @@ -1720,6 +1778,18 @@ bool llama_kv_cache_dsv4_context::apply() { res = res & ctx_raw->apply(); + if (ctx_csa_mem) { + res = res & ctx_csa_mem->apply(); + res = res & ctx_hca_mem->apply(); + res = res & ctx_lid_mem->apply(); + } + + if (ubatches.empty()) { + csa_state->apply_copies(sc_info_csa); + hca_state->apply_copies(sc_info_hca); + lid_state->apply_copies(sc_info_lid); + } + return res; } diff --git a/cpp/llama-kv-cache-dsv4.h b/cpp/llama-kv-cache-dsv4.h index ceeae5a53..66d3445c2 100644 --- a/cpp/llama-kv-cache-dsv4.h +++ b/cpp/llama-kv-cache-dsv4.h @@ -10,6 +10,10 @@ class llama_dsv4_comp_state { public: + using stream_copy_info = llama_kv_cache::stream_copy_info; + + stream_copy_info sc_info; + llama_dsv4_comp_state( const llama_model & model, bool offload, @@ -22,6 +26,8 @@ class llama_dsv4_comp_state { const llama_memory_i::layer_filter_cb & filter); void clear(llama_seq_id seq_id, bool data); + void seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst); + void apply_copies(const stream_copy_info & sc_info) const; uint32_t get_ratio() const; uint32_t get_state_size() const; @@ -44,6 +50,9 @@ class llama_dsv4_comp_state { lm_ggml_tensor * kv; lm_ggml_tensor * score; + + std::vector kv_stream; + std::vector score_stream; }; const uint32_t ratio; @@ -245,6 +254,7 @@ class llama_kv_cache_dsv4_comp_context { class llama_kv_cache_dsv4_context : public llama_memory_context_i { public: using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + using stream_copy_info = llama_kv_cache::stream_copy_info; struct comp_plan { // Per-ubatch recipe for updating compressor state, committing completed @@ -291,7 +301,10 @@ class llama_kv_cache_dsv4_context : public llama_memory_context_i { llama_kv_cache_dsv4_context( llama_kv_cache_dsv4 * kv, llama_context * lctx, - bool optimize); + bool optimize, + stream_copy_info sc_info_csa, + stream_copy_info sc_info_hca, + stream_copy_info sc_info_lid); llama_kv_cache_dsv4_context( llama_kv_cache_dsv4 * kv, @@ -351,9 +364,13 @@ class llama_kv_cache_dsv4_context : public llama_memory_context_i { const std::unique_ptr ctx_hca; const std::unique_ptr ctx_lid; - const llama_dsv4_comp_state * csa_state = nullptr; - const llama_dsv4_comp_state * hca_state = nullptr; - const llama_dsv4_comp_state * lid_state = nullptr; + llama_dsv4_comp_state * csa_state = nullptr; + llama_dsv4_comp_state * hca_state = nullptr; + llama_dsv4_comp_state * lid_state = nullptr; + + stream_copy_info sc_info_csa; + stream_copy_info sc_info_hca; + stream_copy_info sc_info_lid; bool reserve_plans = false; mutable comp_plan reserve_plan_csa; diff --git a/cpp/llama-model.cpp b/cpp/llama-model.cpp index 14582c88f..9c7027bd6 100644 --- a/cpp/llama-model.cpp +++ b/cpp/llama-model.cpp @@ -262,6 +262,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_hunyuan_vl(params); case LLM_ARCH_HUNYUAN_DENSE: return new llama_model_hunyuan_dense(params); + case LLM_ARCH_HY_V3: + return new llama_model_hy_v3(params); case LLM_ARCH_SMOLLM3: return new llama_model_smollm3(params); case LLM_ARCH_OPENAI_MOE: @@ -358,8 +360,10 @@ struct lm_ggml_backend_meta_split_state llama_meta_device_get_split_state(const static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight"); static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight"); - static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight"); - static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias"); + static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight"); + static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias"); + static const std::regex pattern_ffn_gate_weight ("blk\\.\\d*\\.ffn_gate(_exps)?.weight"); + static const std::regex pattern_ffn_gate_bias ("blk\\.\\d*\\.ffn_gate(_exps)?.bias"); static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight"); static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight"); static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias"); @@ -467,10 +471,10 @@ struct lm_ggml_backend_meta_split_state llama_meta_device_get_split_state(const } // FFN - if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight)) { + if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) { return get_tensor_config_impl(LM_GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight"); } - if (std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) { + if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) { return get_tensor_config_impl(LM_GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight"); } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { @@ -554,6 +558,14 @@ struct lm_ggml_backend_meta_split_state llama_meta_device_get_split_state(const LM_GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); return {{n_embd, 1}, {n_embd_gqa, 2}}; } + if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) { + const int64_t n_ff = hparams.n_ff(il); + // some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented + if (tensor->ne[axis] == 2*n_ff) { + return {{n_ff, 2}}; + } + return {{tensor->ne[axis], 1}}; + } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { const int64_t n_ff_exp = hparams.n_ff_exp; LM_GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp); @@ -630,7 +642,8 @@ struct lm_ggml_backend_meta_split_state llama_meta_device_get_split_state(const } // FFN - if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || std::regex_match(tensor_name, pattern_ffn_up_gate_bias) || + if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) || + std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) || std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) { const int64_t blck_size_perf = std::lcm(blck_size, 128); LM_GGML_ASSERT(segments.size() == 1); @@ -2169,7 +2182,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } - if (arch == LLM_ARCH_STEP35 && hparams.n_layer_nextn > 0) { + if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3) && hparams.n_layer_nextn > 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } else { @@ -2525,6 +2538,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_JAIS2: case LLM_ARCH_OPENAI_MOE: case LLM_ARCH_HUNYUAN_DENSE: + case LLM_ARCH_HY_V3: case LLM_ARCH_LFM2: case LLM_ARCH_LFM2MOE: case LLM_ARCH_SMALLTHINKER: diff --git a/cpp/models/deepseek4.cpp b/cpp/models/deepseek4.cpp index 00c1acf83..93ff961d4 100644 --- a/cpp/models/deepseek4.cpp +++ b/cpp/models/deepseek4.cpp @@ -197,22 +197,31 @@ static lm_ggml_tensor * dsv4_hc_affine( return x; } -lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_weighted_sum( +lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( lm_ggml_tensor * x, - lm_ggml_tensor * weights) const { + lm_ggml_tensor * weights, + int il) const { + LM_GGML_ASSERT(x->ne[0] == n_embd); + LM_GGML_ASSERT(x->ne[1] == hparams.dsv4_hc_mult); + const int64_t hc = hparams.dsv4_hc_mult; const int64_t nt = x->ne[2]; - lm_ggml_tensor * acc = nullptr; + if (cparams.fused_dsv4_hc_pre && il >= 0) { + lm_ggml_tensor * result = lm_ggml_dsv4_hc_pre(ctx0, x, weights); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, result, il}); + return result; + } + + lm_ggml_tensor * result = nullptr; for (int64_t ih = 0; ih < hc; ++ih) { lm_ggml_tensor * xh = lm_ggml_view_2d(ctx0, x, n_embd, nt, x->nb[2], ih*x->nb[1]); lm_ggml_tensor * wh = lm_ggml_view_2d(ctx0, weights, 1, nt, weights->nb[1], ih*weights->nb[0]); - lm_ggml_tensor * cur = lm_ggml_mul(ctx0, xh, wh); - acc = acc ? lm_ggml_add(ctx0, acc, cur) : cur; + result = result ? lm_ggml_add(ctx0, result, cur) : cur; } - return acc; + return result; } lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_sinkhorn( @@ -275,11 +284,9 @@ lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( lm_ggml_tensor * scale_pre = dsv4_view_1d(ctx0, hc_scale, 1, 0); lm_ggml_tensor * scale_post = dsv4_view_1d(ctx0, hc_scale, 1, 1); - lm_ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); lm_ggml_tensor * base_pre = dsv4_view_1d(ctx0, hc_base, hc, 0); lm_ggml_tensor * base_post = dsv4_view_1d(ctx0, hc_base, hc, hc); - lm_ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); lm_ggml_tensor * pre = dsv4_view_2d(ctx0, mixes, hc, nt, 0); pre = dsv4_hc_affine(ctx0, pre, scale_pre, base_pre); @@ -293,13 +300,23 @@ lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( *post = lm_ggml_scale(ctx0, *post, 2.0f); cb(*post, "hc_post", il); - *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); - *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); - *comb = lm_ggml_reshape_3d(ctx0, *comb, hc, hc, nt); - *comb = build_hc_sinkhorn(*comb, il); + if (cparams.fused_dsv4_hc_comb) { + *comb = lm_ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + (int32_t) hparams.dsv4_hc_sinkhorn_iters); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + } else { + lm_ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); + lm_ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); + + *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); + *comb = lm_ggml_reshape_3d(ctx0, *comb, hc, hc, nt); + *comb = build_hc_sinkhorn(*comb, il); + } cb(*comb, "hc_comb", il); - return build_hc_weighted_sum(x, pre); + lm_ggml_tensor * result = build_hc_pre(x, pre, il); + return result; } lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_post( @@ -308,7 +325,14 @@ lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_post( lm_ggml_tensor * post, lm_ggml_tensor * comb, int il) const { - LM_GGML_UNUSED(il); + LM_GGML_ASSERT(x->ne[0] == n_embd); + LM_GGML_ASSERT(residual->ne[1] == hparams.dsv4_hc_mult); + + if (cparams.fused_dsv4_hc_post) { + lm_ggml_tensor * result = lm_ggml_dsv4_hc_post(ctx0, x, residual, post, comb); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, result, il}); + return result; + } const int64_t hc = hparams.dsv4_hc_mult; const int64_t nt = x->ne[1]; @@ -320,7 +344,8 @@ lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_post( for (int64_t src = 0; src < hc; ++src) { lm_ggml_tensor * res_src = lm_ggml_view_2d(ctx0, residual, n_embd, nt, residual->nb[2], src*residual->nb[1]); - lm_ggml_tensor * comb_src_dst = lm_ggml_view_2d(ctx0, comb, 1, nt, comb->nb[2], dst*comb->nb[0] + src*comb->nb[1]); + lm_ggml_tensor * comb_src_dst = lm_ggml_view_2d(ctx0, comb, 1, nt, comb->nb[2], + dst*comb->nb[0] + src*comb->nb[1]); cur = lm_ggml_add(ctx0, cur, lm_ggml_mul(ctx0, res_src, comb_src_dst)); } @@ -350,7 +375,7 @@ lm_ggml_tensor * llama_model_deepseek4::graph::build_hc_head( pre = lm_ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); cb(pre, "hc_head_pre", -1); - return build_hc_weighted_sum(x, pre); + return build_hc_pre(x, pre, -1); } lm_ggml_tensor * llama_model_deepseek4::graph::build_hca_compressed_kv_from_state( @@ -435,27 +460,29 @@ lm_ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_ kv_state = dsv4_append_zero_row(ctx0, kv_state, false); score_state = dsv4_append_zero_row(ctx0, score_state, true); - lm_ggml_tensor * prev_idxs = dsv4_view_1d(ctx0, state_read_idxs, ratio*n_blocks, 0); - lm_ggml_tensor * cur_idxs = dsv4_view_1d(ctx0, state_read_idxs, ratio*n_blocks, ratio*n_blocks); + const int64_t n_read = ratio*n_blocks; + + lm_ggml_tensor * kv_rows = lm_ggml_get_rows(ctx0, kv_state, state_read_idxs); + lm_ggml_tensor * score_rows = lm_ggml_get_rows(ctx0, score_state, state_read_idxs); - lm_ggml_tensor * kv_prev = lm_ggml_get_rows(ctx0, kv_state, prev_idxs); - kv_prev = lm_ggml_cont(ctx0, lm_ggml_view_2d(ctx0, kv_prev, n_embd_head, ratio*n_blocks, kv_prev->nb[1], 0)); + lm_ggml_tensor * kv_prev = lm_ggml_cont(ctx0, + lm_ggml_view_2d(ctx0, kv_rows, n_embd_head, n_read, kv_rows->nb[1], 0)); kv_prev = lm_ggml_reshape_3d(ctx0, kv_prev, n_embd_head, ratio, n_blocks); cb(kv_prev, name, il); - lm_ggml_tensor * score_prev = lm_ggml_get_rows(ctx0, score_state, prev_idxs); - score_prev = lm_ggml_cont(ctx0, lm_ggml_view_2d(ctx0, score_prev, n_embd_head, ratio*n_blocks, score_prev->nb[1], 0)); + lm_ggml_tensor * score_prev = lm_ggml_cont(ctx0, + lm_ggml_view_2d(ctx0, score_rows, n_embd_head, n_read, score_rows->nb[1], 0)); score_prev = lm_ggml_reshape_3d(ctx0, score_prev, n_embd_head, ratio, n_blocks); cb(score_prev, name, il); - lm_ggml_tensor * kv_cur = lm_ggml_get_rows(ctx0, kv_state, cur_idxs); - kv_cur = lm_ggml_cont(ctx0, lm_ggml_view_2d(ctx0, kv_cur, n_embd_head, ratio*n_blocks, kv_cur->nb[1], - lm_ggml_row_size(kv_cur->type, n_embd_head))); + lm_ggml_tensor * kv_cur = lm_ggml_cont(ctx0, + lm_ggml_view_2d(ctx0, kv_rows, n_embd_head, n_read, kv_rows->nb[1], + n_read*kv_rows->nb[1] + lm_ggml_row_size(kv_rows->type, n_embd_head))); kv_cur = lm_ggml_reshape_3d(ctx0, kv_cur, n_embd_head, ratio, n_blocks); - lm_ggml_tensor * score_cur = lm_ggml_get_rows(ctx0, score_state, cur_idxs); - score_cur = lm_ggml_cont(ctx0, lm_ggml_view_2d(ctx0, score_cur, n_embd_head, ratio*n_blocks, score_cur->nb[1], - lm_ggml_row_size(score_cur->type, n_embd_head))); + lm_ggml_tensor * score_cur = lm_ggml_cont(ctx0, + lm_ggml_view_2d(ctx0, score_rows, n_embd_head, n_read, score_rows->nb[1], + n_read*score_rows->nb[1] + lm_ggml_row_size(score_rows->type, n_embd_head))); score_cur = lm_ggml_reshape_3d(ctx0, score_cur, n_embd_head, ratio, n_blocks); lm_ggml_tensor * values = lm_ggml_concat(ctx0, kv_prev, kv_cur, 1); diff --git a/cpp/models/hy-v3.cpp b/cpp/models/hy-v3.cpp new file mode 100644 index 000000000..04fad2d25 --- /dev/null +++ b/cpp/models/hy-v3.cpp @@ -0,0 +1,390 @@ +#include "models.h" + +void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + + // HY V3 uses a sigmoid router with expert selection bias by default + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + } + + // NextN/MTP (HY V3): extra decoder block(s) appended beyond the main stack + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); + LM_GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all"); + + switch (hparams.n_layer()) { + case 48: type = LLM_TYPE_30B_A3B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP + // tensors live in a separate file (e.g. user split target/draft). Mark + // MTP tensors NOT_REQUIRED so the trunk loads cleanly. + const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; + const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + auto load_block = [&](int i, int flags) { + auto & layer = layers[i]; + const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / (n_expert_used > 0 ? n_expert_used : 1); + const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, flags); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, flags); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); + + // dense FFN (leading dense blocks, first_k_dense_replace) + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED); + + // MoE routed experts (sigmoid router + expert selection bias) + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, i), {n_expert}, TENSOR_NOT_REQUIRED); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, TENSOR_NOT_REQUIRED); + + // shared expert (always active, no gate) + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED); + }; + + for (int i = 0; i < n_layer; ++i) { + load_block(i, trunk_flags); + } + + // NextN/MTP block(s): a full hy_v3 decoder block plus the NextN projections. + for (int i = n_layer; i < n_layer_all; ++i) { + auto & layer = layers[i]; + + load_block(i, mtp_flags); + + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); + // hy_v3 stores the MTP block's trailing final_layernorm here (applied + // after the decoder block, before the shared LM head). + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED); + } +} + +std::unique_ptr llama_model_hy_v3::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } + return std::make_unique(*this, params); +} + +llama_model_hy_v3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + LM_GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + LM_GGML_ASSERT(n_embd_head == n_rot); + + lm_ggml_tensor * cur; + lm_ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + lm_ggml_tensor * inp_pos = build_inp_pos(); + auto * inp_attn = build_attn_inp_kv(); + lm_ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. + for (int il = 0; il < n_layer; ++il) { + lm_ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + lm_ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); + + Qcur = lm_ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = lm_ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + } + + if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { + cur = lm_ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = lm_ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + lm_ggml_tensor * ffn_inp = lm_ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if (model.layers[il].ffn_gate_inp == nullptr) { + // dense FFN (leading dense blocks) + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_dense_out", il); + } else { + // MoE routed experts (sigmoid gating + expert selection bias) + lm_ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, + hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, model.layers[il].ffn_gate_up_exps, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); + cb(moe_out, "ffn_moe_out", il); + + // shared expert (always active, no gate) + lm_ggml_tensor * sh_out = build_ffn(cur, + model.layers[il].ffn_up_shexp, nullptr, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, nullptr, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, nullptr, model.layers[il].ffn_down_shexp_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(sh_out, "ffn_shared_out", il); + + cur = lm_ggml_add(ctx0, moe_out, sh_out); + cb(cur, "ffn_out", il); + } + + cur = lm_ggml_add(ctx0, cur, ffn_inp); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1); + + // Post-final-norm hidden state: what the MTP draft head's hnorm consumes. + // vLLM feeds the target model's normed output states, and the MTP layer + // itself returns final_layernorm(h), so the chained state is post-norm. + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (!cparams.embeddings_nextn_masked && inp_out_ids) { + cur = lm_ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; + + lm_ggml_build_forward_expand(gf, cur); +} + +// LLM_GRAPH_TYPE_DECODER_MTP draft head for HY V3 (MoE). +// Semantics mirror vLLM's HYV3MultiTokenPredictorLayer (hy_v3_mtp.py): +// enorm(embed) + hnorm(prev_hidden) -> concat(e, h) -> eh_proj -> +// hy_v3 decoder block -> final_layernorm (stored as nextn.shared_head_norm) -> +// shared LM head (the main model's lm_head; the checkpoint has no separate +// MTP head or MTP embeddings). +llama_model_hy_v3::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) + : llm_graph_context(params) { + LM_GGML_ASSERT(hparams.n_layer_nextn > 0 && "HY_V3 MTP requires n_layer_nextn > 0"); + + const int64_t n_embd_head = hparams.n_embd_head_v(); + LM_GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + LM_GGML_ASSERT(n_embd_head == n_rot); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + LM_GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + const auto & layer = model.layers[il]; + + LM_GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + LM_GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + LM_GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + + auto inp = std::make_unique(hparams.n_embd); + + inp->tokens = lm_ggml_new_tensor_1d(ctx0, LM_GGML_TYPE_I32, n_tokens); + lm_ggml_set_input(inp->tokens); + + inp->embd = lm_ggml_new_tensor_2d(ctx0, LM_GGML_TYPE_F32, hparams.n_embd, n_tokens); + lm_ggml_set_input(inp->embd); + lm_ggml_set_name(inp->embd, "mtp_h_input"); + + lm_ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + + lm_ggml_tensor * h_input = inp->embd; + lm_ggml_tensor * tok_embd = lm_ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + cb(tok_embd, "mtp_tok_embd", il); + + res->add_input(std::move(inp)); + + lm_ggml_tensor * inp_pos = build_inp_pos(); + lm_ggml_tensor * inp_out_ids = build_inp_out_ids(); + auto * inp_attn = build_attn_inp_kv(); + + lm_ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + lm_ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + lm_ggml_tensor * concat = lm_ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0); + cb(concat, "mtp_concat", il); + + lm_ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat); + cb(cur, "mtp_eh_proj", il); + + lm_ggml_tensor * inpSA = cur; + + // mtp_block: a full hy_v3 decoder layer (mirrors the trunk graph) + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + { + lm_ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); + + Qcur = lm_ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = lm_ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + cur = build_attn(inp_attn, + layer.wo, layer.wo_b, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "mtp_attn_out", il); + } + + lm_ggml_tensor * ffn_inp = lm_ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "mtp_ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + if (layer.ffn_gate_inp == nullptr) { + cur = build_ffn(cur, + layer.ffn_up, layer.ffn_up_b, layer.ffn_up_s, + layer.ffn_gate, layer.ffn_gate_b, layer.ffn_gate_s, + layer.ffn_down, layer.ffn_down_b, layer.ffn_down_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "mtp_ffn_dense_out", il); + } else { + lm_ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, + hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, layer.ffn_gate_up_exps, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + cb(moe_out, "mtp_ffn_moe_out", il); + + lm_ggml_tensor * sh_out = build_ffn(cur, + layer.ffn_up_shexp, nullptr, layer.ffn_up_shexp_s, + layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s, + layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(sh_out, "mtp_ffn_shared_out", il); + + cur = lm_ggml_add(ctx0, moe_out, sh_out); + cb(cur, "mtp_ffn_out", il); + } + + cur = lm_ggml_add(ctx0, cur, ffn_inp); + cb(cur, "mtp_post_ffn", il); + + // final_layernorm applied after the decoder block, before the shared head. + // The post-norm hidden state seeds the next MTP step (matches vLLM, where + // HYV3MultiTokenPredictorLayer returns final_layernorm(h)). + lm_ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : model.output_norm; + LM_GGML_ASSERT(head_norm_w && "HY_V3 MTP: missing both nextn.shared_head_norm and output_norm"); + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + cur = lm_ggml_get_rows(ctx0, cur, inp_out_ids); + cb(cur, "mtp_shared_head_norm", -1); + + lm_ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + lm_ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; + LM_GGML_ASSERT(head_w && "HY_V3 MTP: missing LM head (nextn.shared_head_head or model.output)"); + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; + lm_ggml_build_forward_expand(gf, cur); +} diff --git a/cpp/models/minimax-m2.cpp b/cpp/models/minimax-m2.cpp index bd33cb7f2..9209d7acf 100644 --- a/cpp/models/minimax-m2.cpp +++ b/cpp/models/minimax-m2.cpp @@ -60,6 +60,8 @@ llama_model_minimax_m2::graph::graph(const llama_model & model, const llm_graph_ lm_ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + lm_ggml_tensor * inpSA = inpL; cur = inpL; diff --git a/cpp/models/models.h b/cpp/models/models.h index f5896fe6d..94741e851 100644 --- a/cpp/models/models.h +++ b/cpp/models/models.h @@ -1187,9 +1187,10 @@ struct llama_model_deepseek4 : public llama_model_base { float kq_scale, int il) const; - lm_ggml_tensor * build_hc_weighted_sum( + lm_ggml_tensor * build_hc_pre( lm_ggml_tensor * x, - lm_ggml_tensor * weights) const; + lm_ggml_tensor * weights, + int il) const; lm_ggml_tensor * build_hc_sinkhorn( lm_ggml_tensor * comb, @@ -1729,6 +1730,22 @@ struct llama_model_hunyuan_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +struct llama_model_hy_v3 : public llama_model_base { + llama_model_hy_v3(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + struct graph_mtp : public llm_graph_context { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_hunyuan_vl : public llama_model_base { llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {} diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 0e2fa390c..d89f60d8a 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -8,7 +8,7 @@ PODS: - hermes-engine (0.82.0): - hermes-engine/Pre-built (= 0.82.0) - hermes-engine/Pre-built (0.82.0) - - llama-rn (0.12.5): + - llama-rn (0.12.6): - boost - DoubleConversion - fast_float @@ -3026,7 +3026,7 @@ SPEC CHECKSUMS: fmt: bf3b0f2427f5c78a3d39ac34a7dbe72faabf986d glog: 5683914934d5b6e4240e497e0f4a3b42d1854183 hermes-engine: 8642d8f14a548ab718ec112e9bebdfdd154138b5 - llama-rn: 37c29804f1196f685f0554c0a257e3f8e144e43a + llama-rn: 961d072edb06c4588fbdc0d6b5541264de7268be RCT-Folly: 846fda9475e61ec7bcbf8a3fe81edfcaeb090669 RCTDeprecation: 22bf66112da540a7d40e536366ddd8557934fca1 RCTRequired: a0ed4dc41b35f79fbb6d8ba320e06882a8c792cf @@ -3097,7 +3097,7 @@ SPEC CHECKSUMS: ReactAppDependencyProvider: c5c4f5280e4ae0f9f4a739c64c4260fe0b3edaf1 ReactCodegen: 374f1c9242fbdd673b460d358b33860c0cc9d926 ReactCommon: 25c7f94aee74ddd93a8287756a8ac0830a309544 - RNAudioAPI: 8f309254a527a858541a692c2ef2db606ad44c14 + RNAudioAPI: 8a9d346fac228321993ac9016b6a6c70fa9cf9fb RNCAsyncStorage: 29f0230e1a25f36c20b05f65e2eb8958d6526e82 RNCClipboard: f538e2ba34c187a6597c2f17c4faa4e1cafae97c RNGestureHandler: f1dd7f92a0faa2868a919ab53bb9d66eb4ebfcf5 @@ -3107,4 +3107,4 @@ SPEC CHECKSUMS: PODFILE CHECKSUM: ca77cacfb81e1a9ba47dd6bc26e1198af40f4f0c -COCOAPODS: 1.15.2 +COCOAPODS: 1.17.0 diff --git a/src/version.ts b/src/version.ts index eefe0c256..1f3e371ae 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,2 +1,2 @@ -export const BUILD_NUMBER = '9982' -export const BUILD_COMMIT = '99f3dc3' +export const BUILD_NUMBER = '10054' +export const BUILD_COMMIT = 'ac2557c'