From 3ba8f10c26915702cf2faad44e6545f002308c6f Mon Sep 17 00:00:00 2001 From: Sanchit Monga Date: Sat, 15 Aug 2026 22:26:30 -0700 Subject: [PATCH 1/2] Add Gemma 4 and Qwen3.6/Qwen3.8 catalog rows Adds text-only GGUF rows for the Gemma 4 family (E2B, E4B, 12B, 26B-A4B MoE, and 31B in both Q4_K_M and Q2_K_XL) and dense/MoE rows for Qwen3.8-27B and Qwen3.6-35B-A3B, plus MLX rows for the two new Qwen releases. Gemma 4 MLX is intentionally not added at any size: mlx-swift-lm 3.31.4 aborts loading every Gemma 4 checkpoint on an asymmetric QK-norm bug already documented in this file. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01DxGfpsMbkNzoQT559b2tXz --- .../Core/Services/ModelCatalogBootstrap.swift | 117 ++++++++++++++++-- 1 file changed, 109 insertions(+), 8 deletions(-) diff --git a/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift b/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift index 024f2f4..4a67805 100644 --- a/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift +++ b/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift @@ -151,6 +151,27 @@ enum ModelCatalogBootstrap { memoryRequirement: 2_800_000_000, supportsThinking: true ) + // Qwen3.8-27B, the newest dense Qwen release (unsloth-published GGUF, + // matching the rest of the Qwen3.x rows in this catalog). + await registerLLM( + id: "qwen3.8-27b-q4_k_m", + name: "Qwen3.8 27B Q4_K_M", + url: "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf", + framework: .llamaCpp, + // 17,106,775,008 B of weights plus KV cache and runtime overhead. + memoryRequirement: 18_800_000_000, + supportsThinking: true + ) + // Qwen3.6-35B-A3B (MoE, 35B total / 3B active, agentic-coding focused). + await registerLLM( + id: "qwen3.6-35b-a3b-q4_k_m", + name: "Qwen3.6 35B-A3B Q4_K_M", + url: "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + framework: .llamaCpp, + // 22,134,528,992 B of weights plus KV cache and runtime overhead. + memoryRequirement: 24_300_000_000, + supportsThinking: true + ) // Exact P0 NVIDIA checkpoint. The pinned llama.cpp fork has native // `nemotron` support; this exact Q4_K_M artifact was load/inference // checked through rcli on macOS before being exposed in the catalog. @@ -261,6 +282,64 @@ enum ModelCatalogBootstrap { framework: .llamaCpp, memoryRequirement: 2_000_000_000 ) + // Gemma 4 family, text-only (unsloth GGUF, no mmproj). Distinct from + // the Gemma 4 E2B/E4B VLM rows above (`ggml-org` repo, decoder+mmproj + // pairs) — those are multimodal registrations; these are plain + // language-only chat models, hence the "-text" suffix on the E4B id to + // avoid colliding with the existing multimodal + // "gemma-4-e4b-it-q4_k_m" id at the same quant level. + await registerLLM( + id: "gemma-4-e2b-it-q4_k_m", + name: "Gemma 4 E2B IT Q4_K_M", + url: "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/main/gemma-4-E2B-it-Q4_K_M.gguf", + framework: .llamaCpp, + // 3,106,738,272 B of weights plus KV cache and runtime overhead. + memoryRequirement: 3_400_000_000 + ) + await registerLLM( + id: "gemma-4-e4b-it-text-q4_k_m", + name: "Gemma 4 E4B IT Q4_K_M", + url: "https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/gemma-4-E4B-it-Q4_K_M.gguf", + framework: .llamaCpp, + // 4,977,171,584 B of weights plus KV cache and runtime overhead. + memoryRequirement: 5_700_000_000 + ) + await registerLLM( + id: "gemma-4-12b-it-q4_k_m", + name: "Gemma 4 12B IT Q4_K_M", + url: "https://huggingface.co/unsloth/gemma-4-12b-it-GGUF/resolve/main/gemma-4-12b-it-Q4_K_M.gguf", + framework: .llamaCpp, + // 7,121,861,440 B of weights plus KV cache and runtime overhead. + memoryRequirement: 8_200_000_000 + ) + await registerLLM( + id: "gemma-4-26b-a4b-it-q4_k_xl", + name: "Gemma 4 26B-A4B IT Q4_K_XL", + url: "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/main/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf", + framework: .llamaCpp, + // 17,010,980,576 B of weights (MoE, 26B total / 4B active) plus KV + // cache and runtime overhead. + memoryRequirement: 18_700_000_000 + ) + // Largest dense Gemma 4. Two quants offered on purpose: Q4_K_M for + // quality, and the smaller UD-Q2_K_XL for devices that cannot fit the + // 4-bit weights. + await registerLLM( + id: "gemma-4-31b-it-q4_k_m", + name: "Gemma 4 31B IT Q4_K_M", + url: "https://huggingface.co/unsloth/gemma-4-31B-it-GGUF/resolve/main/gemma-4-31B-it-Q4_K_M.gguf", + framework: .llamaCpp, + // 18,323,733,440 B of weights plus KV cache and runtime overhead. + memoryRequirement: 20_200_000_000 + ) + await registerLLM( + id: "gemma-4-31b-it-q2_k_xl", + name: "Gemma 4 31B IT Q2_K_XL", + url: "https://huggingface.co/unsloth/gemma-4-31B-it-GGUF/resolve/main/gemma-4-31B-it-UD-Q2_K_XL.gguf", + framework: .llamaCpp, + // 11,774,991,296 B of weights plus KV cache and runtime overhead. + memoryRequirement: 13_500_000_000 + ) logger.info("LLM models registered") #endif @@ -494,14 +573,36 @@ enum ModelCatalogBootstrap { memoryRequirement: 2_400_000_000, supportsThinking: true ) - // NOTE: The MLX Gemma 4 (E2B/E4B) checkpoints are intentionally NOT - // registered. Their attention layers use an asymmetric QK-norm (some - // layers ship `self_attn.q_norm` without a matching `self_attn.k_norm`), - // but mlx-swift-lm 3.31.4's `Gemma4TextAttention` unconditionally loads - // `self_attn.k_norm.weight` and aborts with `keyNotFound` on the first - // such layer — so they download fully and then fail to load. Re-enable - // once mlx-swift-lm makes per-layer k_norm optional. The GGUF (llama.cpp) - // Gemma 4 variants below load fine and remain available. + await registerLLM( + id: "mlx-qwen3.8-27b-4bit", + name: "MLX Qwen3.8 27B 4bit", + url: "https://huggingface.co/mlx-community/Qwen3.8-27B-4bit", + framework: .mlx, + // ~16,054,541,349 B for the whole repo plus KV cache and Metal + // runtime overhead. + memoryRequirement: 17_700_000_000, + supportsThinking: true + ) + await registerLLM( + id: "mlx-qwen3.6-35b-a3b-4bit", + name: "MLX Qwen3.6 35B-A3B 4bit", + url: "https://huggingface.co/mlx-community/Qwen3.6-35B-A3B-4bit", + framework: .mlx, + // ~20,402,204,271 B for the whole repo (MoE, 35B total / 3B + // active) plus KV cache and Metal runtime overhead. + memoryRequirement: 22_400_000_000, + supportsThinking: true + ) + // NOTE: No MLX Gemma 4 checkpoint of ANY size (E2B/E4B/12B/26B-A4B/31B) + // is registered here. Their attention layers use an asymmetric QK-norm + // (some layers ship `self_attn.q_norm` without a matching + // `self_attn.k_norm`), but mlx-swift-lm 3.31.4's `Gemma4TextAttention` + // unconditionally loads `self_attn.k_norm.weight` and aborts with + // `keyNotFound` on the first such layer — so they download fully and + // then fail to load. This is architectural, not per-size, so it blocks + // every Gemma 4 MLX checkpoint until mlx-swift-lm makes per-layer + // k_norm optional. The GGUF (llama.cpp) Gemma 4 variants in the LLM + // section above load fine and remain available at every size. await registerLLM( id: "mlx-qwen2-vl-2b-instruct-4bit", name: "MLX Qwen2-VL 2B Instruct 4bit", From 7aab53f045bf3d073ea41afe324743494699081b Mon Sep 17 00:00:00 2001 From: Sanchit Monga Date: Sat, 15 Aug 2026 22:49:32 -0700 Subject: [PATCH 2/2] Add Muse Glimmer 30B, Granite 4.1, and Nemotron Nano Omni catalog rows Adds Meta Muse Glimmer 30B and NVIDIA Nemotron 3 Nano Omni 30B-A3B Reasoning as VLM rows (GGUF decoder + mmproj), and the IBM Granite 4.1 family (3B/8B/30B, GGUF; 3B/30B also MLX). Live-verified against the pinned mlx-swift-lm 3.31.4 checkout that Muse Glimmer's and Nemotron Omni's declared model_types have no registered VLMModelFactory/LLMModelFactory entry, so their MLX rows are correctly omitted (same class of bug as the existing Gemma 4 MLX note). Also caught and corrected a ~2x undercount in the originally supplied Muse Glimmer MLX repo size against the live HF blobs API. Nemotron-3.5-ASR-Streaming and Supertonic v3 TTS were investigated and NOT added: the vendored sherpa-onnx (1.13.2, per runanywhere-sdks core/VERSIONS at the SDK release this app pins) predates the 1.13.5 needed for the streaming ASR transducer, and while 1.13.2 does add native Supertonic C++ support, the published HF repo ships JSON (unicode_indexer.json, per-speaker voice_styles/*.json) where the vendored loader hard-requires single binary blobs (unicode_indexer.bin, voice.bin) produced by an offline conversion script not present in the repo or this codebase. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01DxGfpsMbkNzoQT559b2tXz --- .../Core/Services/ModelCatalogBootstrap.swift | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift b/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift index 4a67805..d773d89 100644 --- a/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift +++ b/RunAnywhereAI/Core/Services/ModelCatalogBootstrap.swift @@ -340,6 +340,37 @@ enum ModelCatalogBootstrap { // 11,774,991,296 B of weights plus KV cache and runtime overhead. memoryRequirement: 13_500_000_000 ) + // IBM Granite 4.1 family, dense, Apache 2.0 (confirmed via HF + // cardData.license). unsloth GGUF across all three sizes. + await registerLLM( + id: "granite-4.1-3b-q4_k_m", + name: "IBM Granite 4.1 3B Q4_K_M", + url: "https://huggingface.co/unsloth/granite-4.1-3b-GGUF/resolve/main/granite-4.1-3b-Q4_K_M.gguf", + framework: .llamaCpp, + // 2,099,502,400 B of weights plus KV cache and runtime overhead. + memoryRequirement: 2_400_000_000 + ) + await registerLLM( + id: "granite-4.1-8b-q4_k_m", + name: "IBM Granite 4.1 8B Q4_K_M", + url: "https://huggingface.co/unsloth/granite-4.1-8b-GGUF/resolve/main/granite-4.1-8b-Q4_K_M.gguf", + framework: .llamaCpp, + // 5,347,915,136 B of weights plus KV cache and runtime overhead. + // No MLX row for this size: mlx-community only has this 8B in + // bf16/nvfp4/mxfp4/mxfp8 (no clean 4-bit) plus one unofficial + // third-party "-oQ4" repo from a non-reputable quantizer. + memoryRequirement: 6_150_000_000 + ) + // Largest Granite 4.1, desktop-scale. Dense (verified: GGUF metadata + // reports plain `granite` architecture, no expert-routing fields). + await registerLLM( + id: "granite-4.1-30b-q4_k_m", + name: "IBM Granite 4.1 30B Q4_K_M", + url: "https://huggingface.co/unsloth/granite-4.1-30b-GGUF/resolve/main/granite-4.1-30b-Q4_K_M.gguf", + framework: .llamaCpp, + // 17,490,241,472 B of weights plus KV cache and runtime overhead. + memoryRequirement: 19_200_000_000 + ) logger.info("LLM models registered") #endif @@ -593,6 +624,30 @@ enum ModelCatalogBootstrap { memoryRequirement: 22_400_000_000, supportsThinking: true ) + // IBM Granite 4.1, plain dense transformer. Verified against the + // pinned mlx-swift-lm 3.31.4 checkout: both repos' config.json declare + // model_type "granite" (GraniteForCausalLM), which IS a registered + // LLMModelFactory entry (`"granite": create(GraniteConfiguration.self, + // GraniteModel.init)`), so this loads through the ordinary MLX LLM + // path. No MLX row for the 8B size — see the GGUF row's comment. + await registerLLM( + id: "mlx-granite-4.1-3b-4bit", + name: "MLX IBM Granite 4.1 3B 4bit", + url: "https://huggingface.co/mlx-community/granite-4.1-3b-4bit", + framework: .mlx, + // 2,134,391,329 B for the whole repo plus KV cache and Metal + // runtime overhead. + memoryRequirement: 2_350_000_000 + ) + await registerLLM( + id: "mlx-granite-4.1-30b-4bit", + name: "MLX IBM Granite 4.1 30B 4bit", + url: "https://huggingface.co/mlx-community/granite-4.1-30b-4bit", + framework: .mlx, + // 18,041,244,771 B for the whole repo plus KV cache and Metal + // runtime overhead. + memoryRequirement: 19_800_000_000 + ) // NOTE: No MLX Gemma 4 checkpoint of ANY size (E2B/E4B/12B/26B-A4B/31B) // is registered here. Their attention layers use an asymmetric QK-norm // (some layers ship `self_attn.q_norm` without a matching @@ -603,6 +658,31 @@ enum ModelCatalogBootstrap { // every Gemma 4 MLX checkpoint until mlx-swift-lm makes per-layer // k_norm optional. The GGUF (llama.cpp) Gemma 4 variants in the LLM // section above load fine and remain available at every size. + // + // NOTE: No MLX row for `mlx-community/Muse-Glimmer-30B-4bit` either. + // Verified against the pinned mlx-swift-lm 3.31.4 checkout: this + // repo's config.json declares `model_type: "muse_glimmer"`, which is + // NOT among VLMModelFactory's registered types (paligemma, qwen2_vl, + // qwen2_5_vl, qwen3_vl, qwen3_5[_moe], idefics3, gemma3, gemma4[_unified], + // smolvlm, fastvlm, llava_qwen2, pixtral, mistral3, lfm2_vl, glm_ocr — + // no "muse_glimmer" anywhere) — it would download in full and then + // fail to load. Separately, this exact repo's advertised size is + // wrong: the coordinator-supplied ~19.4 GB estimate does not match + // what the HF blobs API reports for this repo (4 safetensors shards + // totaling 39,443,521,145 B, ~39.4 GB — roughly 2x), most likely + // because the vision tower and/or embeddings are kept at higher + // precision even under a nominal "4bit" LLM quant. The GGUF + // (llama.cpp) row below is unaffected and is the only Muse Glimmer + // row in this catalog. + // + // NOTE: No MLX row for + // `mlx-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-4bit` either, + // for the same reason: its config.json declares + // `model_type: "NemotronH_Nano_Omni_Reasoning_V3"`. VLMModelFactory + // has zero Nemotron entries of any kind, and LLMModelFactory's only + // Nemotron entry, `"nemotron_h"`, is a different, non-matching string + // (and is text-only — it has no vision path regardless). The GGUF + // (llama.cpp) row below is the only way this model is exposed here. await registerLLM( id: "mlx-qwen2-vl-2b-instruct-4bit", name: "MLX Qwen2-VL 2B Instruct 4bit", @@ -798,6 +878,55 @@ enum ModelCatalogBootstrap { memoryRequirement: 3_300_000_000, cuaProfile: RunAnywhere.CUA.faraProfile ) + // Meta Muse Glimmer 30B (Meta Superintelligence Labs, Apache 2.0, + // released 2026-08-10). Genuinely vision-capable — unsloth ships a + // real mmproj — so it registers as a VLM row like the Gemma 4 / + // Qwen2.5-VL rows above. UD-Q4_K_XL is unsloth's own top-tier dynamic + // 4-bit quant for this model; the card has no plain Q4_K_M and no + // separate "recommended" override. + await registerMultiFile( + id: "muse-glimmer-30b-q4_k_xl", + name: "Meta Muse Glimmer 30B Q4_K_XL", + files: [ + ("https://huggingface.co/unsloth/Muse-Glimmer-30B-GGUF/resolve/main/Muse-Glimmer-30B-UD-Q4_K_XL.gguf", + "Muse-Glimmer-30B-UD-Q4_K_XL.gguf"), + ("https://huggingface.co/unsloth/Muse-Glimmer-30B-GGUF/resolve/main/mmproj-Muse-Glimmer-30B-Q8_0.gguf", + "mmproj-Muse-Glimmer-30B-Q8_0.gguf") + ], + framework: .llamaCpp, + modality: .multimodal, + // 15,878,222,368 (decoder) + 2,051,685,088 (mmproj) B of weights + // plus KV cache, vision activations, and runtime overhead. + // Desktop-scale (Mac/Windows), like the largest Gemma 4/Granite + // 4.1 rows above; no phone-tier gating exists in this file today, + // so it is registered like every other heavy row and left to the + // existing HardwareTier/recommendation layer at runtime. + memoryRequirement: 20_600_000_000 + ) + // NVIDIA Nemotron 3 Nano Omni 30B-A3B Reasoning (MoE, 31B total / 3B + // active). This has a real mmproj (image projector), so it registers + // as a VLM row here — but ONLY image+text works through this app's + // mmproj/llama.cpp path. The model's "Omni" name markets audio/video + // understanding too; that is NOT exposed by this registration or by + // any code path in this app, so do not describe this row as full + // omni capability anywhere it is surfaced. + await registerMultiFile( + id: "nemotron-3-nano-omni-30b-a3b-reasoning-q4_k_m", + name: "NVIDIA Nemotron 3 Nano Omni 30B-A3B Reasoning Q4_K_M (Image+Text)", + files: [ + ("https://huggingface.co/unsloth/NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF/resolve/main/NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-UD-Q4_K_M.gguf", + "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-UD-Q4_K_M.gguf"), + ("https://huggingface.co/unsloth/NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF/resolve/main/mmproj-F16.gguf", + "mmproj-F16.gguf") + ], + framework: .llamaCpp, + modality: .multimodal, + // 23,887,023,552 (decoder) + 1,587,540,224 (mmproj) B of weights + // plus KV cache, vision activations, and runtime overhead. + // Desktop-scale, same convention as Muse Glimmer above. + memoryRequirement: 28_000_000_000, + supportsThinking: true + ) logger.info("VLM models registered") #endif