Add Gemma 4 vision (gemma4v) support to Flash-MoE fork

src/models/gemma4-iswa.cpp

@@ -276,7 +276,8 @@ ggml_tensor * llm_build_gemma4_iswa::get_per_layer_inputs() {
 
         // Extract and dequantize padding token embedding (row 0)
         ggml_tensor * padding = ggml_view_1d(ctx0, model.tok_embd_per_layer, embd_size, 0);
-        inp_per_layer = ggml_cast(ctx0, padding, GGML_TYPE_F32);
+        inp_per_layer = ggml_cast (ctx0, padding, GGML_TYPE_F32);
+        inp_per_layer = ggml_scale(ctx0, inp_per_layer, sqrtf((float) n_embd_per_layer));
 
         // Reshape to [n_embd_per_layer, n_layer, 1]
         inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, 1);

tools/cli/cli.cpp

@@ -952,6 +952,48 @@ int main(int argc, char ** argv) {
                 cur_msg += marker;
             }
             buffer = params.prompt;
+            // Scan for inline <filename.ext> image patterns in -p text and replace with media markers
+            if (inf.has_inp_image) {
+                static const std::vector<std::string> img_exts = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp"};
+                std::string result;
+                size_t pos = 0;
+                while (pos < buffer.size()) {
+                    size_t open = buffer.find('<', pos);
+                    if (open == std::string::npos) {
+                        result += buffer.substr(pos);
+                        break;
+                    }
+                    result += buffer.substr(pos, open - pos);
+                    size_t close = buffer.find('>', open);
+                    if (close == std::string::npos) {
+                        result += buffer.substr(open);
+                        break;
+                    }
+                    std::string candidate = buffer.substr(open + 1, close - open - 1);
+                    bool is_image = false;
+                    for (const auto & ext : img_exts) {
+                        if (candidate.size() > ext.size() &&
+                                candidate.substr(candidate.size() - ext.size()) == ext) {
+                            is_image = true;
+                            break;
+                        }
+                    }
+                    if (is_image) {
+                        std::string marker = ctx_cli.load_input_file(candidate, true);
+                        if (!marker.empty()) {
+                            console::log("Loaded inline media from '%s'\n", candidate.c_str());
+                            result += marker;
+                        } else {
+                            console::error("inline image not found: '%s'\n", candidate.c_str());
+                            result += buffer.substr(open, close - open + 1);
+                        }
+                    } else {
+                        result += buffer.substr(open, close - open + 1);
+                    }
+                    pos = close + 1;
+                }
+                buffer = result;
+            }
             if (buffer.size() > 500) {
                 console::log("\n> %s ... (truncated)\n", buffer.substr(0, 500).c_str());
             } else {

tools/mtmd/CMakeLists.txt

@@ -5,6 +5,7 @@ find_package(Threads REQUIRED)
 add_library(mtmd
             mtmd.cpp
             mtmd-audio.cpp
+            mtmd-image.cpp
             mtmd.h
             mtmd-helper.cpp
             mtmd-helper.h
@@ -16,7 +17,16 @@ add_library(mtmd
             models/models.h
             models/cogvlm.cpp
             models/conformer.cpp
+            models/dotsocr.cpp
+            models/exaone4_5.cpp
+            models/gemma4a.cpp
+            models/gemma4v.cpp
+            models/gemma4ua.cpp
+            models/gemma4uv.cpp
             models/glm4v.cpp
+            models/granite-speech.cpp
+            models/granite4-vision.cpp
+            models/hunyuanvl.cpp
             models/internvl.cpp
             models/kimivl.cpp
             models/kimik25.cpp
@@ -28,10 +38,16 @@ add_library(mtmd
             models/pixtral.cpp
             models/qwen2vl.cpp
             models/qwen3vl.cpp
+            models/mimovl.cpp
+            models/qwen3a.cpp
+            models/step3vl.cpp
             models/siglip.cpp
             models/whisper-enc.cpp
+            models/deepseekocr.cpp
+            models/deepseekocr2.cpp
             models/mobilenetv5.cpp
             models/youtuvl.cpp
+            models/yasa2.cpp
             )
 
 set_target_properties(mtmd PROPERTIES
@@ -73,17 +89,22 @@ if (NOT MSVC)
     target_compile_options(mtmd PRIVATE -Wno-cast-qual)
 endif()
 
+if (ANDROID)
+    # miniaudio.h defines ma_android_sdk_version() without a prior prototype
+    target_compile_options(mtmd PRIVATE -Wno-missing-prototypes)
+endif()
+
 if (TARGET BUILD_INFO)
     add_dependencies(mtmd        BUILD_INFO)
     add_dependencies(mtmd-helper BUILD_INFO)
 endif()
 
-# if mtmd is linked against common, we throw an error
+# if mtmd is linked against llama-common, we throw an error
 if (TARGET mtmd)
     get_target_property(libs mtmd LINK_LIBRARIES)
-    if (libs AND "common" IN_LIST libs)
+    if (libs AND "llama-common" IN_LIST libs)
         message(FATAL_ERROR "mtmd is designed to be a public library.\n"
-                            "It must not link against common")
+                            "It must not link against llama-common")
     endif()
 endif()
 
@@ -98,11 +119,11 @@ set_target_properties  (${TARGET} PROPERTIES OUTPUT_NAME llama-mtmd-cli)
 if(LLAMA_TOOLS_INSTALL)
     install(TARGETS ${TARGET} RUNTIME)
 endif()
-target_link_libraries  (${TARGET} PRIVATE common mtmd Threads::Threads)
+target_link_libraries  (${TARGET} PRIVATE llama-common mtmd Threads::Threads)
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
 
 # mtmd-debug tool
 add_executable(llama-mtmd-debug debug/mtmd-debug.cpp)
 set_target_properties(llama-mtmd-debug PROPERTIES OUTPUT_NAME llama-mtmd-debug)
-target_link_libraries(llama-mtmd-debug PRIVATE common mtmd Threads::Threads)
+target_link_libraries(llama-mtmd-debug PRIVATE llama-common mtmd Threads::Threads)
 target_compile_features(llama-mtmd-debug PRIVATE cxx_std_17)

tools/mtmd/README.md

@@ -49,6 +49,7 @@ For the following models, you can use `convert_hf_to_gguf.py` with `--mmproj` fl
 - Qwen 2 VL and Qwen 2.5 VL (from [Qwen](https://huggingface.co/Qwen))
 - [Mistral Small 3.1 24B](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503)
 - InternVL 2.5 and InternVL 3 from [OpenGVLab](https://huggingface.co/OpenGVLab) (note: we don't support conversion of `InternVL3-*-hf` model, only non-HF version is supported ; `InternLM2Model` **text** model is not supported)
+- [MiniCPM-V 4.6](https://huggingface.co/openbmb/MiniCPM-V-4_6) ; See the guide [here](../../docs/multimodal/minicpmv4.6.md) - requires the standard `transformers` v5.7.0+ checkpoint
 
 For older models, please refer to the relevant guide for instructions on how to obtain or create them:
 
@@ -60,4 +61,7 @@ NOTE: conversion scripts are located under `tools/mtmd/legacy-models`
 - [MiniCPM-V 2.5](../../docs/multimodal/minicpmv2.5.md)
 - [MiniCPM-V 2.6](../../docs/multimodal/minicpmv2.6.md)
 - [MiniCPM-o 2.6](../../docs/multimodal/minicpmo2.6.md)
+- [MiniCPM-V 4.0](../../docs/multimodal/minicpmv4.0.md)
+- [MiniCPM-o 4.0](../../docs/multimodal/minicpmo4.0.md)
+- [MiniCPM-V 4.5](../../docs/multimodal/minicpmv4.5.md)
 - [IBM Granite Vision](../../docs/multimodal/granitevision.md)

tools/mtmd/clip-graph.h

@@ -11,6 +11,10 @@
 
 #define DEFAULT_INTERPOLATION_MODE (GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS)
 
+struct build_vit_opts {
+    ggml_tensor * attn_mask = nullptr;
+};
+
 struct clip_graph {
     const clip_model & model;
     const clip_hparams & hparams;
@@ -25,13 +29,17 @@ struct clip_graph {
     const int n_patches;
     const int n_embd;
     const int n_head;
+    const int n_head_kv;
     const int d_head;
     const int n_layer;
     const int n_mmproj_embd;
     const float eps;
-    const float kq_scale;
+    float kq_scale; // TODO: maybe move this to hparams
     const clip_flash_attn_type flash_attn_type;
 
+    // TODO [QWEN_VIDEO]: improve this in the future
+    int n_batch = 1;
+
     ggml_context_ptr ctx0_ptr;
     ggml_context * ctx0;
     ggml_cgraph * gf;
@@ -63,7 +71,8 @@ struct clip_graph {
                 norm_type norm_t,
                 ffn_op_type ffn_t,
                 ggml_tensor * learned_pos_embd,
-                std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos);
+                std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
+                const build_vit_opts & opts = {});
 
     // build the input after conv2d (inp_raw --> patches)
     // returns tensor with shape [n_embd, n_patches]
@@ -98,7 +107,8 @@ struct clip_graph {
             ggml_tensor * v_cur,
             ggml_tensor * kq_mask,
             float kq_scale,
-            int il) const;
+            int il,
+            ggml_tensor * sinks = nullptr) const;
 
     // implementation of the 2D RoPE without adding a new op in ggml
     // this is not efficient (use double the memory), but works on all backends