[IN REVIEW] refactor(multimodal): use standalone llm-multimodal crate

.github/CODEOWNERS

@@ -33,7 +33,6 @@
 /crates/kv_index @slin1237
 /crates/mcp @key4ng @CatherineSue @slin1237 @zhoug9127
 /crates/mesh @tonyluj @llfl @slin1237
-/crates/multimodal @slin1237 @CatherineSue
 /crates/protocols @CatherineSue @key4ng
 /crates/reasoning_parser @CatherineSue
 /crates/tokenizer @slin1237 @CatherineSue

.github/labeler.yml

@@ -66,10 +66,6 @@ data-connector:
   - changed-files:
       - any-glob-to-any-file: 'crates/data_connector/**/*'
 
-multimodal:
-  - changed-files:
-      - any-glob-to-any-file: 'crates/multimodal/**/*'
-
 protocols:
   - changed-files:
       - any-glob-to-any-file: 'crates/protocols/**/*'

.github/workflows/nightly-benchmark.yml

@@ -48,7 +48,7 @@ jobs:
           path: |
             bindings/python/dist/*.whl
             clients/python/smg_client/types/_generated.py
-          key: nightly-wheel-${{ runner.os }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'crates/auth/src/**', 'bindings/python/src/**', 'clients/openapi-gen/src/**', 'crates/data_connector/src/**', 'crates/grpc_client/src/**', 'crates/kv_index/src/**', 'crates/mcp/src/**', 'crates/mesh/src/**', 'model_gateway/src/**', 'crates/multimodal/src/**', 'crates/protocols/src/**', 'crates/reasoning_parser/src/**', 'crates/tokenizer/src/**', 'crates/tool_parser/src/**', 'crates/wasm/src/**', 'crates/workflow/src/**') }}
+          key: nightly-wheel-${{ runner.os }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'crates/auth/src/**', 'bindings/python/src/**', 'clients/openapi-gen/src/**', 'crates/data_connector/src/**', 'crates/grpc_client/src/**', 'crates/kv_index/src/**', 'crates/mcp/src/**', 'crates/mesh/src/**', 'model_gateway/src/**', 'crates/protocols/src/**', 'crates/reasoning_parser/src/**', 'crates/tokenizer/src/**', 'crates/tool_parser/src/**', 'crates/wasm/src/**', 'crates/workflow/src/**') }}
           restore-keys: |
             nightly-wheel-${{ runner.os }}-
 

.github/workflows/pr-test-rust.yml

@@ -272,7 +272,7 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/setup-rust
 
-      - name: Verify default multimodal build does not require OpenCV
+      - name: Verify default multimodal dependency build does not require OpenCV
         run: |
           source "$HOME/.cargo/env"
           cargo check -p llm-multimodal
@@ -300,12 +300,6 @@ jobs:
           rustup toolchain install nightly --profile minimal
           cargo +nightly fmt -- --check
 
-      - name: Generate vision golden fixtures
-        run: |
-          python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
-          python -m pip install transformers pillow numpy scipy
-          python crates/multimodal/scripts/generate_vision_golden.py
-
       - name: Run Rust tests
         timeout-minutes: 30
         run: |
@@ -413,6 +407,7 @@ jobs:
         with:
           filters: |
             common:
+              - 'Cargo.toml'
               - 'model_gateway/**'
               - 'crates/protocols/**'
               - 'bindings/**'
@@ -434,7 +429,6 @@ jobs:
               - 'crates/tool_parser/**'
             chat-completions:
               - 'crates/reasoning_parser/**'
-              - 'crates/multimodal/**'
               - 'crates/grpc_client/**'
               - 'grpc_servicer/**'
               - 'e2e_test/chat_completions/**'

.github/workflows/release-crates.yml

@@ -59,8 +59,6 @@ jobs:
             path: crates/mcp
           - crate: smg-grpc-client
             path: crates/grpc_client
-          - crate: llm-multimodal
-            path: crates/multimodal
           - crate: smg-mesh
             path: crates/mesh
     steps:

.pre-commit-config.yaml

@@ -32,7 +32,6 @@ repos:
             src/proto/.*\.proto$|
             src/mesh/proto/.*\.proto$|
             src/tokenizer/chat_template\.rs|
-            crates/multimodal/tests/fixtures/.*|
             target/.*
           )$
 

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["model_gateway", "crates/protocols", "crates/reasoning_parser", "crates/tool_parser", "crates/workflow", "crates/tokenizer", "crates/auth", "crates/mcp", "crates/kv_index", "crates/data_connector", "crates/multimodal", "crates/mm_rdma", "crates/wasm", "crates/mesh", "crates/grpc_client", "bindings/python", "bindings/golang", "clients/rust", "clients/openapi-gen", "crates/mock_worker"]
+members = ["model_gateway", "crates/protocols", "crates/reasoning_parser", "crates/tool_parser", "crates/workflow", "crates/tokenizer", "crates/auth", "crates/mcp", "crates/kv_index", "crates/data_connector", "crates/mm_rdma", "crates/wasm", "crates/mesh", "crates/grpc_client", "bindings/python", "bindings/golang", "clients/rust", "clients/openapi-gen", "crates/mock_worker"]
 resolver = "2"
 
 [workspace.dependencies]
@@ -14,7 +14,7 @@ smg-auth = { version = "1.2.2", path = "crates/auth" }
 smg-mcp = { version = "2.3.2", path = "crates/mcp" }
 kv-index = { version = "1.3.2", path = "crates/kv_index" }
 smg-data-connector = { version = "2.3.2", path = "crates/data_connector", package = "data-connector" }
-llm-multimodal = { version = "1.7.1", path = "crates/multimodal" }
+llm-multimodal = { version = "1.7.1", git = "https://github.com/smg-project/llm-multimodal.git", rev = "7df38e53f99aefaebe86934f010aa8084ec99b2f" }
 smg-wasm = { version = "1.1.3", path = "crates/wasm", package = "smg-wasm" }
 smg-mesh = { version = "1.4.2", path = "crates/mesh", package = "smg-mesh" }
 smg-grpc-client = { version = "1.8.0", path = "crates/grpc_client" }

crates/grpc_client/proto/tokenspeed_scheduler.proto

@@ -10,7 +10,7 @@ import "common.proto";
 // from the cross-engine admin messages in smg.grpc.common (flush/profile).
 // Trimmed to text+multimodal generation (no embed, no LoRA, no hidden-state
 // forwarding). Multimodal carries preprocessed tensors only — media fetch +
-// per-model preprocess happen in the gateway (see crates/multimodal). EPD: a
+// per-model preprocess happen in the gateway (see the llm-multimodal crate). EPD: a
 // prefill request may carry `EncodeBootstrapInfo` so its multimodal embeddings
 // arrive from an encode worker over Mooncake (see tokenspeed_encoder.proto).
 service TokenSpeedScheduler {
@@ -205,7 +205,7 @@ message MultimodalItem {
 }
 
 // Multimodal inputs for vision/audio models. Tensors are produced by the
-// gateway's per-model preprocessor (crates/multimodal); the servicer
+// gateway's per-model preprocessor (llm-multimodal); the servicer
 // only reconstructs them and hands them to the engine — no preprocess.
 message MultimodalInputs {
   // Per-modality itemized payload. Each item owns its tensor payload,