From b95ccbfddca00f56f523518cb18e3a6d24a1f831 Mon Sep 17 00:00:00 2001 From: Pilkyu Choi Date: Wed, 5 Aug 2026 10:30:58 +0900 Subject: [PATCH] Add support for Kanana-2 Tiny (kanana2_tiny) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kakao's kanana-2-1.3b-{base,instruct} use Kanana2TinyForCausalLM. The architecture is Qwen3 except that the rotary embedding differs by attention type: full-attention layers use a yarn-scaled rope for long context, sliding layers keep an unscaled rope and attend within a 1024-token window. The checkpoint expresses this as nested rope_parameters keyed by attention type plus an explicit layer_types array. Without this, loading the checkpoint as qwen3 (the mapping Kakao ships in sglang/config.json) fails at generation — mlx-lm reads rope_theta from the top level of rope_parameters, which is absent — and would collapse both rope variants into one even if it parsed. The per-layer rope dispatch follows gemma3_text, which solves the same problem; layer types are read from layer_types rather than computed from a modulus since the checkpoint states them explicitly. Sliding layers get a RotatingKVCache. Verified against the upstream fp32 implementation on kanana-2-1.3b-instruct: top-50 logit cosine >= 0.99998 and matching argmax on Korean, English, mixed and a 9,478-token prompt. The long prompt is the one that matters — it is past the 4096-token pre-yarn context, so a single-rope implementation would diverge there while passing the short probes. --- mlx_lm/models/kanana2_tiny.py | 240 ++++++++++++++++++++++++++++++++++ tests/test_models.py | 44 +++++++ 2 files changed, 284 insertions(+) create mode 100644 mlx_lm/models/kanana2_tiny.py diff --git a/mlx_lm/models/kanana2_tiny.py b/mlx_lm/models/kanana2_tiny.py new file mode 100644 index 000000000..7683330c6 --- /dev/null +++ b/mlx_lm/models/kanana2_tiny.py @@ -0,0 +1,240 @@ +# Copyright © 2023-2024 Apple Inc. + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import mlx.core as mx +import mlx.nn as nn + +from .activations import swiglu +from .base import BaseModelArgs, create_attention_mask, scaled_dot_product_attention +from .cache import KVCache, RotatingKVCache +from .rope_utils import initialize_rope + + +@dataclass +class ModelArgs(BaseModelArgs): + model_type: str + hidden_size: int + num_hidden_layers: int + intermediate_size: int + num_attention_heads: int + rms_norm_eps: float + vocab_size: int + num_key_value_heads: int + max_position_embeddings: int + head_dim: int + layer_types: List[str] + sliding_window: int + # Absent from the checkpoint config; the weights carry no lm_head, so the + # embedding is tied. Matches the upstream default. + tie_word_embeddings: bool = True + # Rotary parameters are given per attention type, e.g. + # {"full_attention": {"rope_type": "yarn", "rope_theta": 10000, ...}, + # "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}} + rope_parameters: Dict[str, Dict[str, Union[float, str]]] = None + + +class Attention(nn.Module): + """Qwen3 attention, except the rotary embedding follows the layer's type. + + Full-attention layers use a yarn-scaled rope for long context; sliding + layers keep the unscaled rope and only attend within `sliding_window`. + """ + + def __init__(self, args: ModelArgs, layer_type: str): + super().__init__() + + dim = args.hidden_size + self.n_heads = n_heads = args.num_attention_heads + assert args.num_key_value_heads is not None + self.n_kv_heads = n_kv_heads = args.num_key_value_heads + + head_dim = args.head_dim + self.scale = head_dim**-0.5 + + self.q_proj = nn.Linear(dim, n_heads * head_dim, bias=False) + self.k_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False) + self.v_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False) + self.o_proj = nn.Linear(n_heads * head_dim, dim, bias=False) + + self.q_norm = nn.RMSNorm(head_dim, eps=args.rms_norm_eps) + self.k_norm = nn.RMSNorm(head_dim, eps=args.rms_norm_eps) + + params = dict(args.rope_parameters[layer_type]) + base = params.pop("rope_theta") + rope_type = params.get("rope_type", "default") + if rope_type in ("default", "linear"): + # An unscaled rope: passing a scaling config would be a no-op at + # best and misread at worst. + self.rope = initialize_rope(head_dim, base=base, traditional=False) + else: + self.rope = initialize_rope( + head_dim, + base=base, + traditional=False, + scaling_config=params, + max_position_embeddings=args.max_position_embeddings, + ) + + def __call__( + self, + x: mx.array, + mask: Optional[mx.array] = None, + cache: Optional[Any] = None, + ) -> mx.array: + B, L, D = x.shape + + queries, keys, values = self.q_proj(x), self.k_proj(x), self.v_proj(x) + + queries = self.q_norm(queries.reshape(B, L, self.n_heads, -1)).transpose( + 0, 2, 1, 3 + ) + keys = self.k_norm(keys.reshape(B, L, self.n_kv_heads, -1)).transpose( + 0, 2, 1, 3 + ) + values = values.reshape(B, L, self.n_kv_heads, -1).transpose(0, 2, 1, 3) + + if cache is not None: + queries = self.rope(queries, offset=cache.offset) + keys = self.rope(keys, offset=cache.offset) + keys, values = cache.update_and_fetch(keys, values) + else: + queries = self.rope(queries) + keys = self.rope(keys) + + output = scaled_dot_product_attention( + queries, keys, values, cache=cache, scale=self.scale, mask=mask + ) + output = output.transpose(0, 2, 1, 3).reshape(B, L, -1) + return self.o_proj(output) + + +class MLP(nn.Module): + def __init__(self, dim, hidden_dim): + super().__init__() + self.gate_proj = nn.Linear(dim, hidden_dim, bias=False) + self.down_proj = nn.Linear(hidden_dim, dim, bias=False) + self.up_proj = nn.Linear(dim, hidden_dim, bias=False) + + def __call__(self, x) -> mx.array: + return self.down_proj(swiglu(self.gate_proj(x), self.up_proj(x))) + + +class TransformerBlock(nn.Module): + def __init__(self, args: ModelArgs, layer_type: str): + super().__init__() + self.num_attention_heads = args.num_attention_heads + self.hidden_size = args.hidden_size + self.self_attn = Attention(args, layer_type) + self.mlp = MLP(args.hidden_size, args.intermediate_size) + self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps) + self.post_attention_layernorm = nn.RMSNorm( + args.hidden_size, eps=args.rms_norm_eps + ) + self.args = args + + def __call__( + self, + x: mx.array, + mask: Optional[mx.array] = None, + cache: Optional[Any] = None, + ) -> mx.array: + r = self.self_attn(self.input_layernorm(x), mask, cache) + h = x + r + r = self.mlp(self.post_attention_layernorm(h)) + out = h + r + return out + + +class Kanana2TinyModel(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + self.args = args + self.vocab_size = args.vocab_size + self.num_hidden_layers = args.num_hidden_layers + assert self.vocab_size > 0 + self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size) + self.layers = [ + TransformerBlock(args, layer_type) for layer_type in args.layer_types + ] + self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps) + # Index of a layer of each kind, so the two masks are built against a + # cache entry of the matching kind. + self.full_idx = args.layer_types.index("full_attention") + self.sliding_idx = ( + args.layer_types.index("sliding_attention") + if "sliding_attention" in args.layer_types + else None + ) + + def __call__( + self, + inputs: mx.array, + cache=None, + input_embeddings: Optional[mx.array] = None, + ): + if input_embeddings is not None: + h = input_embeddings + else: + h = self.embed_tokens(inputs) + + if cache is None: + cache = [None] * len(self.layers) + + full_mask = create_attention_mask(h, cache[self.full_idx]) + sliding_mask = ( + create_attention_mask( + h, cache[self.sliding_idx], window_size=self.args.sliding_window + ) + if self.sliding_idx is not None + else None + ) + + for layer_type, layer, c in zip(self.args.layer_types, self.layers, cache): + mask = full_mask if layer_type == "full_attention" else sliding_mask + h = layer(h, mask, c) + + return self.norm(h) + + +class Model(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + self.args = args + self.model_type = args.model_type + self.model = Kanana2TinyModel(args) + if not args.tie_word_embeddings: + self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False) + + def __call__( + self, + inputs: mx.array, + cache=None, + input_embeddings: Optional[mx.array] = None, + ): + out = self.model(inputs, cache, input_embeddings) + if self.args.tie_word_embeddings: + out = self.model.embed_tokens.as_linear(out) + else: + out = self.lm_head(out) + return out + + def sanitize(self, weights): + if self.args.tie_word_embeddings: + weights.pop("lm_head.weight", None) + return weights + + @property + def layers(self): + return self.model.layers + + def make_cache(self): + return [ + ( + KVCache() + if layer_type == "full_attention" + else RotatingKVCache(max_size=self.args.sliding_window) + ) + for layer_type in self.args.layer_types + ] diff --git a/tests/test_models.py b/tests/test_models.py index ff9da908a..4dccd3031 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -627,6 +627,50 @@ def test_qwen3(self): model, args.model_type, args.vocab_size, args.num_hidden_layers ) + def test_kanana2_tiny(self): + from mlx_lm.models import kanana2_tiny + + # Alternating pattern as shipped by the checkpoint: three sliding + # layers then one full-attention layer, each type with its own rope. + layer_types = ["sliding_attention"] * 3 + ["full_attention"] + args = kanana2_tiny.ModelArgs( + model_type="kanana2_tiny", + hidden_size=128, + num_hidden_layers=len(layer_types), + intermediate_size=256, + num_attention_heads=4, + num_key_value_heads=2, + rms_norm_eps=1e-5, + vocab_size=10_000, + head_dim=32, + max_position_embeddings=4096, + layer_types=layer_types, + sliding_window=8, + rope_parameters={ + "full_attention": { + "rope_type": "yarn", + "rope_theta": 10000, + "factor": 40.0, + "original_max_position_embeddings": 512, + }, + "sliding_attention": { + "rope_type": "default", + "rope_theta": 10000.0, + }, + }, + ) + model = kanana2_tiny.Model(args) + self.model_test_runner( + model, args.model_type, args.vocab_size, args.num_hidden_layers + ) + + # The two attention types must not share a rope: the full-attention + # layers are yarn-scaled for long context, the sliding ones are not. + sliding_rope = model.layers[0].self_attn.rope + full_rope = model.layers[3].self_attn.rope + self.assertIsNot(sliding_rope, full_rope) + self.assertNotEqual(type(sliding_rope), type(full_rope)) + def test_qwen3_5_family_convert_then_load_norm_not_shift_twice(self): text_config = { "hidden_size": 8,