Skip to content

Commit 6b10873

Browse files
Merge pull request #514 from janhq/update-dev-from-master-2026-05-13-01-07
Sync master with upstream release b9127
2 parents 5192020 + a9883db commit 6b10873

54 files changed

Lines changed: 10465 additions & 6985 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/build.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,8 @@ jobs:
456456
run: |
457457
cd build
458458
# This is using llvmpipe and runs slower than other backends
459-
ctest -L main --verbose --timeout 900
459+
# test-backend-ops is too slow on llvmpipe, skip it
460+
ctest -L main -E test-backend-ops --verbose --timeout 900
460461
461462
ubuntu-24-webgpu-wasm:
462463
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}

.github/workflows/python-type-check.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
uses: actions/setup-python@v6
3232
with:
3333
python-version: "3.11"
34-
pip-install: -r requirements/requirements-all.txt ty==0.0.33
34+
pip-install: -r requirements/requirements-all.txt ty==0.0.35
3535
# - name: Type-check with Pyright
3636
# uses: jakebailey/pyright-action@v2
3737
# with:

common/arg.cpp

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,25 @@ static bool parse_bool_value(const std::string & value) {
435435
// CLI argument parsing functions
436436
//
437437

438+
void common_params_handle_models(common_params & params, llama_example curr_ex) {
439+
auto res = common_params_handle_model(params.model, params.hf_token, params.offline);
440+
if (params.no_mmproj) {
441+
params.mmproj = {};
442+
} else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) {
443+
// optionally, handle mmproj model when -hf is specified
444+
params.mmproj = res.mmproj;
445+
}
446+
// only download mmproj if the current example is using it
447+
for (const auto & ex : mmproj_examples) {
448+
if (curr_ex == ex) {
449+
common_params_handle_model(params.mmproj, params.hf_token, params.offline);
450+
break;
451+
}
452+
}
453+
common_params_handle_model(params.speculative.draft.mparams, params.hf_token, params.offline);
454+
common_params_handle_model(params.vocoder.model, params.hf_token, params.offline);
455+
}
456+
438457
static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
439458
common_params & params = ctx_arg.params;
440459

@@ -588,22 +607,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
588607

589608
// handle model and download
590609
if (!skip_model_download) {
591-
auto res = common_params_handle_model(params.model, params.hf_token, params.offline);
592-
if (params.no_mmproj) {
593-
params.mmproj = {};
594-
} else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) {
595-
// optionally, handle mmproj model when -hf is specified
596-
params.mmproj = res.mmproj;
597-
}
598-
// only download mmproj if the current example is using it
599-
for (const auto & ex : mmproj_examples) {
600-
if (ctx_arg.ex == ex) {
601-
common_params_handle_model(params.mmproj, params.hf_token, params.offline);
602-
break;
603-
}
604-
}
605-
common_params_handle_model(params.speculative.draft.mparams, params.hf_token, params.offline);
606-
common_params_handle_model(params.vocoder.model, params.hf_token, params.offline);
610+
common_params_handle_models(params, ctx_arg.ex);
607611
}
608612

609613
// model is required (except for server)

common/arg.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,5 +129,8 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
129129
// see: https://github.com/ggml-org/llama.cpp/issues/18163
130130
void common_params_add_preset_options(std::vector<common_arg> & args);
131131

132+
// Populate model paths (main model, mmproj, etc) from -hf if necessary
133+
void common_params_handle_models(common_params & params, llama_example curr_ex);
134+
132135
// initialize argument parser context - used by test-arg-parser and preset
133136
common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);

common/preset.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,8 +163,13 @@ void common_preset::merge(const common_preset & other) {
163163
}
164164
}
165165

166-
void common_preset::apply_to_params(common_params & params) const {
166+
void common_preset::apply_to_params(common_params & params, const std::set<std::string> & handled_keys) const {
167167
for (const auto & [opt, val] : options) {
168+
if (!handled_keys.empty()) {
169+
if (!opt.env || handled_keys.find(opt.env) == handled_keys.end()) {
170+
continue;
171+
}
172+
}
168173
// apply each option to params
169174
if (opt.handler_string) {
170175
opt.handler_string(params, val);

common/preset.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ struct common_preset {
4343
void merge(const common_preset & other);
4444

4545
// apply preset options to common_params
46-
void apply_to_params(common_params & params) const;
46+
// optionally specify handled_keys to only apply a subset of options (identified by their env), if empty, apply all options
47+
void apply_to_params(common_params & params, const std::set<std::string> & handled_keys = std::set<std::string>()) const;
4748
};
4849

4950
// interface for multiple presets in one file

convert_hf_to_gguf.py

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2865,8 +2865,12 @@ def __init__(self, *args, **kwargs):
28652865
# fix for SmolVLM2, missing `num_attention_heads` in config.json
28662866
if self.hf_arch == "VLlama3ForCausalLM":
28672867
self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)
2868-
hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
2869-
self.origin_hf_arch = hparams.get('architectures', [None])[0]
2868+
# Mistral consolidated format has no config.json; origin_hf_arch is HF-only.
2869+
if self.is_mistral_format:
2870+
self.origin_hf_arch = None
2871+
else:
2872+
hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
2873+
self.origin_hf_arch = hparams.get('architectures', [None])[0]
28702874

28712875
def set_vocab(self):
28722876
if self.origin_hf_arch == "GlmasrModel":
@@ -9760,6 +9764,73 @@ def prepare_tensors(self):
97609764
raise ValueError(f"Unprocessed experts: {experts}")
97619765

97629766

9767+
@ModelBase.register("MiMoV2ForCausalLM")
9768+
class MiMoV2VisionModel(MmprojModel):
9769+
def __init__(self, *args, **kwargs):
9770+
super().__init__(*args, **kwargs)
9771+
assert self.hparams_vision is not None
9772+
hp = self.hparams_vision
9773+
9774+
hp["image_size"] = hp.get("image_size", 560)
9775+
hp["num_attention_heads"] = hp.get("num_heads", 32)
9776+
hp["num_hidden_layers"] = hp.get("depth", 28)
9777+
9778+
self.n_q_heads = int(hp["num_heads"])
9779+
self.num_kv_heads = int(hp.get("num_key_value_heads", 8))
9780+
self.head_dim = int(hp.get("qk_channels", 64))
9781+
self.spatial_merge_size = int(hp["spatial_merge_size"])
9782+
# MiMoV2 vision RMSNorm: HF uses getattr(config, "rms_norm_eps", 1e-6) and the
9783+
# field is absent from MiMo-V2.5's vision_config
9784+
self.rms_norm_eps = float(hp.get("rms_norm_eps", 1e-6))
9785+
9786+
# fullatt_block_indexes are also reflected in vit_window_attn_types as -1
9787+
self.fullatt_block_indexes = list(hp.get("fullatt_block_indexes") or [])
9788+
self.vit_window_attn_types = list(hp.get("vit_window_attn_types") or [])
9789+
self.visual_token_window_size = int(hp.get("visual_token_window_size", -1))
9790+
self.use_sink = bool(hp.get("use_sink", False))
9791+
9792+
def set_gguf_parameters(self):
9793+
super().set_gguf_parameters()
9794+
9795+
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MIMOVL)
9796+
self.gguf_writer.add_vision_use_silu(True)
9797+
self.gguf_writer.add_vision_head_count_kv(self.num_kv_heads)
9798+
self.gguf_writer.add_vision_spatial_merge_size(self.spatial_merge_size)
9799+
self.gguf_writer.add_uint32(gguf.Keys.ClipVision.WINDOW_SIZE, self.visual_token_window_size)
9800+
self.gguf_writer.add_vision_wa_pattern_mode(self.vit_window_attn_types)
9801+
self.gguf_writer.add_vision_attention_layernorm_eps(self.rms_norm_eps)
9802+
self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))
9803+
self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))
9804+
9805+
def tensor_force_quant(self, name, new_name, bid, n_dims):
9806+
# Sinks must be F32: any sink-style softmax/mask add in ggml requires
9807+
# F32, and we fold sinks into a host-built F32 mask at encode time.
9808+
if new_name.endswith(".attn_sinks"):
9809+
return gguf.GGMLQuantizationType.F32
9810+
return super().tensor_force_quant(name, new_name, bid, n_dims)
9811+
9812+
@classmethod
9813+
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
9814+
name, _ = item
9815+
if not name.startswith("visual."):
9816+
return None
9817+
return super().filter_tensors(item)
9818+
9819+
def modify_tensors(self, data_torch, name, bid):
9820+
# Conv3D patch embed: split along the temporal axis (kt=2) into two Conv2D
9821+
# weights that the existing qwen2vl-style two-Conv2D path consumes.
9822+
if name == "visual.patch_embed.proj.weight":
9823+
_, _, kt, _, _ = data_torch.shape
9824+
if kt != 2:
9825+
raise ValueError(f"unexpected temporal_patch_size: {kt}")
9826+
embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]
9827+
yield (embd_name + ".weight", data_torch[:, :, 0, ...])
9828+
yield (embd_name + ".weight.1", data_torch[:, :, 1, ...])
9829+
return
9830+
9831+
yield from super().modify_tensors(data_torch, name, bid)
9832+
9833+
97639834
@ModelBase.register("Step3p5ForCausalLM")
97649835
class Step35Model(TextModel):
97659836
model_arch = gguf.MODEL_ARCH.STEP35
@@ -13342,16 +13413,20 @@ def set_gguf_parameters(self):
1334213413
self.gguf_writer.add_vision_use_silu(True)
1334313414

1334413415
# spatial_merge_size
13345-
if self.find_vparam(["mm_projector_id"]) == "patch_merge":
13416+
if self.find_vparam(["mm_projector_id"], optional=True) == "patch_merge":
1334613417
self.gguf_writer.add_vision_spatial_merge_size(
1334713418
self.find_vparam(["spatial_merge_size"])
1334813419
)
1334913420

1335013421
def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
1335113422
if name == "vision_language_adapter.w_in.weight":
1335213423
return "mm.1.weight"
13424+
elif name == "vision_language_adapter.w_in.bias":
13425+
return "mm.1.bias"
1335313426
elif name == "vision_language_adapter.w_out.weight":
1335413427
return "mm.2.weight"
13428+
elif name == "vision_language_adapter.w_out.bias":
13429+
return "mm.2.bias"
1335513430
return super().map_tensor_name(name, try_suffixes)
1335613431

1335713432

convert_lora_to_gguf.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,24 @@ def transpose(self, dim0: int, dim1: int) -> LoraTorchTensor:
188188
def swapaxes(self, axis0: int, axis1: int) -> LoraTorchTensor:
189189
return self.transpose(axis0, axis1)
190190

191+
def split(self, split_size: int | Sequence[int], dim: int = 0) -> tuple[LoraTorchTensor, ...]:
192+
shape = self.shape
193+
ndim = len(shape)
194+
if dim < 0:
195+
dim += ndim
196+
if dim == ndim - 1:
197+
A_chunks = self._lora_A.split(split_size, dim=-1)
198+
return tuple(LoraTorchTensor(a, self._lora_B) for a in A_chunks)
199+
elif dim == ndim - 2:
200+
B_chunks = self._lora_B.split(split_size, dim=-2)
201+
return tuple(LoraTorchTensor(self._lora_A, b) for b in B_chunks)
202+
else:
203+
B_chunks = self._lora_B.split(split_size, dim=dim)
204+
if self._lora_A.shape[dim] == 1:
205+
return tuple(LoraTorchTensor(self._lora_A, b) for b in B_chunks)
206+
A_chunks = self._lora_A.split(split_size, dim=dim)
207+
return tuple(LoraTorchTensor(a, b) for a, b in zip(A_chunks, B_chunks))
208+
191209
def to(self, *args, **kwargs):
192210
return LoraTorchTensor(self._lora_A.to(*args, **kwargs), self._lora_B.to(*args, **kwargs))
193211

@@ -230,6 +248,11 @@ def __torch_function__(cls, func: Callable, types, args=(), kwargs=None):
230248
)
231249
else:
232250
raise NotImplementedError
251+
elif func is torch.split:
252+
assert len(args) and len(args) >= 2
253+
tensor, split_size = args[0], args[1]
254+
dim = args[2] if len(args) > 2 else kwargs.get("dim", 0)
255+
return tensor.split(split_size, dim=dim)
233256
else:
234257
raise NotImplementedError
235258

docs/ops.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Legend:
1818
| ACC ||||||| 🟡 |||||
1919
| ADD ||||| 🟡 |||||||
2020
| ADD1 ||||||||||||
21-
| ADD_ID ||||||||| |||
21+
| ADD_ID ||||||||| |||
2222
| ARANGE ||||||||||||
2323
| ARGMAX ||||||||||||
2424
| ARGSORT |||||| 🟡 | 🟡 |||||
@@ -71,7 +71,7 @@ Legend:
7171
| MUL_MAT_HADAMARD ||||||||||||
7272
| MUL_MAT_ID || 🟡 ||| 🟡 | 🟡 | 🟡 || 🟡 | 🟡 ||
7373
| NEG |||| 🟡 |||| 🟡 ||||
74-
| NORM |||||||| 🟡 | |||
74+
| NORM |||||||| 🟡 | |||
7575
| OPT_STEP_ADAMW ||||||||||||
7676
| OPT_STEP_SGD ||||||||||||
7777
| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 ||| 🟡 |||| 🟡 |
@@ -118,5 +118,5 @@ Legend:
118118
| TOP_K ||||||| 🟡 | 🟡 ||||
119119
| TRI ||||||||||||
120120
| TRUNC |||| 🟡 ||| 🟡 | 🟡 ||||
121-
| UPSCALE || 🟡 |||| 🟡 ||| |||
121+
| UPSCALE || 🟡 |||| 🟡 ||| |||
122122
| XIELU ||||||||||||

0 commit comments

Comments
 (0)