qwen2.5-1.5b.yml

# Copyright 2023–2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Qwen 2.5 1.5B Instruct Configuration
# https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct

base_emb_dim: 1536
base_num_query_heads: 12
base_num_kv_heads: 2
base_mlp_dim: 8960
base_num_decoder_layers: 28
head_dim: 128
mlp_activations: ["silu", "linear"]
vocab_size: 151936
decoder_block: "qwen2"
normalization_layer_epsilon: 1e-06
rope_max_timescale: 1000000.0
use_qk_norm: False
# Bias for q, k, v proj.
attention_bias: True
logits_via_embedding: True
normalize_embedding_logits: False
tokenizer_type: "huggingface"