# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/deepseek_v2/modular_deepseek_v2.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_deepseek_v2.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2025 HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from ...configuration_utils import PreTrainedConfig from ...modeling_rope_utils import RopeParameters class DeepseekV2Config(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`DeepseekV2Model`]. It is used to instantiate a DeepSeek model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of DeepSeek-V2-Lite" [deepseek-ai/DeepSeek-V2-Lite"](https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite"). Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: vocab_size (`int`, *optional*, defaults to 32000): Vocabulary size of the DeepSeek model. Defines the number of different tokens that can be represented by the `input_ids` passed when calling [`DeepseekV2Model`]. hidden_size (`int`, *optional*, defaults to 4096): Dimension of the hidden representations. intermediate_size (`int`, *optional*, defaults to 11008): Dimension of the MLP representations. num_hidden_layers (`int`, *optional*, defaults to 32): Number of hidden layers in the Transformer decoder. num_attention_heads (`int`, *optional*, defaults to 32): Number of attention heads for each attention layer in the Transformer decoder. num_key_value_heads (`int`, *optional*): The number of key-value heads used to implement Grouped Query Attention (GQA). If `num_key_value_heads=num_attention_heads`, the model will use Multi-Head Attention (MHA). If `num_key_value_heads=1`, the model will use Multi-Query Attention (MQA). Otherwise, GQA is used. hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): The non-linear activation function (function or string) in the decoder. max_position_embeddings (`int`, *optional*, defaults to 2048): The maximum sequence length that this model might ever be used with. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated normal initializer for initializing all weight matrices. rms_norm_eps (`float`, *optional*, defaults to 1e-06): The epsilon value used by the RMS normalization layers. use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/value attentions (useful for inference optimization). pad_token_id (`int`, *optional*): Padding token ID. bos_token_id (`int`, *optional*, defaults to 1): Beginning-of-sequence token ID. eos_token_id (`int`, *optional*, defaults to 2): End-of-sequence token ID. tie_word_embeddings (`bool`, *optional*, defaults to `False`): Whether to tie input and output embeddings. rope_parameters (`RopeParameters`, *optional*): Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE with longer `max_position_embeddings`. attention_bias (`bool`, *optional*, defaults to `False`): Whether to use a bias in the query, key, value, and output projection layers during self-attention. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout probability applied to attention weights. mlp_bias (`bool`, *optional*, defaults to `False`): Whether to use a bias term in the MLP layers. first_k_dense_replace (`int`, *optional*, defaults to 0): Number of dense layers in the shallow layers before switching to MoE layers. kv_lora_rank (`int`, *optional*, defaults to 512): Rank of the LoRA decomposition for key-value projections. q_lora_rank (`int`, *optional*, defaults to 1536): Rank of the LoRA decomposition for query projections. Specifically, it determines the dimensionality to which the query (q) vectors are compressed before being expanded back to their original size. It reduces computational overhead while maintaining model performance. n_group (`int`, *optional*): Number of groups for routed experts. n_routed_experts (`int`, *optional*, defaults to 64): Number of routed experts (None indicates a dense model). n_shared_experts (`int`, *optional*, defaults to 2): Number of shared experts (None indicates a dense model). qk_nope_head_dim (`int`, *optional*, defaults to 128): The head dimension for the QK (query-key) projections when using NOPE (Neural Operator Position Encoding). qk_rope_head_dim (`int`, *optional*, defaults to 64): The head dimension for QK projections when using RoPE. routed_scaling_factor (`float`, *optional*, defaults to 1.0): Scaling factor for routed experts in MoE models. topk_group (`int`, *optional*): Number of selected groups per token for expert selection. topk_method (`str`, *optional*, defaults to `"greedy"`): The method used for selecting top-k experts in the routed gate mechanism. norm_topk_prob (`bool`, *optional*, defaults to `False`): Whether to renormalize the router probabilities when `top_k > 1`. This flag is kept for backward compatibility with previously released checkpoints and runtimes relying on the legacy DeepSeek config. v_head_dim (`int`, *optional*, defaults to 128): The dimension of value projections in the attention layers. num_experts_per_tok (`int`, *optional*): The number of experts selected per token. If `None`, the model behaves as a dense Transformer. moe_intermediate_size (`int`, *optional*, defaults to 1407): Dimension of the MoE (Mixture of Experts) representations. ```python >>> from transformers import DeepseekV2Model, DeepseekV2Config >>> # Initializing a DeepSeek-V2 style configuration >>> configuration = DeepseekV2Config() >>> # Accessing the model configuration >>> model = DeepseekV2Model(configuration) >>> print(model.config) ``` """ model_type = "deepseek_v2" keys_to_ignore_at_inference = ["past_key_values"] base_model_tp_plan = { "layers.*.self_attn.q_proj": "colwise", "layers.*.self_attn.q_b_proj": "colwise", "layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj", "layers.*.self_attn.kv_b_proj": "colwise", "layers.*.self_attn.o_proj": "rowwise", "layers.*.mlp.experts.gate_up_proj": "packed_colwise", "layers.*.mlp.experts.down_proj": "rowwise", "layers.*.mlp.experts": "moe_tp_experts", "layers.*.mlp.shared_experts.gate_proj": "colwise", "layers.*.mlp.shared_experts.up_proj": "colwise", "layers.*.mlp.shared_experts.down_proj": "rowwise", "layers.*.mlp.gate_proj": "colwise", "layers.*.mlp.up_proj": "colwise", "layers.*.mlp.down_proj": "rowwise", } base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } def __init__( self, vocab_size: int | None = 32000, hidden_size: int | None = 4096, intermediate_size: int | None = 11008, num_hidden_layers: int | None = 32, num_attention_heads: int | None = 32, num_key_value_heads: int | None = None, hidden_act: str | None = "silu", max_position_embeddings: int | None = 2048, initializer_range: float | None = 0.02, rms_norm_eps: int | None = 1e-6, use_cache: bool | None = True, pad_token_id: int | None = None, bos_token_id: int | None = 1, eos_token_id: int | None = 2, tie_word_embeddings: bool | None = False, rope_parameters: RopeParameters | dict[str, RopeParameters] | None = None, attention_bias: bool | None = False, attention_dropout: float | None = 0.0, mlp_bias: bool | None = False, first_k_dense_replace: int | None = 0, kv_lora_rank: int | None = 512, q_lora_rank: int | None = 1536, n_group: int | None = None, n_routed_experts: int | None = 64, n_shared_experts: int | None = 2, qk_nope_head_dim: int | None = 128, qk_rope_head_dim: int | None = 64, routed_scaling_factor: float | None = 1.0, topk_group: int | None = None, topk_method: str | None = "greedy", norm_topk_prob: bool | None = False, v_head_dim: int | None = 128, num_experts_per_tok: int | None = None, moe_intermediate_size: int | None = 1407, **kwargs, ): self.first_k_dense_replace = first_k_dense_replace self.kv_lora_rank = kv_lora_rank self.q_lora_rank = q_lora_rank self.n_group = n_group self.n_routed_experts = n_routed_experts self.n_shared_experts = n_shared_experts self.qk_nope_head_dim = qk_nope_head_dim self.qk_rope_head_dim = qk_rope_head_dim self.routed_scaling_factor = routed_scaling_factor self.topk_group = topk_group self.topk_method = topk_method self.norm_topk_prob = norm_topk_prob self.v_head_dim = v_head_dim self.num_experts_per_tok = num_experts_per_tok self.moe_intermediate_size = moe_intermediate_size self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads # for backward compatibility if num_key_value_heads is None: num_key_value_heads = num_attention_heads self.num_key_value_heads = num_key_value_heads self.hidden_act = hidden_act self.initializer_range = initializer_range self.rms_norm_eps = rms_norm_eps self.use_cache = use_cache self.attention_bias = attention_bias self.attention_dropout = attention_dropout self.mlp_bias = mlp_bias self.head_dim = qk_rope_head_dim self.rope_parameters = rope_parameters self.tie_word_embeddings = tie_word_embeddings self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id self.eos_token_id = eos_token_id super().__init__(**kwargs) __all__ = ["DeepseekV2Config"]