
    qi4                     `    d dl mZ d dlmZ ddlmZmZ  G d de      Z G d de      ZddgZ	y	)
   )PreTrainedConfig)RopeParameters   )CONFIG_MAPPING
AutoConfigc            /       |    e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d&de	dz  de	dz  de	de	dz  de	dz  de	dz  de
dz  de	dz  dedz  de	dz  dedz  de	dz  de	dz  de	dz  dedz  deee
ef   z  dz  dedz  dedz  d edz  d!e	dz  d"e	d#e	d$e	f. fd%Z xZS )'AriaTextConfigau  
    This class handles the configuration for the text component of the Aria model.
    Instantiating a configuration with the defaults will yield a similar configuration to that of the model of the Aria
    [rhymes-ai/Aria](https://huggingface.co/rhymes-ai/Aria) architecture.
    This class extends the LlamaConfig to include additional parameters specific to the Mixture of Experts (MoE) architecture.

    Args:
        vocab_size (`int`, *optional*, defaults to 32000):
            Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`LlamaModel`]
        hidden_size (`int`, *optional*, defaults to 4096):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 4096):
            The size of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens,
            Llama 2 up to 4096, CodeLlama up to 16384.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 2):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 1):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 2):
            End of stream token id.
        pretraining_tp (`int`, *optional*, defaults to 1):
            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
            document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to
            understand more about it. This value is necessary to ensure exact reproducibility of the pretraining
            results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232).
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        attention_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        mlp_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
        head_dim (`int`, *optional*):
            The attention head dimension. If None, it will default to hidden_size // num_heads
        moe_num_experts (`int`, *optional*, defaults to 8):
            The number of experts in the MoE layer.
        moe_topk (`int`, *optional*, defaults to 2):
            The number of top experts to route to for each token.
        moe_num_shared_experts (`int`, *optional*, defaults to 2):
            The number of shared experts.
    	aria_textpast_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormtext_configN
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actmax_position_embeddingsinitializer_rangerms_norm_eps	use_cachebos_token_ideos_token_idpretraining_tptie_word_embeddingsrope_parametersattention_biasattention_dropoutmlp_biashead_dimmoe_num_expertsmoe_topkmoe_num_shared_expertsc                    || _         || _        || _        || _        || _        || _        || _        || _         || _        || _        ||}|| _	        || _
        |	| _        |
| _        || _        || _        || _        || _        || _        ||n| j                  | j                  z  | _        || _        || _        || _        || _        || _        t1        | d  di | y )N )r   r*   r+   r,   r   r   r   r   r   r   r   r   r   r#   r    r&   r'   r(   r)   r%   r$   pad_token_idr!   r"   super__init__)selfr   r   r   r   r   r   r   r   r   r   r    r/   r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   kwargs	__class__s                             ]/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/aria/configuration_aria.pyr1   zAriaTextConfig.__init__s   s    8 "3. &<#$'>$&!2!2#6  &"5#6 $!2(,",!2 $,$8d>N>NRVRjRj>j.#6 ((("6"    )i }     r7       r8   Nsilui   {Gz?gư>Tr      r   r;   FNFg        FN   r   r   )__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planbase_config_keyintstrfloatboolr   dictr1   __classcell__r4   s   @r5   r	   r	      s   EN J#4"5%.%.%.%.1:/81: &(9:#%568IJ!"_$56
 $O "'"&!%(**,*.!'.2*.#'!%#$#$%&+0MQ&+*- %# &'3;#$J;# 4Z;# 	;#
 :;# !4Z;# !4Z;# $J;# "%t;# !4<;# Dj;# $;;# Dj;# Dj;#  d
!;#" "D[#;#$ ($sN/B*CCdJ%;#& t';#( !4<);#* ++;#, *-;#. /;#0 1;#2 !$3;# ;#r6   r	   c                   z     e Zd ZdZdZddiZeedZ	 	 	 	 	 	 	 dde	ded	e
dz  de	dz  d
edz  dedz  f fdZ xZS )
AriaConfiga  
    This class handles the configuration for both vision and text components of the Aria model,
    as well as additional parameters for image token handling and projector mapping.
    Instantiating a configuration with the defaults will yield a similar configuration to that of the model of the Aria
    [rhymes-ai/Aria](https://huggingface.co/rhymes-ai/Aria) architecture.

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vision_config (`AriaVisionConfig` or `dict`, *optional*):
            Configuration for the vision component.
        vision_feature_layer (`int`, *optional*, defaults to -1):
            The index of the layer to select the vision feature.
        text_config (`AriaTextConfig` or `dict`, *optional*):
            Configuration for the text component.
        projector_patch_to_query_dict (`dict`, *optional*):
            Mapping of patch sizes to query dimensions.
        image_token_index (`int`, *optional*, defaults to 9):
            Index used to represent image tokens.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated normal initializer for initializing all weight matrices.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
    ariaimage_token_idimage_token_index)r   vision_configNvision_feature_layerr   projector_patch_to_query_dictr   r$   c                    || _         |ddd}|j                         D 	
ci c]  \  }	}
t        |	      t        |
       c}
}	| _        t	        | j                  j                               | _        || _        t        |t              rd|d<   t        |d      di |}n|t        d          }|| _        || _        t        |t              rd|v rt        di |}n|
t               }|| _        || _        t!        | D  di | y c c}
}	w )N      )i  i$  idefics3_visionrA   r.   )rQ   itemsrF   rT   maxvalues'max_value_projector_patch_to_query_dictrS   
isinstancerJ   r   rR   r   r	   r   r$   r0   r1   )r2   rR   rS   r   rT   rQ   r   r$   r3   kvr4   s              r5   r1   zAriaConfig.__init__   s    "3 )0-) JgIlIlIn-oAc!fc!fn-o*7:4;];];d;d;f7g4$8!mT**;M,'*=+FGX-XM"*+<=?M*!2k4(\[-H(7;7K (*K&#6 "6") .ps   D)NNN	   r:   F)r=   r>   r?   r@   rA   attribute_mapr	   r   sub_configsrF   rJ   rH   rI   r1   rK   rL   s   @r5   rN   rN      s    4 J-M #1:NK $&&*59()*.+0(# "(# $	(#
 (,d{(# :(# !4<(# "D[(# (#r6   rN   N)
configuration_utilsr   modeling_rope_utilsr   autor   r   r	   rN   __all__r.   r6   r5   <module>rh      s=   ( 4 1 -U#% U#pI#! I#X )
*r6   