
    qi                     8    d dl mZ d dlmZ  G d de      ZdgZy)   )PreTrainedConfig)RopeParametersc            ,       h    e Zd ZdZdZdgZdddddddZdgdgfd	d
gd	gfd	gd	gfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"dedz  dedz  dedz  dedz  dedz  dedz  de	dz  dedz  de
dz  de
dz  dedz  dedz  dedz  dedz  dedz  dedz  de
dz  dedz  dedz  d eee	ef   z  dz  f( fd!Z xZS )#Jais2Configa
  
    This is the configuration class to store the configuration of a [`Jais2Model`]. It is used to instantiate a Jais2
    model according to the specified arguments, defining the model architecture.
    [inceptionai/Jais-2-8B-Chat](https://huggingface.co/inceptionai/Jais-2-8B-Chat).

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 150272):
            Vocabulary size of the Jais2 model.
        hidden_size (`int`, *optional*, defaults to 3328):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 26624):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 26):
            Number of attention heads for each attention layer.
        num_key_value_heads (`int`, *optional*):
            Number of key_value heads for Grouped Query Attention.
        hidden_act (`str`, *optional*, defaults to `"relu2"`):
            The non-linear activation function in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether to return last key/values attentions.
        pad_token_id (`int`, *optional*):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 0):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 150024):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings.
        attention_bias (`bool`, *optional*, defaults to `True`):
            Whether to use a bias in the query, key, value and output projection layers.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        mlp_bias (`bool`, *optional*, defaults to `True`):
            Whether to use a bias in up_proj, down_proj and gate_proj layers.
        head_dim (`int`, *optional*):
            The attention head dimension.
        rope_parameters (`dict`, *optional*):
            The RoPE parameters.
    jais2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormN
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actmax_position_embeddingsinitializer_rangelayer_norm_eps	use_cachepad_token_idbos_token_ideos_token_idtie_word_embeddingsattention_biasattention_dropoutmlp_biashead_dimrope_parametersc                 z   || _         || _        || _        || _        || _        || _        ||}|| _        || _        |	| _        || _	        || _
        || _        || _        ||n| j                  | j
                  z  | _        || _        || _        || _        || _        || _        t'        | P  di | |
| _        y )N )r   r   r   r   r   r   r   r   r   r   r!   r"   r#   r$   r%   r    r   r   r   super__init__r   )selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   kwargs	__class__s                         _/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/jais2/configuration_jais2.pyr)   zJais2Config.__init___   s    0 %'>$&!2!2#6  &"5#6 $!2",!2 $,$8d>N>NRVRjRj>j.#6 ((("6",    )i K i   i h         Nrelu2i    g{Gz?gh㈵>TN    iJ FTg        TNN)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planintstrfloatboolr   dictr)   __classcell__)r,   s   @r-   r   r      s   1f J#4"5 &/%.%.%. )"+ &(9:#%568IJ!"_$56 "("&(-(**,*.!(.2*.'+!%#'#$#)+0&**- $#MQ+2-$J2- 4Z2- :	2-
 :2- !4Z2- !4Z2- $J2- "%t2- !4<2- 2- $;2- Dj2- Dj2- Dj2-  "D[!2-" t#2-$ !4<%2-& +'2-( *)2-* ($sN/B*CCdJ+2- 2-r.   r   N)configuration_utilsr   modeling_rope_utilsr   r   __all__r'   r.   r-   <module>rD      s&   , 4 1w-" w-t /r.   