
    IuiB                     x    d dl Z d dlZd dlmZmZ  G d de j                        Z G d dej                        Z	y)    N)common_spec
model_specc                       e Zd ZdZdZdZdZy)RotaryScalingTypezRoPE scaling type.r         N)__name__
__module____qualname____doc__LinearSuLlama3     R/opt/pipecat/venv/lib/python3.12/site-packages/ctranslate2/specs/attention_spec.pyr   r   	   s    F	
BFr   r   c                   8    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZy)MultiHeadAttentionSpecNc                 |   t         j                  | _        |rt        j                  |      | _        t        |rdnd      D cg c]  }t        j                          c}| _        |r6t        j                  |      | _	        t        j                  |      | _
        |rd | _        d | _        |rd | _        d | _        |rd | _        d | _        d | _        |dk7  r)t%        j&                  d      j)                  |      | _        |dk7  r)t%        j&                  d      j)                  |      | _        |t%        j&                  d      j)                  |      | _        || _        t%        j&                  d      j)                  |
      | _        |)t%        j&                  d      j)                  |      | _        |t6        j8                  u r*t%        j&                  d      j)                  |	      | _        nA|t6        j<                  u rd | _        d | _         n |t6        jB                  u rd | _"        d | _#        |)t%        j&                  d      j)                  |      | _$        |)t%        j&                  d      j)                  |      | _%        |*t%        j&                  d      j)                  |      | _&        y y c c}w )N)rms_normr      r   int32float32int8)'r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrange
LinearSpeclinearq_normk_normrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_window)selfself_attentionrelative_positionrelative_asymmetric_positionr&   r   r0   r1   r3   r4   r2   r.   r/   r9   r:   r;   qk_normqk_norm_rmshas_norm_s                       r   __init__zMultiHeadAttentionSpec.__init__   sR   * (00)77JDO.3AQ.O
)*K""$
 %33[IDK%33[IDK*.D',0D)"+/D(37D0'59D2.2D+/3D,+q046HHW4E4J4J05D1 #a'+-88G+<+A+A',D( ! hhw/44Z@DO%6D"!xx	277DD".+-88F+;+@+@AT+U("&7&>&>>-/XXi-@-E-E).* %(9(<(<<26/370$(9(@(@@.2+/3,# " 1 6 6| DDHHW-228<DM%"$((7"3"8"8"HD &m
s   J9)FFFFFNTNr   i'  r   r   NNNFTT)r	   r
   r   rD   r   r   r   r   r      sA     %* % )* !'PIr   r   )
enumnumpyr+   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r   r   r   <module>rJ      s4      5 QIZ11 QIr   