
    IuiF                    0
   d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlZ		 d dl
Z
d dlZd dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZmZ ej4                  j6                  ej4                  j8                  ej4                  j8                  ej4                  j6                  ej4                  j8                  ej4                  j:                  ej4                  j<                  ej4                  j>                  ej4                  j>                  d	Z ejB                  jD                  ejB                  jF                  ejB                  jH                  ejB                  jF                  dZ%ejL                  jN                  ejL                  jP                  dZ)i Z*d	 Z+ G d
 de      Z, G d de jZ                        Z. e+d       G d de.             Z/ e+d       G d de/             Z0 e+d       G d de/             Z1 e+d       G d de/             Z2 e+d       G d de/             Z3 e+d       G d de/             Z4 e+d        G d! d"e.             Z5 e+d#       G d$ d%e.             Z6 e+d&       G d' d(e.             Z7 e+d)       G d* d+e.             Z8 e+d,       G d- d.e.             Z9 e+d/       G d0 d1e/             Z: e+d2       G d3 d4e/             Z; e+d5       G d6 d7e/             Z< e+d8       G d9 d:e.             Z= e+d;       G d< d=e=             Z> e+d>       G d? d@e.             Z? e+dA       G dB dCe.             Z@ e+dD       G dE dFe.             ZA e+dG       G dH dIe.             ZB e+dJ       G dK dLe.             ZC e+dM       e+dN       G dO dPe.                    ZD e+dQ       G dR dSe.             ZE e+dT       G dU dVe.             ZF e+dW       G dX dYe.             ZG e+dZ       G d[ d\e.             ZH e+d]       G d^ d_e.             ZI e+d`       G da dbe.             ZJ e+dc       G dd dee.             ZK e+df       G dg dheK             ZL e+di       G dj dke.             ZM e+dl       G dm dne.             ZN e+do       G dp dqe.             ZO e+dr       G ds dte.             ZP e+du       G dv dwe.             ZQdx ZReSdyk(  r eR        g dzg d{g d|g d}g d~g dg dg dg dg dg ddZT e+d       G d de.             ZUy# e$ r Y w xY w)    N)ListOptional)utils)	Converter)attention_speccommon_spec
model_spectransformer_specwav2vec2_specwav2vec2bert_specwhisper_spec)	gelu	gelu_fastgelu_newgelu_pythongelu_pytorch_tanh
quick_gelurelusiluswish)linearsullama3longrope)gemmgemvc                       fd}|S )z5Registers a model loader for this configuration name.c                 $     |        t         <   | S N)_MODEL_LOADERS)clsconfig_names    U/opt/pipecat/venv/lib/python3.12/site-packages/ctranslate2/converters/transformers.py	decoratorz"register_loader.<locals>.decorator<   s    &)e{#
     )r"   r$   s   ` r#   register_loaderr'   9   s     r%   c                   r    e Zd ZdZ	 	 	 	 	 	 ddedee   deee      dedee   ded	efd
Zd Z	d Z
d Zd Zy)TransformersConverterz/Converts models from Hugging Face Transformers.Nmodel_name_or_pathactivation_scales
copy_filesload_as_float16revisionlow_cpu_mem_usagetrust_remote_codec                 f    || _         || _        || _        || _        || _        || _        || _        y)a  Initializes the converter.

        Arguments:
          model_name_or_path: Name of the pretrained model to download, or path to the
            directory containing the pretrained model.
          activation_scales: Path to the pre-computed activation scales. Models may
            use them to rescale some weights to smooth the intermediate activations
            and improve the quantization accuracy. See
            https://github.com/mit-han-lab/smoothquant.
          copy_files: List of filenames to copy from the Hugging Face model to the
            converted model directory.
          load_as_float16: Load the model weights as float16. More precisely, the model
            will be loaded with ``from_pretrained(..., dtype=torch.float16)``.
          revision: Revision of the model to download from the Hugging Face Hub.
          low_cpu_mem_usage: Enable the flag ``low_cpu_mem_usage`` when loading the model
            with ``from_pretrained``.
          trust_remote_code: Allow converting models using custom code.
        N)_model_name_or_path_activation_scales_copy_files_load_as_float16	_revision_low_cpu_mem_usage_trust_remote_code)selfr*   r+   r,   r-   r.   r/   r0   s           r#   __init__zTransformersConverter.__init__F   s;    8 $6 "3% /!"3"3r%   c                    t        j                         5  t        j                  j	                  | j
                  | j                        }|j                  j                  }t        j                  |      }|<t        d|ddj                  t        t        j                                     d      t        t        |j                         }t        j"                  }d| j$                  rt         j&                  nt        |dd       xs t        |dd       i}| j(                  r| j(                  |d<   | j*                  r| j*                  |d	<   | j                  r| j                  |d
<    | j,                  || j
                  fi |}i }| j                  r| j                  |d
<    | j.                  || j
                  fi |}	 |||	      }
| j0                  r3t        j2                  | j0                  d      }|j5                  |
|       | j6                  r1| j6                  D ]"  }|
j9                  | j;                  |             $ |
cd d d        S # 1 sw Y   y xY w)N)r0   z8No conversion is registered for the model configuration z  (supported configurations are: , )dtypetorch_dtyper.   r/   r0   cpu)map_location)torchno_gradtransformers
AutoConfigfrom_pretrainedr2   r8   	__class____name__r    get
ValueErrorjoinsortedkeysgetattrarchitecture_nameAutoTokenizerr5   float16r6   r7   
load_modelload_tokenizerr3   loadsmooth_activationr4   register_fileget_model_file)r9   configr"   loadermodel_classtokenizer_classkwargsmodeltokenizer_kwargs	tokenizerspecr+   filenames                r#   _loadzTransformersConverter._loadj   s0   ]]_ 8	!,,<<((D<S<S = F !**33K#''4F~  #DIIf^5H5H5J.K$LN  ",0H0HIK*88O ,, MM $7 <v}d;F ~~%)^^z"&&.2.E.E*+&&.2.E.E*+#DOOK1I1ITVTE!&&8<8O8O !45+++!9!9=MI %+D&&$)JJ++%%! ((/@A $ 0 0 FH&&t':':8'DEF q8	 8	 8	s   H3IIc                 (     |j                   |fi |S r   rF   )r9   rZ   r*   r\   s       r#   rR   z TransformersConverter.load_model   s    *{**+=HHHr%   c                 (     |j                   |fi |S r   rd   )r9   r[   r*   r\   s       r#   rS   z$TransformersConverter.load_tokenizer   s    .../ALVLLr%   c                    t         j                  j                  | j                        r+t         j                  j	                  | j                  |      }n"	 t        j                  | j                  |      }|t         j                  j                  |      st        d|d| j                        |S # t
        j                  j                  $ r d }Y _w xY w)N)repo_idra   zFile z does not exist in model )ospathisdirr2   rK   huggingface_hubhf_hub_downloadr   EntryNotFoundErrorisfilerJ   )r9   ra   ri   s      r#   rW   z$TransformersConverter.get_model_file   s    77==11277<< 8 8(CD&66 44x <rww~~d3T557 
  #((;; s   !B5 5CC)NNFNFF)rH   
__module____qualname____doc__strr   r   boolr:   rb   rR   rS   rW   r&   r%   r#   r)   r)   C   s    9
 ,0*. %"&"'"'"4"4 $C="4 T#Y'	"4
 "4 3-"4  "4  "4H9vIMr%   r)   c                       e Zd ZdZed        Zej                  d        Zd Z	d Z
d Zd Zd Zej                  j                   fd	Zd
 Zd Zd Zd Zy)ModelLoaderzRBase class for loading Transformers models into a CTranslate2 model specification.c                      y r   r&   r9   s    r#   rO   zModelLoader.architecture_name   s    r%   c                     t               r   NotImplementedErrorr9   r]   s     r#   get_model_speczModelLoader.get_model_spec   s    !##r%   c                     | j                  |      }| j                  |j                  ||       | j                  ||      }| j	                  ||       |S r   )r|   
set_configrX   get_vocabularyset_vocabulary)r9   r]   r_   r`   tokenss        r#   __call__zModelLoader.__call__   sM    ""5)UI6$$UI6D&)r%   c                     t        |j                         j                         d       D cg c]  \  }}|	 c}}S c c}}w )Nc                     | d   S N   r&   )items    r#   <lambda>z,ModelLoader.get_vocabulary.<locals>.<lambda>   s
    Q r%   )key)rL   	get_vocabitems)r9   r]   r_   token_s        r#   r   zModelLoader.get_vocabulary   sE     ###%++-3G
q 
 	
 
s   >c                      y r   r&   r9   r`   r   s      r#   r   zModelLoader.set_vocabulary       r%   c                      y r   r&   r9   rX   r]   r_   s       r#   r~   zModelLoader.set_config   r   r%   c                 H    |j                   |_        |j                  |_        y r   weightgammabiasbetar9   r`   modules      r#   set_layer_normzModelLoader.set_layer_norm   s    ]]
KK	r%   c                    |t         j                  j                  k(  r|j                  |_        n3|j                  |_        |j
                  |_        |j                  |_        t        |t        j                        r!|j                  j                  dd      |_        t        |d      r|j                  |j                  |_        y y y )Nr   r   r   )r   QuantizationCT2r   qweightscalesweight_scaleqzerosweight_zero
isinstancerD   Conv1D	transposehasattrr   )r9   r`   r   
quant_types       r#   
set_linearzModelLoader.set_linear   s    11555 --DK ..DK &D%}}Dfl112++//15DK66"v{{'>DI (?"r%   c                 &    |j                   |_         y r   )r   r   s      r#   set_embeddingszModelLoader.set_embeddings   s    mmr%   c                 t    |j                   |_        t        |dd      }|dkD  r|j                  |d  |_        y y )Noffsetr   r   	encodingsrN   r9   r`   r   r   s       r#   set_position_encodingsz"ModelLoader.set_position_encodings   s9    1-A:!^^FG4DN r%   c                     t        d      )Nz7No activation smoothing logic is defined for this modelry   )r9   r`   r+   s      r#   rU   zModelLoader.smooth_activation   s    !E
 	
r%   c           	         t        |dd       }|r|j                  d      xs |j                  d      }|dk(  rd }nIt        j                  |      }|2t        d|ddj	                  t        j                                     |j                  dd	      }|j                  d
|      }nd }d	}t        |d
|      }|||fS )Nrope_scalingtype	rope_typedefaultRoPE scaling type 'T' is not yet implemented. The following RoPE scaling types are currently supported: r<   factorr   
rope_theta)rN   rI   _SUPPORTED_ROPE_SCALINGrz   rK   rM   )r9   rX   default_rope_thetar   r   rotary_scaling_typerotary_scaling_factorr   s           r#   get_rotary_paramszModelLoader.get_rotary_params   s    v~t<$((0QL4D4D[4QII%&*#&=&A&A)&L#&.- %dii0G0L0L0N&OQ 
 %1$4$4Xq$A!%)),8JKJ"&$%! 7IJJ"$9:EEr%   N)rH   ro   rp   rq   propertyrO   abcabstractmethodr|   r   r   r   r~   r   r   r   r   r   r   r   rU   r   r&   r%   r#   ru   ru      su    \  	$ $
  3>2J2J2N2N $$5

Fr%   ru   
BartConfigc                   Z     e Zd Zed        Zd Z fdZd Zd Zd Z	d Z
d
dZd	 Z xZS )
BartLoaderc                      y)NBartForConditionalGenerationr&   rw   s    r#   rO   zBartLoader.architecture_name  s    -r%   c                 $   t         j                  j                  |j                  j                  |j                  j
                  f|j                  j                  |j                  j                  t        |j                  j                     t        |j                  dd            }| j                  |j                  |j                  j                         | j                  |j                  |j                  j                         | j!                  |j                  j"                  |j$                         t        |dd       }|J|j'                         j)                         dk7  r)|j+                         |j                  j"                  _        |S )Nnormalize_embeddingTpre_norm
activationlayernorm_embeddingfinal_logits_biasr   )r
   TransformerSpecfrom_configrX   encoder_layersdecoder_layersencoder_attention_headsnormalize_before_SUPPORTED_ACTIVATIONSactivation_functionrN   set_encoderencoderr]   set_decoderdecoderr   
projectionlm_headnonzeronumelsqueezer   )r9   r]   r`   r   s       r#   r|   zBartLoader.get_model_spec  s!   //;;\\((%,,*E*EFLL00\\22-ell.N.NO '6KT R < 
 	u{{':':;u{{':':;//?#E+>E(->-F-F-H-N-N-PTU-U+<+D+D+FDLL##(r%   c                     t         |   ||      }|j                  j                  t	        |      k  r|d |j                  j                   }|S r   )superr   rX   
vocab_sizelenr9   r]   r_   r   rG   s       r#   r   zBartLoader.get_vocabulary2  sF    'y9<<""S[05ell556Fr%   c                 H    |j                  |       |j                  |       y r   register_source_vocabularyregister_target_vocabularyr   s      r#   r   zBartLoader.set_vocabulary8      ''/''/r%   c                     |j                   |_         |j                  |_        |j                  |_        |j                  |j                  j
                        |_        y r   )	bos_token	eos_token	unk_tokenconvert_ids_to_tokensrX   decoder_start_token_iddecoder_start_tokenr   s       r#   r~   zBartLoader.set_config<  sL    $..$..$..%.%D%DLL//&
"r%   c                 H   | j                  ||       t        |j                  |j                        D ]  \  }}| j	                  |j
                  |j                  d       | j                  |j
                  j                  |j                         | j                  |j                  j                  |j                         | j                  |j                  j                  |j                         | j                  |j                  j                  |j                           y NTself_attention)set_common_layersziplayerlayersset_attentionr   	self_attnr   
layer_normself_attn_layer_normr   ffnlinear_0fc1linear_1fc2final_layer_norm)r9   r`   r   
layer_specr   s        r#   r   zBartLoader.set_encoderD  s    tW-!$TZZ!@ 	SJ))#  
 ))44**
 OOJNN33UYY?OOJNN33UYY?
 9 95;Q;QR	Sr%   c                    | j                  ||       t        |j                  |j                        D ]R  \  }}| j	                  |j
                  |j                  d       | j                  |j
                  j                  |j                         t        |d      rX| j	                  |j                  |j                  d       | j                  |j                  j                  |j                         | j                  |j                  j                   |j"                         | j                  |j                  j$                  |j&                         | j                  |j                  j                  |j(                         U y )NTr   encoder_attnF)r   r   r   r   r   r   r   r   r   r   r   	attentionr  encoder_attn_layer_normr   r   r   r   r  r  r  )r9   r`   r   r  r   s        r#   r   zBartLoader.set_decoderV  s;   tW-!$TZZ!@ 	SJ))#  
 ))44**
 un-""((&&#( # 
 ##((3311
 OOJNN33UYY?OOJNN33UYY?
 9 95;Q;QR1	Sr%   c                 R   t        d      D cg c]  }t        j                          }}| j                  |d   |j                         | j                  |d   |j
                         | j                  |d   |j                         |r$t        j                  |j                  d   |       nLt        j                  |j                  d   |d d        t        j                  |j                  d   |dd         | j                  |j                  d   |j                         y c c}w )N   r   r      )ranger   
LinearSpecr   q_projk_projv_projr   fuse_linearr   out_projr9   r`   r  r   r   split_layerss         r#   r   zBartLoader.set_attentions  s    :?(CQ..0CCQ)9)9:Q)9)9:Q)9)9:dkk!nl;dkk!nl2A.>?dkk!nl12.>?B););< Ds   D$c                 z   dd l }t        |d      s>|j                  j                  r%|j	                  |j                  j
                        nd}n|j                  }||_        | j                  |j                  |j                         | j                  t        |j                  t              r|j                  d   n|j                  |j                         t        |d      r&| j!                  |j"                  |j"                         t        |d      r'| j!                  |j$                  |j$                         y y )Nr   embed_scale      ?r   r   )mathr   rX   scale_embeddingsqrtd_modelr  scale_embeddingsr   position_encodingsembed_positionsr   r   
embeddingslistembed_tokensr   r   r   )r9   r`   r   r  r  s        r#   r   zBartLoader.set_common_layers  s    v}- ==00 		&--//0  !,,K +##D$;$;V=S=ST doot4 "__	
 6<(1B1BC601 8 8&:T:TU 2r%   F)rH   ro   rp   r   rO   r|   r   r   r~   r   r   r   r   __classcell__rG   s   @r#   r   r     s@    . .&0
S$S:=Vr%   r   MarianConfigc                   N     e Zd Zed        Z fdZd Z fdZ fdZd Z	 xZ
S )MarianMTLoaderc                      y)NMarianMTModelr&   rw   s    r#   rO   z MarianMTLoader.architecture_name  s    r%   c                     d|j                   _        d|j                   _        t        |   |      }| j                  |       |S NF)rX   r   r   r   r|   _remove_pad_weights)r9   r]   r`   rG   s      r#   r|   zMarianMTLoader.get_model_spec  s=    (-%+0(w%e,  &r%   c                 j    |j                   |_         |j                  |_        |j                   |_        y r   )r   r   r   r   s       r#   r~   zMarianMTLoader.set_config  s.    $..$.. &/%8%8"r%   c                 4    d|_         t        | 	  ||       y NT)start_from_zero_embeddingr   r   r9   r`   r   rG   s      r#   r   zMarianMTLoader.set_decoder  s    )-&D'*r%   c                 X    t         |   ||      }|d   dk(  r|j                          |S )Nr  z<pad>)r   r   popr   s       r#   r   zMarianMTLoader.get_vocabulary  s0     'y9": JJLr%   c                    |j                   j                  d   |j                  j                  |j                  j                  g}|d   j                  j
                  d   dz
  }|D ]  }|j                  j
                  d   |dz   k(  r|j                  d d |_        t        |t        j                        sQ|j                         sb|j                  j
                  d   |dz   k(  s|j                  d d |_
         y )Nr   r   r  )r   r   r   r   r   shaper   r   r  has_biasr   )r9   r`   vocab_specsnew_vocab_size
vocab_specs        r#   r-  z"MarianMTLoader._remove_pad_weights  s    LL##A&LL##LL##
 %Q..44Q7!;% 	7J  &&q)^a-??$.$5$5cr$:
!:{'='=>'')OO))!,0BB",//#2"6
	7r%   )rH   ro   rp   r   rO   r|   r~   r   r   r-  r$  r%  s   @r#   r(  r(    s.     9+7r%   r(  M2M100Configc                   >     e Zd Zed        Z fdZd Z fdZ xZS )M2M100Loaderc                      y)NM2M100ForConditionalGenerationr&   rw   s    r#   rO   zM2M100Loader.architecture_name  s    /r%   c                 f    d|j                   _        d|j                   _        t        |   |      S )NTF)rX   r   r   r   r|   )r9   r]   rG   s     r#   r|   zM2M100Loader.get_model_spec  s+    (,%+0(w%e,,r%   c                 @    |j                   |j                  d  |_        y r   )weightsr   r   r   s      r#   r   z#M2M100Loader.set_position_encodings  s    8r%   c                    t         |   ||      }|d   |j                  k(  r*|j                  |j                  |j                                |j                  j                  dg       D ]  }||vs|j                  |        t        |d|j                  j                  t        |      z
        }|dkD  r|t        |      D cg c]  }d|z  	 c}z  }|S c c}w )Nr  additional_special_tokensnum_madeup_wordsr   zmadeupword%d)r   r   r   insertunk_token_idr4  special_tokens_maprI   appendrN   rX   r   r   r  )r9   r]   r_   r   r   rE  irG   s          r#   r   zM2M100Loader.get_vocabulary  s    'y9 ":,,,MM)00&**,?11556QSUV 	%EF"e$	% #)5<<+B+BS[+P
 a59I3JKa~)KKF Ls   C)	rH   ro   rp   r   rO   r|   r   r   r$  r%  s   @r#   r=  r=    s)    0 0-
9 r%   r=  MBartConfigc                   "    e Zd Zed        Zd Zy)MBartLoaderc                      y)NMBartForConditionalGenerationr&   rw   s    r#   rO   zMBartLoader.architecture_name  s    .r%   c                     |j                   |_         |j                  |_        |j                  |_        t        |j                  dd       dv rd |_        y |j                  |_        y )Nr[   )MBartTokenizerN)r   r   r   rN   rX   r   r   s       r#   r~   zMBartLoader.set_config  s[    $..$..$.. 5<<!2D9=UU)-F&)2)<)<F&r%   NrH   ro   rp   r   rO   r~   r&   r%   r#   rM  rM    s    / /	=r%   rM  PegasusConfigc                   "    e Zd Zed        Zd Zy)PegasusLoaderc                      y)NPegasusForConditionalGenerationr&   rw   s    r#   rO   zPegasusLoader.architecture_name      0r%   c                     |j                   |_        |j                  |_        |j                  |_        |j                   |_        y r   )	pad_tokenr   r   r   r   r   s       r#   r~   zPegasusLoader.set_config  s:    $..$..$..%.%8%8"r%   NrR  r&   r%   r#   rU  rU  	  s    1 19r%   rU  	OPTConfigc                   V     e Zd Zed        Zd Zd Zd Zd Z fdZ	d Z
 fdZ xZS )		OPTLoaderc                      y)NOPTForCausalLMr&   rw   s    r#   rO   zOPTLoader.architecture_name      r%   c                    t         j                  j                  |j                  j                  |j                  j
                  |j                  j                  t        |j                  j                     |j                  j                  |j                  j                  k7        }| j                  |j                  |j                  j                         | j                  |j                  j                  |j                          |S )N)r   r   project_in_out)r
   TransformerDecoderModelSpecr   rX   num_hidden_layersnum_attention_headsdo_layer_norm_beforer   r   word_embed_proj_dimhidden_sizer   r   r]   r   r   r   r9   r]   r`   s      r#   r|   zOPTLoader.get_model_spec  s    ;;GGLL**LL,,\\66-ell.N.NO <<;;u||?W?WW H 
 	u{{':':;//?r%   c                 v   t        |j                  j                        D ]  \  }}d|z  }t        j                  |j
                  j                  |j
                  j                  d   |d|z            t        j                  |j                  j                  |j                  j                  |d|z             y )Nzmodel.decoder.layers.%dr   z%s.self_attn.q_projz%s.fc1)
	enumerater   r   r   rU   r   r   r   r   r   )r9   r`   r+   rJ  r   layer_scopes         r#   rU   zOPTLoader.smooth_activation)  s    !$,,"4"45 	HAu3a7K##$$//$$++A.!"7+"EF ##		$$		""!(["89	r%   c                 &    |j                  |       y r   register_vocabularyr   s      r#   r   zOPTLoader.set_vocabulary9        (r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   r   r   r   r   s       r#   r~   zOPTLoader.set_config<  ,    $..$..$..r%   c                 T   t         |   ||       |j                  &| j                  |j                  |j                         |j                  &| j                  |j                  |j                         |j
                  '| j                  |j                  |j
                         y y r   )r   r   
project_inr   project_outr  r   r   r2  s      r#   r   zOPTLoader.set_decoderA  s    D'*)OODOOW-?-?@*OOD,,g.A.AB##/1I1IJ 0r%   c                     d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         y r,  )r  r   r  r  r   r   r"  r   s      r#   r   zOPTLoader.set_common_layersK  s@     %##D$;$;V=S=STDOOV-@-@Ar%   c                     t         |   ||      }d}t        |      dz  dk7  r=dj                  |      }||vr|j	                  |       |dz  }t        |      dz  dk7  r=|S )Nr      zmadeupword{:04d}r   )r   r   r   formatrI  )r9   r]   r_   r   rJ  symbolrG   s         r#   r   zOPTLoader.get_vocabularyP  so    'y9&kAo"'..q1FV#f%FA	 &kAo" r%   )rH   ro   rp   r   rO   r|   rU   r   r~   r   r   r   r$  r%  s   @r#   r]  r]    s?        )/
KB

 
r%   r]  GPTBigCodeConfigc                   F     e Zd Zed        Zd Zd Z fdZd Zd Z	 xZ
S )GPTBigCodeMHALoaderc                      y)NGPTBigCodeForCausalLMr&   rw   s    r#   rO   z%GPTBigCodeMHALoader.architecture_name_  s    &r%   c                    t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                     d      }| j                  |j                  |j                         | j                  |j                  j                  |j                         |S )NT)r   r   multi_query_attentionr
   rc  r   rX   n_layern_headr   r   r   r   transformerr   r   r   ri  s      r#   r|   z"GPTBigCodeMHALoader.get_model_specc  s    ;;GGLL  LL-ell.N.NO"& H 
 	u'8'89//?r%   c                 &    |j                  |       y r   rn  r   s      r#   r   z"GPTBigCodeMHALoader.set_vocabularyp  rp  r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S Nz<extra_id_%d>r   r   rX   r   r   r  rI  r9   r]   r_   r   	extra_idsrJ  rG   s         r#   r   z"GPTBigCodeMHALoader.get_vocabularys  X    'y9LL++c&k9	y! 	/AMM/A-.	/ r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zGPTBigCodeMHALoader.set_config|  rs  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         | j                  |j                  |j                         t        |j                  |j                        D ]T  \  }}| j                  |j                  j                  |j                         | j                  |j                  j                   d   |j"                  j$                         | j                  |j                  j                   d   |j"                  j&                         | j                  |j(                  j                  |j*                         | j                  |j(                  j,                  |j.                  j0                         | j                  |j(                  j2                  |j.                  j&                         W y NFr   r   r  r   r   wter   r  wper   r   ln_fr   r   hr   ln_1r   r   attnc_attnc_projr   ln_2r   mlpc_fcr  r9   r`   r   r  r   s        r#   r   zGPTBigCodeMHALoader.set_decoder  K    %DOOVZZ8##D$;$;VZZHDOOV[[9!$TZZ!: 	GJ
 9 9 D DejjQOOJ55<<Q?ARARSOOJ55<<Q?ARARS
 9 95::FOOJNN33UYY^^DOOJNN33UYY5E5EF	Gr%   )rH   ro   rp   r   rO   r|   r   r   r~   r   r$  r%  s   @r#   r~  r~  ]  s/    ' ')/
Gr%   r~  
GPT2Configc                   4    e Zd Zed        Zd Zd Zd Zd Zy)
GPT2Loaderc                      y)NGPT2LMHeadModelr&   rw   s    r#   rO   zGPT2Loader.architecture_name       r%   c                 ~   t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                           }| j                  |j                  |j                         | j                  |j                  j                  |j                         |S )NT)r   r   r  ri  s      r#   r|   zGPT2Loader.get_model_spec  s    ;;GGLL  LL-ell.N.NO	 H 
 	u'8'89//?r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zGPT2Loader.set_vocabulary  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zGPT2Loader.set_config  rs  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         | j                  |j                  |j                         t        |j                  |j                        D ]T  \  }}| j                  |j                  j                  |j                         | j                  |j                  j                   d   |j"                  j$                         | j                  |j                  j                   d   |j"                  j&                         | j                  |j(                  j                  |j*                         | j                  |j(                  j,                  |j.                  j0                         | j                  |j(                  j2                  |j.                  j&                         W y r  r  r  s        r#   r   zGPT2Loader.set_decoder  r  r%   N	rH   ro   rp   r   rO   r|   r   r~   r   r&   r%   r#   r  r    s)    ! !
)/
Gr%   r  
GPTJConfigc                   4    e Zd Zed        Zd Zd Zd Zd Zy)
GPTJLoaderc                      y)NGPTJForCausalLMr&   rw   s    r#   rO   zGPTJLoader.architecture_name  r  r%   c           
         t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                     |j                  j                  ddd      }| j                  |j                  |j                  |j                  j                  |j                  j
                         | j                  |j                  j                  |j                         |S NTFr   r   
rotary_dimrotary_interleaveparallel_residualshared_layer_norm)r
   rc  r   rX   r  r  r   r   r  r   r   r  r   r   r   ri  s      r#   r|   zGPTJLoader.get_model_spec  s    ;;GGLL  LL-ell.N.NO||..#"" H 	
 	LLLL##LL		
 	//?r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zGPTJLoader.set_vocabulary  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zGPTJLoader.set_config  rs  r%   c                 0   d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]  \  }}| j	                  |j                  |j                         |j                  j                  j                  }|j                  j                  j                  }|j                  j                   j                  }	t#        j$                  |||      }t#        j$                  |||      }t'        j(                  |||	f      |j*                  j,                  d   _        | j/                  |j*                  j,                  d   |j                  j0                         | j/                  |j2                  j4                  |j6                  j8                         | j/                  |j2                  j:                  |j6                  j<                          y r  )r  r   r   r  r   r   r  r   r   r  r  r  r  r  r   r  r  r   permute_for_sliced_rotaryrB   catr   r   r   r  r   r   r  fc_inr  fc_out)
r9   r`   r   r  	num_headsr  r   qwkwvws
             r#   r   zGPTJLoader.set_decoder  ss    %DOOVZZ8DOOV[[9!$TZZ!: 	GJ
 < <ejjI""))B""))B""))B00Y
KB00Y
KB9>BB<9PJ%%,,Q/6OOJ55<<Q?ATATUOOJNN33UYY__EOOJNN33UYY5E5EF	Gr%   Nr  r&   r%   r#   r  r    s)    ! !*)/
Gr%   r  CodeGenConfigc                   F     e Zd Zed        Zd Z fdZd Zd Zd Z	 xZ
S )CodeGenLoaderc                      y)NCodeGenForCausalLMr&   rw   s    r#   rO   zCodeGenLoader.architecture_name      #r%   c           
         t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                     |j                  j                  ddd      }d}t        |j                  d      r|j                  j                  dv rd}| j                  |j                  |j                  |j                  j                  |j                  j
                  |j                  j                  |       | j                  |j                  j                   |j"                         |S )	NTFr     head_dim)      ry  )mp_num)r
   rc  r   rX   r  r  r   r   r  r   r  r   r   r  n_embdr   r   r   )r9   r]   r`   r  s       r#   r|   zCodeGenLoader.get_model_spec  s    ;;GGLL  LL-ell.N.NO||..#"" H 	
 5<<,1F1F*1T FLLLL##LLLL 	 	
 	//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zCodeGenLoader.get_vocabulary  sX    'y9LL++c&k9	y! 	/AMM/A-.	/ r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zCodeGenLoader.set_vocabulary   rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zCodeGenLoader.set_config#  rs  r%   c           
      "   d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        j                  d|dz        j                  dd      j                  j                         j                         }||z  }t        j                  |D 	cg c]!  }	t        j                  |	|z  |	dz   |z        # c}	      }
t        |j                   |j"                        D ]  \  }}| j	                  |j$                  |j&                         |j(                  j*                  j,                  }||
d d f   }|j/                  dd      \  }}}t1        j2                  |||      }t1        j2                  |||      }t        j                  |||f      |j4                  j6                  d   _        | j9                  |j4                  j6                  d   |j(                  j:                         | j9                  |j<                  j>                  |j@                  jB                         | j9                  |j<                  jD                  |j@                  jF                          y c c}	w )NFr   r
  r  r   dim)$r  r   r   r  r   r   r  nparangereshapeTflattentolistrB   r  r   r   r  r  r  r  qkv_projr   chunkr   r  r   r   r   r  r   r   r  r  r  r  )r9   r`   r   r  r  	embed_dimr  base_permutation	local_dimrJ  permutationr  r   r  new_qkv_projr  r  r  s                     r#   r   zCodeGenLoader.set_decoder(  s    %DOOVZZ8DOOV[[999Q
3;;BBDDLLNUUW'	iiGWX!U\\!i-!a%9)<=X
 "%TZZ!: 	GJ
 < <ejjI zz**11H $KN3L%++A1+5JBB 00Y
KB00Y
KB9>BB<9PJ%%,,Q/6OOJ55<<Q?ATATUOOJNN33UYY__EOOJNN33UYY5E5EF-	G Ys   ;&JrH   ro   rp   r   rO   r|   r   r   r~   r   r$  r%  s   @r#   r  r    s/    $ $<)/
!Gr%   r  GPTNeoXConfigc                   F     e Zd Zed        Zd Z fdZd Zd Zd Z	 xZ
S )GPTNeoXLoaderc                      y)NGPTNeoXForCausalLMr&   rw   s    r#   rO   zGPTNeoXLoader.architecture_nameN  r  r%   c                 n   t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                     t        |j                  j                  |j                  j                  |j                  j
                  z  z        d|j                  j                  d      }| j                  |j                  |j                  |j                  j
                         | j                  |j                  j                   |j"                         |S r  )r
   rc  r   rX   rd  re  r   
hidden_actint
rotary_pctrh  use_parallel_residualr   r   gpt_neoxr   r   	embed_outri  s      r#   r|   zGPTNeoXLoader.get_model_specR  s    ;;GGLL**LL,,-ell.E.EF''<<++u||/O/OOQ $#ll@@# H 
 	u~~u||7W7WX//Ar%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zGPTNeoXLoader.get_vocabularye  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zGPTNeoXLoader.set_vocabularyn  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zGPTNeoXLoader.set_configq  rs  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]l  \  }}t        |d      rM| j	                  |j                  |j                         | j	                  |j                  |j                         n`| j	                  |j                  j
                  |j                         | j	                  |j                   j
                  |j                         |j"                  j$                  j&                  }|j"                  j$                  j(                  }|j+                  |dd|j,                  d         j/                  dd      j+                  d|j,                  d         }|j+                  |dd      j/                  dd      j+                  d      }||j                  j0                  d   _        ||j                  j0                  d   _        | j3                  |j                  j0                  d   |j"                  j4                         | j3                  |j                   j6                  |j8                  j:                         | j3                  |j                   j<                  |j8                  j>                         o y )NFinput_layer_normr
  r  r   r   ) r  r   r   embed_inr   r   r  r   r   r   r   r  input_layernormpost_attention_layer_normpost_attention_layernormr   r   r  query_key_valuer   r   r  r6  swapaxesr   r   denser   r  dense_h_to_4hr  dense_4h_to_h)r9   r`   r   r  r  r   qkv_wqkv_bs           r#   r   zGPTNeoXLoader.set_decoderv  s*    %DOOV__=DOOV-D-DE!$TZZ!? 	NJz#56##J$?$?AVAVW##88%:X:X ##--88%:O:O ##NN--u/M/M OO33::EOO3388E iBB@!QU[[_- 
 MM)Q3<<QBJJ2NE9>J%%,,Q/67<J%%,,Q/4OOJ55<<Q?AVAVWOOJNN33UYY5L5LMOOJNN33UYY5L5LM=	Nr%   r  r%  s   @r#   r  r  L  s/    $ $&)/
#Nr%   r  WhisperConfigc                   f     e Zd Zed        Zd Zd Zd Z fdZd Z	 fdZ
 fdZd	 Zd
 Z xZS )WhisperLoaderc                      y)NWhisperForConditionalGenerationr&   rw   s    r#   rO   zWhisperLoader.architecture_name  rX  r%   c                    t        j                  |j                  j                  |j                  j                  |j                  j
                  |j                  j                        }| j                  |j                  |j                  j                         | j                  |j                  |j                  j                         | j                  |j                  j                  |j                         |S r   )r   WhisperSpecrX   r   r   r   decoder_attention_headsr   r   r]   r   r   r   r   proj_outri  s      r#   r|   zWhisperLoader.get_model_spec  s    ''LL''LL00LL''LL00	
 	u{{':':;u{{':':;//@r%   c                 |    g d}t        |dg       }|sg S |D cg c]  }||vr|j                  |       c}S c c}w )N)z<|endoftext|>z<|startoftranscript|>z<|translate|>z<|transcribe|>z<|startoflm|>z<|startofprev|>z<|nocaptions|>z<|notimestamps|>rD  )rN   convert_tokens_to_ids)r9   r_   non_lang_special_tokensadditional_tokensr   s        r#   _get_lang_ids_from_tokenizerz*WhisperLoader._get_lang_ids_from_tokenizer  sZ    	#
 $I/JBO I +
33 ++E2
 	
 
s   9c                    t        |dd       }|t|j                  |_        |j                  |_        t        |d      r|j                  |_        t        |d      rt        |j                  j                               |_
        nZ|j                  j                  |_        |j                  j                  |_        t        j                  |j                        |_        t        |dd       | j                  |      |_
        |j                  g|j                  j                   }|j                  j"                  }t%        t'        j(                  t+        |dz  |      t+        |                  |_        y y )Ngeneration_configalignment_heads
lang_to_idlang_idsr  )rN   suppress_tokenssuppress_idsbegin_suppress_tokenssuppress_ids_beginr   r  rL   r  valuesr  rX   _WHISPER_ALIGNMENT_HEADSrI   name_or_pathr  r   r  r!  	itertoolsproductr  )r9   rX   r]   r_   
gen_config
num_layersr  s          r#   r~   zWhisperLoader.set_config  s/   U$7>
!","<"<F(2(H(HF%z#45)3)C)C&z<0"()>)>)E)E)G"H"',,">">F(-(J(JF%%=%A%A%BTBT%UF"6:t,4"??	JFO!!)44J<<I%)!!*/:6)$&F"	 *r%   c           	          t         |   ||      }|j                  d t        |j                  j
                  t        |      z
        D               |S )Nc              3   ,   K   | ]  }d |dz  z    yw)z<|%.2f|>g{Gz?Nr&   ).0rJ  s     r#   	<genexpr>z/WhisperLoader.get_vocabulary.<locals>.<genexpr>  s!      
 !d(#
s   )r   r   extendr  rX   r   r   r   s       r#   r   zWhisperLoader.get_vocabulary  sN    'y9 	 
5<<22S[@A
 	

 r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zWhisperLoader.set_vocabulary  rp  r%   c                     | j                  |j                  |j                         | j                  |j                  |j                         t        |   ||       y r   )
set_conv1dconv1conv2r   r   )r9   r`   r   rG   s      r#   r   zWhisperLoader.set_encoder  s>    

GMM2

GMM2D'*r%   c                 r    | j                  |j                  |j                         t        |   ||       y r   )r   r   r"  r   r   r2  s      r#   r   zWhisperLoader.set_decoder  s,    DOOW-A-ABD'*r%   c                     | j                  |j                  |j                         | j                  |j                  |j                         y r   )r   r  r  r   r   r   s      r#   r   zWhisperLoader.set_common_layers  s8    ##D$;$;V=S=STDOOV->->?r%   c                 H    |j                   |_         |j                  |_        y r   r   r   r   s      r#   r#  zWhisperLoader.set_conv1d  s    mmKK	r%   )rH   ro   rp   r   rO   r|   r  r~   r   r   r   r   r   r#  r$  r%  s   @r#   r   r     sC    1 1
,:	)+
+@ r%   r   Wav2Vec2Configc                   ^     e Zd Zed        Zd Zd Zd Zd Zd Z	d Z
d Z fd	Zd
 Z xZS )Wav2Vec2Loaderc                      y)NWav2Vec2ForCTCr&   rw   s    r#   rO   z Wav2Vec2Loader.architecture_name  r`  r%   c                 *   t        |j                  j                  dd      }t        j                  |j                  j                  j
                  |j                  j                  j                  j                  |j                  j                  j                  j                  |j                  j                  j                  d   |      }|j                  j                  j                  D ]u  }|j                  |_        |j                  |_        |j"                  j$                  |_        |j"                  j(                  |_        |j"                  j,                  |_        w | j1                  |j                  ||j                  j                         |S Nreturn_hiddenFr   )rN   wav2vec2rX   r   Wav2Vec2Specnum_feat_extract_layersr   rd  re  r   r   r6  r   r  r   r   r   feed_forwardintermediate_act_fnactivation_fnintermediate_denser   output_denser  r   )r9   r]   r1  r`   r   s        r#   r|   zWav2Vec2Loader.get_model_spec	  s%    5 5N))NN!!99NN""));;NN""))==MM  &&q)
 ^^++22 	8E#ooEO).)9)9E&"'"4"4"H"HE**==EI**77EI	8 	uenn.C.CDr%   c                      y r   r&   r   s       r#   r~   zWav2Vec2Loader.set_config      r%   c                 "    |j                         S r   r   r9   r]   r_   s      r#   r   zWav2Vec2Loader.get_vocabulary!      ""$$r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zWav2Vec2Loader.set_vocabulary$  rp  r%   c                    |j                   d   j                  j                  |j                  j                  _        |j                   d   j                  j                  |j                  j                  _        | j                  |j                  j                  |j                   d   j                         t        |j                  |j                   dd        D ]u  \  }}|j                  j                  |j                  _        |j                  j                  |j                  _        | j                  |j                  |j                         w y )Nr   r   )	conv_layersconvr   feat_layer0r   r   r   r   
feat_layer)r9   r`   feature_extractor
spec_layermodule_layers        r#   set_feature_extractorz$Wav2Vec2Loader.set_feature_extractor'  s   '8'D'DQ'G'L'L'S'S$%6%B%B1%E%J%J%O%O"''):)F)Fq)I)T)T	
 ),OO.::12>)
 	P$J &2%6%6%=%=JOO"#/#4#4#9#9JOO 
 5 5|7N7NO	Pr%   c                     | j                  |j                  |j                         | j                  |j                  |j
                         y r   r   fp_layer_normr   r   fp_projectionr   r9   r`   feature_projections      r#   set_feature_projectionz%Wav2Vec2Loader.set_feature_projection4  :    D..0B0M0MN**,>,I,IJr%   c                    |j                   j                  j                  j                  j	                         |j                   j                  j                  _        |j                   j                  j
                  j	                         |j                   j                  j
                  _        |j                   j                         D ]!  }|j                  j	                         |_        # |j                  t        j                  dd|j                  f             |j                   j                  j                  |j                   j                  _        |j                   j                  j
                  |j                   j                  _        y r   )
pos_conv_embedrC  r   datafloatr   
parametersrB   randnrh  )r9   r`   r   rX   params        r#   set_pos_conv_embedz!Wav2Vec2Loader.set_pos_conv_embed8  s    ""''..3399; 	##**/ 180F0F0K0K0P0P0V0V0X##((-++668 	,E))+EJ	,u{{Aq&2D2D+EFG*1*@*@*E*E*L*L  '(/(>(>(C(C(H(H  %r%   c                    | j                  ||j                  j                         | j                  ||j                  j                         | j                  ||j                  j                  |       t        | !  ||j                  j                         t        |j                  j                  dd      }|s'| j                  |j                  |j                         y y Nr1  F)rI  r2  rF  rP  rO  rY  r   r   r   rN   rX   r   r   )r9   r`   r]   rX   r1  rG   s        r#   r   zWav2Vec2Loader.set_encoderE  s    ""4)I)IJ##D%..*K*KLenn&<&<fED%.."8"89 5 5NOODLL%--8 r%   c                 P    | j                  |j                  |j                         y r   )r   r   r   s      r#   r   z Wav2Vec2Loader.set_common_layersN  s    DOOV->->?r%   )rH   ro   rp   r   rO   r|   r~   r   r   rI  rP  rY  r   r   r$  r%  s   @r#   r,  r,    sF       *%)PKI9@r%   r,  Wav2Vec2BertConfigc                   b    e Zd Zed        Zd Zd Zd Zd Zd Z		 ddZ
d	 Zd
 Zd Zd Zd Zy)Wav2Vec2BertLoaderc                      y)NWav2Vec2BertForCTCr&   rw   s    r#   rO   z$Wav2Vec2BertLoader.architecture_nameT  r  r%   c                 j   t        |j                  j                  dd      }t        j                  |j                  j                  j
                  |j                  j                  j                  |j                  j                  j                  d   |      }| j                  |j                  |       |S r0  )rN   wav2vec2_bertrX   r   Wav2Vec2BertSpecnum_adapter_layersrd  r   r   r6  r   r   )r9   r]   r1  r`   s       r#   r|   z!Wav2Vec2BertLoader.get_model_specX  s     3 3 : :OUS 11&&99&&88MM  &&q)	
 	u-r%   c                      y r   r&   r   s       r#   r~   zWav2Vec2BertLoader.set_configc  r;  r%   c                 "    |j                         S r   r=  r>  s      r#   r   z!Wav2Vec2BertLoader.get_vocabularyf  r?  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   z!Wav2Vec2BertLoader.set_vocabularyi  rp  r%   c                     | j                  |j                  |j                         | j                  |j                  |j
                         y r   rK  rN  s      r#   rP  z)Wav2Vec2BertLoader.set_feature_projectionl  rQ  r%   Nc                    t        d      D cg c]  }t        j                          }}| j                  |d   |j                         | j                  |d   |j
                         | j                  |d   |j                         t        j                  |j                  d   |       | j                  |j                  d   |j                         |s|rn|j                  j                  |_        t        j                  d      j!                  |      |_        t        j                  d      j!                  |      |_        y y c c}w )Nr
  r   r   r  r  int32)r  r   r  r   linear_qlinear_klinear_vr   r  r   
linear_outdistance_embeddingr   !relative_asymmetric_position_keysr  r>   r   relative_left_max_positionrelative_right_max_position)r9   r`   r  left_max_positionright_max_positionr   r  s          r#   r   z Wav2Vec2BertLoader.set_attentionp  s    ;@(CQ..0CCQ););<Q););<Q););<$++a.,7B)=)=> 25>5Q5Q5X5XD2.0hhw.?.D.DEV.WD+/1xx/@/E/E"0D, !3 Ds   Ec                 `   t        ||      D ]  \  }}| j                  |j                  |j                         | j	                  |j
                  j                  |j                  j                         | j	                  |j
                  j                  |j                  j                         | j                  |j                  |j                  ||       | j                  |j                  |j                         | j                  |j                   |j"                  j$                         | j'                  |j(                  |j"                  j*                         | j'                  |j,                  |j"                  j.                         | j                  |j0                  |j"                  j2                         | j'                  |j4                  |j"                  j6                         | j                  |j8                  |j:                         | j	                  |j<                  j                  |j>                  j                         | j	                  |j<                  j                  |j>                  j                         | j                  |j@                  |jB                          y r   )"r   r   enc_ffn1_layer_normffn1_layer_normr   enc_ffn1r   ffn1r8  r  r9  r   enc_attnr   enc_attn_layer_normr   enc_conv_layer_normconv_moduler   r#  enc_conv_pointwise_conv1pointwise_conv1enc_conv_depthwise_convdepthwise_convenc_conv_depthwise_layer_normdepthwise_layer_normenc_conv_pointwise_conv2pointwise_conv2enc_ffn2_layer_normffn2_layer_normenc_ffn2ffn2enc_final_layer_normr  )r9   spec_layersr   rt  ru  slayerr   s          r#   set_wav2vec2bert_encoderz+Wav2Vec2BertLoader.set_wav2vec2bert_encoder  s    !f5 	UMFE : :E<Q<QROOFOO44ejj6S6STOOFOO44ejj6M6MN2CEW  : :E<V<VW**E,=,=,H,H OO//1B1B1R1R OO..0A0A0P0P 44!!66 OO//1B1B1R1R  : :E<Q<QROOFOO44ejj6S6STOOFOO44ejj6M6MN ; ;U=S=ST7	Ur%   c                    t        ||      D ]^  \  }}| j                  |j                  |j                         | j	                  |j
                  |j                         | j                  |j                  |j                         | j	                  |j                  |j                         | j                  |j                  |j                         | j                  |j                  |j                         | j!                  |j"                  j$                  |j&                  j(                         | j!                  |j"                  j*                  |j&                  j,                         a y r   )r   r   adpt_residual_layer_normresidual_layer_normr#  adpt_residual_convresidual_convadpt_attn_layer_normr   adpt_attn_convself_attn_convr   adpt_attn_layerr   adpt_ffn_layer_normffn_layer_normr   adpt_ffnr   r   r8  r  r9  )r9   r  r   r  r   s        r#   set_wav2vec2bert_adapterz+Wav2Vec2BertLoader.set_wav2vec2bert_adapter  s	    f5 
	NMFE//1J1J OOF55u7J7JK ; ;U=W=WXOOF1153G3GHv55uG : :E<P<PQOOFOO44eii6R6RSOOFOO44eii6L6LM
	Nr%   c                 H   | j                  ||j                  j                         | j                  |j                  |j                  j
                  j                  |j                  j                  j                  |j                  j                  j                         | j                  |j                  |j                  j                  j                         t        |j                  j                  dd      }|s'| j                  |j                  |j                         y y r[  )rP  rc  rO  r  r   r   r   rX   left_max_position_embeddingsright_max_position_embeddingsr  adapter_layersadapterrN   r   r   )r9   r`   r]   r1  s       r#   r   zWav2Vec2BertLoader.set_encoder  s    ##D%*=*=*P*PQ%%''..&&CC&&DD		
 	%%!4!4!<!<!C!C	
   3 3 : :OUSOODLL%--8 r%   c                 b    |j                   |_         |j                  |j                  |_        y y r   r)  r   s      r#   r#  zWav2Vec2BertLoader.set_conv1d  s'    mm;;"DI #r%   c                 b    |j                   |_        |j                  |j                  |_        y y r   r   r   s      r#   r   z!Wav2Vec2BertLoader.set_layer_norm  s'    ]]
;;"DI #r%   )NN)rH   ro   rp   r   rO   r|   r~   r   r   rP  r   r  r  r   r#  r   r&   r%   r#   r_  r_  R  sV    $ $	%)K
 KO U@N9$
$r%   r_  T5Configc                   h     e Zd Zed        Zd Z fdZd Zd ZddZ	d Z
d Zd	 Zdd
Zd Z xZS )T5Loaderc                      y)NT5ForConditionalGenerationr&   rw   s    r#   rO   zT5Loader.architecture_name  s    +r%   c           	         t         j                  j                  |j                  j                  |j                  j
                  f|j                  j                  dt        |j                  j                     |j                  j                  dd      }| j                  |j                  |j                         | j                  |j                  |j                  d       | j                  |j                  j                  |j                         |j                  j                   r(|j                  j"                  dz  |j                  _        |S )NT)r   r   ffn_glurelative_attention_biasrms_norm)
is_decoderg      )r
   r   r   rX   r  num_decoder_layersr  r   dense_act_fnis_gated_act	set_stackr   r   r   r   r   tie_word_embeddingsr  scale_outputsri  s      r#   r|   zT5Loader.get_model_spec  s    //;;\\$$ell&E&EFLL""-ell.G.GHLL--$( < 
 	t||U]]3t||U]]tD//?<<++).)=)=t)CDLL&r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zT5Loader.get_vocabulary  r  r%   c                 H    |j                  |       |j                  |       y r   r   r   s      r#   r   zT5Loader.set_vocabulary  r   r%   c                    |j                   |_        |j                  |_        |j                  |_        t	        |j
                  d      r+|j                  |j
                  j                        |_        y |j                   |_        y )Nr   )	rZ  r   r   r   r   rX   r   r   r   r   s       r#   r~   zT5Loader.set_config  sk    $..$..$..5<<!9:)2)H)H33*F& *3)<)<F&r%   c                    | j                  |j                  |j                         | j                  t	        |j
                  t              r|j
                  d   n|j
                  |j                         d|_        t        t        |j                  |j                              D ]  \  }\  }}| j                  |j                  |j                  d          |dkD  rO|j                  d   j                  }|j                  |j                  _        |j                   |j                  _        |r)| j#                  |j$                  |j                  d          | j'                  |j(                  |j                  d           y )Nr   Fr   r  )r   r   r  r   r   r   r!  r"  r  rk  r   r   blockset_self_attentionr   r  relative_attention_max_distanceset_cross_attentionr  set_ffnr   )r9   r`   r   r  rJ  r  r  first_self_attentions           r#   r  zT5Loader.set_stack  s5   DOOV-D-DE doot4 "__	
 !&&/DJJ0M&N 	:"A"
E##J$=$=u{{1~N1u'+zz!}'C'C$(@@ ))A )HH ))I (()=)=u{{1~NLLR9!	:r%   c                    t        |d      ra| j                  |j                  |j                  j                         | j                  |j
                  |j                  j                         n0| j                  |j                  |j                  j                         | j                  |j                  |j                  j                         | j                  |j                  |j                         y )Nlinear_0_noact)r   r   r   DenseReluDensewi_0r  wi_1wir  wor   r   r   s      r#   r  zT5Loader.set_ffn  s    4)*OODMM6+@+@+E+EFOOD//1F1F1K1KLOODMM6+@+@+C+CDv'<'<'?'?@DOOV->->?r%   c                     | j                  ||j                  d       | j                  |j                  |j                         y r   )r   SelfAttentionr   r   r   s      r#   r  zT5Loader.set_self_attention"  s7    4!5!5dKDOOV->->?r%   c                     | j                  ||j                         | j                  |j                  |j                         y r   )r   EncDecAttentionr   r   r   s      r#   r  zT5Loader.set_cross_attention&  s2    4!7!78DOOV->->?r%   c                    d|_         t        d      D cg c]  }t        j                          }}| j	                  |d   |j
                         | j	                  |d   |j                         | j	                  |d   |j                         |r$t        j                  |j                  d   |       nLt        j                  |j                  d   |d d        t        j                  |j                  d   |dd         | j	                  |j                  d   |j                         |j                  rO|j                  j                  |_        t        j                   d      j#                  |j$                        |_        y y c c}w )Nr  r
  r   r   r  r  rk  )queries_scaler  r   r  r   qkvr   r  r   ohas_relative_attention_biasr  r   r  r>   r   r  r  s         r#   r   zT5Loader.set_attention*  s,    :?(CQ..0CCQ5Q5Q5dkk!nl;dkk!nl2A.>?dkk!nl12.>?B500+4+L+L+S+SD(3588G3D3I3I994D0 1 Ds   Fc                 &    |j                   |_        y r   r   r   r9   r`   r   s      r#   r   zT5Loader.set_layer_norm@      &&
r%   r#  )rH   ro   rp   r   rO   r|   r   r   r~   r  r  r  r  r   r   r$  r%  s   @r#   r  r    sJ    , ,(0	=:>@@@,'r%   r  	MT5Configc                       e Zd Zed        Zy)	MT5Loaderc                      y)NMT5ForConditionalGenerationr&   rw   s    r#   rO   zMT5Loader.architecture_nameF  s    ,r%   N)rH   ro   rp   r   rO   r&   r%   r#   r  r  D  s    - -r%   r  BloomConfigc                   L     e Zd Zed        Zd Z fdZd Zd Zd Z	d Z
 xZS )BloomLoaderc                      y)NBloomForCausalLMr&   rw   s    r#   rO   zBloomLoader.architecture_nameM      !r%   c           	      ~   t         j                  j                  |j                  j                  |j                  j
                  dt        j                  j                  ddd      }| j                  |j                  |j                         | j                  |j                  j                  |j                         |S )NT)r   r   r   alibialibi_use_positive_positions)r
   rc  r   rX   r  r  r   
ActivationGELUTanhr   r   r  r   r   r   ri  s      r#   r|   zBloomLoader.get_model_specQ  s    ;;GGLL  LL"--66 $)- H 
 	u'8'89//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zBloomLoader.get_vocabulary`  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zBloomLoader.set_vocabularyi  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zBloomLoader.set_configl  rs  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         | j	                  |j                  |j                         t        |j                  |j                        D ]i  \  }}| j	                  |j                  j                  |j                         | j                  |j                  j                  d   |j                  j                   |j                  j"                         | j%                  |j                  j                  d   |j                  j&                         | j	                  |j(                  j                  |j*                         | j%                  |j(                  j,                  |j.                  j0                         | j%                  |j(                  j2                  |j.                  j4                         l y r  )r  r   r   word_embeddingsr   r   word_embeddings_layernormr   r  r   r   r  r   r  set_qkv_linearr   r  r  r   r  r   r  r   r  r  r  r  r  s        r#   r   zBloomLoader.set_decoderq  s    %DOOV-C-CDD44f6V6VWDOOV[[9!$TZZ!: 	NJ))44e6K6K ))003$$44$$..
 OO))003U5I5I5O5O ))5+I+I OOJNN33UYY5L5LMOOJNN33UYY5L5LM#	Nr%   c                 `   |j                   }|j                  |dd|j                  d         }|j                  dd      }|j                  d|j                  d         }|j                  }|j                  |dd      }|j                  dd      }|j                  d      }||_         ||_        y )Nr
  r  r   r   )r   r  r6  r   r   )r9   r`   r   r  r   r   s         r#   r  zBloomLoader.set_qkv_linear  s    	1b&,,r2BC!!!Q'FLL$45{{||Iq"-~~a#||B	r%   )rH   ro   rp   r   rO   r|   r   r   r~   r   r  r$  r%  s   @r#   r  r  K  s4    " ")/
N2r%   r  	MPTConfigc                   L     e Zd Zed        Zd Z fdZd Zd Zd Z	d Z
 xZS )	MPTLoaderc                      yNAutoModelForCausalLMr&   rw   s    r#   rO   zMPTLoader.architecture_name      %r%   c                    t         j                  j                  |j                  j                  |j                  j
                  dt        j                  j                  d      }| j                  |j                  |j                         |S )NT)r   r   r  )r
   rc  r   rX   n_layersn_headsr   r  GELUr   r   r  ri  s      r#   r|   zMPTLoader.get_model_spec  sn    ;;GGLL!!LL  "--22 H 
 	u'8'89r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zMPTLoader.get_vocabulary  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zMPTLoader.set_vocabulary  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zMPTLoader.set_config  rs  r%   c                    | j                  |j                  |j                         | j                  |j                  |j
                         d|_        |j                  j                  |j                  _        t        |j                  |j                        D ]T  \  }}| j                  |j                  j                  |j                         | j                  |j                  j                  d   |j                   j"                         | j                  |j                  j                  d   |j                   j$                         | j                  |j&                  j                  |j(                         | j                  |j&                  j*                  |j&                  j,                         | j                  |j&                  j.                  |j&                  j0                         W y r  )r   r   r  r   r   norm_fr  r   r   r   r   blocksr   norm_1r   r   r  Wqkvr  r   norm_2r   up_projr  	down_projr  s        r#   r   zMPTLoader.set_decoder  sG   DOOVZZ8DOOV]]; %!%!7!7!$TZZ!? 	JJ
 9 9 D DellSOOJ55<<Q?QOOJ55<<Q?ATATU
 9 95<<HOOJNN33UYY5F5FGOOJNN33UYY5H5HI	Jr%   c                 n    |j                   |_        t        j                  |j                        |_        y r   )r   r   rB   
zeros_liker   r   s      r#   r   zMPTLoader.set_layer_norm  s#    ]]
$$TZZ0	r%   )rH   ro   rp   r   rO   r|   r   r   r~   r   r   r$  r%  s   @r#   r  r    s4    & &
)/
J 1r%   r  GemmaConfigc                   L     e Zd Zed        Zd Z fdZd Zd Zd Z	d Z
 xZS )GemmaLoaderc                      y)NGemmaForCausalLMr&   rw   s    r#   rO   zGemmaLoader.architecture_name  r  r%   c                    |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }t        |j                   dd      }t        j
                  j                  |||dk(  rt        j                  j                  nt        j                  j                  dddddt        |j                   dd	      ||j                   j                  
      }| j                  |j                  |j                         | j                  |j                  j                   |j"                         |j                   j$                  dz  |j                  j&                  _        |S )Nnum_key_value_headshidden_activationr   r   Tr   Fr   '  )	r   r   r  r  r  r  rotary_basenum_heads_kvr        ?rX   rd  re  rN   r
   rc  r   r   r  r  r  r  r   r   r]   r   r   r   rh  r   multiply_by_sqrt_depthr9   r]   r  r  r  activation_configr`   s          r#   r|   zGemmaLoader.get_model_spec  s2   \\33
LL44	u||-BIN9$L#LL-/B
  ;;GG %. &&++ ++44#lEB%\\** H 
$ 	u{{3//?9>9Q9QSV9V6r%   c                 *   t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |j                  j                  t	        |      k  r|d |j                  j                   }|S r  r  r  s         r#   r   zGemmaLoader.get_vocabulary      'y9LL++c&k9	y! 	/AMM/A-.	/<<""S[05ell556Fr%   c                 &    |j                  |       y r   rn  r   s      r#   r   zGemmaLoader.set_vocabulary  rp  r%   c                     |j                   |_         |j                  |_        |j                  |_        |j                  j                  |_        y r   r   r   r   rX   rms_norm_epslayer_norm_epsilonr   s       r#   r~   zGemmaLoader.set_config  >    $..$..$..$)LL$=$=!r%   c                 4    |j                   |_        d|_        y r0  r   r   layer_norm_use_residualr  s      r#   r   zGemmaLoader.set_layer_norm      &&
'+$r%   c                 $   d|_         d|_        | j                  |j                  |j                         | j                  |j                  |j                         t        |j                  |j                        D ]  \  }}| j                  |j                  j                  |j                         | j                  |j                  j                  |j                         |j                  j                   j"                  }|j                  j$                  j"                  }|j                  j&                  j"                  }|j                  j(                  j"                  }t+        j,                  |||g      |j                  j.                  d   _        ||j                  j.                  d   _        | j1                  |j                  j2                  |j4                  j6                         | j1                  |j                  j8                  |j4                  j:                         | j1                  |j                  j<                  |j4                  j>                         tA        |d       tA        |d       tC        jD                           y NTFr   r   r   r  )#r  r1  r   r   r"  r   r   normr   r   r   r   r  r   r  r   r  r   r  r  o_projrB   r  r   r   r   r  	gate_projr  r  r  r   delattrgccollect	r9   r`   r   r  r   wqwkwvr  s	            r#   r   zGemmaLoader.set_decoder  s    $).&DOOV-@-@ADOOV[[9!$TZZ!? 	J))44e6K6K ))5+I+I ''..B''..B''..B''..B9>BB<9PJ%%,,Q/69;J%%,,Q/6OOJNN33UYY5H5HIOOJNN99599;L;LMOOJNN33UYY5H5HIE;'E5!JJL-	r%   rH   ro   rp   r   rO   r|   r   r   r~   r   r   r$  r%  s   @r#   r  r    s4    " "!F	)>,r%   r  Gemma2Configc                   L     e Zd Zed        Zd Z fdZd Zd Zd Z	d Z
 xZS )Gemma2Loaderc                      y)NGemma2ForCausalLMr&   rw   s    r#   rO   zGemma2Loader.architecture_name3      "r%   c                    |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }t        |j                   dd      }t        j
                  j                  |||dk(  rt        j                  j                  nt        j                  j                  dddddt        |j                   dd	      ||j                   j                  d
      }| j                  |j                  |j                         | j                  |j                  j                   |j"                         |j                   j$                  dz  |j                  j&                  _        |S )Nr	  r
  r   r   Tr   Fr   r  )
r   r   r  r  r  r  r  r  r  pre_post_layer_normr  r  r  s          r#   r|   zGemma2Loader.get_model_spec7  s5   \\33
LL44	u||-BIN9$L#LL-/B
  ;;GG %. &&++ ++44#lEB%\\** $! H 
& 	u{{3//?9>9Q9QSV9V6r%   c                 *   t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |j                  j                  t	        |      k  r|d |j                  j                   }|S r  r  r  s         r#   r   zGemma2Loader.get_vocabulary[  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zGemma2Loader.set_vocabularyf  rp  r%   c                     |j                   |_         |j                  |_        |j                  |_        |j                  j                  |_        y r   r  r   s       r#   r~   zGemma2Loader.set_configi  r  r%   c                 4    |j                   |_        d|_        y r0  r  r  s      r#   r   zGemma2Loader.set_layer_normo  r  r%   c                    d|_         d|_        | j                  |j                  |j                         | j                  |j                  |j                         t        |j                  |j                        D ]J  \  }}| j                  |j                  |j                         | j                  |j                  |j                         | j                  |j                  |j                          | j                  |j"                  |j$                         |j&                  j(                  j*                  }|j&                  j,                  j*                  }|j&                  j.                  j*                  }|j&                  j0                  j*                  }t3        j4                  |||g      |j6                  j8                  d   _        ||j6                  j8                  d   _        | j;                  |j<                  j>                  |j@                  jB                         | j;                  |j<                  jD                  |j@                  jF                         | j;                  |j<                  jH                  |j@                  jJ                         tM        |d       tM        |d       tO        jP                          M y r   ))r  r1  r   r   r"  r   r   r!  r   r   r   r  r  r  r  pre_feedforward_layer_normpre_feedforward_layernormpost_feedforward_layer_normpost_feedforward_layernormr   r  r   r  r  r"  rB   r  r   r   r   r   r   r  r#  r  r  r  r   r$  r%  r&  r'  s	            r#   r   zGemma2Loader.set_decoders  s    $).&DOOV-@-@ADOOV[[9!$TZZ!? 	J
 ; ;U=R=RS44e6T6T 55u7V7V 668X8X ''..B''..B''..B''..B9>BB<9PJ%%,,Q/69;J%%,,Q/6OOJNN33UYY5H5HIOOJNN99599;L;LMOOJNN33UYY5H5HIE;'E5!JJL;	r%   r+  r%  s   @r#   r.  r.  1  s4    # #"H	)>,#r%   r.  LlamaConfigc                   x     e Zd Zed        Zd Z fdZd Zd Zd Z	e
j                  j                  fdZ xZS )LlamaLoaderc                      y)NLlamaForCausalLMr&   rw   s    r#   rO   zLlamaLoader.architecture_name  r  r%   c                 ~   |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }| j	                  |j                   d      \  }}}t        |j                   dd       }|rd }	|j
                  dk(  rt        j                  |j                        }	|	<t        d|j
                  ddj                  t        j                                     |j                  }
|j                  }nt        j                  j                   }	d }
d }t"        j$                  j'                  ||t        j(                  j*                  dddd	d
|||||	|
|      }| j-                  |j.                  |j0                  |	       | j3                  |j.                  j4                  |j6                         t        |j                   dd       }|t8        j:                  j<                  k(  rC|j.                  j>                  D ]*  }|d   |j@                  _!        |d   |j@                  _"        , |S )Nr	  r  quantization_configawqQuantization type 'T' is not yet implemented. The following Quantization types are currently supported: r<   Tr   Fr   r   r  r  r  r  r   r   r  r  r   quant_group_size
quant_bitsr   low_freq_factorhigh_freq_factor)#rX   rd  re  rN   r   quant_method_SUPPORTED_QUANTIZATIONrI   versionrz   rK   rM   
group_sizebitsr   r   r   r
   rc  r   r  SWISHr   r   r]   r   r   r   r   RotaryScalingTypeLlama3r   r   rotary_low_freq_factorrotary_high_freq_factor)r9   r]   r  r  r  r   r   r   rC  r   rH  rI  r`   r   r   s                  r#   r|   zLlamaLoader.get_model_spec  s   \\33
LL44	u||-BIN9$LAEAWAWLL&B
>2J &ell4I4PJ"//584889L9T9TU
!) ,88		"9">">"@A	   3==,11J$1155J#J;;GG"--33# 3"7"%!-! H 
$ 	u{{J?//? u||^TB."B"B"I"II++ >J%?$$; @L&@$$<	 r%   c                 *   t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |j                  j                  t	        |      k  r|d |j                  j                   }|S r  r  r  s         r#   r   zLlamaLoader.get_vocabulary  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zLlamaLoader.set_vocabulary  rp  r%   c                     |j                   |_         |j                  |_        |j                  |j                  nd|_        |j                  j                  |_        y N r  r   s       r#   r~   zLlamaLoader.set_config  sP    $..$..#,#6#6#BI 	 %*LL$=$=!r%   c                 &    |j                   |_        y r   r  r  s      r#   r   zLlamaLoader.set_layer_norm  r  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]  \  }}| j	                  |j                  j
                  |j                         | j	                  |j                  j
                  |j                         t        d      D cg c]  }t        j                           }}| j#                  |d   |j$                  j&                  |       | j#                  |d   |j$                  j(                  |       | j#                  |d   |j$                  j*                  |       |t        j,                  j.                  k(  r.t1        j2                  |j                  j4                  d   |       nO|t        j,                  j6                  k(  rdnd}t1        j8                  |j                  j4                  d   ||       | j#                  |j                  j4                  d   |j$                  j:                  |       | j#                  |j                  j<                  |j>                  j@                  |       | j#                  |j                  jB                  |j>                  jD                  |       | j#                  |j                  jF                  |j>                  jH                  |       tK        |d       tK        |d       tM        jN                           y c c}w 	NFr
  r   r   r   r  r   r  (r  r   r   r"  r   r   r!  r   r   r   r   r  r   r  r  r   r  r   r   r  r  r  r   r   r   r  r   AWQ_GEMMfuse_linear_prequantr"  r   r  r#  r  r  r  r   r$  r%  r&  	r9   r`   r   r   r  r   r   r  cc_dims	            r#   r   zLlamaLoader.set_decoder  s    %DOOV-@-@ADOOV[[9!$TZZ!? ,	J))44e6K6K ))5+I+I ?DAhGK224GLGOOQ!7!7J   OOQ!7!7J   OOQ!7!7J   [55999!!*";";"B"B1"E|T(K,D,D,M,MMST**--44Q7v OO))003&&%   OO'')<)<   OO--uyy/@/@Z   OO'')<)<   E;'E5!JJLY,	 H   (MrH   ro   rp   r   rO   r|   r   r   r~   r   r   r   r   r   r$  r%  s   @r#   r?  r?    sF    " "@D	)>' 4?3K3K3O3O 1r%   r?  Gemma3TextConfigGemma3Configc                   x     e Zd Zed        Zd Z fdZd Zd Zd Z	e
j                  j                  fdZ xZS )Gemma3Loaderc                      y)NGemma3ForCausalLMr&   rw   s    r#   rO   zGemma3Loader.architecture_name1  r1  r%   c                 n   |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }|j                   j                  }t        |j                   dd      }t        |j                   dd      }t        |j                   dd      }t        |j                   dd	      }	t        |j                   d
d       }
t        |j                   dd       }|ra|j
                  dk(  rt        j                  |j                        }t        d|j
                  z        |j                  }|j                  }nt        j                  j                  }d }d }t        j                   j#                  |||dk(  rt        j$                  j&                  nt        j$                  j(                  ddd|d||||	d|||d      }|
| _        t-        |
      D ]  \  }}|j.                  j0                  |   }|dk(  rgt3        j4                  d      j7                  |      |j8                  _        t3        j4                  d      j7                  d      |j8                  _        |dk(  st3        j4                  d      j7                  |      |j8                  _        t3        j4                  d      j7                  |	      |j8                  _         | j?                  |j.                  |j@                  |       | jC                  |j.                  jD                  |jF                         |S )Nr	  r
  r   r   @B rope_local_base_freqr  sliding_windowi   layer_typesrC  rD  z.Quantization type '%s' is not yet implemented.r   TF)r   r   r  r  r  r  r  r  r  ro  r3  r   rH  rI  qk_normfull_attentionfloat32rk  r   sliding_attention)$rX   rd  re  rN   r  rL  rM  rI   rN  rz   rO  rP  r   r   r   r
   rc  r   r  r  r  _layer_typesrk  r   r   r  r>   r   r   r  ro  r   r]   r   r   r   )r9   r]   r  r  r  r  r  r   rn  ro  rp  rC  r   rH  rI  r`   rJ  
layer_typer   s                      r#   r|   zGemma3Loader.get_model_spec5  s   \\33
LL44	u||-BIN9$L<<((#LL-/B

 U\\<C
&LL0& 

 !/?FellM4@%ell4I4P"//584889L9T9TU
!)D)667   3==,11J$1155J#J  ;;GG %. &&++ ++44#,%) $!-!+ H 
2 ( '{3 	MAzLL&&q)E--3588I3F3K3KJ3W$$068hhw6G6L6LQ6O$$3223588I3F3K3K(4$$0 79hhw6G6L6L"7$$3	 	u{{J?//?r%   c                 *   t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |j                  j                  t	        |      k  r|d |j                  j                   }|S r  r  r  s         r#   r   zGemma3Loader.get_vocabulary  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zGemma3Loader.set_vocabulary  rp  r%   c                     |j                   |_         |j                  |_        t        |d      r<t        |j                  t
              r"|j                  j                         rd|_        y |j                  |_        y )Nchat_templatez<end_of_turn>)r   r   r   r   rz  rr   stripr   r   s       r#   r~   zGemma3Loader.set_config  sb    $..$.. I/922C8''--/.F(22Fr%   c                 4    |j                   |_        d|_        y r0  r  r  s      r#   r   zGemma3Loader.set_layer_norm  r  r%   c                    d|_         d|_        | j                  |j                  |j                         | j                  |j                  |j                         t        |j                  |j                        D ]v  \  }}| j                  |j                  |j                         | j                  |j                  |j                         | j                  |j                  |j                          | j                  |j"                  |j$                         | j                  |j&                  j(                  |j*                  j(                         | j                  |j&                  j,                  |j*                  j,                         t/        d      D cg c]  }t1        j2                          }}| j5                  |d   |j*                  j6                  |       | j5                  |d   |j*                  j8                  |       | j5                  |d   |j*                  j:                  |       |t0        j<                  j>                  k(  r.tA        jB                  |j&                  jD                  d   |       nO|t0        j<                  jF                  k(  rdnd}tA        jH                  |j&                  jD                  d   ||       | j5                  |j&                  jD                  d   |j*                  jJ                  |       | j5                  |jL                  jN                  |jP                  jR                  |       | j5                  |jL                  jT                  |jP                  jV                  |       | j5                  |jL                  jX                  |jP                  jZ                  |       t]        |d       t]        |d	       t_        j`                          y y c c}w )
NTFr
  r   r^  r   r  r   r  )1r  r1  r   r   r"  r   r   r!  r   r   r   r  r  r  r  r9  r:  r;  r<  r   q_normr   k_normr  r   r  r   r  r  r  r   r   r   r  r   r`  ra  r"  r   r   r  r#  r  r  r  r   r$  r%  r&  rb  s	            r#   r   zGemma3Loader.set_decoder  s2    $).&DOOV-@-@ADOOV[[9!$TZZ!? >	J
 ; ;U=R=RS44e6T6T 55u7V7V 668X8X
 ))00%//2H2H ))00%//2H2H
 ?DAhGK224GLGOOQ!7!7J   OOQ!7!7J   OOQ!7!7J   [55999!!*";";"B"B1"E|T(K,D,D,M,MMST**--44Q7v OO))003&&%   OO'')<)<   OO--uyy/@/@Z   OO'')<)<   E;'E5!JJL}>	0 Hs   O6re  r%  s   @r#   ri  ri  .  sI     # #Sj	)3, 4?3K3K3O3O Dr%   ri  MistralConfigc                   x     e Zd Zed        Zd Z fdZd Zd Zd Z	e
j                  j                  fdZ xZS )MistralLoaderc                      y)NMistralForCausalLMr&   rw   s    r#   rO   zMistralLoader.architecture_name  r  r%   c                    |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }t        |j                   dd      }| j	                  |j                   d      \  }}}t        |j                   dd       }	|	r|	j
                  dk(  rt        j                  |	j                        }

<t        d|	j
                  dd	j                  t        j                                     |	j                  }|	j                  }nt        j                  j                   }
d }d }t"        j$                  j'                  ||t        j(                  j*                  d
d
d
dd||||||
|||j                   j,                        }| j/                  |j0                  |j2                  |
       | j5                  |j0                  j6                  |j8                         |S )Nr	  ro  r   r  rC  rD  rE  rF  r<   TF)r   r   r  r  r  r  r   r   r  r  ro  r   rH  rI  r  r^  )rX   rd  re  rN   r   rL  rM  rI   rN  rz   rK   rM   rO  rP  r   r   r   r
   rc  r   r  rQ  r  r   r   r]   r   r   r   )r9   r]   r  r  r  ro  r   r   r   rC  r   rH  rI  r`   s                 r#   r|   zMistralLoader.get_model_spec  s   \\33
LL44	u||-BIN9$L /?CAEAWAWLL&B
>2J &ell4I4P"//584889L9T9TU
!) ,88		"9">">"@A	   3==,11J$1155J#J;;GG"--33# 3"7"%)!-!\\**# H 
( 	u{{zJ//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zMistralLoader.get_vocabulary0  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zMistralLoader.set_vocabulary9  rp  r%   c                     |j                   |_         |j                  |_        |j                  |_        |j                  j                  |_        y r   r  r   s       r#   r~   zMistralLoader.set_config<  r  r%   c                 &    |j                   |_        y r   r  r  s      r#   r   zMistralLoader.set_layer_normB  r  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]  \  }}| j	                  |j                  j
                  |j                         | j	                  |j                  j
                  |j                         t        d      D cg c]  }t        j                           }}| j#                  |d   |j$                  j&                  |       | j#                  |d   |j$                  j(                  |       | j#                  |d   |j$                  j*                  |       |t        j,                  j.                  k(  r.t1        j2                  |j                  j4                  d   |       nO|t        j,                  j6                  k(  rdnd}t1        j8                  |j                  j4                  d   ||       | j#                  |j                  j4                  d   |j$                  j:                  |       | j#                  |j                  j<                  |j>                  j@                  |       | j#                  |j                  jB                  |j>                  jD                  |       | j#                  |j                  jF                  |j>                  jH                  |       tK        |d       tK        |d       tM        jN                           y c c}w r]  r_  rb  s	            r#   r   zMistralLoader.set_decoderE  s    %DOOV-@-@ADOOV[[9!$TZZ!? +	J))44e6K6K ))5+I+I ?DAhGK224GLGOOQ!7!7J   OOQ!7!7J   OOQ!7!7J   [55999!!*";";"B"B1"E|T(K,D,D,M,MMST**--44Q7v OO))003&&%   OO'')<)<   OO--uyy/@/@Z   OO'')<)<   E;'E5!JJLW+	 Hrd  re  r%  s   @r#   r  r    sE    $ $8t)>' 4?3K3K3O3O 0r%   r  Qwen2Configc                   x     e Zd Zed        Zd Z fdZd Zd Zd Z	e
j                  j                  fdZ xZS )Qwen2Loaderc                      y)NQwen2ForCausalLMr&   rw   s    r#   rO   zQwen2Loader.architecture_namez  r  r%   c                    |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }| j	                  |j                   d      \  }}}t        |j                   dd       }|rd }	|j
                  dk(  rt        j                  |j                        }	|	<t        d|j
                  ddj                  t        j                                     |j                  }
|j                  }nt        j                  j                   }	d }
d }t"        j$                  j'                  ||t        j(                  j*                  dddd	d
|||||	|
|      }| j-                  |j.                  |j0                  |	       | j3                  |j.                  j4                  |j6                         |S )Nr	  r  rC  rD  rE  rF  r<   Tr   FrG  )rX   rd  re  rN   r   rL  rM  rI   rN  rz   rK   rM   rO  rP  r   r   r   r
   rc  r   r  rQ  r   r   r]   r   r   r   )r9   r]   r  r  r  r   r   r   rC  r   rH  rI  r`   s                r#   r|   zQwen2Loader.get_model_spec~  s   \\33
LL44	u||-BIN9$LAEAWAWLL&B
>2J
 &ell4I4PJ"//584889L9T9TU
!) ,88		"9">">"@A	   3==,11J$1155J#J;;GG"--33# 3"7"%!-! H 
$ 	u{{J?//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zQwen2Loader.get_vocabulary  sV    'y9LL++c&k9	y! 	/AMM/A-.	/r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zQwen2Loader.set_vocabulary  rp  r%   c                     |j                   |j                   n|j                  |_         |j                  |_        |j                  |j                  nd|_        |j                  j
                  |_        y rY  r   rZ  r   r   rX   r  r  r   s       r#   r~   zQwen2Loader.set_config  n     "". $$ 	
 %..#,#6#6#BI 	 %*LL$=$=!r%   c                 &    |j                   |_        y r   r  r  s      r#   r   zQwen2Loader.set_layer_norm  r  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]  \  }}| j	                  |j                  j
                  |j                         | j	                  |j                  j
                  |j                         t        d      D cg c]  }t        j                           }}| j#                  |d   |j$                  j&                  |       | j#                  |d   |j$                  j(                  |       | j#                  |d   |j$                  j*                  |       |t        j,                  j.                  k(  r.t1        j2                  |j                  j4                  d   |       nO|t        j,                  j6                  k(  rdnd}t1        j8                  |j                  j4                  d   ||       | j#                  |j                  j4                  d   |j$                  j:                  |       | j#                  |j                  j<                  |j>                  j@                  |       | j#                  |j                  jB                  |j>                  jD                  |       | j#                  |j                  jF                  |j>                  jH                  |       tK        |d       tK        |d       tM        jN                           y c c}w r]  r_  rb  s	            r#   r   zQwen2Loader.set_decoder  s    %DOOV-@-@ADOOV[[9!$TZZ!? -	J))44e6K6K ))5+I+I ?DAhGK224GLGOOQ!7!7J   OOQ!7!7J   OOQ!7!7J   [55999!!*";";"B"B1"E|T(K,D,D,M,MMST**--44Q7v OO))003&&%   OO'')<)<   OO--uyy/@/@Z   OO'')<)<   E;'E5!JJL[-	 Hrd  re  r%  s   @r#   r  r  x  sE    " "6p)
>' 4?3K3K3O3O 2r%   r  Qwen3Configc                   x     e Zd Zed        Zd Z fdZd Zd Zd Z	e
j                  j                  fdZ xZS )Qwen3Loaderc                      y)NQwen3ForCausalLMr&   rw   s    r#   rO   zQwen3Loader.architecture_name	  r  r%   c                    |j                   j                  }|j                   j                  }t        |j                   d|      }t        |j                   d|j                   j                  |z        }||k(  rd }| j                  |j                   d      \  }}}t        |j                   dd       }	|	rd }
|	j                  dk(  rt        j                  |	j                        }
|
<t        d|	j                  ddj                  t        j                                     |	j                  }|	j                  }nt        j                   j"                  }
d }d }t$        j&                  j)                  ||t        j*                  j,                  d	d	d	|j                   j.                  d
|||||d	|
||      }| j1                  |j2                  |j4                  |
       | j7                  |j2                  j8                  |j:                         |S )Nr	  r  rm  rC  rD  rE  rF  r<   TF)r   r   r  r  r  r  r   r   r  r  r  rq  r   rH  rI  )rX   rd  re  rN   rh  r   rL  rM  rI   rN  rz   rK   rM   rO  rP  r   r   r   r
   rc  r   r  rQ  r  r   r   r]   r   r   r   )r9   r]   r  r  r  r  r   r   r   rC  r   rH  rI  r`   s                 r#   r|   zQwen3Loader.get_model_spec	  s   \\33
LL44	u||-BINLL*ell&>&>)&K
 9$LAEAWAWLL)B
>2J &ell4I4PJ"//584889L9T9TU
!) ,88		"9">">"@A	   3==,11J$1155J#J;;GG"--33||,,# 3"7"%!-!# H 
( 	u{{J?//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zQwen3Loader.get_vocabularyG	  sV    'y9LL++c&k9	y! 	/AMM/A-.	/r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zQwen3Loader.set_vocabularyN	  rp  r%   c                     |j                   |j                   n|j                  |_         |j                  |_        |j                  |j                  nd|_        |j                  j
                  |_        y rY  r  r   s       r#   r~   zQwen3Loader.set_configQ	  r  r%   c                 &    |j                   |_        y r   r  r  s      r#   r   zQwen3Loader.set_layer_norm]	  r  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        t        |j                  |j                              D ]A  \  }\  }}| j	                  |j                  j
                  |j                         | j	                  |j                  j
                  |j                         | j	                  |j                  j                  |j                   j                         | j	                  |j                  j"                  |j                   j"                         t%        d      D cg c]  }t'        j(                          }}| j+                  |d   |j                   j,                  |       | j+                  |d   |j                   j.                  |       | j+                  |d   |j                   j0                  |       |t&        j2                  j4                  k(  r.t7        j8                  |j                  j:                  d   |       nO|t&        j2                  j<                  k(  rdnd}	t7        j>                  |j                  j:                  d   ||	       | j+                  |j                  j:                  d   |j                   j@                  |       | j+                  |j                  jB                  |jD                  jF                  |       | j+                  |j                  jH                  |jD                  jJ                  |       | j+                  |j                  jL                  |jD                  jN                  |       tQ        |d       tQ        |d       tS        jT                          D y c c}w r]  )+r  r   r   r"  r   r   r!  rk  r   r   r   r   r  r   r  r~  r   r  r  r   r  r   r  r  r  r   r   r   r  r   r`  ra  r"  r   r  r#  r  r  r  r   r$  r%  r&  )
r9   r`   r   r   	layer_idxr  r   r   r  rc  s
             r#   r   zQwen3Loader.set_decoder`	  s    %DOOV-@-@ADOOV[[9.7DJJ8V.W 4	*I*
E))44e6K6K ))5+I+I ))00%//2H2H ))00%//2H2H ?DAhGK224GLGOOQ!7!7J   OOQ!7!7J   OOQ!7!7J   [55999!!*";";"B"B1"E|T(K,D,D,M,MMST**--44Q7v OO))003&&%   OO'')<)<   OO--uyy/@/@Z   OO'')<)<   E;'E5!JJLi4	 Hs   (Ore  r%  s   @r#   r  r  	  sE    " ":x)
>' 4?3K3K3O3O 9r%   r  MixFormerSequentialConfigc                   F     e Zd Zed        Zd Z fdZd Zd Zd Z	 xZ
S )MixFormerSequentialLoaderc                      yr  r&   rw   s    r#   rO   z+MixFormerSequentialLoader.architecture_name	  r  r%   c           
         t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                     |j                  j                  ddd      }| j                  |j                  |j                         | j                  |j                  j                  |j                  d   j                         |S )NTFr  r  r   r   r  r  r  r  r  )r
   rc  r   rX   r  r  r   r   r  r   r   r   r   r   r   ri  s      r#   r|   z(MixFormerSequentialLoader.get_model_spec	  s    ;;GG||++ll))-ell.N.NO||..#"" H 	
 	u||4//b1A1H1HIr%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   z(MixFormerSequentialLoader.get_vocabulary	  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   z(MixFormerSequentialLoader.set_vocabulary	  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   z$MixFormerSequentialLoader.set_config	  rs  r%   c                 $   d|_         | j                  |j                  |d   j                         | j	                  |j
                  |d   j                         t        |j                  |dd       D ]  \  }}| j	                  |j                  |j                         | j                  |j                  j                  d   |j                  j                         | j                  |j                  j                  d   |j                  j                         | j                  |j                   j"                  |j$                  j&                         | j                  |j                   j(                  |j$                  j*                          y )NFr   r  r   )r  r   r   r  r   r   lnr   r   r  r   r   r   mixerr  r  r   r   r  r   r  r  r  s        r#   r   z%MixFormerSequentialLoader.set_decoder	  s    %DOOVAY]];DOOVBZ]];!$TZZ"!> 	DJ
 < <ehhGOOJ55<<Q?AQAQROOJ55<<Q?AUAUVOOJNN33UYY]]COOJNN33UYY]]C	Dr%   r  r%  s   @r#   r  r  	  s/    & & )/

Dr%   r  	PhiConfigc                   F     e Zd Zed        Zd Z fdZd Zd Zd Z	 xZ
S )	PhiLoaderc                      yr  r&   rw   s    r#   rO   zPhiLoader.architecture_name	  r  r%   c           
      6   t         j                  j                  |j                  j                  |j                  j
                  dt        |j                  j                     |j                  j                  ddd      }| j                  |j                  |j                         | j                  |j                  j                  |j                  j                         | j!                  |j                  j"                  |j                  j$                         |S )NTFr  )r
   rc  r   rX   r  r  r   r   r  r   r   r  r   r   r   r   r   r   r  ri  s      r#   r|   zPhiLoader.get_model_spec	  s    ;;GG||++ll))-ell.N.NO||..#"" H 	
 	u'8'89//1E1EFDLL33U]]5E5EFr%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zPhiLoader.get_vocabulary	  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zPhiLoader.set_vocabulary	  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zPhiLoader.set_config	  rs  r%   c                    d|_         | j                  |j                  |j                  j                         t        |j                  |j                        D ]  \  }}| j                  |j                  |j                         | j                  |j                  j                  d   |j                  j                         | j                  |j                  j                  d   |j                  j                          | j                  |j"                  j$                  |j&                  j(                         | j                  |j"                  j*                  |j&                  j,                          y r  )r  r   r   embdr  r   r   r  r   r  r  r   r   r   r  r  r  r   r   r  r   r  r  r  s        r#   r   zPhiLoader.set_decoder	  s     %DOOV[[__=!$TZZ!: 	DJ
 < <ehhGOOJ55<<Q?AQAQROOJ55<<Q?AUAUVOOJNN33UYY]]COOJNN33UYY]]C	Dr%   r  r%  s   @r#   r  r  	  s/    & &")/
	Dr%   r  
Phi3Configc                   ~     e Zd Zed        Zd Z fdZd Zd Zd Z	d Z
ej                  j                  fdZ xZS )	
Phi3Loaderc                      yr  r&   rw   s    r#   rO   zPhi3Loader.architecture_name
  r  r%   c                    |j                   j                  }|j                   j                  }t        |j                   d|      }||k(  rd }t        |j                   dd      }t        |j                   dd      }t        |j                   dd       }|rat        j                  |d         }|j                  dd      }	|9t        d	|d   d
dj                  t        j                                     d }d}	t        |j                   dd       }
|
rd }|
j                  dk(  rt        j                  |
j                        }|<t        d|
j                  ddj                  t        j                                     |
j                  }|
j                  }nt        j                  j                   }d }d }t"        j$                  j'                  ||t        j(                  j*                  ddddd||	t        |j                   dd      ||||||      }| j-                  |j.                  |j0                  |       | j3                  |j.                  j4                  |j6                         |S )Nr	   original_max_position_embeddingsr   max_position_embeddingsr   r   r   r   r   r   r<   rC  rD  rE  rF  TFr   r  )r   r   r  r  r  r  r   r   r  r  r  r  r   rH  rI  )rX   rd  re  rN   r   rI   rz   rK   rM   rL  rM  rN  rO  rP  r   r   r   r
   rc  r   r  rQ  r   r   r]   r   r   r   )r9   r]   r  r  r  r  r  r   r   r   rC  r   rH  rI  r`   s                  r#   r|   zPhi3Loader.get_model_spec

  sH   \\33
LL44	u||-BIN9$L+2LL<a,
( #*%,,8QST"Uu||^TB"9"="=l6>R"S$0$4$4Xq$A!"*) $F+TYY7N7S7S7U-VX  #'$%! &ell4I4PJ"//584889L9T9TU
!) ,88		"9">">"@A	   3==,11J$1155J#J;;GG"--33# 3"7lEB-M$;%!-!# H 
( 	u{{J?//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zPhi3Loader.get_vocabularyS
  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zPhi3Loader.set_vocabulary\
  rp  r%   c                 j    |j                   |_         |j                  |_        |j                  |_        y r   rr  r   s       r#   r~   zPhi3Loader.set_config_
  rs  r%   c                 &    |j                   |_        y r   r  r  s      r#   r   zPhi3Loader.set_layer_normd
  r  r%   c                     t        j                  |t         j                        |_        t        j                  |t         j                        |_        y )N)r>   )rB   tensorrs  rotary_scaling_long_factorrotary_scaling_short_factor)r9   r`   r  r  s       r#   set_rotary_embeddingsz Phi3Loader.set_rotary_embeddingsg
  s:     +0,,&emm+
' ,1<<'u}},
(r%   c                 J   d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]  \  }}| j	                  |j                  j
                  |j                         | j	                  |j                  j
                  |j                         | j                  |j                  j                  d   |j                   j"                  |       | j                  |j                  j                  d   |j                   j$                  |       |j                   j&                  j(                  y|j                   j&                  j*                  Y| j-                  |j                  |j                   j&                  j(                  |j                   j&                  j*                         |t.        j0                  j2                  k(  rl|j4                  j6                  j8                  j;                  dd      \  }}||j                  j<                  _        ||j                  j>                  _        n>|j4                  j6                  j@                  j;                  dd      \  }}	|j4                  j6                  jB                  j;                  dd      \  }
}|j4                  j6                  jD                  j;                  dd      \  }}||j                  j<                  _        |
|j                  j<                  _#        ||j                  j<                  _$        |	|j                  j>                  _        ||j                  j>                  _#        ||j                  j>                  _$        | j                  |j                  jJ                  |j4                  jL                  |       tO        |d       tO        |d       tQ        jR                           y )	NFr   r^  r   r  r  r   r  )*r  r   r   r"  r   r   r!  r   r   r   r   r  r   r  r   r   r   r  r"  
rotary_emblong_factorshort_factorr  r   r   r   r  gate_up_projr   r  r   r  r   r   r   r   r   r  r   r$  r%  r&  )r9   r`   r   r   r  r   r#  r  gate_qweight
up_qweightgate_scales	up_scalesgate_qzeros	up_qzeross                 r#   r   zPhi3Loader.set_decoderq
  s)    %DOOV-@-@ADOOV[[9!$TZZ!? 7	J))44e6K6K ))5+I+I OO))003((%  
 OO))003&&%   **66BOO..;;G**--OO..::OO..;; [55999%*YY%;%;%B%B%H%HPQ%H%R"	71:
''.7>
--4 ,199+A+A+I+I+O+O1 ,P ,(j */)?)?)F)F)L)LQTU)L)V&Y).)?)?)F)F)L)LQTU)L)V&Y1=
''.7B
''46A
''37A
--4=F
--:<E
--9OO'')<)<   E;'E5!JJLo7	r%   )rH   ro   rp   r   rO   r|   r   r   r~   r   r  r   r   r   r   r$  r%  s   @r#   r  r  
  sK    & &GR)/
'
 4?3K3K3O3O <r%   r  RWConfigc                   T     e Zd Zed        Zd Zd Z fdZd Zd Z	d Z
d	dZ xZS )
RWLoaderc                      yr  r&   rw   s    r#   rO   zRWLoader.architecture_name
  r  r%   c                     |j                   j                  | _        |j                   j                  | _        t        |j                   dd       | _        d| _        y )N	n_head_kvnum_kv)rX   r  _num_layersr  
_num_headsrN   _num_heads_kv_num_kv_attrr{   s     r#   get_falcon_speczRWLoader.get_falcon_spec
  sB     <<//,,--$U\\;E$r%   c                 T   | j                  |       t        |j                  dd      rd}n| j                  }t        j
                  j                  | j                  | j                  dt        j                  j                  |j                  j                  dd|j                  j                  rdnd d|j                  j                  |dk(  |      }| j                  |j                   |j"                         | j%                  |j                   j&                  |j(                         |S )Nmulti_queryFr   Tr   )
r   r   r  r  scale_alibir  r  r  r  r  )r  rN   rX   r  r
   rc  r   r  r  r   r  r  r  rotaryparallel_attnr   r   r  r   r   r   )r9   r]   r  r`   s       r#   r|   zRWLoader.get_model_spec
  s    U#5<<6L--L;;GGOO"--22,,$$)-!LL//qT##ll88*a/% H 
 	u'8'89//?r%   c                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zRWLoader.get_vocabulary
  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zRWLoader.set_vocabulary
  rp  r%   c                 j    |j                   |_        |j                   |_         |j                   |_        y r   )r   r   r   r   s       r#   r~   zRWLoader.set_config
  rs  r%   c                    d|_         | j                  |j                  |j                         | j	                  |j
                  |j                         t        |j                  |j                        D ]t  \  }}t        |d      rM| j	                  |j                  |j                         | j	                  |j                  |j                         nt        |d      r'| j	                  |j                  |j                          n`| j	                  |j"                  j
                  |j                          | j	                  |j$                  j
                  |j&                         t)        |j"                  | j*                        }|dk(  r>| j-                  |j"                  j.                  d   |j"                  j0                         nn| j3                  |j"                  j.                  d   |j"                  j0                  |j"                  j4                  ||j"                  j4                  k  r|nd        | j-                  |j"                  j.                  d   |j"                  j6                         | j-                  |j$                  j8                  |j:                  j<                         | j-                  |j$                  j>                  |j:                  j@                         w y )NFln_attnr  r   r   )!r  r   r   r  r   r   r  r   r   r  r   r  r  r  ln_mlpr  r  r   r   r  rN   r  r   r   r  r  r  r  r   r  r  r  r  )r9   r`   r   r  r   r  s         r#   r   zRWLoader.set_decoder
  s1    %DOOV-C-CDDOOV[[9!$TZZ!: !	NJui(##J$?$?O##J$H$H%,,W%89##J$@$@%BWBWX##--88%:O:O ##NN--u/M/M U1143D3DEF{--44Q7((88
 ##--44Q7((88((22$u';';'E'EEF4	 OO))003U5I5I5O5O OOJNN33UYY5L5LMOOJNN33UYY5L5LMC!	Nr%   c                    |j                   }|S|j                  |dd|j                  d         }|j                  dd      }|j                  d|j                  d         }n|j                  d   ||dz  z   z  }|j                  d||z  dz   ||j                  d         }|j	                  ||z  ddgd      \  }}}	t        j                  |j                  ||z  d      |j                  ||z  d      |	j                  ||z  d      g      }||_         |j                  |j                  }
|7|
j                  |dd      }
|
j                  dd      }
|
j                  d      }
n|
j                  d||z  dz         }
|
j	                  ||z  ddgd      \  }}}	t        j                  |j                  ||z        |j                  ||z        |	j                  ||z        g      }
|
|_        y y )Nr
  r  r   r   r  r  )r   r  r6  r   splitrB   r  r   )r9   r`   r   r  r  r   r  r  r  r  r   s              r#   r  zRWLoader.set_qkv_linear  s   >^^Iq"fll26FGF%%a+F^^BR(89F||A9vz+ABH^^I'!+Xv||B7GF llI$7A#>AlFGAq!YYIIi(2B7IIfx/4IIfx/4F ;;";;D~||Iq"5~~a+||B'||B	V(;a(?J**i6&91a%@a*H1ayy		)h"67		&8"34		&8"34 DI% #r%   r   )rH   ro   rp   r   rO   r  r|   r   r   r~   r   r  r$  r%  s   @r#   r  r  
  s:    & &%6)/
&NP)r%   r  FalconConfigc                       e Zd Zd Zy)FalconLoaderc                     |j                   j                  | _        |j                   j                  | _        t        |j                   dd       | _        d| _        y )Nnum_kv_heads)rX   rd  r  re  r  rN   r  r  r{   s     r#   r  zFalconLoader.get_falcon_spec>  sB     <<99,,::$U\\>4H*r%   N)rH   ro   rp   r  r&   r%   r#   r  r  <  s    +r%   r  DistilBertConfigc                   .    e Zd Zed        Zd Zd Zd Zy)DistilBertLoaderc                      y)NDistilBertModelr&   rw   s    r#   rO   z"DistilBertLoader.architecture_nameG  r  r%   c                    t        j                  |j                  j                  |j                  j                  dt
        |j                  j                     d      }t        j                  |      }d|j                  _	        | j                  |j                  j                  d   |j                  j                         | j                  |j                  j                  |j                  j                         | j!                  |j                  j"                  |j                  j$                         t'        |j                  j(                  |j*                  j(                        D ]  \  }}t-        d      D cg c]  }t/        j0                          }}| j3                  |d   |j4                  j6                         | j3                  |d   |j4                  j8                         | j3                  |d   |j4                  j:                         t=        j>                  |j@                  jB                  d   |       | j3                  |j@                  jB                  d   |j4                  jD                         | j!                  |j@                  jF                  |jH                         | j3                  |jJ                  jL                  |jJ                  jN                         | j3                  |jJ                  jP                  |jJ                  jR                         | j!                  |jJ                  jF                  |jT                          |S c c}w )NFTr   r   r
  r   r  )+r
   TransformerEncoderSpecrX   r  r  r   r   TransformerEncoderModelSpecr   r  r   r   r  r   r  position_embeddingsr   r   	LayerNormr   r   r  r  r   r  r   r  q_link_linv_linr   r  r   r   out_linr   sa_layer_normr   r   lin1r  lin2output_layer_normr9   r]   encoder_specr`   r  r   r   r  s           r#   r|   zDistilBertLoader.get_model_specK  sw   '>>LL!!LL  -ell.E.EF $
  ;;
 ).%LL##A&(8(8(H(H	
 	##LL++U-=-=-Q-Q	
 	LL,,e.>.>.H.H	
 "%T\\%7%79J9J9P9P!Q 	TJ>CAhGK224GLGOOLOU__-B-BCOOLOU__-B-BCOOLOU__-B-BCj77>>qA<POO))003U__5L5L ))44e6I6I OOJNN33UYY^^DOOJNN33UYY^^D
 9 95;R;RS!	T$ # Hs   =Mc                 &    |j                  |       y r   rn  r   s      r#   r   zDistilBertLoader.set_vocabularyw  rp  r%   c                 4    |j                   |_         d|_        y )Ng-q=)r   r  r   s       r#   r~   zDistilBertLoader.set_configz  s    $..$)!r%   N)rH   ro   rp   r   rO   r|   r   r~   r&   r%   r#   r  r  E  s$    ! !*X)*r%   r  
BertConfigc                   @     e Zd Zed        Zd Z fdZd Zd Z xZ	S )
BertLoaderc                      y)N	BertModelr&   rw   s    r#   rO   zBertLoader.architecture_name  s    r%   c           	         |j                   j                  dk(  sJ t        j                  |j                   j                  |j                   j
                  dt        |j                   j                     ddt        j                  j                        }t        j                  |dt        j                  j                        }d|j                  _        | j!                  |j                  j"                  d   |j"                  j$                         | j!                  |j                  j"                  d   |j"                  j&                         | j)                  |j                  j*                  |j"                  j,                         | j/                  |j                  j0                  |j"                  j2                         | j5                  |j6                  |j8                  j:                         t=        |j                  j>                  |j                  j>                        D ]/  \  }}tA        d	      D cg c]  }t        jB                          }}| j5                  |d   |jD                  jF                  jH                         | j5                  |d   |jD                  jF                  jJ                         | j5                  |d   |jD                  jF                  jL                         tO        jP                  |jR                  jT                  d   |       | j5                  |jR                  jT                  d   |jD                  jV                  j:                         | j/                  |jR                  jX                  |jD                  jV                  j2                         | j5                  |jZ                  j\                  |j^                  j:                         | j5                  |jZ                  j`                  |jV                  j:                         | j/                  |jZ                  jX                  |jV                  j2                         2 |S c c}w 
NabsoluteFTr  )r   r   r   num_source_embeddingsembeddings_merge)pooling_layerpooling_activationr   r   r
  )1rX   position_embedding_typer
   r  rd  re  r   r  r   EmbeddingsMergeADDr  r  Tanhr   r  r   r   r  token_type_embeddingsr   r  r  r   r   r  r   pooler_densepoolerr  r   r   r  r  r  r9   queryr   valuer   r  r   r   outputr   r   r   intermediater  r  s           r#   r|   zBertLoader.get_model_spec  s+   ||33zAAA'>>LL**LL,,-ell.E.EF $"#(88<<
  ;;*55::
 ).%LL##A&(8(8(H(H	
 	LL##A&(8(8(N(N	
 	##LL++U-=-=-Q-Q	
 	LL,,e.>.>.H.H	
 	))5<<+=+=>!$T\\%7%79L9L!M 	SJ>CAhGK224GLGOOLOU__-A-A-G-GHOOLOU__-A-A-E-EFOOLOU__-A-A-G-GHj77>>qA<POO))003U__5K5K5Q5Q ))44eoo6L6L6V6V OOJNN33U5G5G5M5MNOOJNN33U\\5G5GH
 9 95<<;Q;QR!	S$ # Hs   :Qc                     t         |   ||      }|j                  j                  t	        |      z
  }t        |      D ]  }|j                  d|z          |S r  r  r  s         r#   r   zBertLoader.get_vocabulary  r  r%   c                 &    |j                  |       y r   rn  r   s      r#   r   zBertLoader.set_vocabulary  rp  r%   c                 \    |j                   |_         |j                  j                  |_        y r   r   rX   layer_norm_epsr  r   s       r#   r~   zBertLoader.set_config  "    $..$)LL$?$?!r%   )
rH   ro   rp   r   rO   r|   r   r   r~   r$  r%  s   @r#   r  r    s+     6p)@r%   r  XLMRobertaConfigc                   4    e Zd Zed        Zd Zd Zd Zd Zy)XLMRobertaLoaderc                      y)N#XLMRobertaForSequenceClassificationr&   rw   s    r#   rO   z"XLMRobertaLoader.architecture_name  s    4r%   c           	      r	   |j                   j                  dk(  sJ t        j                  |j                   j                  |j                   j
                  dt        |j                   j                     ddt        j                  j                        }|j                  j                  d}nd}t        j                  ||t        j                  j                        }d|j                   _        | j%                  |j                   j&                  d   |j                  j&                  j(                         | j%                  |j                   j&                  d   |j                  j&                  j*                         | j-                  |j                   j.                  |j                  j&                  j0                         | j3                  |j                   j4                  |j                  j&                  j6                         |r:| j9                  |j:                  |j                  j                  j<                         t?        |j                   j@                  |j                  j                   j@                        D ]/  \  }}tC        d	      D cg c]  }t        jD                          }}| j9                  |d   |jF                  jH                  jJ                         | j9                  |d   |jF                  jH                  jL                         | j9                  |d   |jF                  jH                  jN                         tQ        jR                  |jT                  jV                  d   |       | j9                  |jT                  jV                  d   |jF                  jX                  j<                         | j3                  |jT                  jZ                  |jF                  jX                  j6                         | j9                  |j\                  j^                  |j`                  j<                         | j9                  |j\                  jb                  |jX                  j<                         | j3                  |j\                  jZ                  |jX                  j6                         2 |S c c}w r  )2rX   r  r
   r  rd  re  r   r  r   r  r  robertar  r  r  r  r   r  r   r   r  r  r   r  r  r   r   r  r   r  r  r   r   r  r  r  r9   r  r   r  r   r  r   r   r  r   r   r   r  r  	r9   r]   r  r  r`   r  r   r   r  s	            r#   r|   zXLMRobertaLoader.get_model_spec  se   ||33zAAA'>>LL**LL,,-ell.E.EF $"#(88<<
 =='!M M;;'*55::
 ).%LL##A&(@(@(P(P	
 	LL##A&(@(@(V(V	
 	##LL++MM$$88	
 	LL,,emm.F.F.P.P	
 OOD--u}}/C/C/I/IJ!$T\\%7%79N9N9T9T!U 	SJ>CAhGK224GLGOOLOU__-A-A-G-GHOOLOU__-A-A-E-EFOOLOU__-A-A-G-GHj77>>qA<POO))003U__5K5K5Q5Q ))44eoo6L6L6V6V OOJNN33U5G5G5M5MNOOJNN33U\\5G5GH
 9 95<<;Q;QR!	S$ # Hs   R4c                 &    |j                  |       y r   rn  r   s      r#   r   zXLMRobertaLoader.set_vocabulary  rp  r%   c                 \    |j                   |_         |j                  j                  |_        y r   r#  r   s       r#   r~   zXLMRobertaLoader.set_config  r%  r%   c                 z    |j                   |_        t        |dd      }|dkD  r|j                  |dz   d  |_        y y Npadding_idxr   r   r   r   s       r#   r   z'XLMRobertaLoader.set_position_encodings  =    2A:!^^FQJL9DN r%   N	rH   ro   rp   r   rO   r|   r   r~   r   r&   r%   r#   r(  r(    s*    5 5<|)@:r%   r(  RobertaConfigc                   4    e Zd Zed        Zd Zd Zd Zd Zy)RobertaLoaderc                      y)NRobertaModelr&   rw   s    r#   rO   zRobertaLoader.architecture_name"  s    r%   c           	         |j                   j                  dk(  sJ t        j                  |j                   j                  |j                   j
                  dt        |j                   j                     ddt        j                  j                        }|j                  d}nd}t        j                  ||t        j                  j                        }d|j                  _        | j#                  |j                  j$                  d   |j$                  j&                         | j#                  |j                  j$                  d   |j$                  j(                         | j+                  |j                  j,                  |j$                  j.                         | j1                  |j                  j2                  |j$                  j4                         |r0| j7                  |j8                  |j                  j:                         t=        |j                  j>                  |j                  j>                        D ]/  \  }}tA        d	      D cg c]  }t        jB                          }}| j7                  |d   |jD                  jF                  jH                         | j7                  |d   |jD                  jF                  jJ                         | j7                  |d   |jD                  jF                  jL                         tO        jP                  |jR                  jT                  d   |       | j7                  |jR                  jT                  d   |jD                  jV                  j:                         | j1                  |jR                  jX                  |jD                  jV                  j4                         | j7                  |jZ                  j\                  |j^                  j:                         | j7                  |jZ                  j`                  |jV                  j:                         | j1                  |jZ                  jX                  |jV                  j4                         2 |S c c}w r  1rX   r  r
   r  rd  re  r   r  r   r  r  r  r  r  r  r   r  r   r   r  r  r   r  r  r   r   r  r   r  r  r   r   r  r  r  r9   r  r   r  r   r  r   r   r  r   r   r   r  r  r-  s	            r#   r|   zRobertaLoader.get_model_spec&  C   ||33zAAA'>>LL**LL,,-ell.E.EF $"#(88<<
 <<!M M;;'*55::
 ).%LL##A&(8(8(H(H	
 	LL##A&(8(8(N(N	
 	##LL++00	
 	LL,,e.>.>.H.H	
 OOD--u||/A/AB!$T\\%7%79L9L!M 	SJ>CAhGK224GLGOOLOU__-A-A-G-GHOOLOU__-A-A-E-EFOOLOU__-A-A-G-GHj77>>qA<POO))003U__5K5K5Q5Q ))44eoo6L6L6V6V OOJNN33U5G5G5M5MNOOJNN33U\\5G5GH
 9 95<<;Q;QR!	S$ # H   Q.c                 &    |j                  |       y r   rn  r   s      r#   r   zRobertaLoader.set_vocabularyd  rp  r%   c                 \    |j                   |_         |j                  j                  |_        y r   r#  r   s       r#   r~   zRobertaLoader.set_configg  r%  r%   c                 z    |j                   |_        t        |dd      }|dkD  r|j                  |dz   d  |_        y y r1  r   r   s       r#   r   z$RobertaLoader.set_position_encodingsk  r3  r%   Nr4  r&   r%   r#   r7  r7     s*     <|)@:r%   r7  CamembertConfigc                   4    e Zd Zed        Zd Zd Zd Zd Zy)CamembertLoaderc                      y)NCamembertModelr&   rw   s    r#   rO   z!CamembertLoader.architecture_namet  r`  r%   c           	         |j                   j                  dk(  sJ t        j                  |j                   j                  |j                   j
                  dt        |j                   j                     ddt        j                  j                        }|j                  d}nd}t        j                  ||t        j                  j                        }d|j                  _        | j#                  |j                  j$                  d   |j$                  j&                         | j#                  |j                  j$                  d   |j$                  j(                         | j+                  |j                  j,                  |j$                  j.                         | j1                  |j                  j2                  |j$                  j4                         |r0| j7                  |j8                  |j                  j:                         t=        |j                  j>                  |j                  j>                        D ]/  \  }}tA        d	      D cg c]  }t        jB                          }}| j7                  |d   |jD                  jF                  jH                         | j7                  |d   |jD                  jF                  jJ                         | j7                  |d   |jD                  jF                  jL                         tO        jP                  |jR                  jT                  d   |       | j7                  |jR                  jT                  d   |jD                  jV                  j:                         | j1                  |jR                  jX                  |jD                  jV                  j4                         | j7                  |jZ                  j\                  |j^                  j:                         | j7                  |jZ                  j`                  |jV                  j:                         | j1                  |jZ                  jX                  |jV                  j4                         2 |S c c}w r  r;  r-  s	            r#   r|   zCamembertLoader.get_model_specx  r<  r=  c                 &    |j                  |       y r   rn  r   s      r#   r   zCamembertLoader.set_vocabulary  rp  r%   c                 \    |j                   |_         |j                  j                  |_        y r   r#  r   s       r#   r~   zCamembertLoader.set_config  r%  r%   c                 z    |j                   |_        t        |dd      }|dkD  r|j                  |dz   d  |_        y y r1  r   r   s       r#   r   z&CamembertLoader.set_position_encodings  r3  r%   Nr4  r&   r%   r#   rC  rC  r  s*       <|)@:r%   rC  c            	      X   t        j                  t         j                        } | j                  ddd       | j                  dd       | j                  d	d
d       | j                  dd       | j                  ddd       | j                  ddd       t	        j
                  |        | j                         }t        |j                  |j                  |j                  |j                  dv |j                  |j                  |j                        }|j                  |       y )N)formatter_classz--modelTzaName of the pretrained model to download, or path to a directory containing the pretrained model.)requiredhelpz--activation_scaleszPath to the pre-computed activation scales. Models may use them to rescale some weights to smooth the intermediate activations and improve the quantization accuracy. See https://github.com/mit-han-lab/smoothquant.)rM  z--copy_files+zWList of filenames to copy from the Hugging Face model to the converted model directory.)nargsrM  z
--revisionz<Revision of the model to download from the Hugging Face Hub.z--low_cpu_mem_usage
store_truezNEnable the flag low_cpu_mem_usage when loading the model with from_pretrained.)actionrM  z--trust_remote_codez*Allow converting models using custom code.)rQ   int8_float16)r+   r,   r-   r.   r/   r0   )argparseArgumentParserArgumentDefaultsHelpFormatteradd_argumentr   declare_arguments
parse_argsr)   r]   r+   r,   quantizationr.   r/   r0   convert_from_args)parserargs	converters      r#   mainr^    sG   $$ >>F F	   :   	   K   ]  
 9   'D%

00??))-HH0000I %r%   __main__))r   r   )r  r   )r     r
  r   r
  r   r
  r  r
  r
  r
  r  ))r  r  ra  rc  rd  re  )r
  r`  )rd  r     r`  r   )r`  r`  )r`  rg  )rb  )r  r  )r  r
  rf  rh  )r`  r  )r`  r  )r`     ))ri  ri  rg  r   )rg  r
  )rg  ry  )ry  r  )ry  r`  ry  rg  	   r   )rm  r  )rm  ry  )rm  
   )rn  r   )rn  r   )rn  r  )rn  r
  )rn  ri  )rn     ro  r  ro  r  )
)r`  r
  )r`  rm  )ry  r   )ry  r  rk  )ry  ry  rl  )rm  rg  )rm  rm  )rn  r`  )rq  )   r   )rr     )rr  rr     r  )   r   )rv  r  )rv  rm  )   rs  rw  rr  )   rg  )ry  rn  )ry  ru     r   )r{  r
  )r{  rm  )r{  rr  )   rs  ))   ru  rt  )ru  ru  rv  r   rz  )   r  )	)rm     rp  rq  )ro  rw  )   rg  )r  ro  )r  rw  )r  r  )r  ru  ))rn  rs  )r}  rw  )rv  ro  )rv  rs  )rv  r}  )rw  ru  )rw  rv  )ry  r  )ry  ro  )ry  r  r  ro  )r|  r  )r|  r
  )r  r
  )r  rm  )r  rs  )r  r`  )r  rg  )r  r}  )   r`  )   r   )r  rs  )   ru  )
rj  )rn  rw  )rs  ry  )r}  rs  r~  rx  r  )r|  r  )   r   )r  ri  )zopenai/whisper-tiny.enzopenai/whisper-tinyzopenai/whisper-base.enzopenai/whisper-basezopenai/whisper-small.enzopenai/whisper-smallzopenai/whisper-medium.enzopenai/whisper-mediumzopenai/whisper-largezopenai/whisper-large-v2zopenai/whisper-large-v3T5GemmaConfigc                       e Zd Zed        Zd Zd Zd Zd Ze	j                  j                  fdZe	j                  j                  fdZy)	T5GemmaLoaderc                      y)NT5GemmaForConditionalGenerationr&   rw   s    r#   rO   zT5GemmaLoader.architecture_name  rX  r%   c                 @    |j                   j                  dz   |_        y )Nr  )r   rT  r   r  s      r#   r   zT5GemmaLoader.set_layer_norm  s    &&++c1
r%   c                 R   |j                   j                  }|j                   j                  }t        |j                   dd      }|j                  }t        |d|      }||k(  rd }t        j                  |j                  |j                  dt        |j                     dd|j                  dt        |dd      |d||j                        }|j                  }t        |d|      }	|	|k(  rd }	t        j                  |j                  |j                  dt        |j                     ddd|j                  dt        |dd      |dd|	|j                  	      }
t        j                  ||
      }| j                  |j                  |j                  j                  |       | j                  |j                  |j                  j                  |t         j"                  j$                         | j'                  |j                  j(                  |j                  j                  j*                         |S )
Nro  i   r	  TFr   r  )r   r   r  r  r  r  r  ro  r3  r  r  )r   r   r  r  with_encoder_attentionr  r  r  ro  r3   external_pre_post_encoder_layersr  r  )rX   r   r   rN   re  r
   r  rd  r   r
  r  TransformerDecoderSpecr   r   r]   r   r   r   r   r   r   r"  )r9   r]   encoder_configdecoder_configro  encoder_num_headsencoder_num_heads_kvr   decoder_num_headsdecoder_num_heads_kvr   r`   s               r#   r|   zT5GemmaLoader.get_model_spec  s   ---- /?F*>>&13D 
  #44#' "99,,..-n.N.NO%..#eD) $-#,,
  +>>&13D 
  #44#' "99,,..-n.N.NO#'%..#eD) $-1-#,,
$  //Au{{':':NKLLKK$$((		
 	//1D1D1Q1QRr%   c                 H    |j                  |       |j                  |       y r   r   r   s      r#   r   zT5GemmaLoader.set_vocabulary  r   r%   c                 v   |j                   |_         |j                  |_        |j                  |_        t        |j                  d      r&|j                  j
                  j                  |_        n9t        |j                  d      r|j                  j                  |_        nd|_        |j                   |_        y )Nr   r  gư>)	r   r   r   r   rX   r   r  r  r   r   s       r#   r~   zT5GemmaLoader.set_config  s    $..$..$..5<<+(-(<(<(I(IF%U\\>2(-(A(AF%(,F%%.%8%8"r%   c                    d|_         t        |j                  t              r|j                  d   n|j                  }| j	                  ||j
                         |j                  dz  |_        | j                  |j                  |j                         |}t        t        |j                  |j                              D ]  \  }\  }}	| j                  |j                  |	j                          | j                  |j"                  |	j$                         t'        d      D 
cg c]  }
t)        j*                          }}
| j-                  |d   |	j.                  j0                  |       | j-                  |d   |	j.                  j2                  |       | j-                  |d   |	j.                  j4                  |       t7        j8                  |j:                  j<                  d   |       | j-                  |j:                  j<                  d   |	j.                  j>                  |       | j                  |j@                  |	jB                         | j                  |jD                  |	jF                         | j-                  |jH                  jJ                  |	jL                  jN                  |       | j-                  |jH                  jP                  |	jL                  jR                  |       | j-                  |jH                  jT                  |	jL                  jV                  |       tY        |	d       tY        |	d	       t[        j\                           y c c}
w )
NTr   r  r
  r^  r   r  r   r  )/r  r   r   r!  r   r"  rh  r  r   r   r!  rk  r   r   r   r  pre_self_attn_layernormr  post_self_attn_layernormr  r   r  r   r   r  r  r  r   r  r   r   r"  r9  r:  r;  r<  r   r   r  r#  r  r  r  r   r$  r%  r&  )r9   r`   r   r  r   encoder_emb_specr   rJ  r  r   r   qkv_split_layerss               r#   r   zT5GemmaLoader.set_encoder  s    !% #-T__d"CDOOA 	 	,g.B.BC2@2L2Lc2Q/DOOW\\:&/DJJ0N&O 1	"A"
E++U-J-J 44e6T6T
 CH(KQ 6 6 8KKOO #U__%;%;
   OO #U__%;%;
   OO #U__%;%;
   j77>>qACSTOO))003&&%   55u7V7V 668X8X
 OO'')<)<   OO--uyy/@/@Z   OO'')<)<  
 E;'E5!JJLc1	  Ls   !Mc                 	   d|_         d|_        | j                  |j                  |j                         |j
                  dz  |j                  _        | j                  |j                  |j                         t        t        |j                  |j                              D ]  \  }\  }}| j                  |j                  |j                         | j                  |j                   |j"                         t%        d      D cg c]  }t'        j(                          }	}| j+                  |	d   |j,                  j.                  |       | j+                  |	d   |j,                  j0                  |       | j+                  |	d   |j,                  j2                  |       t5        j6                  |j8                  j:                  d   |	       | j+                  |j8                  j:                  d   |j,                  j<                  |       | j                  |j>                  |j@                         | j                  |jB                  |jD                         | j+                  |jF                  j:                  d   |jH                  j.                  |       t%        d      D cg c]  }t'        j(                          }
}| j+                  |
d   |jH                  j0                  |       | j+                  |
d   |jH                  j2                  |       t5        j6                  |jF                  j:                  d   |
       | j+                  |jF                  j:                  d   |jH                  j<                  |       | j                  |jJ                  |jL                         | j                  |jN                  |jP                         | j+                  |jR                  jT                  |jV                  jX                  |       | j+                  |jR                  jZ                  |jV                  j\                  |       | j+                  |jR                  j^                  |jV                  j`                  |       tc        |d	       tc        |d
       tc        |d       te        jf                           y c c}w c c}w )NTFr  r
  r   r^  r   r  r   
cross_attnr  )4r  r1  r   r   r"  rh  r  r   r   r!  rk  r   r   r   r  r  r  r  r  r   r  r   r   r  r  r  r   r  r   r   r"  )external_pre_encoder_attention_layer_normpre_cross_attn_layernorm*external_post_encoder_attention_layer_normpost_cross_attn_layernormr  r  r9  r:  r;  r<  r   r   r  r#  r  r  r  r   r$  r%  r&  )r9   r`   r   r  r   rJ  r  r   r   r  kv_split_layerss              r#   r   zT5GemmaLoader.set_decoder&  s    !%).&DOOV-@-@A1?1K1KS1P.DOOV[[9&/DJJ0N&O Y	"A"
E++U-J-J 44e6T6T
 CH(KQ 6 6 8KKOO #U__%;%;
   OO #U__%;%;
   OO #U__%;%;
   j77>>qACSTOO))003&&%   DD..
 EE// OO$$++A.  ''%   BGqJA{557JOJOO"  ''%  
 OO"  ''%  
 j2299!<oN OO$$++A.  ''%   55u7V7V 668X8X
 OO'')<)<   OO--uyy/@/@Z   OO'')<)<  
 E;'E<(E5!JJLsY	  LH Ks   S'SN)rH   ro   rp   r   rO   r   r|   r   r~   r   r   r   r   r   r&   r%   r#   r  r    sZ    1 12BH09 9D8P8P8T8T?D 8C7O7O7S7Scr%   r  )Vr   rS  r%  r  rh   typingr   r   numpyr  rk   rB   rD   ImportErrorctranslate2.convertersr    ctranslate2.converters.converterr   ctranslate2.specsr   r   r	   r
   r   r   r   r  r  r  GELUSigmoidRELUrQ  r   rR  LinearSurS  r   r   r`  AWQ_GEMVrM  r    r'   r)   ABCru   r   r(  r=  rM  rU  r]  r~  r  r  r  r  r   r,  r_  r  r  r  r  r  r.  r?  ri  r  r  r  r  r  r  r  r  r  r  r(  r7  rC  r^  rH   r  r  r&   r%   r#   <module>r     s   
  	  	 ! 	 ) 6   ""''''00&&//))..$//88((44""''""((##))
  ..55

*
*
-
-..550033	  $$--$$-- 
 yI yxWF#'' WFt @V @V @VF  47Z 47 !47n  :  !B =* =  =" !	9J 	9 "	9 C
 C CL #$/G+ /G %/Gd %G %G %GP 5G 5G 5Gp !VGK VG "VGr !LNK LN "LN^ !c J c  "c L !"K@Z K@ #K@\ %&q$ q$ 'q$h y'{ y' y'x - - - J+ J  JZ 41 41 41n \+ \  \~  d; d !dN Q+ Q  Qh #$ }; } ! %}@ !DK D "DN I+ I  IX S+ S  Sl ,-0D 0D .0Df 0D 0D 0Df h h hV H{ H HV  +8 + !+ #$6*{ 6* %6*r K@ K@ K@\ #$N:{ N: %N:b !N:K N: "N:b "#N:k N: $N:b7&t zF	 LF	 *!( V
 2 c} B !BK B "Bqk  		s   T TT