qi7ddlmZddlmZddlmZddlmZmZddl m Z m Z m Z m Z mZe r ddlZddlmZe rdd lmZe Ze resddlZej,eZGd d eZGd d ej j4ZGddej8ZeddZ ddee dzfdZ!y)) lru_cache)ACT2FN) ConversionOps)get_module_from_nameshould_convert_module)is_accelerate_availableis_fbgemm_gpu_availableis_torch_availableis_torch_xpu_availableloggingN)nn)init_empty_weightsc eZdZdZ ddeeejeejzfdejjdzdeeejffdZ y)FbgemmFp8Quantizec||_yN) hf_quantizer)selfrs V/opt/pipecat/venv/lib/python3.12/site-packages/transformers/integrations/fbgemm_fp8.py__init__zFbgemmFp8Quantize.__init__-s (N input_dictmodelreturnc zt|jd\}}|d}ddlm}t ||\}}t ||r|dk(r~|j dd} | j} | jd| d} t| \} } | j| }|j dd}| j| dd| d}n|dk(r|j dd} | j} | jd| d} t| \} } | j| }|j dd}| j| d| dd}nJt|\}}tjj|j|jdd}|tjj|diS) Nrr)FbgemmFp8Llama4TextExperts gate_up_proj down_proj_scale)tupleitems integrationsrr isinstance transposeshapereshapequantize_fp8_per_rowtorchr Parameterview)rrrkwargs target_keyvaluermodule tensor_nametransposed_paramoriginal_shapeflattened_paramnew_value_flatweight_scale_flat new_value weight_scales rconvertzFbgemmFp8Quantize.convert0s "*"2"2"45a8 Ea=25*E f8 9n,$)??1a#8 "2!7!7"2":":2~b?Q"R5I4Y1 1+22>B %//15 0889JA~^_O`a  +$)??1a#8 "2!7!7"2":":2~b?Q"R5I4Y1 1+22>B %//15 0889JN[\L]_`a &:5&A #I| 88--l.?.? @R@RST@UWX.YZLEHH..y9j\;PR^__rr) __name__ __module__ __qualname__rdictstrr+TensorlistrModuler:rrrr,sh) )-2`ellT%,,-???@2`xx%2` c5<< 2`rrc<eZdZejffd ZdZxZS)FbgemmFp8Linearcrt||||||_||_tj j t j||f||_tj j t j|dftj|_ |jdt jdgtjd|rRtj j t j|jtj|_ yd|_ y)Ndtyperinput_scale_ubF persistent)superr in_features out_featuresr+rr,zerosweightfloat32r9register_bufferfloatbias)rrMrNrTrH __class__s rrzFbgemmFp8Linear.__init__fs lD9&(hh((lK5PX])^_ !HH..u{{L!;LTYTaTa/bc -u{{A3ekk/R_de **5;;8I8IRWR_R_+`aDIDIrcg|jddd}t|jd|jdj|j\}}|j j tj}tretj||jj|jd|j|j|j}nUtj j"j%||j||d}|j||jzn|}|j |j&}|j)|}~~|S)Nr )scale_ub)scale_ascale_b out_dtyperTTuse_fast_accum)r(r*r- contiguousrIr9tor+rQ_is_torch_xpu_available _scaled_mmrPt unsqueezerHrTopsfbgemmf8f8bf16_rowwisedevicer))rx output_shape x_quantizedx_scaleweight_scale_float32outputs rforwardzFbgemmFp8Linear.forwardts0*"*r*  4AFF2qwwr{4K4V4V4Xcgcvcvw W $0033EMMB "%% ))"-,..0''YY FYY%%66T[['3GX\7F,099+@Vdii'fF188$ -  r)r;r<r=r+ float8_e4m3fnrrm __classcell__rUs@rrErEes>C>Q>Q rrEc<eZdZejffd ZdZxZS)rcnt||j|_|j|_|j |_|j|_t|j|_ tjjtj|j|j d|j zftj|_tjjtj|jd|j dzftj |_tjjtj|j|j |j ftj|_tjjtj|j|j dftj |_|j)dtjdgtj*dy)NrrGrrIFrJ)rLrnum_local_experts num_expertsintermediate_size hidden_size expert_dimr hidden_actact_fnr+rr,rOrnrrQgate_up_proj_scaler!down_proj_scalerRrS)rconfigrHrUs rrz#FbgemmFp8Llama4TextExperts.__init__s !33!'!9!9!--00V../ !HH.. KK))4+;+;Q=PQY^YlYl m #((("4"4 KK))1doo.AB%-- X# ++ KK))4??DPQVVWY[\]hhjlln+11 %%$%%a(221a89K9LMXXZ\\^(22266q9!<=O=PQVVWY[\]hhjlln+11 yy''88$%%a(221a89L:LMXXZ .q1!45H6HINNrSTU``b#' 9YY%%66$%%a(221a89K9LMXXZ .q1!45G5HINNrSTU``b#' 7T[[..I3G S]_c_r_r3s 0 &*&:&:&=&=emm&L #& % 0 0'NN1%//15@@B+55b93A6;;BBMMOQQS+11 ! !& 0 0 A A'NN1%//15@@B#+A.33B:EEG#' !B! +KNA@ +B"nn]%9%9: D$4$455r)r;r<r=r+rQrrmrorps@rrrs%*]]f0P6rrr)maxsizectrddlm}|djStj j jS)Nr get_kernelzkernels-community/fp8-fbgemm)r_ hub_kernelsrr*r+rcrdrs rget_quantize_fp8_per_rowrs0+89NNN 99   0 00rmodules_to_not_convertcttad}|rinddi}|jD]\}}t||sd} t d5|j j dk(r;t|jd|j} t| xs |j} nYt|tjr?t|j|j|j dufi|} | j#dddd| |j%|| d}|st&j)d|S#1swY;xYw) a A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules. This will enable running your models using high performance fp8 kernel from FBGEMM library. Parameters: model (`torch.nn.Module`): Input model or `torch.nn.Module` as the function is run recursively. modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`): Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons. quantization_config (`FbgemmFp8Config`): The quantization config object that contains the quantization parameters. pre_quantized (`book`, defaults to `False`): Whether the model is pre-quantized or not FrHNT)include_buffersLlama4TextExperts text_configzYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)rr* named_modulesrrrUr;getattrr|rr&rLinearrErMrNrTrequires_grad_ set_submoduleloggerwarning) rrquantization_config pre_quantizedtp_planhas_been_replaced module_kwargs module_namer1 new_modulers rreplace_with_fbgemm_fp8_linearr s;$45'Bgt_M$224! V$[2HI   5 1((,?? &ellM5<<P 7 8Su||T FBII.,&&''KKt+$  ))%0 1"    K4 5!8   L= 1 1s B.D..D7 )NNFN)" functoolsr activationsrcore_model_loadingrquantizers.quantizers_utilsrrutilsr r r r r r+r acceleraterr_fbgemm_gpu.experimental.gen_ai fbgemm_gpu get_loggerr;rrrrErBrrrAr?rrCrrrs .U-02%<)   H %6` 6`r,ehhoo,^i6i6X 111tx:#'9t#3:r