qi ddlZddlZddlZddlZddlmZddlmcmZ ddlm Z ddl m Z ddlmZddlmZddlmZddlmZdd lmZmZmZmZmZmZdd lmZmZdd l m!Z!m"Z"m#Z#d d l$m%Z%e#jLe'Z(GddejRZ*GddejRZ+GddejRZ,GddejRZ-GddejRZ.GddeZ/GddeZ0GddejRZ1GddejRZ2Gd d!ejRZ3e!Gd"d#eZ4Gd$d%eZ5Gd&d'eZ6Gd(d)eZ7Gd*d+ejRZ8Gd,d-ejRZ9Gd.d/ejRZ: dKd0e;e;Gd?d@e4ZEe!GdAdBe4ZFGdCdDejRZGGdEdFejRZHe!dG;GdHdIe4ZIgdJZJy)LN)CrossEntropyLoss)initialization)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)GradientCheckpointingLayer)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel*get_torch_context_manager_or_global_device)auto_docstringis_peft_availablelogging) WavLMConfigc$eZdZfdZdZxZS)WavLMSamePadLayercPt||dzdk(rd|_yd|_yNrr)super__init__num_pad_remove)selfnum_conv_pos_embeddings __class__s Z/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/wavlm/modeling_wavlm.pyrzWavLMSamePadLayer.__init__&s) #:Q#>!#CacV|jdkDr|ddddd|j f}|SNr)rr hidden_statess r"forwardzWavLMSamePadLayer.forward*s6    ")!Q0F43F3F2F0F*FGMr#__name__ __module__ __qualname__rr( __classcell__r!s@r"rr%sKr#rc$eZdZfdZdZxZS)WavLMPositionalConvEmbeddingct|tj|j|j|j |j dz|j |_tjj}ttjjdr$tjjj}tr(ddl }|jj|jj d5||jdd|_dddt|jdrU|jjj j"}|jjj j$}n,|jj&}|jj(}|jj+|||jj+||n||jdd|_t-|j |_t0|j2|_y#1swY'xYw) Nr) kernel_sizepaddinggroups weight_normr) modifier_rankweight)namedimparametrizations)rrnnConv1d hidden_sizer num_conv_pos_embedding_groupsconvutilsr5hasattrr:r deepspeedzeroGatheredParametersr7 original0 original1weight_gweight_vregister_external_parameterrr3rfeat_extract_activation activation)rconfigr5rBrGrHr!s r"rz%WavLMPositionalConvEmbedding.__init__1s II      6622a777   hh** 288,,m <((33??K % ' 224993C3CST2U I' aH  Ityy"459955<<FF9955<<FF99--99-- NN 6 6tX F NN 6 6tX F#DIIH!DDI()G)GH  !?!?@ I Is IIc|jdd}|j|}|j|}|j|}|jdd}|SNrr) transposer?r3rKr&s r"r(z$WavLMPositionalConvEmbedding.forwardRsV%//15  -0  ]3  6 %//15 r#r)r.s@r"r0r00sABr#r0c$eZdZfdZdZxZS)WavLMFeatureProjectionc4t|tj|jd|j |_tj|jd|j|_ tj|j|_ y)Neps) rrr; LayerNormconv_dimlayer_norm_eps layer_normLinearr= projectionDropoutfeat_proj_dropoutdropoutrrLr!s r"rzWavLMFeatureProjection.__init__^sf ,,vr':@U@UV))FOOB$79K9KLzz&":":; r#cp|j|}|j|}|j|}||fSN)rYr[r^)rr'norm_hidden_statess r"r(zWavLMFeatureProjection.forwardds:!__];(:;  ]3 000r#r)r.s@r"rQrQ]s < 1r#rQceZdZdZ ddedededededef fd Z dd ejd ejd zd ejd zd ede ejejd ze ejd zff dZ d ejd ejejzdejd ede ejejff dZdededejfdZdejdejfdZxZS)WavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper embed_dim num_headsr^ num_buckets max_distancehas_relative_position_biasct|||_||_||_||z|_|j |z|jk7rt d|jd|d|j dz|_tj|||_ tj|||_ tj|||_ tj|||_ ||_||_tj t#j$d|jdd|_tj|j d|_|r0tj*|j|j|_yy)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rrrerfr^head_dim ValueErrorscalingr;rZk_projv_projq_projout_projrgrh Parametertorchonesgru_rel_pos_constgru_rel_pos_linear Embeddingrel_attn_embed)rrerfr^rgrhrir!s r"rzWavLMAttention.__init__os7 "" !Y. MMI %$.. 8MdnnM]$YKr3 }}d* ii 95 ii 95 ii 95  )Y7 &(!#ejjDNNAq.Q!R"$))DMM1"= %"$,,t/?/?"PD  &r#Nr'attention_mask position_biasoutput_attentionsreturnc|j\}}}|S|j||}|jdj|dddj ||j z||}|j |j dd|j dfz} | jdddd} |j| } | j | j dddzjd} tj| jdd\} } | | |jzd z zd z} | j ||j zdd|z}|j d||f}|j||||\}}|||fS) z'Attention layer with relative attentionNrrrSrr)rr9?g@)size compute_bias unsqueezerepeatviewrfshapepermuterwsumrtsigmoidchunkrvtorch_multi_head_self_attention)rr'rzr{r|indexbsztgt_len_gated_hidden_statesrelative_position_projgate_agate_b gate_outputgated_position_bias attn_output attn_weightss r"r(zWavLMAttention.forwards(,,.Wa   --gw?M''*11#q!Q?DDS4>>EY[bdkl  ,001D1DSb1IT^^]_L`1`a199!Q1E"&!8!89L!M!7!DDQBDO)?)? ?# EFL *..sT^^/CRKm[166GW7MN$($H$H >+>@Q% ! \L-77r#rcX|jddx}x}}||jdnd}dx} } d} tj||||j|j t jdgt j|jj|jj|jjf| | | |j|jj|jj|j |||d|jj|jj|jj\} } | jdd} | C| dddfj#| j$dd|j fz| j$ddz} | | fS)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)use_separate_proj_weight q_proj_weight k_proj_weight v_proj_weight)rOneFmulti_head_attention_forwardrerfrtemptycatrqbiasrorpr^rrr7training broadcast_tor)rr'rzrr|querykeyvaluekey_padding_maskbias_kbias_v add_zero_attnrrs r"rz.WavLMAttention.torch_multi_head_self_attentions,55a;;;e3A3M>,,Q/SW %&$B$B    NN NN KK  IIt{{'')9)94;;;K;KL M    LL MM MM   MM   %)++,,++,,++,,+% ! \2"++Aq1  #(40==""2A&$..)::\=O=OPQPR=SSLL((r# query_length key_lengthctj|tjdddf}tj|tjdddf}||z }|j|}|j |j j j}|j |}|jgd}|S)Ndtype)rrr) rtarangelong_relative_positions_buckettoryr7devicer)rrrcontext_positionmemory_positionrelative_positionrelative_position_bucketvaluess r"rzWavLMAttention.compute_biass << EJJG4P,,zDT1WM+.>>#'#B#BCT#U #;#>#>t?R?R?Y?Y?`?`#a $$%=> * r#relative_positionsc$|jdz}|dkDjtj|z}tj|}|dz}||k}tj |j |z }|tj |j|z z }|||z z}||zjtj}tj|tj||dz }|tj|||z }|Sr) rgrrtrabslogfloatmathrhmin full_likewhere)rrrgrelative_buckets max_exactis_smallrelative_positions_if_largerelative_position_if_larges r"rz)WavLMAttention._relative_positions_buckets &&!+ .266uzzB[P"YY'9:1$ % 1&+ii0B0H0H0JY0V&W#&ADHHTM^M^ajMjDk&k#&A[S\E\&]#&/2M&M%Q%QRWR\R\%]"%*YY &8RT_bcTc(d& " EKK2DF`aar#)i@i TNNFr)r*r+r,__doc__intrboolrrtTensortupler( FloatTensor LongTensor BoolTensorrrrr-r.s@r"rdrdlsG +/"Q"Q"Q "Q  "Q  "Q%)"QN/3-1"' '8||'8 t+'8||d* '8  '8 u||U\\D0% 2E2LL M'8R5)((5)((5+;+;;5)#.. 5)  5) u  %"3"33 4 5)n#%BSBS U=N=N SXSdSd r#rdc$eZdZfdZdZxZS)WavLMFeedForwardct|tj|j|_tj |j|j|_ t|jtrt|j|_n|j|_tj |j|j|_tj|j |_yra)rrr;r\activation_dropoutintermediate_dropoutrZr=intermediate_sizeintermediate_dense isinstance hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutr_s r"rzWavLMFeedForward.__init__s $&JJv/H/H$I!"$))F,>,>@X@X"Y f'' -'-f.?.?'@D $'-'8'8D $IIf&>&>@R@RS jj)>)>?r#c|j|}|j|}|j|}|j|}|j |}|Sra)rrrrrr&s r"r(zWavLMFeedForward.forward sX// > 00? 11-@ ))-8 ++M: r#r)r.s@r"rrs @r#rc2eZdZddedeffd ZddZxZS)WavLMEncoderLayerrLrict|t|j|j|j |j |j||_tj|j|_ tj|j|j|_t!||_tj|j|j|_yN)rerfr^rgrhrirTrrrdr=num_attention_headsattention_dropoutrgmax_bucket_distance attentionr;r\rr^rVrXrYr feed_forwardfinal_layer_normrrLrir!s r"rzWavLMEncoderLayer.__init__+ '((00,,**33'A  zz&"7"78 ,,v'9'9v?T?TU,V4 " V-?-?VEZEZ [r#c|}|j|||||\}}}|j|}||z}|j|}||j|z}|j |}||f}|r||fz }|S)Nrzr{r|r)rr^rYrr) rr'rzr{r|r attn_residualroutputss r"r(zWavLMEncoderLayer.forward:s% 59^^ )'/ 6D6 2 |] ]3 % 5  6 %(9(9-(HH --m<  -0    &Gr#Trr*r+r,rrrr(r-r.s@r"rr*s \{ \ \r#rc2eZdZddedeffd ZddZxZS) WavLMEncoderLayerStableLayerNormrLrict|t|j|j|j |j |j||_tj|j|_ tj|j|j|_t!||_tj|j|j|_yrrrs r"rz)WavLMEncoderLayerStableLayerNorm.__init__Trr#c|}|j|}|j||||\}}}|j|}||z}||j|j |z}||f}|r||fz }|S)N)rzr{r|)rYrr^rr)rr'rzr{r|rrrs r"r(z(WavLMEncoderLayerStableLayerNorm.forwardcs%  6 59^^ )'/ 6D6 2 |]  ]3 % 5 %(9(9$:O:OP]:^(__  -0    &Gr#r)NNFrr.s@r"rrSs \{ \ \r#rc.eZdZfdZ ddZxZS) WavLMEncoderc t|||_t||_t j |j|j|_ t j|j|_ t jt|jDcgc]}t!||dk(c}|_d|_ycc}wNrTr)riF)rrrLr0pos_conv_embedr;rVr=rXrYr\rr^ ModuleListrangenum_hidden_layersrlayersgradient_checkpointingrrLir!s r"rzWavLMEncoder.__init__ys  :6B,,v'9'9v?T?TUzz&"7"78 mmUZ[a[s[sUt uPQ v16 K u  ',# v!C c|rdnd}|rdnd}|5|jdjdd|jd}d||<|j|} || z}|j |}|j |}t xs t|} d} t|jD]y\} } |r||fz}tjg}|jxr | dkDxr||jjk}|r| r| ||| || }|dd\}} |rd}|sq|dfz}{|r||fz}|std|||fDSt!||| S) NrSrrrrNNNc3&K|] }|| ywrar.0vs r" z'WavLMEncoder.forward..mq_`_lmlast_hidden_stater' attentions)rrrrrYr^rr enumeraterrtrandrrL layerdroprr rr'rzr|output_hidden_states return_dictall_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddings synced_gpusr{rlayerdropout_probabilityskip_the_layer layer_outputss r"r(zWavLMEncoder.forwards#7BD$5b4  %$2$<$.rrr)rrrrr^rrrrrtrrrLrrYrr rs r"r(z#WavLMEncoderStableLayerNorm.forwards#7BD$5b4  %$2$<$AQAQ Q+00o1Mt`d`m`moqr !oob)..z?BO J&&r#) r*r+r,rr staticmethodr;r(r-r.s@r"r*r* s& * "'r#r*ceZdZUeed<dZdZdZdZdZ dZ dZ e jdZdd e jezd edzfd Z dd ed e jfdZy)WavLMPreTrainedModelrLwavlm input_valuesaudioTFc t|trutj|jj ddtj |jjtj|jyt|trtj|jj ddtjd|jjd|jjzz ztj |jjdyt|t"rtjd|j$j&z }tj|j$j | |tj|j$j| |yt|t(j*rctj|j d|j,j.|j tj |jyyt|t(j0t(j2fr?tj |jtj4|j yt|t(j6rtj8|j |j`tj|j:|j|jdzz }tj|j| |yyy)zInitialize the weightsrr)r5stdrr)abN)rr*initnormal_r2r7zeros_runiform_r1r0r?rsqrtr2 in_channels constant_rQr[ in_featuresr;rZrLinitializer_rangerV GroupNormones_r<kaiming_normal_r4)rmoduleks r" _init_weightsz"WavLMPreTrainedModel._init_weights^s* f8 9 LL++22! D KK**// 0 MM&,, -  < = LL "" !v{{'>'>q'AFKKD[D['["\]]  NN6;;++Q /  6 7 !f//;;; MM&++00QB! <  * LLSdkk6S6S T{{& FKK(' r|| < = KK $ JJv}} %  *   /{{&IIfmmv/A/AFDVDVWXDY/YZ[ fkkaR15'+r#N input_lengths add_adaptercT||jjn|}d}t|jj|jjD]\}}||||}|rBt |jj D] }||d|jj}"|S)zH Computes the output length of the convolutional layers c>tj||z |ddzS)Nfloor) rounding_moder)rtdiv input_lengthr2strides r"_conv_out_lengthzOWavLMPreTrainedModel._get_feat_extract_output_lengths.._conv_out_lengths"99\K7wWZ[[ [r#r)rLrgzip conv_kernel conv_stridernum_adapter_layersadapter_stride)rrfrgrpr2rors r" _get_feat_extract_output_lengthsz5WavLMPreTrainedModel._get_feat_extract_output_lengthss 2=1Ddkk--+  \ $'t{{'>'> @W@W#X Q K,]KPM Q 4;;99: _ 04;;C]C] ^  _r#feature_vector_lengthrzc|jddddf}|j||}|jtj}|j d}tj ||f|j|j}d|tj|j d|j|dz f<|jdgjdjdgj}|S)NrSrrgr)rrr)r) cumsumrvrrtrrzerosrrrflipr)rrwrzrgnon_padded_lengthsoutput_lengthsrGs r""_get_feature_vector_attention_maskz7WavLMPreTrainedModel._get_feature_vector_attention_masks ,22r2:1b5A>>?Q_j>k'**5::6#))!,  . /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOr#ra)r*r+r,r__annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attnrtno_gradrerrrrvrrr#r"rOrOSs $O&*# NU]]_66Be>N>NQT>Tcgjncn,Y]%(:?:J:Jr#rOc&eZdZdfd ZdZxZS)WavLMNoLayerNormConvLayercdt||dkDr|j|dz nd|_|j||_t j |j|j|j||j||j|_ t|j|_ y)Nrrr2ror)rrrW in_conv_dim out_conv_dimr;r<rrrs conv_biasr?rrJrKrrLlayer_idr!s r"rz"WavLMNoLayerNormConvLayer.__init__s .5s#h!$5f$=#hs)rrrr=r;rZprojrVproj_layer_normrrrtrrr_s `r"rzWavLMAdapter.__init__+s   $ $(:(: : &"4"4f6O6OPDI#%<<0I0I#JD /3 3DI,mm#huVMfMfGg#hh ))r#ch|j.|j"|j|}|j|}|jdd}|jD]D}tj j }|j r||jkDs=||}F|jdd}|SrN)rrrOrnprandomrr)rr'rlayerdrop_probs r"r(zWavLMAdapter.forward8s 99 T%9%9%E IIm4M 00?M%//15 [[ 5EYY--/N==^dnn%D %m 4  5 &//15 r#r)r.s@r"rr*s  *r#rr mask_prob mask_lengthrz min_masksr}c|\}dkr tdkDrtdddtjjdj fd}|-|j j djnt|Dcgc]}c}}tj|ft } g} |} | d k(r| S|D]} || } tjjtj| dz z | d }t|d k(rdz }n|d }tj|tj| | z tj |zg}| j#|tj$| } tj&| dddddf|| f} | j)|| z} tjddddf}tj&||| fj)|| z}| |z} | j+dz kDrdz | | dz kD<tj,| | dd| Scc}w) an Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `ct|zz z}t|}|zkDrz}|dz z |krt|dz z d}|S)z;Given input length, compute how many spans should be maskedrr)rmax)rnnum_masked_spanepsilonrrrrHs r"compute_num_masked_spanz6_compute_mask_indices..compute_num_masked_spanosoi,6DwNOoy9 [ (? :-DL'/DL/5/A/A|F+t  r#c8|jjyz Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rrrs r"freeze_feature_encoderz!WavLMModel.freeze_feature_encoders 113r#Nr'mask_time_indicesrzct|jdds|S|j\}}}|)|jj |j ||<n|jj dkDr|jrt||f|jj |jj||jj}tj||jtj}|jj |j ||<|jjdkDr|jrt||f|jj|jj |jj"}tj||jtj}|dddfj%d|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://huggingface.co/papers/1904.08779). apply_spec_augmentTNr)rrrzr)rr)rrrrS)getattrrLrrrrrrrmask_time_lengthmask_time_min_masksrttensorrrrmask_feature_lengthmask_feature_min_masksexpand)rr'rrzrGrHr=mask_feature_indicess r"_mask_hidden_stateszWavLMModel._mask_hidden_statesst{{$8$? 4A3E3E3G0 O[  (/3/E/E/H/HI\I\/]M+ , [[ ' '! + 5_-++44 KK88-++99 ! !& ->}G[G[chcmcm n /3/E/E/H/HI\I\/]M+ , ;; ( (1 ,#8[)++77 KK;;++<< $ $)<<0D]MaMainisis#t #74#@#G#GO]_#` 23M. /r#rQr|rrr}c H||n|jj}||n|jj}||n|jj}|j |}|j dd}|!|j |jd|d}|j|\} }|j| ||} |j| ||||} | d} |j|j| } |s | |f| ddzSt| || j| j S) a/ mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict masked extracted features in *config.proj_codevector_dim* space. NrrFry)rrzrzr|rrr)rextract_featuresr'r)rLr|ruse_return_dictrrOrrrrrrWavLMBaseModelOutputr'r) rrQrzrr|rrkwargsrr'encoder_outputss r"r(zWavLMModel.forwardsb 2C1N-TXT_T_TqTq$8$D $++JjJj &1%uEN+/*A*ABR*S' '00 ->~1 ,, )/!5# ' (* << # LL7M!#34qr7JJ J#+-)77&11   r#)NNNNNNN)r*r+r,rrrrtrrrrrrrrr(r-r.s@r"rrs{(47;26 ,((,!,,t3,((4/ ,\/36:)-,0#'8 llT)8  t+8 !,,t3 8  $; 8 #Tk 8 D[8  % %8 8 r#rrzm WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). ) custom_introceZdZddedzffd ZdZdZdZe dde jdzde jdzd e dzd e dzd e dzd e jdzd e e zfdZxZS) WavLMForCTCN target_langct||t||_t j |j |_||_|jtd|jdt|dr|jr |jn |j}t j ||j|_|j%y)a/ target_lang (`str`, *optional*): Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or adapter..bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by default. NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.rg)rrrrPr;r\ final_dropoutr^r vocab_sizermr!rArgrr=rZlm_headr)rrLrrr!s r"rzWavLMForCTC.__init__Ss  ' zz&"6"67 &    $00@AHH *1)GFL^L^F % %djdvdv yy!3V5F5FG  r#c 8ttjdk(ry|j}|&t |j ddt d|d|-t |j ddtjdy||j|dyy) a' This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. metaNadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T) force_load) rrtrrrrLrmloggerinfo load_adapter)rrrs r" tie_weightszWavLMForCTC.tie_weightsps 6 75<<;O O  &&  "wt{{ 1` a classification loss is computed (Cross-Entropy). NTrrrrSrrrr)rLrr6rPr-rtstackr;r@rCr7rrr9r5rrrrr;rr:r r'r)rrQrzr|rrrrrr' norm_weights pooled_output padding_maskexpand_padding_maskr!r loss_fctr2s r"r(z&WavLMForSequenceClassification.forwards0&1%->r1a-HHMMRSMTM#AJM}5  !)..1.5MBB=CVCVWXCY[ijL"."8"8"<"C"CAq-J]J]^_J`"a 25M.. /)--!-4|7G7GA7G7N7S7STVXY7ZZM/  ')HFKKDKK,B,BCV[[QS_UDY)F)G!HHF)-)9TGf$ Ev E'!//))   r#r)r*r+r,rrrrrtrrrr r(r-r.s@r"r4r4s":(/3)-,0#'&*C llT)C  t+C  $; C #Tk C D[ C  t#C  ) )C C r#r4ceZdZfdZdZdZe d dejdzdejdzdejdzde dzd e dzd e dzd e e zfd Z xZ S) WavLMForAudioFrameClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_|j |_|j%y)Nrgz_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrrArgrmrrPrr6r;rsrtrur7rZr=r:r;rr<s r"rz)WavLMForAudioFrameClassification.__init__Ps  6= )f.@.@q  ' --1  ( (!#ejj.Dz.Q!RD ))F$6$68I8IJ ++ r#cL|jjjyrrrs r"rz7WavLMForAudioFrameClassification.freeze_feature_encoder`rr#cP|jjD] }d|_ yrrrs r"rz2WavLMForAudioFrameClassification.freeze_base_modelgrr#NrQrzrr|rrr}c ||n|jj}|jjrdn|}|j|||||}|jjrr|t} t j | d} tjj|jd} | | jdddzjd} n|d} |j| } d} |\t} | | jd|jt j |jd|jd} |s| f|tdz}|St#| | |j$|j& S) rANTrrrrSr)axisr)rLrr6rPr-rtrBr;r@rCr7rrr;rr:rDr r'r)rrQrzrr|rrrrr'rCr!r rGr2s r"r(z(WavLMForAudioFrameClassification.forwardosh0&1%->r1a-HHMMRSMTM#AJM/  ')HFKKDOOtuDY)F)G!HHFM$!//))   r#r)r*r+r,rrrrrtrrrr r(r-r.s@r"rIrINs :(/3&*)-,0#': llT):  t+:  t# :  $; : #Tk : D[:  & &: : r#rIc&eZdZdfd ZdZxZS) AMSoftmaxLossct|||_||_||_t j tj||d|_ t j|_ y)NT)r) rrscalemarginr:r;rsrtrandnr7rr )r input_dimr:rRrSr!s r"rzAMSoftmaxLoss.__init__sQ   $ll5;;y*#EUYZ '') r#c|j}tjj|jd}tjj|d}t j ||}||jz }tjj||j}|jt j|j||z}|j||}|S)Nrrr)flattenr;r@ normalizer7rtmmrSone_hotr:rRrrr ) rr'rr7 cos_thetapsionehotr!r s r"r(zAMSoftmaxLoss.forwards!((!(< // 1/E HH]F3 $++%&&vt?ekk&++-iHHyy( r#)g>@g?r)r.s@r"rPrPs * r#rPcXeZdZdfd Zdej dej fdZxZS) TDNNLayerct||dkDr|j|dz n|j||_|j||_|j ||_|j||_tj|j|j z|j|_ tj|_ y)Nrr)rrtdnn_dimrr tdnn_kernelr2 tdnn_dilationdilationr;rZkernelReLUrKrs r"rzTDNNLayer.__init__s +BFD]D]!^))F$=$=v?X?XY&v'@'@&BSBST Rs>FcL|jjjyrrrs r"rz&WavLMForXVector.freeze_feature_encoderrr#cP|jjD] }d|_ yrrrs r"rz!WavLMForXVector.freeze_base_modelrr#rfcVd}|jjD] }|||d}|S)z? Computes the output length of the TDNN layers c||z |zdzS)Nrrrms r"rpzBWavLMForXVector._get_tdnn_output_lengths.._conv_out_lengths!;.69A= =r#r)rLrb)rrfrpr2s r"_get_tdnn_output_lengthsz(WavLMForXVector._get_tdnn_output_lengths s:  >  ;;22 LK,]KKM Lr#NrQrzr|rrrr}c ||n|jj}|jjrdn|}|j|||||}|jjrr|t} t j | d} tjj|jd} | | jdddzjd} n|d} |j| } |jD] } | | } |%| jd} | j!d} n|j#|jd}|j%|}g} g} t'|D]U\}}| j)| |d|fjd| j)| |d|fj!dWt j | } t j | } t j*| | gd}|j-|}|j/|}d}||j1||}|s||f|tdz}||f|zS|St3||||j4|j6S) rANTrrrrSr)r r! embeddingsr'r)rLrr6rPr-rtrBr;r@rCr7rrr9rpr5rTrvrxrrrrr;rrrr'r)rrQrzr|rrrrrr'rC tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrlengthstatistic_poolingoutput_embeddingsr!r r2s r"r(zWavLMForXVector.forwards0&1%->r1a-HHMMRSMTM#AJM}5 )) 6J&}5M 6  !)..1.5M(,,,3L*.*O*OP^PbPbghPbPi*j '"&"?"?@["\ ML&':; J 6$$]1gvg:%>%C%C%C%JK##M!WfW*$=$A$Aa$A$HI J"KK 6M ;;|4L!II}l&CL 223DE!23  >>&&1D/07;X;Y3ZZF)-)9TGf$ Ev E(!//))   r#r)r*r+r,rrrrtrrrxrrrrrr(r-r.s@r"rnrns &:( e6F6F6L /3)-,0#'&*P llT)P  t+P  $; P #Tk P D[ P  t#P   P P r#rn)rIrr4rnrrOr%)Krrjnumpyrrttorch.nnr;torch.nn.functionalr@rrrrW activationsrintegrations.deepspeedrintegrations.fsdprmodeling_layersr modeling_outputsr r r r rrmodeling_utilsrrr@rrrconfiguration_wavlmr get_loggerr*r Modulerr0rQrdrrrrr%r*rOrrrrrrrrrrndarrayrrrr-rr4rIrPr_rn__all__rr#r"rs  %&!@79Z??,   H % *299*Z 1RYY 1c RYYc Lryy0&2&R"'A"JG 299G TH "))H VC'C'LS?SSl :*8680#"))#L $299F/3 t c?ttt$$t+ t  t ZZ tn/C %C C L!" K &K  K \ e %9e  e P[ ';[ [ |BII. @ C *C  C L r#