
    qi#                        d dl mZ d dlZd dlZd dlmZ d dlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&  e$jN                  e(      Z) G d dejT                        Z+ G d dejT                        Z, G d de      Z- G d de      Z. G d de      Z/ G d dejT                        Z0 G d dejT                        Z1	 	 dId ejT                  d!ejd                  d"ejd                  d#ejd                  d$ejd                  dz  d%e3dz  d&e3d'e e"   fd(Z4 G d) d*ejT                        Z5 G d+ d,ejT                        Z6 G d- d.e      Z7 G d/ d0ejT                        Z8 G d1 d2ejT                        Z9 G d3 d4e      Z: G d5 d6ejT                        Z;e# G d7 d8e             Z<	 	 dJd9e=e>e>f   d:e3d;e>d$ej~                  dz  d<e>d=ej                  fd>ZAe# G d? d@e<             ZBdZC e#dAB       G dC dDe<             ZD e#dEB       G dF dGe<             ZEg dHZFy)K    )CallableN)CrossEntropyLoss   )initialization)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)create_bidirectional_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputCausalLMOutputSequenceClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel*get_torch_context_manager_or_global_device)Unpack)TransformersKwargsauto_docstringlogging   )HubertConfigc                   $     e Zd Z fdZd Z xZS )HubertPositionalConvEmbeddingc                    t         |           t        j                  |j                  |j                  |j
                  |j
                  dz  |j                        | _        d | _        |j                  r&t        j                  |j                        | _        nt        j                  j                  }t        t        j                  j                  d      r$t        j                  j                  j                  }t               r(dd l}|j"                  j%                  | j                  j&                  d      5   || j                  dd      | _        d d d        t        | j                  d      rU| j                  j                  j&                  j(                  }| j                  j                  j&                  j*                  }n,| j                  j,                  }| j                  j.                  }|j"                  j1                  | |       |j"                  j1                  | |       n || j                  dd      | _        t3        |j
                        | _        t6        |j8                     | _        y # 1 sw Y   'xY w)	N   )kernel_sizepaddinggroupsweight_normr   modifier_rankweight)namedimparametrizations)super__init__nnConv1dhidden_sizenum_conv_pos_embeddingsnum_conv_pos_embedding_groupsconv
batch_normconv_pos_batch_normBatchNorm1dutilsr    hasattrr&   r   	deepspeedzeroGatheredParametersr#   	original0	original1weight_gweight_vregister_external_parameterHubertSamePadLayerr   r   feat_extract_activation
activation)selfconfigr    r4   r9   r:   	__class__s         \/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/hubert/modeling_hubert.pyr(   z&HubertPositionalConvEmbedding.__init__.   s   II6622a777
	 %% nnV-?-?@DO((..Krxx00-@ hh77CC)+ ^^66tyy7G7GWX6Y M +DIIH! LDIM499&89#yy99@@JJH#yy99@@JJH#yy11H#yy11H::4J::4J'		aH	)&*H*HI !?!?@M Ms   ?I??J	c                     |j                  dd      }| j                  | j                  |      }| j                  |      }| j                  |      }| j	                  |      }|j                  dd      }|S )Nr   r   )	transposer/   r.   r   r>   r?   hidden_statess     rB   forwardz%HubertPositionalConvEmbedding.forwardS   sn    %//15??& OOM:M		-0]36%//15    __name__
__module____qualname__r(   rG   __classcell__rA   s   @rB   r   r   -   s    #AJ	rH   r   c                   $     e Zd Z fdZd Z xZS )r<   c                 P    t         |           |dz  dk(  rd| _        y d| _        y )Nr   r   r   )r'   r(   num_pad_remove)r?   r,   rA   s     rB   r(   zHubertSamePadLayer.__init__`   s)    #:Q#>!#CarH   c                 V    | j                   dkD  r|d d d d d | j                    f   }|S Nr   )rQ   rE   s     rB   rG   zHubertSamePadLayer.forwardd   s6    ")!Q0F43F3F2F0F*FGMrH   rI   rN   s   @rB   r<   r<   _   s    KrH   r<   c                   &     e Zd Zd fd	Zd Z xZS )HubertNoLayerNormConvLayerc                 d   t         |           |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                        | _
        t        |j                     | _        y )Nr   r   r   stridebias)r'   r(   conv_dimin_conv_dimout_conv_dimr)   r*   conv_kernelconv_stride	conv_biasr.   r   r=   r>   r?   r@   layer_idrA   s      rB   r(   z#HubertNoLayerNormConvLayer.__init__k   s    <DqL6??8a<8a"OOH5II**84%%h/!!
	 !!?!?@rH   c                 J    | j                  |      }| j                  |      }|S N)r.   r>   rE   s     rB   rG   z"HubertNoLayerNormConvLayer.forwardy   s$    		-06rH   r   rI   rN   s   @rB   rU   rU   j   s    ArH   rU   c                   &     e Zd Zd fd	Zd Z xZS )HubertLayerNormConvLayerc                    t         |           |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                        | _
        t        j                  | j                  d      | _        t        |j                     | _        y )Nr   r   rW   T)elementwise_affine)r'   r(   rZ   r[   r\   r)   r*   r]   r^   r_   r.   	LayerNorm
layer_normr   r=   r>   r`   s      rB   r(   z!HubertLayerNormConvLayer.__init__   s    <DqL6??8a<8a"OOH5II**84%%h/!!
	 ,,t'8'8TR !?!?@rH   c                     | j                  |      }|j                  dd      }| j                  |      }|j                  dd      }| j                  |      }|S )N)r.   rD   rj   r>   rE   s     rB   rG   z HubertLayerNormConvLayer.forward   sV    		-0%//B76%//B76rH   rd   rI   rN   s   @rB   rf   rf      s    ArH   rf   c                   &     e Zd Zd fd	Zd Z xZS )HubertGroupNormConvLayerc                    t         |           |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                        | _
        t        |j                     | _        t        j                  | j                  | j                  d      | _        y )Nr   r   rW   T)
num_groupsnum_channelsaffine)r'   r(   rZ   r[   r\   r)   r*   r]   r^   r_   r.   r   r=   r>   	GroupNormrj   r`   s      rB   r(   z!HubertGroupNormConvLayer.__init__   s    <DqL6??8a<8a"OOH5II**84%%h/!!
	 !!?!?@,,$2C2CRVRcRclpqrH   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rc   )r.   rj   r>   rE   s     rB   rG   z HubertGroupNormConvLayer.forward   s2    		-066rH   rd   rI   rN   s   @rB   ro   ro      s    r rH   ro   c                   .     e Zd ZdZ fdZd Zd Z xZS )HubertFeatureEncoderz.Construct the features from raw audio waveformc           	         t         |           |j                  dk(  rDt        |d      gt	        |j
                  dz
        D cg c]  }t        ||dz          c}z   }nV|j                  dk(  r.t	        |j
                        D cg c]  }t        ||       }}nt        d|j                   d      t        j                  |      | _        d| _        d	| _        y c c}w c c}w )
Ngroupr   )ra   r   layerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r'   r(   feat_extract_normro   rangenum_feat_extract_layersrU   rf   
ValueErrorr)   
ModuleListconv_layersgradient_checkpointing_requires_grad)r?   r@   ir   rA   s       rB   r(   zHubertFeatureEncoder.__init__   s    ##w.3FQGHLQRXRpRpstRtLuLGH*6AEBL K %%0QVW]WuWuQvwA3FQGwKw01I1I0JJst  ==5&+#"L xs   C"	C'c                 J    | j                         D ]	  }d|_         d| _        y NF)
parametersrequires_gradr   r?   params     rB   _freeze_parametersz'HubertFeatureEncoder._freeze_parameters   s(    __& 	(E"'E	(#rH   c                     |d d d f   }| j                   r| j                  rd|_        | j                  D ]
  } ||      } |S )NT)r   trainingr   r   )r?   input_valuesrF   
conv_layers       rB   rG   zHubertFeatureEncoder.forward   sP    $QW- 4==*.M'** 	6J&}5M	6 rH   )rJ   rK   rL   __doc__r(   r   rG   rM   rN   s   @rB   rw   rw      s    8#"$

rH   rw   c                   $     e Zd Z fdZd Z xZS )HubertFeatureProjectionc                 n   t         |           |j                  | _        | j                  r3t        j                  |j
                  d   |j                        | _        t        j                  |j
                  d   |j                        | _
        t        j                  |j                        | _        y )Nrm   eps)r'   r(   feat_proj_layer_normr)   ri   rZ   layer_norm_epsrj   Linearr+   
projectionDropoutfeat_proj_dropoutdropoutr?   r@   rA   s     rB   r(   z HubertFeatureProjection.__init__   s}    $*$?$?!$$ ll6??2+>FDYDYZDO))FOOB$79K9KLzz&":":;rH   c                     | j                   r| j                  |      }| j                  |      }| j                  |      }|S rc   )r   rj   r   r   rE   s     rB   rG   zHubertFeatureProjection.forward   s;    $$ OOM:M6]3rH   rI   rN   s   @rB   r   r      s    <rH   r   modulequerykeyvalueattention_maskscalingr   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |d      }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nrm         r   r   r%   )pr   r   )
sizetorchmatmulrD   r)   
functionalsoftmaxr   r   
contiguous)
r   r   r   r   r   r   r   r   attn_weightsattn_outputs
             rB   eager_attention_forwardr      s     **R.D( <<s}}Q':;gEL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$rH   c                   (    e Zd ZdZ	 	 	 	 	 ddedededededed	edz  f fd
Z	 	 	 dde	j                  de	j                  dz  de	j                  dz  dedz  dee   dee	j                  e	j                  dz  ee	j                     dz  f   fdZ xZS )HubertAttentionz=Multi-headed attention from 'Attention Is All You Need' paperN	embed_dim	num_headsr   
is_decoderrY   	is_causalr@   c                 
   t         |           || _        || _        || _        ||z  | _        || _        | j
                  |z  | j                  k7  rt        d| j                   d| d      | j
                  dz  | _        || _	        || _
        t        j                  |||      | _        t        j                  |||      | _        t        j                  |||      | _        t        j                  |||      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r   )rY   )r'   r(   r   r   r   head_dimr@   r~   r   r   r   r)   r   k_projv_projq_projout_proj)	r?   r   r   r   r   rY   r   r@   rA   s	           rB   r(   zHubertAttention.__init__	  s     	""!Y.MMI%$..8MdnnM]$YKr3  }}d*$"ii	94@ii	94@ii	94@		)YTBrH   rF   key_value_statesr   output_attentionsr   returnc                    |du}|j                   dd \  }}|r|j                   d   n|}	||d| j                  f}
||	d| j                  f} | j                  |      j                  |
 j	                  dd      }|r|n|} | j                  |      j                  | j	                  dd      } | j                  |      j                  | j	                  dd      }t        j                  | j                  j                  t              } || ||||f| j                  sdn| j                  | j                  |d|\  }}|j                  ||d      j!                         }| j#                  |      }||dfS )z#Input shape: Batch x Time x ChannelNrm   r   r           )r   r   r   )shaper   r   viewrD   r   r   r   get_interfacer@   _attn_implementationr   r   r   r   reshaper   r   )r?   rF   r   r   r   r   is_cross_attentionbsztgt_lensrc_lenq_input_shapekv_input_shapequery_statescurrent_states
key_statesvalue_statesattention_interfacer   r   s                      rB   rG   zHubertAttention.forward(  s    .T9 %**3B/W/A"((+wgr4==9wDMM: 7t{{=166FPPQRTUV-?)]5T[[055~FPPQRTUV
7t{{>277HRRSTVWX(?(M(MKK,,.E)
 %8
%
  $}}C$,,LL/
%
 
%
!\ "))#w;FFHmmK0L$..rH   )r   FTFN)NNF)rJ   rK   rL   r   intfloatboolr   r(   r   Tensorr   r   tuplerG   rM   rN   s   @rB   r   r     s    G  &*CC C 	C
 C C C t#CD 15.2).1/||1/  ,,-1/ t+	1/
  $;1/ -.1/ 
u||U\\D0%2E2LL	M1/rH   r   c                   $     e Zd Z fdZd Z xZS )HubertFeedForwardc                    t         |           t        j                  |j                        | _        t        j                  |j                  |j                        | _	        t        |j                  t              rt        |j                     | _        n|j                  | _        t        j                  |j                  |j                        | _        t        j                  |j                         | _        y rc   )r'   r(   r)   r   activation_dropoutintermediate_dropoutr   r+   intermediate_sizeintermediate_dense
isinstance
hidden_actstrr   intermediate_act_fnoutput_densehidden_dropoutoutput_dropoutr   s     rB   r(   zHubertFeedForward.__init__]  s    $&JJv/H/H$I!"$))F,>,>@X@X"Yf''-'-f.?.?'@D$'-'8'8D$IIf&>&>@R@RS jj)>)>?rH   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S rc   )r   r   r   r   r   rE   s     rB   rG   zHubertFeedForward.forwardj  sX    //>00?11-@))-8++M:rH   rI   rN   s   @rB   r   r   \  s    @rH   r   c                   &     e Zd Z fdZddZ xZS )HubertEncoderLayerc                    t         |           t        |j                  |j                  |j
                  d|      | _        t        j                  |j                        | _
        t        j                  |j                  |j                        | _        t        |      | _        t        j                  |j                  |j                        | _        y )NFr   r   r   r   r@   r   )r'   r(   r   r+   num_attention_headsattention_dropout	attentionr)   r   r   r   ri   r   rj   r   feed_forwardfinal_layer_normr   s     rB   r(   zHubertEncoderLayer.__init__u  s    (((00,,
 zz&"7"78,,v'9'9v?T?TU-f5 "V-?-?VEZEZ [rH   c                     |}| j                  |||      \  }}}| j                  |      }||z   }| j                  |      }|| j                  |      z   }| j	                  |      }|f}|r||fz  }|S Nr   r   )r   r   rj   r   r   r?   rF   r   r   attn_residualr   _outputss           rB   rG   zHubertEncoderLayer.forward  s    %)-.L] *8 *
&|Q ]3%56%(9(9-(HH--m< "&GrH   r   rI   rN   s   @rB   r   r   t  s    \rH   r   c                   r     e Zd Z fdZ	 	 	 	 d	dej
                  dej                  dz  dededef
dZ xZ	S )
HubertEncoderc                    t         |           || _        t        |      | _        t        j                  |j                  |j                        | _	        t        j                  |j                        | _        t        j                  t        |j                        D cg c]  }t!        |       c}      | _        d| _        y c c}w Nr   F)r'   r(   r@   r   pos_conv_embedr)   ri   r+   r   rj   r   r   r   r   r|   num_hidden_layersr   layersr   r?   r@   r   rA   s      rB   r(   zHubertEncoder.__init__  s    ;FC,,v'9'9v?T?TUzz&"7"78mmvOgOgIh$iA%7%?$ij&+# %j   !CNrF   r   r   output_hidden_statesreturn_dictc                    |rdnd }|rdnd }|5|j                  d      j                  dd|j                  d         }d|| <   t        | j                  ||      }| j                  |      }	||	j                  |j                        z   }| j                  |      }| j                  |      }t               xs t        |       }
| j                  D ]j  }|r||fz   }t        j                  g       }| j                  xr || j                  j                   k  }|r|
r ||||      }|d   }|rd}|sb|d   fz   }l |r||fz   }|st#        d	 |||fD              S t%        |||
      S )N rm   r   r   r   r@   inputs_embedsr   r   NNc              3   &   K   | ]	  }||  y wrc   r  .0vs     rB   	<genexpr>z(HubertEncoder.forward.<locals>.<genexpr>       mq_`_lm   last_hidden_staterF   
attentions)	unsqueezerepeatr   r
   r@   r   todevicerj   r   r   r	   r   r   randr   	layerdropr   r   r?   rF   r   r   r   r  all_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddingssynced_gpusrz   dropout_probabilityskip_the_layerlayer_outputss                  rB   rG   zHubertEncoder.forward  s    #7BD$5b4%$2$<$<R$@$G$G1mNaNabcNd$e!45M0012;;')
 #11-@%(;(>(>}?S?S(TT6]302R6LT6R[[ 	PE#$58H$H! #(**R.!]]Z/BT[[EZEZ/ZN![ %!.Te! !.a 0 , &9]1=M<O&O#'	P*   1]4D Dm]4EGZ$[mmm++*
 	
rH   NFFT)
rJ   rK   rL   r(   r   tensorr   r   rG   rM   rN   s   @rB   r   r     sX    , /3"'%* ;
||;
 t+;
  	;

 #;
 ;
rH   r   c                   >     e Zd Z fdZdej
                  fdZ xZS )HubertAttnAdapterLayerc                    t         |           |j                  | _        |j                  | _        t        j                  | j
                        | _        t        j                  | j
                  | j                        | _
        t        j                         | _        t        j                  | j                  | j
                        | _        y)z
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)r'   r(   adapter_attn_dim	input_dimr+   
hidden_dimr)   ri   normr   linear_1ReLUact_fnlinear_2r   s     rB   r(   zHubertAttnAdapterLayer.__init__  s    
 	00 ,,LL1			$//4>>Bggi		$..$//BrH   rF   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rc   )r(  r)  r+  r,  rE   s     rB   rG   zHubertAttnAdapterLayer.forward  s@    		-0m4M2m4rH   )rJ   rK   rL   r(   r   FloatTensorrG   rM   rN   s   @rB   r#  r#    s    CU%6%6 rH   r#  c                   f     e Zd Z fdZ	 	 ddej
                  dej
                  dz  defdZ xZS )!HubertEncoderLayerStableLayerNormc                    t         |           t        |j                  |j                  |j
                  d|      | _        t        j                  |j                        | _
        t        j                  |j                  |j                        | _        t        |      | _        t        j                  |j                  |j                        | _        t#        |dd       t%        |      | _        y d | _        y )NFr   r   r%  )r'   r(   r   r+   r   r   r   r)   r   r   r   ri   r   rj   r   r   r   getattrr#  adapter_layerr   s     rB   r(   z*HubertEncoderLayerStableLayerNorm.__init__  s    (((00,,
 zz&"7"78,,v'9'9v?T?TU-f5 "V-?-?VEZEZ [6-t4@!7!?D!%DrH   NrF   r   r   c                 $   |}| j                  |      }| j                  |||      \  }}}| j                  |      }||z   }|| j                  | j	                  |            z   }| j
                  || j                  |      z   }|f}|r||fz  }|S r   )rj   r   r   r   r   r3  r   s           rB   rG   z)HubertEncoderLayerStableLayerNorm.forward  s     &6)-.L] *8 *
&|Q ]3%5%(9(9$:O:OP]:^(__))D,>,>},MMM "&GrH   r   )	rJ   rK   rL   r(   r   r   r   rG   rM   rN   s   @rB   r0  r0    s>    &, /3"'	|| t+  	rH   r0  c                   .     e Zd Z fdZ	 	 	 	 ddZ xZS )HubertEncoderStableLayerNormc                    t         |           || _        t        |      | _        t        j                  |j                  |j                        | _	        t        j                  |j                        | _        t        j                  t        |j                        D cg c]  }t!        |       c}      | _        d| _        y c c}w r   )r'   r(   r@   r   r   r)   ri   r+   r   rj   r   r   r   r   r|   r   r0  r   r   r   s      rB   r(   z%HubertEncoderStableLayerNorm.__init__(  s    ;FC,,v'9'9v?T?TUzz&"7"78mm@EfF^F^@_`1.v6`
 ',# ar   c                    |rdnd }|rdnd }|5|j                  d      j                  dd|j                  d         }d|| <   t        | j                  ||      }| j                  |      }	||	z   }| j                  |      }t               xs t        |       }
| j                  D ]j  }|r||fz   }t        j                  g       }| j                  xr || j                  j                  k  }|r|
r ||||      }|d   }|rd}|sb|d   fz   }l | j                  |      }|r||fz   }|st        d	 |||fD              S t!        |||
      S )Nr  rm   r   r   r   r  r   r  c              3   &   K   | ]	  }||  y wrc   r  r  s     rB   r  z7HubertEncoderStableLayerNorm.forward.<locals>.<genexpr>k  r  r  r  )r  r  r   r
   r@   r   r   r   r	   r   r   r  r   r  rj   r   r   r  s                  rB   rG   z$HubertEncoderStableLayerNorm.forward3  s    #7BD$5b4%$2$<$<R$@$G$G1mNaNabcNd$e!45M0012;;')
 #11-@%(;;]302R6LT6R[[ 	PE#$58H$H! #(**R.!]]Z/BT[[EZEZ/ZN![ !&!.Te! !.a 0 , &9]1=M<O&O#)	P, 6 1]4D Dm]4EGZ$[mmm++*
 	
rH   r   rI   rN   s   @rB   r6  r6  '  s    	, "=
rH   r6  c                       e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZ ej                         d        Zd	ej                   ez  fd
Zdedej                   fdZy)HubertPreTrainedModelr@   hubertr   audior   ParametrizedConv1dTc                    t        |t        j                        rct        j                  |j
                  d| j                  j                         |j                   t        j                  |j                         yyt        |t        j                  t        j                  t        j                  f      rt        j                  |j                         t        j                  |j
                         t        |dd      ^t        j                  |j                         t        j                  |j                          t        j                  |j"                         yyt        |t        j$                        rt'               rddl}t+        |d      rht+        |d      r\|j,                  j/                  |j0                  |j2                  gd      5  t        j4                  |j
                         ddd       no|j,                  j/                  |j
                  d      5  t        j4                  |j
                         ddd       nt        j4                  |j
                         |j                   t        j                  |j                         yyt        |t6              r-t+        |d	      r t        j8                  |j:                         yyt        |t<              rHt+        |d
      r;t        j>                  |j@                  d| j                  jB                  dz   z         yyy# 1 sw Y   xY w# 1 sw Y   xY w)zInitialize the weightsr   )meanstdNrunning_meanr   r:   r9   r!   masked_spec_embedlayer_weightsg      ?r   )"r   r)   r   initnormal_r#   r@   initializer_rangerY   zeros_ri   rt   r1   ones_r2  rB  running_varnum_batches_trackedr*   r   r4   r3   r5   r6   r:   r9   kaiming_normal_HubertModeluniform_rC  HubertForSequenceClassification	constant_rD  r   )r?   r   r4   s      rB   _init_weightsz#HubertPreTrainedModel._init_weights  sF    fbii(LLSdkk6S6ST{{&FKK( 'r||R^^ LMKK$JJv}}%v~t4@F//0

6--.F667 A 		*)+ 6:.76:3N"::FOOV__;]mn:o <,,V]];< < #::6==XY:Z <,,V]];< < $$V]]3{{&FKK( ',v23f667 4 ?@v/v33SDKK<Y<Y\]<]5^_ 0 A< << <s    L0% L<0L9<Minput_lengthsc                     d }t        | j                  j                  | j                  j                        D ]  \  }} ||||      } |S )zH
        Computes the output length of the convolutional layers
        c                 >    t        j                  | |z
  |d      dz   S )Nfloor)rounding_moder   )r   div)input_lengthr   rX   s      rB   _conv_out_lengthzPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length  s"     99\K7wWZ[[[rH   )zipr@   r]   r^   )r?   rR  rY  r   rX   s        rB    _get_feat_extract_output_lengthsz6HubertPreTrainedModel._get_feat_extract_output_lengths  sQ    
	\
 $'t{{'>'>@W@W#X 	QK,]KPM	Q rH   feature_vector_lengthr   c                    | j                  |j                  d            j                  t        j                        }|j
                  d   }t        j                  ||f|j                  |j                        }d|t        j                  |j
                  d   |j                        |dz
  f<   |j                  dg      j                  d      j                  dg      j                         }|S )Nrm   r   )dtyper  r   )r  )r[  sumr  r   longr   zerosr^  r  arangeflipcumsumr   )r?   r\  r   output_lengths
batch_sizes        rB   "_get_feature_vector_attention_maskz8HubertPreTrainedModel._get_feature_vector_attention_mask  s    >>~?Q?QRT?UVYYZ_ZdZde#))!,
./~7K7KTbTiTi
 uv^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOrH   N)rJ   rK   rL   r   __annotations__base_model_prefixmain_input_nameinput_modalities_no_split_modulessupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attnr   no_gradrQ  
LongTensorr   r[  rg  r  rH   rB   r;  r;  s  s     $O-/CD&*#NU]]_!` !`Fe>N>NQT>T 
 
]b]m]m 
rH   r;  r   	mask_probmask_length	min_masksr   c                    | \  }dk  rt        d      kD  rt        d d d      t        j                  j                  d      j	                         fd}|-|j                         j                  d      j                         nt        |      D cg c]  } c}}t        j                  |ft        	      }	g }
 |      }|d
k(  r|	S |D ]  } ||      }t        j                  j                  t        j                  |dz
  z
        |d      }t        |      d
k(  rdz
  }n|d
   }t        j                  |t        j                  ||z
  t        j                   	      |z  g      }|
j#                  |        t        j$                  |
      }
t        j&                  |
dddddf   ||f      }
|
j)                  ||z        }
t        j                        ddddf   }t        j&                  |||f      j)                  ||z        }|
|z   }
|
j+                         dz
  kD  rdz
  |
|
dz
  kD  <   t        j,                  |	|
dd       |	S c c}w )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `c                     t        | z  z  z         }t        |      }|z  kD  rz  }| dz
  z
  |k  rt        | dz
  z
  d      }|S )z;Given input length, compute how many spans should be maskedr   r   )r   max)rX  num_masked_spanepsilonrt  rs  ru  sequence_lengths     rB   compute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_span  so    i,6DwNOoy9 [(?:-<O ;?+o=!,+/"BAFOrH   Nrm   r^  r   F)replace)r~   nprandomr  itemdetachr_  tolistr|   ra  r   choicerb  lenconcatenateonesint32appendarraybroadcast_tor   ry  put_along_axis)r   rs  rt  r   ru  rf  r}  r   rR  spec_aug_maskspec_aug_mask_idxsmax_num_masked_spanrX  rz  spec_aug_mask_idxdummy_mask_idxoffsetsr{  r|  s    `` `            @@rB   _compute_mask_indicesr    s   0 #(JQABB_$]^i]j&&7q:
 	
 iinnQ$$&G $ % 	##B'..0',Z'89!o9  HHj/:$GM1/Ba% 51,? II,,IIlkAo67RW - 
  !Q& -q0N.q1NNN(;o(MUWU]U] ^ao op
 	!!"34/52 "45 1a:&5H+(V ,33J@SVa@ab ii$T4]3Goog
4G'UV^^'+5G ,g5 /A"55GVYZGZ-!0CCD m%7B?w :s   $	I+c                   &    e Zd Zdef fdZ	 	 ddej                  dej                  dz  dej                  dz  fdZe		 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  deez  fd       Z xZS )rM  r@   c                    t         |   |       || _        t        |      | _        t        |      | _        |j                  dkD  s|j                  dkD  rEt        j                  t        j                  |j                        j                               | _        |j                   rt#        |      | _        nt'        |      | _        | j)                          y Nr   )r'   r(   r@   rw   feature_extractorr   feature_projectionmask_time_probmask_feature_probr)   	Parameterr   r   r+   rN  rC  do_stable_layer_normr6  encoderr   	post_initr   s     rB   r(   zHubertModel.__init__8  s     !5f!="9&"A   3&&*B*BS*H%'\\%,,v?Q?Q2R2[2[2]%^D"&&7?DL(0DL 	rH   NrF   mask_time_indicesr   c                    t        | j                  dd      s|S |j                         \  }}}|)| j                  j	                  |j
                        ||<   n| j                  j                  dkD  r| j                  rt        ||f| j                  j                  | j                  j                  || j                  j                        }t        j                  ||j                  t        j                        }| j                  j	                  |j
                        ||<   | j                  j                  dkD  r| j                  rt        ||f| j                  j                  | j                  j                   | j                  j"                        }t        j                  ||j                  t        j                        }|dddf   j%                  d|d      }d||<   |S )	z
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        apply_spec_augmentTNr   )rs  rt  r   ru  )r  r^  )rs  rt  ru  rm   )r2  r@   r   rC  r  r^  r  r   r  mask_time_lengthmask_time_min_masksr   r!  r  r   r  mask_feature_lengthmask_feature_min_masksexpand)r?   rF   r  r   rf  r|  r+   mask_feature_indicess           rB   _mask_hidden_stateszHubertModel._mask_hidden_statesJ  s    t{{$8$?   4A3E3E3G0
O[(/3/E/E/H/HI\I\/]M+,[[''!+ 5_-++44 KK88-++99! !&->}G[G[chcmcm n/3/E/E/H/HI\I\/]M+,;;((1,#8[)++77 KK;;++<<	$  $)<<0D]MaMainisis#t #74#@#G#GO]_#` 23M./rH   r   r   r   r  r   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  |      }|j                  dd      }|| j                  |j                  d   |      }| j                  |      }	| j                  |	|      }	| j                  |	||||      }
|
d   }	|s	|	f|
dd z   S t        |	|
j                  |
j                        S )a1  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.

        Example:

        ```python
        >>> from transformers import AutoProcessor, HubertModel
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(example):
        ...     example["speech"] = example["audio"]["array"]
        ...     return example


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```Nr   r   )r  r   r   r   r  r   r  )r@   r   r   use_return_dictr  rD   rg  r   r  r  r  r   rF   r  )r?   r   r   r  r   r   r  r   extract_featuresrF   encoder_outputss              rB   rG   zHubertModel.forwardx  s,   J 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]11,?+55a;%!DDEUE[E[\]E^`noN//0@A00Rc0d,,)/!5# ' 
 (*!#oab&999+)77&11
 	
rH   r  NNNNN)rJ   rK   rL   r   r(   r   r.  rr  r  r   r   r   r   r   rG   rM   rN   s   @rB   rM  rM  6  s    | * 7;26	,((, !,,t3, ((4/	,\  /36:)-,0#'E
llT)E
 t+E
 !,,t3	E

  $;E
 #TkE
 D[E
 
	 E
 E
rH   rM  zn
    Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )custom_introc                        e Zd Zddedz  f fdZd Zd Zd Ze	 	 	 	 	 dde	j                  dz  de	j                  dz  d	edz  d
edz  dedz  de	j                  dz  deez  fd       Z xZS )HubertForCTCNtarget_langc                    t         |   |       t        |      | _        t	        j
                  |j                        | _        || _        |j                  t        d| j                   d      t        |d      r|j                  r|j                  n|j                  }t	        j                   ||j                        | _        | j%                          y)a0  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`HubertForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `HubertForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.add_adapter)r'   r(   rM  r<  r)   r   final_dropoutr   r  
vocab_sizer~   rA   r3   r  output_hidden_sizer+   r   lm_headr  )r?   r@   r  r  rA   s       rB   r(   zHubertForCTC.__init__  s     	 !&)zz&"6"67&$00@ AH H  *1)GFL^L^F%%djdvdv 	 yy!3V5F5FG 	rH   c                 8   t               t        j                  d      k(  ry| j                  }|&t	        | j
                  dd      t        d| d      |-t	        | j
                  dd      t        j                  d       y|| j                  |d       yy)	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        metaNr%  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)
force_load)
r   r   r  r  r2  r@   r~   loggerinfoload_adapter)r?   r   r  s      rB   tie_weightszHubertForCTC.tie_weights  s     675<<;OO &&"wt{{<NPT'U']:;-Gtuvv WT[[:Ld%S%_KKCD$kd; %rH   c                 L    | j                   j                  j                          yz
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        Nr<  r  r   r?   s    rB   freeze_feature_encoderz#HubertForCTC.freeze_feature_encoder      
 	%%88:rH   c                 P    | j                   j                         D ]	  }d|_         yz
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FNr<  r   r   r   s     rB   freeze_base_modelzHubertForCTC.freeze_base_model  (    
 [[++- 	(E"'E	(rH   r   r   r   r   r  labelsr   c           
         ||n| j                   j                  }|I|j                         | j                   j                  k\  r"t	        d| j                   j                         | j                  |||||      }|d   }	| j                  |	      }	| j                  |	      }
d}|b||n$t        j                  |t        j                        }| j                  |j                  d            j                  t        j                        }|dk\  }|j                  d      }|j                  |      }t        j                   j#                  |
dt        j$                        j'                  dd      }t        j(                  j*                  j-                  d	
      5  t        j                   j/                  ||||| j                   j0                  | j                   j2                  | j                   j4                        }ddd       |s|
f|t6        d z   }||f|z   S |S t9        ||
|j:                  |j<                        S # 1 sw Y   ExY w)a  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r  r   r~  rm   )r%   r^  r   F)enabled)blank	reductionzero_infinitylosslogitsrF   r  )r@   r  ry  r  r~   r<  r   r  r   	ones_liker`  r[  r_  r  masked_selectr)   r   log_softmaxfloat32rD   backendscudnnflagsctc_losspad_token_idctc_loss_reductionctc_zero_infinity_HIDDEN_STATES_START_POSITIONr   rF   r  )r?   r   r   r   r   r  r  r   r   rF   r  r  rR  labels_masktarget_lengthsflattened_targets	log_probsoutputs                     rB   rG   zHubertForCTC.forward  s'   $ &1%<k$++B]B]&**,$++2H2H"HCDKKDZDZC[\]]++)/!5#  
  
]3m, #1"<%//R^fkfpfpBq  !AA.BTBTUWBXY\\]b]g]ghM !A+K(__R0N & 4 4[ A 11&b1V``abdefI%%++E+: 	}}--%!"++22"kk<<"&++"?"? . 	 Y)F)G!HHF)-)9TGf$EvEfG4I4IV]VhVh
 	
	 	s   A#IIrc   r  )rJ   rK   rL   r   r(   r  r  r  r   r   r   r   r   r   rG   rM   rN   s   @rB   r  r    s    C$J :<0;(  /3)-,0#'&*E
llT)E
 t+E
  $;	E

 #TkE
 D[E
 t#E
 
	E
 E
rH   r  z
    Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                        e Zd Z fdZd Zd Ze	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  de	dz  d	e	dz  d
ej                  dz  de
ez  fd       Z xZS )rO  c                    t         |   |       t        |d      r|j                  rt	        d      t        |      | _        |j                  dz   }|j                  r0t        j                  t        j                  |      |z        | _        t        j                  |j                  |j                         | _        t        j                  |j                   |j$                        | _        | j)                          y )Nr  z]Sequence classification does not support the use of Hubert adapters (config.add_adapter=True)r   )r'   r(   r3   r  r~   rM  r<  r   use_weighted_layer_sumr)   r  r   r  rD  r   r+   classifier_proj_size	projector
num_labels
classifierr  )r?   r@   
num_layersrA   s      rB   r(   z(HubertForSequenceClassification.__init__^  s     6=)f.@.@o  "&)--1
((!#ejj.Dz.Q!RD6#5#5v7R7RS))F$?$?ARARS 	rH   c                 L    | j                   j                  j                          yr  r  r  s    rB   r  z6HubertForSequenceClassification.freeze_feature_encodero  r  rH   c                 P    | j                   j                         D ]	  }d|_         yr  r  r   s     rB   r  z1HubertForSequenceClassification.freeze_base_modelv  r  rH   Nr   r   r   r   r  r  r   c                 <   ||n| j                   j                  }| j                   j                  rdn|}| j                  |||||      }| j                   j                  rr|t           }	t        j                  |	d      }	t        j                  j                  | j                  d      }
|	|
j                  ddd      z  j                  d      }	n|d   }	| j                  |	      }	||	j                  d      }n| j                  |	j                   d   |      }|j#                  d      j%                  dd|	j                   d         }d	|	| <   |	j                  d      |j                  d      j                  dd      z  }| j'                  |      }d}|Ft)               } ||j                  d| j                   j*                        |j                  d            }|s|f|t        d z   }||f|z   S |S t-        |||j.                  |j0                  
      S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`HubertProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTr  r   r   rm   r   r   r   r  )r@   r  r  r<  r  r   stackr)   r   r   rD  r   r_  r  r@  rg  r   r  r  r  r   r  r   rF   r  )r?   r   r   r   r   r  r  r   r   rF   norm_weightspooled_outputpadding_maskexpand_padding_maskr  r  loss_fctr  s                     rB   rG   z'HubertForSequenceClassification.forward~  s   0 &1%<k$++B]B]'+{{'I'ItOc++)/!5#  
 ;;--#$ABM!KK1=M==001C1C0LL*\->->r1a-HHMMRSMTM#AJM}5!)..1.5MBB=CVCVWXCY[ijL"."8"8"<"C"CAq-J]J]^_J`"a25M../)--!-4|7G7GA7G7N7S7STVXY7ZZM/')HFKKDKK,B,BCV[[QS_UDY)F)G!HHF)-)9TGf$EvE'!//))	
 	
rH   r  )rJ   rK   rL   r(   r  r  r   r   r   r   r   r   rG   rM   rN   s   @rB   rO  rO  W  s    ";(  /3)-,0#'&*C
llT)C
 t+C
  $;	C

 #TkC
 D[C
 t#C
 
)	)C
 C
rH   rO  )r  rO  rM  r;  r  rS   )Gcollections.abcr   numpyr  r   torch.nnr)   r    r   rE  activationsr   integrations.deepspeedr   integrations.fsdpr	   masking_utilsr
   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   modeling_utilsr   r   r   processing_utilsr   r2   r   r   r   configuration_hubertr   
get_loggerrJ   r  Moduler   r<   rU   rf   ro   rw   r   r   r   r   r   r   r   r   r#  r0  r6  r;  r   r   rr  ndarrayr  rM  r  r  rO  __all__r  rH   rB   <module>r	     s  * %    % & ! @ 7 6 B 9 Y Y r r & @ @ . 
		H	%/BII /d !; *9 69 0#299 #Lbii 0 !%II%<<% 
% <<	%
 LL4'% T\% % '(%8S/bii S/l		 0!3 !HE
BII E
PRYY 2+(B +\I
299 I
X HO H H^ /3tc?tt t $$t+	t
 t ZZtn G
' G
 G
T !"  
K
( K

K
\ e
&; e
e
P frH   