
    qiK                     J   d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZmZ dd	lmZmZmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0m1Z1m2Z2 ddl3m4Z4  e,jj                  e6      Z7e) G d de$             Z8 G d dejr                        Z: G d dejr                        Z; ed       G d dejr                               Z< G d d ejr                        Z=d! Z> ed"      dEd#       Z?d$ej                  d%eAd&ej                  fd'ZB	 dFd(ejr                  d)ej                  d*ej                  d+ej                  d,ej                  dz  d-eCd.eCd/e&e(   fd0ZD ee?       G d1 d2ejr                               ZE G d3 d4ejr                        ZF G d5 d6e      ZG G d7 d8e8      ZH G d9 d:e      ZI G d; d<e8      ZJ e)d=>       G d? d@e8             ZK e)dA>       G dB dCe8e4             ZLg dDZMy)G    )Callable)OptionalN)nn   )initialization)ACT2FN)CacheDynamicCacheEncoderDecoderCache)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_bidirectional_maskcreate_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torchdynamo_compilinglogging)maybe_autocast   )	DiaConfigDiaDecoderConfigDiaEncoderConfig)DiaGenerationMixinc                   N     e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZ fdZ xZS )DiaPreTrainedModelconfigmodelT	input_idsDiaEncoderLayerDiaDecoderLayerc                 &   t         |   |       t        |t              rqt	        j
                  | j                  j                  t        j                        | j                  j                  z  }t        j                  |j                  |       y y )Ndtype)super_init_weights
isinstanceDiaMultiChannelEmbeddingtorcharanger)   num_channelslong
vocab_sizeinitcopy_offsets)selfmoduler<   	__class__s      V/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/dia/modeling_dia.pyr2   z DiaPreTrainedModel._init_weights@   sb    f%f67ll4;;#;#;5::NQUQ\Q\QgQggGJJv~~w/ 8    )__name__
__module____qualname__r#   __annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraphmain_input_name_no_split_modulesr2   __classcell__r?   s   @r@   r(   r(   4   sG    &*#N!!O*,=>0 0rA   r(   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )r4   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r)   c                 ~   t         |           t        j                  |j                  |j
                  z  |j                        | _        |j                  | _        |j
                  | _        t        j                  |j
                  t        j                        |j                  z  }| j                  d|d       y )Nr/   r<   F
persistent)r1   __init__r   	Embeddingr9   r7   hidden_sizeembedr5   r6   r8   register_buffer)r=   r)   r<   r?   s      r@   rT   z!DiaMultiChannelEmbedding.__init__U   s    \\&"3"3f6I6I"I6K]K]^
!--"//,,v22%**EHYHYYYEBrA   audio_codesreturnc                 "   || j                   j                  |j                        z   j                  d      }| j	                  |      j                  |j                  d   |j                  d   d| j                        }|j                  d      S )Nr"   r      dim)	r<   todevicesqueezerW   viewshaperV   sum)r=   rY   tokensembedss       r@   forwardz DiaMultiChannelEmbedding.forward]   su    0B0B CCLLQOF#((a+:K:KA:NPRTXTdTdezzaz  rA   )
rB   rC   rD   __doc__r$   rT   r5   Tensorrh   rN   rO   s   @r@   r4   r4   G   s2    C/ C!5<< !ELL !rA   r4   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )DiaMLPc                 *   t         |           || _        t        j                  |j
                  d|j                  z  d      | _        t        j                  |j                  |j
                  d      | _        t        |j                     | _        y )Nr]   Fbias)r1   rT   r)   r   LinearrV   intermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fnr=   r)   r?   s     r@   rT   zDiaMLP.__init__d   sp    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56rA   hidden_statesrZ   c                     | j                  |      }|j                  dd      \  }}|| j                  |      z  }| j                  |      S )Nr]   r\   r^   )rr   chunkru   rs   )r=   rw   	up_statesgates       r@   rh   zDiaMLP.forwardl   sL    %%m4	#//!/4i 2 24 88	~~i((rA   )rB   rC   rD   rT   r5   FloatTensorrh   rN   rO   s   @r@   rl   rl   c   s'    7)U%6%6 )5;L;L )rA   rl   RMSNormc                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	
DiaRMSNormepsrZ   Nc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z9
        DiaRMSNorm is equivalent to T5LayerNorm
        N)r1   rT   r   	Parameterr5   onesweightvariance_epsilon)r=   rV   r   r?   s      r@   rT   zDiaRMSNorm.__init__w   s1     	ll5::k#:; #rA   rw   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )Nr]   r\   T)keepdim)	r0   r`   r5   float32powmeanrsqrtr   r   )r=   rw   input_dtypevariances       r@   rh   zDiaRMSNorm.forward   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::rA   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler   rd   r   )r=   s    r@   
extra_reprzDiaRMSNorm.extra_repr   s*    ))*+6$2G2G1HIIrA   )gư>)
rB   rC   rD   floatrT   r5   rj   rh   r   rN   rO   s   @r@   r   r   u   s7    $ $$ $;U\\ ;ell ;JrA   r   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )DiaRotaryEmbeddinginv_freqNr)   c                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultr   FrR   original_inv_freq)r1   rT   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr)   rope_parametersr   compute_default_rope_parametersr   attention_scalingrX   clone)r=   r)   ra   rope_init_fnr   r?   s        r@   rT   zDiaRotaryEmbedding.__init__   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUrA   ra   ztorch.deviceseq_lenrZ   ztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimNg      ?r   r]   r/   )ra   r0   )	r   getattrrV   num_attention_headsr5   r6   int64r`   r   )r)   ra   r   baser_   attention_factorr   s          r@   r   z2DiaRotaryEmbedding.compute_default_rope_parameters   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))rA   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r\   r"   mpscpuF)device_typeenabledr]   r^   r/   )r   r   expandrd   r`   ra   r3   typestrr!   	transposer5   catcosr   sinr0   )
r=   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r@   rh   zDiaRotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$N)NNN)rB   rC   rD   r5   rj   rE   r#   rT   staticmethodr   intr   r   r   no_gradr   rh   rN   rO   s   @r@   r   r      s    llVy V  #'+/"*D *(* t* 
~u$	%	* *: U]]_<  <rA   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr\   r]   r^   )rd   r5   r   )r   x1x2s      r@   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''rA   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   unsqueeze_dimq_embedk_embeds          r@   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGrA   rw   n_reprZ   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)rd   r   reshape)rw   r   batchnum_key_value_headsslenr   s         r@   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTrA   r>   querykeyvalueattention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nr]   r   r\   )r_   r0   )ptrainingr"   )r   num_key_value_groupsr5   matmulr   r   
functionalsoftmaxr   r`   r0   r   r   
contiguous)r>   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               r@   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$rA   c                   $    e Zd ZdZddeez  dedef fdZ	 	 	 	 dde	j                  dee	j                  e	j                  f   dz  d	e	j                  dz  d
edz  de	j                  dz  dee   dee	j                  e	j                  f   fdZ xZS )DiaSelfAttention=Multi-headed attention from 'Attention Is All You Need' paperr)   	layer_idx	is_causalc                    t         |           || _        || _        |j                  | _        | j                  j
                  | _        | j                  j                  xs | j                  | _        | j                  | j                  z  | _        t        |d|j                  | j                  z        | _
        d| _        d| _        || _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        y )Nr   r"           Frn   )r1   rT   r)   r   rV   r   	num_headsr   r   r   r   r   attention_dropoutr   r   rp   q_projk_projv_projo_proj)r=   r)   r   r   r?   s       r@   rT   zDiaSelfAttention.__init__  sF   "!--88#';;#B#B#Tdnn $(NNd6N6N$N!
F4F4F$..4XY!$"ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]rA   Nrw   position_embeddingsr   past_key_valuescache_positionr   rZ   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
| j                  |      j                  |      j	                  dd      }|\  }}t        |	|
||      \  }	}
|'|||d}|j                  |
|| j                  |      \  }
}t        j                  | j                  j                  t              } || |	|
||f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )Nr\   r"   r]   )r   r   r   r   )r   r   )rd   r   r   rc   r   r   r   r   updater   r   get_interfacer)   _attn_implementationr   r   r   r   r   r   r   )r=   rw   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsattention_interfacer   r   s                     r@   rh   zDiaSelfAttention.forward'  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S#7jRUWZ#[ j&#&snUL'6'='=j,X\XfXfht'u$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((rA   )F)NNNN)rB   rC   rD   ri   r%   r$   r   boolrT   r5   rj   r   r	   
LongTensorr   r   rh   rN   rO   s   @r@   r   r     s    G^/2BB ^s ^_c ^* IM.2(,26))||)) #5<<#=>E)) t+	))
 )) ((4/)) +,)) 
u||U\\)	*))rA   r   c                        e Zd ZdZdedef fdZ	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  f   fdZ xZS )DiaCrossAttentionr   r)   r   c                 f   t         |           || _        || _        |j                  | _        |j
                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  | j                  z  | _
        |j                  | _        d| _        d| _        d| _        t!        j"                  | j                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j                  | j                  z  | j                  d      | _        y )Nr"   r   Frn   )r1   rT   r)   r   rV   cross_hidden_sizecross_num_attention_headsr   cross_num_key_value_headsr   r   cross_head_dimr   r   r   r   r   rp   r   r   r   r   r=   r)   r   r?   s      r@   rT   zDiaCrossAttention.__init__V  s?   "!--!'!9!9>>#';;#H#H $(NNd6N6N$N!--!$ii 0 0$..4==2PW\]ii 6 68P8PSWS`S`8`glmii 6 68P8PSWS`S`8`glmii >@P@PW\]rA   Nrw   cross_attention_statesr   r   r   rZ   c                 F   |j                   d d }g |d| j                  }g |j                   d d d| j                  }| j                  |      j                  |      j	                  dd      }	|%|j
                  j                  | j                        nd}
|]|
r[|j                  j                  | j                     j                  }|j                  j                  | j                     j                  }n| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }|C|j                  j                  ||| j                        \  }}d|j
                  | j                  <   t        j                   | j"                  j$                  t&              } || |	|||fd| j(                  i|\  }}|j+                  g |d      j-                         }| j/                  |      }||fS )Nr\   r"   r]   FTr   )rd   r   r   rc   r   
is_updatedgetr   cross_attention_cachelayerskeysvaluesr   r   r   r   r   r)   r   r   r   r   r   r   )r=   rw   r  r   r   r   r   r   cross_shaper   r  r   r   r   r   r   s                   r@   rh   zDiaCrossAttention.forwardi  s    $))#2.88b8$--8M.44Sb9M2Mt}}M{{=166|DNNqRSTGVGb_//33DNNChm
&:(>>EEdnnUZZJ*@@GGW^^L%;<AA+NXXYZ\]^J;;'=>CCKPZZ[\^_`L*+:+P+P+W+W NN,(
L >B**4>>:(?(M(MKK,,.E)
 %8%
 LL%
 %
!\ "))*<K*<*<=HHJkk+.L((rA   NN)rB   rC   rD   ri   r$   r   rT   r5   rj   r   r   r   r   rh   rN   rO   s   @r@   r  r  S  s    G^/ ^C ^. /36:1)||1) !&1) t+	1)
 -t31) -.1) 
u||U\\D00	11)rA   r  c                        e Zd Zdedef fdZ	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	e
   d	eej                  ej                  dz  f   f
d
Z xZS )r,   r)   r   c                     t         |           t        |j                  |j                        | _        t        ||d      | _        t        |j                  |j                        | _        t        |      | _
        y )Nr   Fr   )r1   rT   r   rV   norm_epspre_sa_normr   self_attentionpost_sa_normrl   mlpr
  s      r@   rT   zDiaEncoderLayer.__init__  s\    %f&8&8fooN.vyER&v'9'9vO&>rA   Nrw   r   r   r   rZ   c                     |}| j                  |      } | j                  |f||d|\  }}||z   }|}| j                  |      }| j                  |      }	||	z   }||fS )N)r   r   )r  r  r  r  )
r=   rw   r   r   r   residualnormed_statesself_attn_outputself_attn_weightsmlp_outs
             r@   rh   zDiaEncoderLayer.forward  s     !((7.Ad.A.A/
 3)/
 	/
++ !#33 ))-8((=) 7*///rA   r  )rB   rC   rD   r%   r   rT   r5   rj   r   r   r   rh   rN   rO   s   @r@   r,   r,     s    "/ "C " IM.2	0||0 #5<<#=>E0 t+	0
 -.0 
u||U\\D00	10rA   r,   c                        e Zd Zdef fdZee	 	 	 ddej                  dej                  dz  de	dz  de	dz  de
e   d	eez  fd
              Z xZS )
DiaEncoderr)   c           	         t         |   |       || _        t        j                  |j
                  |j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w Nr  r)   )r1   rT   r)   r   rU   r9   rV   	embedding
ModuleListrangenum_hidden_layersr,   r  r   r  normr   
rotary_emb	post_initr
  s      r@   rT   zDiaEncoder.__init__  s     f&7&79K9KLmmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   -CNr+   r   output_attentionsoutput_hidden_statesr   rZ   c                    | j                  |      }t        j                  |j                  d   |j                        d d d f   }t        | j                  ||      }| j                  ||      }|rdnd }	|rdnd }
| j                  D ](  }|r|	|fz   }	 ||f|||d|}|d   }|s |
|d   fz   }
* | j                  |      }|r|	|fz  }	t        ||	|
	      S )
Nr\   ra   )r)   inputs_embedsr   r    )r   r   r   r   r"   last_hidden_staterw   
attentions)r)  r5   r6   rd   ra   r   r)   r.  r  r-  r   )r=   r+   r   r0  r1  r   rw   r   r   encoder_statesall_attentionsencoder_layerlayer_outputss                r@   rh   zDiaEncoder.forward  s    y1
 ||IOOB$7	@P@PQRVXYRYZ2;;')

 #oom,oW30d![[ 	FM#!/=2B!B)-)$7	
 M *!,M !/=3C2E!E	F  		-0}..N+>Vd
 	
rA   )NFF)rB   rC   rD   r%   rT   r   r   r5   rj   r  r   r   r   r   rh   rN   rO   s   @r@   r%  r%    s    /   /3).,10
<<0
 t+0
  $;	0

 #Tk0
 -.0
 
5	 0
  0
rA   r%  c                   l    e Zd Zdedef fdZ	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  dej                  dz  deej                  ej                  dz  ej                  dz  f   fdZ xZS )r-   r)   r   c                    t         |           |j                  | _        t	        ||d      | _        t        ||      | _        t        |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |      | _        y )NTr  r  )r1   rT   rV   	embed_dimr   r  r  cross_attentionr   r  r  pre_ca_normpre_mlp_normrl   r  r
  s      r@   rT   zDiaDecoderLayer.__init__  s    ++.vyDQ0C%f&8&8fooN%f&8&8fooN&v'9'9vO&>rA   Nrw   r   r   encoder_hidden_statesencoder_attention_maskr   r   rZ   c                 d   |}	t        |	t              r|	j                  }	|}
| j                  |      } | j                  ||||	fd|i|\  }}|
|z   }|}
| j                  |      } | j                  ||f||d|\  }}|
|z   }|}
| j                  |      }| j                  |      }|
|z   }|||fS )Nr   )r   r   )	r3   r   self_attention_cacher  r  rB  rA  rC  r  )r=   rw   r   r   rD  rE  r   r   r   self_attn_cacher  r   r!  r"  cross_statescross_attn_weightsr#  s                    r@   rh   zDiaDecoderLayer.forward  s    *o':;-BBO ((7.Ad.A.A 	/
 *	/
 	/
++ !#33 ((7+?4+?+?!,
 2+	,

 ,
(( !</ ))-8((=) 7*/1CCCrA   )NNNNNN)rB   rC   rD   r$   r   rT   r5   rj   r   r   r  rh   rN   rO   s   @r@   r-   r-     s    "/ "C " IM.2596:6:26-D||-D #5<<#=>E-D t+	-D
  %||d2-D !&t 3-D -t3-D ((4/-D 
u||U\\D0%,,2EE	F-DrA   r-   c                   ,    e Zd ZdZdef fdZee	 	 	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dedz  dej                  dz  deez  fd              Z xZS )
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.r)   c           	         t         |   |       |j                  | _        |j                  | _        t	        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w r'  )r1   rT   r7   r9   r4   
embeddingsr   r*  r+  r,  r-   r  r   rV   r  r-  r   r.  r/  r
  s      r@   rT   zDiaDecoder.__init__?  s     "// ++26:mmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   )CNr+   r   r   rD  rE  r   r0  r1  r   rZ   c
           	         |j                         dd \  }}||j                         nd}|	%t        j                  |||z   |j                        }	|	|	dddf   }| j                  |      }|1t               s'||z   }t        j                  |||j                        }t        | j                  |||	|      }t        | j                  |||      }| j                  ||      }|rdnd}|rdnd}|r|dnd}| j                  D ]8  }|r||fz  } |||||f|||	|d	|
}|d   }|s$||d
   fz   }|0||d   fz   }: | j                  |      }|r||fz  }t        |||||      S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        Nr\   r   r3  )r)   r4  r   r   r   )r)   r4  r   rD  r5  r6  )rE  r   r   r   r"   r]   )r8  r   rw   r9  cross_attentions)sizeget_seq_lengthr5   r6   ra   rN  r   r   r   r)   r   r.  r  r-  r   )r=   r+   r   r   rD  rE  r   r0  r1  r   r   
batch_size
seq_lengthpast_key_values_lengthrw   mask_seq_lengthr   all_hidden_statesall_self_attnsall_cross_attentionslayerr=  s                         r@   rh   zDiaDecoder.forwardL  s   , "+!1#2!6
JETE`!?!?!Afg!"\\&(>(KT]TdTdN )$'2L 	2!*B*D4zAO"ZZ
OIL\L\]N+;;'))+
 ";;;'1"7	"
 #oom,oW"6BD0d&7<Q<]rdh[[ 	VE#!m%55!! $% (> /-) M *!,M !/=3C2E!E(4+?=QRCSBU+U(/	V2 		-0-!118+++%1
 	
rA   )NNNNNFFN)rB   rC   rD   ri   r$   rT   r   r   r5   rj   r  r|   r   r  r   r   rh   rN   rO   s   @r@   rL  rL  <  s   7/   15.2:>:>6:).,126\
<<\
 &&-\
 t+	\

  %0047\
 !& 0 04 7\
 -t3\
  $;\
 #Tk\
 ((4/\
 
3U	:\
  \
rA   rL  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )custom_introc                   N    e Zd Zdef fdZee	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
z  dz  d
edz  dedz  dedz  dedz  dej                  dz  de
ez  fd              Z xZS )DiaModelr)   c                     t         |   |       || _        t        |j                        | _        t        |j                        | _        | j                          y r   )
r1   rT   r)   r%  encoder_configencoderrL  decoder_configdecoderr/  rv   s     r@   rT   zDiaModel.__init__  sE     !&"7"78!&"7"78rA   Nr+   r   decoder_input_idsdecoder_position_idsdecoder_attention_maskencoder_outputsr   	use_cacher0  r1  r   rZ   c                    ||t        d      |	|	n| j                  j                  }	|
|
n| j                  j                  }
||n| j                  j                  }| j
                  r%| j                  r|rt        j                  d       d}|r6|4t        t        | j                        t        | j                              }| | j                  d|||	|
d|}nGt        |t              s7t        |d   t        |      dkD  r|d   ndt        |      d	kD  r|d	   nd
      }|d   j                  d   d| j                  j                   j"                  }}}|Ct%        j&                  |d|f| j                  j                   j(                  | j*                        }|j,                  d	k(  r#|j/                  |||      j1                  dd	      } | j2                  d||||d   |||	|
||d
|}t5        |j6                  |j8                  |j:                  |j<                  |j>                  |d   |j:                  |j<                        S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr(  )r+   r   r0  r1  r   r"   r]   r7  r\   )rQ  
fill_valuera   )
r+   r   r   rD  rE  r   r0  r1  rg  r   )r8  r   decoder_hidden_statesdecoder_attentionsrP  encoder_last_hidden_staterD  encoder_attentionsr6  ) 
ValueErrorr)   r0  r1  rg  is_gradient_checkpointingr   loggerwarning_oncer   r
   r`  r3   r   lenrd   ra  r7   r5   fullbos_token_idra   ndimr   r   rb  r   r8  r   rw   r9  rP  )r=   r+   r   rc  rd  re  rf  r   rg  r0  r1  r   r   bszr   channelsdecoder_outputss                    r@   rh   zDiaModel.forward  su   N !8j  2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	))dmm##p "	01,dkk2RT`hlhshsTtuO"*dll #-"3%9	
 O O_=-"1!"4474H14Loa0RV14_1E1I?1-tO #2!"4":":1"=r4;;C]C]CjCjhW$ %

1h'DKK4N4N4[4[dhdodo! !!Q& 1 9 9#x Q [ [\]_` a&$,, 
'-1"1!"4#1+/!5)
 
 "-??+;;"1"?"?.99,==&5a&8"1"?"?.99	
 		
rA   )NNNNNNNNNNN)rB   rC   rD   r#   rT   r   r   r5   r  r   r   r   r  r   rh   rN   rO   s   @r@   r]  r]    s5   y   .226598<:>:>6:!%)-,026k
##d*k
 ((4/k
 !++d2	k

 $..5k
 !& 0 04 7k
 )5047k
 -t3k
 $;k
  $;k
 #Tkk
 ((4/k
 
#	#k
  k
rA   r]  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   v    e Zd ZdZdZdef fdZee	 	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  deez  dz  dedz  dedz  dedz  dedz  de	j                  dz  de	j                  dz  deez  fd              Z xZS )DiaForConditionalGenerationr*   )audior)   c                 |   t         |   |       || _        t        |      | _        |j
                  j                  | _        |j
                  j                  | _        t        j                  |j
                  j                  | j                  | j                  z  d      | _        d| _        | j                          y )NFrn   ForMaskedLM)r1   rT   r)   r]  r*   ra  r7   r9   r   rp   rV   logits_dense	loss_typer/  rv   s     r@   rT   z$DiaForConditionalGeneration.__init__3  s     f%
"11>> //::II!!--0A0ADOO0S[`
 ' 	rA   Nr+   r   rc  rd  re  rf  r   rg  r0  r1  labelsr   rZ   c                 ^    | j                   d	|||||||||	|
|d|}|d   }|j                  d   }| j                  |      j                  |d| j                  | j
                  f      j                  dd      j                         j                  || j                  z  d| j
                        }d}|  | j                  d	||| j
                  d|}t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                   	      S )
a  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r+   r   rc  rd  re  rf  r   rg  r0  r1  r   r   r\   r"   r]   N)logitsr  r9   )	lossr  r   rj  rk  rP  rl  rD  rm  r6  )r*   rd   r~  rc   r7   r9   r   r   loss_functionr   r   rj  rk  rP  rl  rD  rm  )r=   r+   r   rc  rd  re  rf  r   rg  r0  r1  r  r   r   outputsr8  rS  audio_logitsr  s                      r@   rh   z#DiaForConditionalGeneration.forwardB  sH   X $** 
)/!5#9++/!5)
 
 $AJ&,,Q/
 /0T:r4#4#4dooFGYq!_Z\T*t000"dooF 	 %4%%o\&UYUdUdohnoD#33")"?"?&99$55&-&G&G")"?"?&99

 
	
rA   )NNNNNNNNNNNN)rB   rC   rD   rF   output_modalitiesr#   rT   r   r   r5   r  r   r   r   r  r   rh   rN   rO   s   @r@   rz  rz  *  s[     "y   .226598<:>:>6:!%)-,0*.26R
##d*R
 ((4/R
 !++d2	R

 $..5R
 !& 0 04 7R
 )5047R
 -t3R
 $;R
  $;R
 #TkR
   4'R
 ((4/R
 
	 R
  R
rA   rz  )r]  r(   rz  )r"   )r   )Ncollections.abcr   typingr   r5   r    r   r:   activationsr   cache_utilsr	   r
   r   integrationsr   r   r   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r    utils.genericr!   configuration_diar#   r$   r%   generation_diar&   
get_loggerrB   rp  r(   Moduler4   rl   r   r   r   r   rj   r   r   r   r   r   r  r,   r%  r-   rL  r]  rz  __all__r6  rA   r@   <module>r     sm  * %    & ! C C f f J B 9  L F & l l + L L . 
		H	% 0 0 0$!ryy !8)RYY )$ Y'J J (J(>< ><B( *+ ,2	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%2 )*>)ryy >) +>)BG)		 G)T00 0B@
# @
F8D0 8Dvn
# n
b 
u
! u

u
p 
g
"46H g

g
T LrA   