
    qiu                        d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z-  e j\                  e/      Z0e G d de             Z1 G d dejd                        Z3 G d de'      Z4 G d de#      Z5 G d de$      Z6 G d d e"      Z7 G d! d"ejd                        Z8 G d# d$e      Z9 G d% d&e1      Z: G d' d(e      Z; G d) d*e1      Z< ed+,       G d- d.e1             Z= ed/,       G d0 d1e1e-             Z>g d2Z?y)3zPyTorch Dia model.    )CallableN)nn   )initialization)DynamicCacheEncoderDecoderCache)create_bidirectional_maskcreate_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringcan_return_tupleis_torchdynamo_compilinglogging   )LlamaAttentionLlamaRMSNormLlamaRotaryEmbeddingeager_attention_forward)Phi3MLP   )	DiaConfigDiaDecoderConfigDiaEncoderConfig)DiaGenerationMixinc                   N     e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZ fdZ xZS )DiaPreTrainedModelconfigmodelT	input_idsDiaEncoderLayerDiaDecoderLayerc                 &   t         |   |       t        |t              rqt	        j
                  | j                  j                  t        j                        | j                  j                  z  }t        j                  |j                  |       y y )Ndtype)super_init_weights
isinstanceDiaMultiChannelEmbeddingtorcharanger%   num_channelslong
vocab_sizeinitcopy_offsets)selfmoduler8   	__class__s      U/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/dia/modular_dia.pyr.   z DiaPreTrainedModel._init_weights=   sb    f%f67ll4;;#;#;5::NQUQ\Q\QgQggGJJv~~w/ 8    )__name__
__module____qualname__r   __annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraphmain_input_name_no_split_modulesr.   __classcell__r;   s   @r<   r$   r$   1   sG    &*#N!!O*,=>0 0r=   r$   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )r0   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r%   c                 ~   t         |           t        j                  |j                  |j
                  z  |j                        | _        |j                  | _        |j
                  | _        t        j                  |j
                  t        j                        |j                  z  }| j                  d|d       y )Nr+   r8   F)
persistent)r-   __init__r   	Embeddingr5   r3   hidden_sizeembedr1   r2   r4   register_buffer)r9   r%   r8   r;   s      r<   rO   z!DiaMultiChannelEmbedding.__init__R   s    \\&"3"3f6I6I"I6K]K]^
!--"//,,v22%**EHYHYYYEBr=   audio_codesreturnc                 "   || j                   j                  |j                        z   j                  d      }| j	                  |      j                  |j                  d   |j                  d   d| j                        }|j                  d      S )Nr   r   r   )dim)	r8   todevicesqueezerR   viewshaperQ   sum)r9   rT   tokensembedss       r<   forwardz DiaMultiChannelEmbedding.forwardZ   su    0B0B CCLLQOF#((a+:K:KA:NPRTXTdTdezzaz  r=   )
r>   r?   r@   __doc__r    rO   r1   Tensorra   rJ   rK   s   @r<   r0   r0   D   s2    C/ C!5<< !ELL !r=   r0   c                       e Zd Zy)DiaMLPNr>   r?   r@    r=   r<   re   re   `       r=   re   c                       e Zd Zy)
DiaRMSNormNrf   rg   r=   r<   rj   rj   d   rh   r=   rj   c                       e Zd Zy)DiaRotaryEmbeddingNrf   rg   r=   r<   rl   rl   h   rh   r=   rl   c                   ,    e Zd ZdZddeez  dedefdZy)DiaSelfAttention=Multi-headed attention from 'Attention Is All You Need' paperr%   	layer_idx	is_causalc                    t         j                  j                  |        || _        || _        |j
                  | _        | j                  j                  | _        | j                  j                  xs | j                  | _        | j                  | j                  z  | _	        t        |d|j
                  | j                  z        | _        d| _        d| _        || _        t        j                  | j
                  | j                  | j                  z  d      | _        t        j                  | j
                  | j                  | j                  z  d      | _        t        j                  | j
                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j
                  d      | _        y )Nhead_dimr           Fbias)r   ModulerO   r%   rp   rQ   num_attention_heads	num_headsnum_key_value_headsnum_key_value_groupsgetattrrs   scalingattention_dropoutrq   Linearq_projk_projv_projo_proj)r9   r%   rp   rq   s       r<   rO   zDiaSelfAttention.__init__o   sL   
		4 "!--88#';;#B#B#Tdnn $(NNd6N6N$N!
F4F4F$..4XY!$"ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]r=   N)F)	r>   r?   r@   rb   r!   r    intboolrO   rg   r=   r<   rn   rn   l   s+    G^/2BB ^s ^_c ^r=   rn   c                        e Zd ZdZdedef fdZ	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  f   fdZ xZS )DiaCrossAttentionro   r%   rp   c                 f   t         |           || _        || _        |j                  | _        |j
                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  | j                  z  | _
        |j                  | _        d| _        d| _        d| _        t!        j"                  | j                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j                  | j                  z  | j                  d      | _        y )Nr   rt   Fru   )r-   rO   r%   rp   rQ   cross_hidden_sizecross_num_attention_headsry   cross_num_key_value_headsrz   r{   cross_head_dimrs   r}   r~   rq   r   r   r   r   r   r   r9   r%   rp   r;   s      r<   rO   zDiaCrossAttention.__init__   s?   "!--!'!9!9>>#';;#H#H $(NNd6N6N$N!--!$ii 0 0$..4==2PW\]ii 6 68P8PSWS`S`8`glmii 6 68P8PSWS`S`8`glmii >@P@PW\]r=   Nhidden_statescross_attention_statesattention_maskpast_key_valueskwargsrU   c                 F   |j                   d d }g |d| j                  }g |j                   d d d| j                  }| j                  |      j                  |      j	                  dd      }	|%|j
                  j                  | j                        nd}
|]|
r[|j                  j                  | j                     j                  }|j                  j                  | j                     j                  }n| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }|C|j                  j                  ||| j                        \  }}d|j
                  | j                  <   t        j                   | j"                  j$                  t&              } || |	|||fd| j(                  i|\  }}|j+                  g |d      j-                         }| j/                  |      }||fS )NrW   r   r   FTr}   )r]   rs   r   r\   	transpose
is_updatedgetrp   cross_attention_cachelayerskeysvaluesr   r   updater   get_interfacer%   _attn_implementationr   r}   reshape
contiguousr   )r9   r   r   r   r   r   input_shapehidden_shapecross_shapequery_statesr   
key_statesvalue_statesattention_interfaceattn_outputattn_weightss                   r<   ra   zDiaCrossAttention.forward   s    $))#2.88b8$--8M.44Sb9M2Mt}}M{{=166|DNNqRSTGVGb_//33DNNChm
&:(>>EEdnnUZZJ*@@GGW^^L%;<AA+NXXYZ\]^J;;'=>CCKPZZ[\^_`L*+:+P+P+W+W NN,(
L >B**4>>:(?(M(MKK,,.E)
 %8%
 LL%
 %
!\ "))*<K*<*<=HHJkk+.L((r=   NN)r>   r?   r@   rb   r    r   rO   r1   rc   r   r   r   tuplera   rJ   rK   s   @r<   r   r      s    G^/ ^C ^. /36:1)||1) !&1) t+	1)
 -t31) -.1) 
u||U\\D00	11)r=   r   c                        e Zd Zdedef fdZ	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	e
   d	eej                  ej                  dz  f   f
d
Z xZS )r(   r%   rp   c                     t         |           t        |j                  |j                        | _        t        ||d      | _        t        |j                  |j                        | _        t        |      | _
        y )NepsFrq   )r-   rO   rj   rQ   norm_epspre_sa_normrn   self_attentionpost_sa_normre   mlpr   s      r<   rO   zDiaEncoderLayer.__init__   s\    %f&8&8fooN.vyER&v'9'9vO&>r=   Nr   position_embeddingsr   r   rU   c                     |}| j                  |      } | j                  |f||d|\  }}||z   }|}| j                  |      }| j                  |      }	||	z   }||fS )N)r   r   )r   r   r   r   )
r9   r   r   r   r   residualnormed_statesself_attn_outputself_attn_weightsmlp_outs
             r<   ra   zDiaEncoderLayer.forward   s     !((7.Ad.A.A/
 3)/
 	/
++ !#33 ))-8((=) 7*///r=   r   )r>   r?   r@   r!   r   rO   r1   rc   r   r   r   ra   rJ   rK   s   @r<   r(   r(      s    "/ "C " IM.2	0||0 #5<<#=>E0 t+	0
 -.0 
u||U\\D00	10r=   r(   c                        e Zd Zdef fdZee	 	 	 ddej                  dej                  dz  de	dz  de	dz  de
e   d	eez  fd
              Z xZS )
DiaEncoderr%   c           	         t         |   |       || _        t        j                  |j
                  |j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w Nr   r%   )r-   rO   r%   r   rP   r5   rQ   	embedding
ModuleListrangenum_hidden_layersr(   r   rj   r   normrl   
rotary_emb	post_initr   s      r<   rO   zDiaEncoder.__init__   s     f&7&79K9KLmmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   -CNr'   r   output_attentionsoutput_hidden_statesr   rU   c                    | j                  |      }t        j                  |j                  d   |j                        d d d f   }t        | j                  ||      }| j                  ||      }|rdnd }	|rdnd }
| j                  D ](  }|r|	|fz   }	 ||f|||d|}|d   }|s |
|d   fz   }
* | j                  |      }|r|	|fz  }	t        ||	|
	      S )
NrW   rZ   )r%   inputs_embedsr   position_idsrg   )r   r   r   r   r   last_hidden_stater   
attentions)r   r1   r2   r]   rZ   r	   r%   r   r   r   r   )r9   r'   r   r   r   r   r   r   r   encoder_statesall_attentionsencoder_layerlayer_outputss                r<   ra   zDiaEncoder.forward   s    y1
 ||IOOB$7	@P@PQRVXYRYZ2;;')

 #oom,oW30d![[ 	FM#!/=2B!B)-)$7	
 M *!,M !/=3C2E!E	F  		-0}..N+>Vd
 	
r=   )NFF)r>   r?   r@   r!   rO   r   r   r1   rc   r   r   r   r   r   ra   rJ   rK   s   @r<   r   r      s    /   /3).,10
<<0
 t+0
  $;	0

 #Tk0
 -.0
 
5	 0
  0
r=   r   c                   l    e Zd Zdedef fdZ	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  dej                  dz  deej                  ej                  dz  ej                  dz  f   fdZ xZS )r)   r%   rp   c                    t         |           |j                  | _        t	        ||d      | _        t        ||      | _        t        |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |      | _        y )NTr   r   )r-   rO   rQ   	embed_dimrn   r   r   cross_attentionrj   r   r   pre_ca_normpre_mlp_normre   r   r   s      r<   rO   zDiaDecoderLayer.__init__1  s    ++.vyDQ0C%f&8&8fooN%f&8&8fooN&v'9'9vO&>r=   Nr   r   r   encoder_hidden_statesencoder_attention_maskr   cache_positionrU   c                 d   |}	t        |	t              r|	j                  }	|}
| j                  |      } | j                  ||||	fd|i|\  }}|
|z   }|}
| j                  |      } | j                  ||f||d|\  }}|
|z   }|}
| j                  |      }| j                  |      }|
|z   }|||fS )Nr   )r   r   )	r/   r   self_attention_cacher   r   r   r   r   r   )r9   r   r   r   r   r   r   r   r   self_attn_cacher   r   r   r   cross_statescross_attn_weightsr   s                    r<   ra   zDiaDecoderLayer.forward;  s    *o':;-BBO ((7.Ad.A.A 	/
 *	/
 	/
++ !#33 ((7+?4+?+?!,
 2+	,

 ,
(( !</ ))-8((=) 7*/1CCCr=   )NNNNNN)r>   r?   r@   r    r   rO   r1   rc   r   r   
LongTensorra   rJ   rK   s   @r<   r)   r)   0  s    "/ "C " IM.2596:6:26-D||-D #5<<#=>E-D t+	-D
  %||d2-D !&t 3-D -t3-D ((4/-D 
u||U\\D0%,,2EE	F-Dr=   r)   c                   ,    e Zd ZdZdef fdZee	 	 	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dedz  dej                  dz  deez  fd              Z xZS )
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.r%   c           	         t         |   |       |j                  | _        |j                  | _        t	        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w r   )r-   rO   r3   r5   r0   
embeddingsr   r   r   r   r)   r   rj   rQ   r   r   rl   r   r   r   s      r<   rO   zDiaDecoder.__init__n  s     "// ++26:mmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   )CNr'   r   r   r   r   r   r   r   r   rU   c
           	         |j                         dd \  }}||j                         nd}|	%t        j                  |||z   |j                        }	|	|	dddf   }| j                  |      }|1t               s'||z   }t        j                  |||j                        }t        | j                  |||	|      }t        | j                  |||      }| j                  ||      }|rdnd}|rdnd}|r|dnd}| j                  D ]8  }|r||fz  } |||||f|||	|d	|
}|d   }|s$||d
   fz   }|0||d   fz   }: | j                  |      }|r||fz  }t        |||||      S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        NrW   r   r   )r%   r   r   r   r   )r%   r   r   r   r   rg   )r   r   r   r   r   r   )r   r   r   r   cross_attentions)sizeget_seq_lengthr1   r2   rZ   r   r   onesr
   r%   r	   r   r   r   r   )r9   r'   r   r   r   r   r   r   r   r   r   
batch_size
seq_lengthpast_key_values_lengthr   mask_seq_lengthr   all_hidden_statesall_self_attnsall_cross_attentionslayerr   s                         r<   ra   zDiaDecoder.forward{  s   , "+!1#2!6
JETE`!?!?!Afg!"\\&(>(KT]TdTdN )$'2L 	2!*B*D4zAO"ZZ
OIL\L\]N+;;'))+
 ";;;'1"7	"
 #oom,oW"6BD0d&7<Q<]rdh[[ 	VE#!m%55!! $% (> /-) M *!,M !/=3C2E!E(4+?=QRCSBU+U(/	V2 		-0-!118+++%1
 	
r=   )NNNNNFFN)r>   r?   r@   rb   r    rO   r   r   r1   rc   r   FloatTensorr   r   r   r   ra   rJ   rK   s   @r<   r   r   k  s   7/   15.2:>:>6:).,126\
<<\
 &&-\
 t+	\

  %0047\
 !& 0 04 7\
 -t3\
  $;\
 #Tk\
 ((4/\
 
3U	:\
  \
r=   r   z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )custom_introc                   N    e Zd Zdef fdZee	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
z  dz  d
edz  dedz  dedz  dedz  dej                  dz  de
ez  fd              Z xZS )DiaModelr%   c                     t         |   |       || _        t        |j                        | _        t        |j                        | _        | j                          y )N)
r-   rO   r%   r   encoder_configencoderr   decoder_configdecoderr   r9   r%   r;   s     r<   rO   zDiaModel.__init__  sE     !&"7"78!&"7"78r=   Nr'   r   decoder_input_idsdecoder_position_idsdecoder_attention_maskencoder_outputsr   	use_cacher   r   r   rU   c                    ||t        d      |	|	n| j                  j                  }	|
|
n| j                  j                  }
||n| j                  j                  }| j
                  r%| j                  r|rt        j                  d       d}|r6|4t        t        | j                        t        | j                              }| | j                  d|||	|
d|}nGt        |t              s7t        |d   t        |      dkD  r|d   ndt        |      d	kD  r|d	   nd
      }|d   j                  d   d| j                  j                   j"                  }}}|Ct%        j&                  |d|f| j                  j                   j(                  | j*                        }|j,                  d	k(  r#|j/                  |||      j1                  dd	      } | j2                  d||||d   |||	|
||d
|}t5        |j6                  |j8                  |j:                  |j<                  |j>                  |d   |j:                  |j<                        S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   )r'   r   r   r   r   r   r   r   rW   )r   
fill_valuerZ   )
r'   r   r   r   r   r   r   r   r  r   )r   r   decoder_hidden_statesdecoder_attentionsr   encoder_last_hidden_stater   encoder_attentionsrg   ) 
ValueErrorr%   r   r   r  is_gradient_checkpointingtrainingloggerwarning_oncer   r   r   r/   r   lenr]   r   r3   r1   fullbos_token_idrZ   ndimr   r   r   r   r   r   r   r   r   )r9   r'   r   r   r  r  r  r   r  r   r   r   r   bszseq_lenchannelsdecoder_outputss                    r<   ra   zDiaModel.forward  su   N !8j  2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	))dmm##p "	01,dkk2RT`hlhshsTtuO"*dll #-"3%9	
 O O_=-"1!"4474H14Loa0RV14_1E1I?1-tO #2!"4":":1"=r4;;C]C]CjCjhW$ %

1h'DKK4N4N4[4[dhdodo! !!Q& 1 9 9#x Q [ [\]_` a&$,, 
'-1"1!"4#1+/!5)
 
 "-??+;;"1"?"?.99,==&5a&8"1"?"?.99	
 		
r=   )NNNNNNNNNNN)r>   r?   r@   r   rO   r   r   r1   r   r   r   r   r   r   ra   rJ   rK   s   @r<   r   r     s5   y   .226598<:>:>6:!%)-,026k
##d*k
 ((4/k
 !++d2	k

 $..5k
 !& 0 04 7k
 )5047k
 -t3k
 $;k
  $;k
 #Tkk
 ((4/k
 
#	#k
  k
r=   r   zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   v    e Zd ZdZdZdef fdZee	 	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  deez  dz  dedz  dedz  dedz  dedz  de	j                  dz  de	j                  dz  deez  fd              Z xZS )DiaForConditionalGenerationr&   )audior%   c                 |   t         |   |       || _        t        |      | _        |j
                  j                  | _        |j
                  j                  | _        t        j                  |j
                  j                  | j                  | j                  z  d      | _        d| _        | j                          y )NFru   ForMaskedLM)r-   rO   r%   r   r&   r   r3   r5   r   r   rQ   logits_dense	loss_typer   r   s     r<   rO   z$DiaForConditionalGeneration.__init__b  s     f%
"11>> //::II!!--0A0ADOO0S[`
 ' 	r=   Nr'   r   r   r  r  r  r   r  r   r   labelsr   rU   c                 ^    | j                   d	|||||||||	|
|d|}|d   }|j                  d   }| j                  |      j                  |d| j                  | j
                  f      j                  dd      j                         j                  || j                  z  d| j
                        }d}|  | j                  d	||| j
                  d|}t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                   	      S )
a  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r'   r   r   r  r  r  r   r  r   r   r   r   rW   r   r   N)logitsr  r5   )	lossr!  r   r  r  r   r	  r   r
  rg   )r&   r]   r  r\   r3   r5   r   r   loss_functionr   r   r  r  r   r	  r   r
  )r9   r'   r   r   r  r  r  r   r  r   r   r  r   r   outputsr   r   audio_logitsr"  s                      r<   ra   z#DiaForConditionalGeneration.forwardq  sH   X $** 
)/!5#9++/!5)
 
 $AJ&,,Q/
 /0T:r4#4#4dooFGYq!_Z\T*t000"dooF 	 %4%%o\&UYUdUdohnoD#33")"?"?&99$55&-&G&G")"?"?&99

 
	
r=   )NNNNNNNNNNNN)r>   r?   r@   rB   output_modalitiesr   rO   r   r   r1   r   r   r   r   r   r   ra   rJ   rK   s   @r<   r  r  Y  s[     "y   .226598<:>:>6:!%)-,0*.26R
##d*R
 ((4/R
 !++d2	R

 $..5R
 !& 0 04 7R
 )5047R
 -t3R
 $;R
  $;R
 #TkR
   4'R
 ((4/R
 
	 R
  R
r=   r  )r   r$   r  )@rb   collections.abcr   r1   r    r   r6   cache_utilsr   r   masking_utilsr	   r
   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   llama.modeling_llamar   r   r   r   phi3.modeling_phi3r   configuration_diar   r    r!   generation_diar"   
get_loggerr>   r  r$   rw   r0   re   rj   rl   rn   r   r(   r   r)   r   r   r  __all__rg   r=   r<   <module>r7     su    $   & < J B 9  G & X X  ) L L . 
		H	% 0 0 0$!ryy !8	W 		 		- 	^~ ^,G)		 G)T00 0B@
# @
F8D0 8Dvn
# n
b 
u
! u

u
p 
g
"46H g

g
T Lr=   