
    qi9*                        d Z ddlmZ ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ  ej>                  e       Z!dZ"dZ# G d dejH                        Z% G d de      Z&d&dZ' G d dejH                        Z( G d de      Z) G d de      Z* G d d e      Z+ G d! d"e      Z, G d# d$e      Z-g d%Z.y)'zPyTorch Phi-3 model.    )CallableN)nn   )ACT2FN)Cache)GenerationMixin)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONS)Unpack)logging   )MistralDecoderLayerMistralForCausalLM MistralForSequenceClassificationMistralForTokenClassificationMistralPreTrainedModeleager_attention_forwardrotate_half)PhiRotaryEmbedding   )
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Phi3MLPc                 *   t         |           || _        t        j                  |j
                  d|j                  z  d      | _        t        j                  |j                  |j
                  d      | _        t        |j                     | _        y )Nr   Fbias)super__init__configr   Linearhidden_sizeintermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fn)selfr   	__class__s     W/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/phi3/modular_phi3.pyr   zPhi3MLP.__init__1   sp    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56    hidden_statesreturnc                     | j                  |      }|j                  dd      \  }}|| j                  |      z  }| j                  |      S )Nr   dim)r#   chunkr&   r$   )r'   r+   	up_statesgates       r)   forwardzPhi3MLP.forward9   sL    %%m4	#//!/4i 2 24 88	~~i((r*   )__name__
__module____qualname__r   torchFloatTensorr4   __classcell__r(   s   @r)   r   r   0   s'    7)U%6%6 )5;L;L )r*   r   c                       e Zd Zy)Phi3RotaryEmbeddingNr5   r6   r7    r*   r)   r=   r=   B       r*   r=   c                 `   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }	}t        j                  ||z  t	        |      |z  z   |gd      }
t        j                  ||z  t	        |      |z  z   |	gd      }|
|fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r.   .Nr/   )	unsqueezeshaper8   catr   )qkcossinunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               r)   apply_rotary_pos_embrQ   F   s    $ --
&C
--
&C2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6Eii%#++e*<s*BCVLRTUGii%#++e*<s*BCVLRTUGGr*   c                   >    e Zd ZdZddededz  f fdZ	 	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ej                  dz  dee   de	ej                  ej                  dz  e	ej                     dz  f   fdZ xZS )Phi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   	layer_idxc                 |   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        |j                  | j                  z  d|j                  | j                  z  z  z   }t        j                  |j                  | j                  z  |j
                  d      | _        t        j                  |j
                  |d      | _        y )Nhead_dimg      Tr   Fr   )r   r   r   rT   getattrr!   num_attention_headsrV   num_key_value_headsnum_key_value_groupsscalingattention_dropout	is_causalr   r    o_projqkv_proj)r'   r   rT   op_sizer(   s       r)   r   zPhi3Attention.__init__g   s    "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9,,t}}<qFD^D^aeananDn?ooii : :T]] JFL^L^ejk		&"4"4gEJr*   r+   position_embeddingsattention_maskpast_key_valuescache_positionkwargsr,   c           
         |j                   d d }g |d| j                  }| j                  |      }	| j                  j                  | j                  z  }
|	dd |
f   }|	d|
|
| j
                  | j                  z  z   f   }|	d|
| j
                  | j                  z  z   d f   }|j                  |      j                  dd      }|j                  |      j                  dd      }|j                  |      j                  dd      }|\  }}t        ||||      \  }}|'|||d}|j                  ||| j                  |      \  }}t        j                  | j                  j                  t              } || ||||f| j                  sdn| j                   | j"                  t%        | j                  dd       d|\  }} |j&                  g |d j)                         }| j+                  |      }||fS )	Nr.   .r   r   )rH   rG   rd   g        sliding_window)dropoutr[   rg   )rC   rV   r_   r   rX   rY   view	transposerQ   updaterT   r
   get_interface_attn_implementationr   trainingr\   r[   rW   reshape
contiguousr^   )r'   r+   ra   rb   rc   rd   re   input_shapehidden_shapeqkv	query_posquery_states
key_statesvalue_statesrG   rH   cache_kwargsattention_interfaceattn_outputattn_weightss                       r)   r4   zPhi3Attention.forwardv   s    $))#2.88b8$--8mmM*KK33dmmC	3

?+i)d6N6NQUQ^Q^6^*^^^_
3	D,D,Dt}},T T VVW#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j&#&snUL'6'='=j,X\XfXfht'u$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL"4;;0@$G
%
 
%
!\ *k));;;;FFHkk+.L((r*   )N)NN)r5   r6   r7   __doc__r   intr   r8   Tensortupler   
LongTensorr   r	   r4   r:   r;   s   @r)   rS   rS   d   s    GKz KcDj K( )-260)||0) #5<<#=>0) t+	0)
 0) ((4/0) -.0) 
u||U\\D0%2E2LL	M0)r*   rS   c                   d    e Zd Zdedef fdZ	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
ej                  dz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )Phi3DecoderLayerr   rT   c                    t         |   ||       || _        t        ||      | _        t        |      | _        t        j                  |j                        | _
        t        j                  |j                        | _        y )N)r   rT   )r   r   r   rS   	self_attnr   mlpr   Dropoutresid_pdropresid_attn_dropoutresid_mlp_dropout)r'   r   rT   r(   s      r)   r   zPhi3DecoderLayer.__init__   s`    +&f	J6?"$**V-?-?"@!#F,>,>!?r*   Nr+   rb   position_idsrc   	use_cacherd   ra   re   r,   c                     |}	| j                  |      } | j                  d|||||||d|\  }}
|	| j                  |      z   }|}	| j                  |      }| j	                  |      }|	| j                  |      z   }|S )N)r+   rb   r   rc   r   rd   ra   r?   )input_layernormr   r   post_attention_layernormr   r   )r'   r+   rb   r   rc   r   rd   ra   re   residualself_attn_weightss              r)   r4   zPhi3DecoderLayer.forward   s     !,,];+94>> 	,
')%+) 3	,
 	,
(( !4#:#:=#II 55mD/ 4#9#9-#HHr*   )NNNFNN)r5   r6   r7   r   r}   r   r8   r~   r   r   boolr   r   r	   r9   r4   r:   r;   s   @r)   r   r      s    @z @c @ /304(,!&26HL|| t+ &&-	
  $; ((4/ #5<<#=>E -. 
u  %(9(95;L;L(L"MPT"TT	Ur*   r   c                       e Zd ZdZy)Phi3PreTrainedModelz0.0.5N)r5   r6   r7   _versionr?   r*   r)   r   r      s    Hr*   r   c                   "    e Zd Z	 	 	 	 	 	 	 ddZy)Phi3ForCausalLMNc	                    |r_t        | j                  d      rI|j                  d   | j                  j                  dz   k\  r |d   }
|
| j                  j                  k  rd }t	        j
                  | f||||||||d|	}|S )N original_max_position_embeddingsr   r   )	input_idsrc   rb   inputs_embedsrd   r   r   logits_to_keep)hasattrr   rC   r   r   prepare_inputs_for_generation)r'   r   rc   rb   r   rd   r   r   r   re   past_lengthmodel_inputss               r)   r   z-Phi3ForCausalLM.prepare_inputs_for_generation   s    $ %GH"dkk&R&RUV&VV(+KdkkJJJ"&&DD
+)')%)
 
 r*   )NNNNNTN)r5   r6   r7   r   r?   r*   r)   r   r      s     &r*   r   c                       e Zd Zy)Phi3ForSequenceClassificationNr>   r?   r*   r)   r   r     r@   r*   r   c                       e Zd Zy)Phi3ForTokenClassificationNr>   r?   r*   r)   r   r     r@   r*   r   )r   	Phi3Modelr   r   r   )r   )/r|   collections.abcr   r8   r   activationsr   cache_utilsr   
generationr   modeling_flash_attention_utilsr	   modeling_utilsr
   processing_utilsr   utilsr   mistral.modeling_mistralr   r   r   r   r   r   r   phi.modeling_phir   configuration_phi3r   
get_loggerr5   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCModuler   r=   rQ   rS   r   r   r   r   r   __all__r?   r*   r)   <module>r      s     $   !   ) B 5 &    2 * 
		H	%8 )bii )$	, 	<B)BII B)J'* 'T0 '( 'T	$D 		!> 	r*   