
    qi                     B   d Z ddlmZ ddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,  e%jZ                  e.      Z/ G d dej`                        Z1	 	 dBdej`                  dejd                  dejd                  dejd                  dejd                  dz  de3dz  de3dee#   fdZ4 G d d ej`                        Z5 G d! d"ej`                        Z6 G d# d$ej`                        Z7 G d% d&ej`                        Z8e$ G d' d(e             Z9e e$d)*       G d+ d,e"                    Z:e$ G d- d.e9             Z; e$d/*       G d0 d1e9             Z< G d2 d3ej`                        Z= G d4 d5ej`                        Z>e$ G d6 d7e9             Z? e$d8*       G d9 d:e9             Z@e$ G d; d<e9             ZAe$ G d= d>e9             ZBe$ G d? d@e9             ZCg dAZDy)CzPyTorch ALBERT model.    )Callable)	dataclassN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FN)create_bidirectional_mask)BaseModelOutputBaseModelOutputWithPoolingMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs   )AlbertConfigc                        e Zd ZdZdef fdZ	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  f
d
Z
 xZS )AlbertEmbeddingszQ
    Construct the embeddings from word, position and token_type embeddings.
    configc                    t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  |j
                        | _	        t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        | j#                  dt%        j&                  |j                        j)                  d      d       | j#                  dt%        j*                  | j,                  j/                         t$        j0                        d       y )	N)padding_idxepsposition_idsr   F)
persistenttoken_type_ids)dtype)super__init__r   	Embedding
vocab_sizeembedding_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandzerosr(   sizelongselfr#   	__class__s     \/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/albert/modeling_albert.pyr/   zAlbertEmbeddings.__init__6   s   !||F,=,=v?T?Tbhbubuv#%<<0N0NPVPePe#f %'\\&2H2H&J_J_%`"f&;&;AVAVWzz&"<"<= 	ELL)G)GHOOPWXej 	 	
 	ekk$*;*;*@*@*B%**Ubg 	 	
    N	input_idsr,   r(   inputs_embedsreturnc                    ||j                         }n|j                         d d }|\  }}|| j                  d d d |f   }|t        | d      rT| j                  j	                  |j
                  d   d      }t        j                  |d|      }|j	                  ||      }n:t        j                  |t        j                  | j                  j                        }|| j                  |      }| j                  |      }	||	z   }
| j                  |      }|
|z   }
| j                  |
      }
| j                  |
      }
|
S )Nr*   r,   r   r   )dimindex)r-   device)rC   r(   hasattrr,   rA   shaper?   gatherrB   rD   rP   r4   r8   r6   r9   r=   )rF   rJ   r,   r(   rK   input_shape
batch_size
seq_lengthbuffered_token_type_idsr8   
embeddingsr6   s               rH   forwardzAlbertEmbeddings.forwardG   sG     #..*K',,.s3K!,
J,,Q^<L
 !t-.*.*=*=*D*D\EWEWXYEZ\^*_'*/,,7NTU]i*j'!8!?!?
J!W!&[

SWSdSdSkSk!l  00;M $ : :> J"%::
"66|D"55
^^J/
\\*-
rI   )NNNN)__name__
__module____qualname____doc__r    r/   r?   
LongTensorFloatTensorTensorrY   __classcell__rG   s   @rH   r"   r"   1   s    
| 
& .2260426'##d*' ((4/' &&-	'
 ((4/' 
'rI   r"   modulequerykeyvalueattention_maskscalingr=   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |d      }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr*            r	   rN   )ptrainingr   )
rC   r?   matmul	transposer   
functionalsoftmaxr=   ro   
contiguous)
rc   rd   re   rf   rg   rh   r=   ri   attn_weightsattn_outputs
             rH   eager_attention_forwardrw   r   s     **R.D( <<s}}Q':;gEL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$rI   c                        e Zd Zdef fdZ	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fdZ xZS )
AlbertAttentionr#   c                 $   t         |           |j                  |j                  z  dk7  r1t	        |d      s%t        d|j                   d|j                         || _        |j                  | _        |j                  | _        |j                  |j                  z  | _        | j                  | j                  z  | _        | j                  dz  | _	        t        j                  |j                        | _        t        j                  |j                        | _        t        j                   |j                  | j                        | _        t        j                   |j                  | j                        | _        t        j                   |j                  | j                        | _        t        j                   |j                  |j                        | _        t        j*                  |j                  |j,                        | _        d| _        y )Nr   r2   zThe hidden size (z6) is not a multiple of the number of attention heads (rk   r&   F)r.   r/   hidden_sizenum_attention_headsrQ   
ValueErrorr#   attention_head_sizeall_head_sizerh   r   r;   attention_probs_dropout_probattention_dropoutr<   output_dropoutLinearrd   re   rf   denser9   r:   	is_causalrE   s     rH   r/   zAlbertAttention.__init__   s    : ::a?PVXhHi#F$6$6#7 8 4457  #)#=#= !--#)#5#59S9S#S !558P8PP//5!#F,O,O!P jj)C)CDYYv1143E3EF
99V//1C1CDYYv1143E3EF
YYv1163E3EF
f&8&8f>S>STrI   Nhidden_statesrg   ri   rL   c                    |j                   d d }g |d| j                  } | j                  |      j                  | j	                  dd      } | j                  |      j                  | j	                  dd      } | j                  |      j                  | j	                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  sdn| j                  j                  | j                  d|\  }
} |
j                   g |d j#                         }
| j%                  |
      }
| j'                  |
      }
| j)                  ||
z         }
|
|fS )Nr*   r   rl           )r=   rh   )rR   r~   rd   viewrq   re   rf   r   get_interfacer#   _attn_implementationrw   ro   r   rn   rh   reshapert   r   r   r9   )rF   r   rg   ri   rT   hidden_shapequery_layer	key_layervalue_layerattention_interfacerv   ru   s               rH   rY   zAlbertAttention.forward   s}    $))#2.CCbC$*B*BC 5djj/44lCMMaQRS0DHH]+00,?II!QO	4djj/44lCMMaQRS(?(M(MKK,,.E)
 %8	%
  $}}C$2H2H2J2JLL	%
 	%
!\ *k));;;;FFHjj-))+6nn][%@AL((rI   NrZ   r[   r\   r    r/   r?   r`   r_   r   r   tuplerY   ra   rb   s   @rH   ry   ry      sf    | < 48")||") ))D0") +,	")
 
u||U\\)	*")rI   ry   c                        e Zd Zdef fdZ	 ddej                  dej                  dz  dee	   de
ej                  ej                  f   fdZd	ej                  dej                  fd
Z xZS )AlbertLayerr#   c                    t         |           || _        |j                  | _        d| _        t        j                  |j                  |j                        | _	        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t         |j"                     | _        t        j&                  |j(                        | _        y )Nr   r&   )r.   r/   r#   chunk_size_feed_forwardseq_len_dimr   r9   r{   r:   full_layer_layer_normry   	attentionr   intermediate_sizeffn
ffn_outputr   
hidden_act
activationr;   r<   r=   rE   s     rH   r/   zAlbertLayer.__init__   s    '-'E'E$%'\\&2D2D&J_J_%`"(099V//1I1IJ))F$<$<f>P>PQ !2!23zz&"<"<=rI   Nr   rg   ri   rL   c                      | j                   ||fi |\  }}t        | j                  | j                  | j                  |      }| j                  ||z         }|S r   )r   r   ff_chunkr   r   r   )rF   r   rg   ri   attention_output_r   s          rH   rY   zAlbertLayer.forward   se     -dnn]NUfU!.MM((	

 22:@P3PQrI   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   )rF   r   r   s      rH   r   zAlbertLayer.ff_chunk   s3    XX./
__Z0
__Z0
rI   r   )rZ   r[   r\   r    r/   r?   r`   r_   r   r   r   rY   r   ra   rb   s   @rH   r   r      s    >| >  48|| ))D0 +,	
 
u||U\\)	*" %,, rI   r   c                        e Zd Zdef fdZ	 d
dej                  dej                  dz  dee	   de
ej                  e
ej                     z  df   fd	Z xZS )AlbertLayerGroupr#   c                     t         |           t        j                  t	        |j
                        D cg c]  }t        |       c}      | _        y c c}w r   )r.   r/   r   
ModuleListrangeinner_group_numr   albert_layersrF   r#   r   rG   s      rH   r/   zAlbertLayerGroup.__init__   s=    ]]vOeOeIf+gAK,?+gh+gs   ANr   rg   ri   rL   .c                 T    t        | j                        D ]  \  }} |||fi |} |S r   )	enumerater   )rF   r   rg   ri   layer_indexalbert_layers         rH   rY   zAlbertLayerGroup.forward   s<     *343E3E)F 	R%K(Q&QM	RrI   r   r   rb   s   @rH   r   r      sr    i| i 48|| ))D0 +,	
 
u||eELL1136	7rI   r   c            
       z     e Zd Zdef fdZ	 d	dej                  dej                  dz  dee	   de
ez  fdZ xZS )
AlbertTransformerr#   c                     t         |           || _        t        j                  |j
                  |j                        | _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        y c c}w r   )r.   r/   r#   r   r   r2   r{   embedding_hidden_mapping_inr   r   num_hidden_groupsr   albert_layer_groupsr   s      rH   r/   zAlbertTransformer.__init__  sf    +-99V5J5JFL^L^+_(#%==TYZ`ZrZrTs1tq2B62J1t#u 1ts   ,BNr   rg   ri   rL   c                 $   | j                  |      }t        | j                  j                        D ]R  }t	        || j                  j                  | j                  j
                  z  z        } | j                  |   ||fi |}T t        |      S )N)last_hidden_state)r   r   r#   num_hidden_layersintr   r   r   )rF   r   rg   ri   i	group_idxs         rH   rY   zAlbertTransformer.forward  s     88Gt{{445 	AA!>!>A^A^!^_`I?D44Y? M		 ??rI   r   )rZ   r[   r\   r    r/   r?   r`   r_   r   r   r   r   rY   ra   rb   s   @rH   r   r     s`    v| v 48@||@ ))D0@ +,	@
 
5	 @rI   r   c                   \    e Zd ZeZdZdZdZdZdZ	e
edZ ej                         d        Zy)AlbertPreTrainedModelalbertT)r   
attentionsc                 f   t        |t        j                        rct        j                  |j
                  d| j                  j                         |j                   t        j                  |j                         yyt        |t        j                        rt        j                  |j
                  d| j                  j                         |j                  Et        |j
                  dd      s-t        j                  |j
                  |j                            yyyt        |t        j                        r?t        j                  |j                         t        j                  |j
                         yt        |t              r t        j                  |j                         yt        |t               ryt        j"                  |j$                  t'        j(                  |j$                  j*                  d         j-                  d             t        j                  |j.                         yy)zInitialize the weights.r   )meanstdN_is_hf_initializedFr*   r)   )
isinstancer   r   initnormal_weightr#   initializer_rangebiaszeros_r0   r%   getattrr9   ones_AlbertMLMHeadr"   copy_r(   r?   r@   rR   rA   r,   )rF   rc   s     rH   _init_weightsz#AlbertPreTrainedModel._init_weights0  sa    fbii(LLSdkk6S6ST{{&FKK( '-LLSdkk6S6ST!!-gfmmMach6iFMM&*<*<=> 7j--KK$JJv}}%.KK$ 01JJv**ELL9L9L9R9RSU9V,W,^,^_f,ghKK--. 2rI   N)rZ   r[   r\   r    config_classbase_model_prefix_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr   ry   _can_record_outputsr?   no_gradr    rI   rH   r   r   #  sN    L N"&$%
 U]]_/ /rI   r   z2
    Output type of [`AlbertForPreTraining`].
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   y)AlbertForPreTrainingOutputa  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the masked language modeling loss and the next sequence prediction
        (classification) loss.
    prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    sop_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
        Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
        before SoftMax).
    Nlossprediction_logits
sop_logitsr   r   )rZ   r[   r\   r]   r   r?   r_   __annotations__r   r   r   r   r   r   rI   rH   r   r   F  s}    	 &*D%

d
")26u((4/6+/J!!D(/59M5**+d2926Je''(4/6rI   r   c                   R    e Zd ZeZdZddedef fdZdej                  fdZ
dej                  ddfd	Zeee	 	 	 	 	 dd
ej                   dz  dej"                  dz  dej                   dz  dej                   dz  dej"                  dz  dee   deez  fd                     Z xZS )AlbertModelr   r#   add_pooling_layerc                 f   t         |   |       || _        t        |      | _        t        |      | _        |rIt        j                  |j                  |j                        | _
        t        j                         | _        nd| _
        d| _        |j                  | _        | j                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r.   r/   r#   r"   rX   r   encoderr   r   r{   poolerTanhpooler_activationr   attn_implementation	post_init)rF   r#   r   rG   s      rH   r/   zAlbertModel.__init__d  s    
 	 *62(0))F$6$68J8JKDK%'WWYD"DK%)D"#)#>#>  	rI   rL   c                 .    | j                   j                  S r   rX   r4   rF   s    rH   get_input_embeddingsz AlbertModel.get_input_embeddingsz  s    ...rI   rf   Nc                 &    || j                   _        y r   r   )rF   rf   s     rH   set_input_embeddingsz AlbertModel.set_input_embeddings}  s    */'rI   rJ   rg   r,   r(   rK   ri   c                 >   |d u |d uz  rt        d      | j                  ||||      }t        | j                  ||      } | j                  ||fd|i|}|d   }	| j
                  '| j                  | j                  |	d d df               nd }
t        |	|
      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r(   r,   rK   )r#   rK   rg   r(   r   )r   pooler_output)r}   rX   r   r#   r   r   r   r   )rF   rJ   rg   r,   r(   rK   ri   embedding_outputencoder_outputssequence_outputpooled_outputs              rH   rY   zAlbertModel.forward  s     -t";<YZZ??L_l + 
 3;;*)
 '$,,
 &
 	
 *!,VZVaVaVm..t{{?1a4;P/QRsw)-'
 	
rI   )T)NNNNN)rZ   r[   r\   r    r   r   boolr/   r   r0   r   r   r   r   r   r?   r^   r_   r   r   r   r   rY   ra   rb   s   @rH   r   r   _  s   L |  ,/bll /0",, 04 0   .237260426$
##d*$
 ))D0$
 ((4/	$

 &&-$
 ((4/$
 +,$
 
$e	+$
    $
rI   r   z
    Albert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a
    `sentence order prediction (classification)` head.
    c                       e Zd ZdddZdef fdZdej                  fdZdej                  dd	fd
Z	dej                  fdZee	 	 	 	 	 	 	 ddej                  d	z  dej                   d	z  dej                  d	z  dej                  d	z  dej                   d	z  dej                  d	z  dej                  d	z  dee   deez  fd              Z xZS )AlbertForPreTraining(albert.embeddings.word_embeddings.weightpredictions.biaszpredictions.decoder.weightzpredictions.decoder.biasr#   c                     t         |   |       t        |      | _        t	        |      | _        t        |      | _        | j                          y r   )	r.   r/   r   r   r   predictionsAlbertSOPHeadsop_classifierr   rE   s     rH   r/   zAlbertForPreTraining.__init__  sB     !&)(0+F3 	rI   rL   c                 .    | j                   j                  S r   r   decoderr   s    rH   get_output_embeddingsz*AlbertForPreTraining.get_output_embeddings      '''rI   new_embeddingsNc                 &    || j                   _        y r   r  rF   r  s     rH   set_output_embeddingsz*AlbertForPreTraining.set_output_embeddings  s    #1 rI   c                 B    | j                   j                  j                  S r   r   rX   r4   r   s    rH   r   z)AlbertForPreTraining.get_input_embeddings      {{%%555rI   rJ   rg   r,   r(   rK   labelssentence_order_labelri   c           	          | j                   |f||||dd|}	|	dd \  }
}| j                  |
      }| j                  |      }d}|u|st               } ||j	                  d| j
                  j                        |j	                  d            } ||j	                  dd      |j	                  d            }||z   }t        ||||	j                  |	j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        sentence_order_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
            (see `input_ids` docstring) Indices should be in `[0, 1]`. `0` indicates original order (sequence A, then
            sequence B), `1` indicates switched order (sequence B, then sequence A).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, AlbertForPreTraining
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")
        >>> model = AlbertForPreTraining.from_pretrained("albert/albert-base-v2")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)
        >>> # Batch size 1
        >>> outputs = model(input_ids)

        >>> prediction_logits = outputs.prediction_logits
        >>> sop_logits = outputs.sop_logits
        ```Trg   r,   r(   rK   return_dictNrl   r*   )r   r   r   r   r   )
r   r   r  r   r   r#   r1   r   r   r   )rF   rJ   rg   r,   r(   rK   r  r  ri   outputsr   r   prediction_scores
sop_scores
total_lossloss_fctmasked_lm_losssentence_order_losss                     rH   rY   zAlbertForPreTraining.forward  s   N $++
))%'
 
 *1!& ,,_=((7

"6"B')H%&7&<&<RAWAW&XZ`ZeZefhZijN"*:??2q+ACWC\C\]_C`"a'*==J)/!!//))
 	
rI   NNNNNNN)rZ   r[   r\   _tied_weights_keysr    r/   r   r   r  r
  r0   r   r   r   r?   r^   r_   r   r   r   r   rY   ra   rb   s   @rH   r   r     sE    'Q$6
| (ryy (2BII 2$ 26bll 6  .237260426*.8<A
##d*A
 ))D0A
 ((4/	A

 &&-A
 ((4/A
   4'A
 $..5A
 +,A
 
$e	+A
  A
rI   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r   r#   c                    t         |           t        j                  |j                  |j
                        | _        t        j                  t        j                  |j                              | _
        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        |j                      | _        y )Nr&   )r.   r/   r   r9   r2   r:   	Parameterr?   rB   r1   r   r   r{   r   r  r   r   r   rE   s     rH   r/   zAlbertMLMHead.__init__  s    f&;&;AVAVWLLV->->!?@	YYv1163H3HI
yy!6!68I8IJ !2!23rI   r   rL   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|}|S r   )r   r   r9   r  )rF   r   r  s      rH   rY   zAlbertMLMHead.forward  sF    

=16}5]3)  rI   	rZ   r[   r\   r    r/   r?   r`   rY   ra   rb   s   @rH   r   r     s*    4| 4!U\\ !ell !rI   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r   r#   c                     t         |           t        j                  |j                        | _        t        j                  |j                  |j                        | _	        y r   )
r.   r/   r   r;   classifier_dropout_probr=   r   r{   
num_labels
classifierrE   s     rH   r/   zAlbertSOPHead.__init__%  sB    zz&"@"@A))F$6$68I8IJrI   r   rL   c                 J    | j                  |      }| j                  |      }|S r   )r=   r%  )rF   r   dropout_pooled_outputlogitss       rH   rY   zAlbertSOPHead.forward+  s%     $] ;!67rI   r   rb   s   @rH   r   r   $  s,    K| KU\\ ell rI   r   c                   ~    e Zd ZdddZ fdZdej                  fdZdej                  ddfd	Zdej                  fd
Z
ee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dee   deez  fd              Z xZS )AlbertForMaskedLMr   r   r   c                     t         |   |       t        |d      | _        t	        |      | _        | j                          y NF)r   )r.   r/   r   r   r   r   r   rE   s     rH   r/   zAlbertForMaskedLM.__init__8  s7     !&EB(0 	rI   rL   c                 .    | j                   j                  S r   r  r   s    rH   r  z'AlbertForMaskedLM.get_output_embeddingsA  r  rI   r  Nc                 \    || j                   _        |j                  | j                   _        y r   )r   r  r   r	  s     rH   r
  z'AlbertForMaskedLM.set_output_embeddingsD  s$    #1  . 3 3rI   c                 B    | j                   j                  j                  S r   r  r   s    rH   r   z&AlbertForMaskedLM.get_input_embeddingsH  r  rI   rJ   rg   r,   r(   rK   r  ri   c           
      :    | j                   d|||||dd|}|d   }	| j                  |	      }
d}|Ft               } ||
j                  d| j                  j
                        |j                  d            }t        ||
|j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoTokenizer, AlbertForMaskedLM

        >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")
        >>> model = AlbertForMaskedLM.from_pretrained("albert/albert-base-v2")

        >>> # add mask_token
        >>> inputs = tokenizer("The capital of [MASK] is Paris.", return_tensors="pt")
        >>> with torch.no_grad():
        ...     logits = model(**inputs).logits

        >>> # retrieve index of [MASK]
        >>> mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0]
        >>> predicted_token_id = logits[0, mask_token_index].argmax(axis=-1)
        >>> tokenizer.decode(predicted_token_id)
        'france'
        ```

        ```python
        >>> labels = tokenizer("The capital of France is Paris.", return_tensors="pt")["input_ids"]
        >>> labels = torch.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100)
        >>> outputs = model(**inputs, labels=labels)
        >>> round(outputs.loss.item(), 2)
        0.81
        ```
        TrJ   rg   r,   r(   rK   r  r   Nr*   r   r(  r   r   r   )	r   r   r   r   r#   r1   r   r   r   )rF   rJ   rg   r,   r(   rK   r  ri   r  sequence_outputsr  r  r  s                rH   rY   zAlbertForMaskedLM.forwardK  s    ^ $++ 
))%'
 
 #1: ,,-=>')H%&7&<&<RAWAW&XZ`ZeZefhZijN$!//))	
 	
rI   NNNNNN)rZ   r[   r\   r  r/   r   r   r  r
  r0   r   r   r   r?   r^   r_   r   r   r   r   rY   ra   rb   s   @rH   r*  r*  1  s%    'Q$6
(ryy (4BII 4$ 46bll 6  .237260426*.D
##d*D
 ))D0D
 ((4/	D

 &&-D
 ((4/D
   4'D
 +,D
 
%	D
  D
rI   r*  z
    Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                       e Zd Zdef fdZee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e
e   deez  fd              Z xZS )AlbertForSequenceClassificationr#   c                 N   t         |   |       |j                  | _        || _        t	        |      | _        t        j                  |j                        | _	        t        j                  |j                  | j                  j                        | _        | j                          y r   )r.   r/   r$  r#   r   r   r   r;   r#  r=   r   r{   r%  r   rE   s     rH   r/   z(AlbertForSequenceClassification.__init__  st      ++!&)zz&"@"@A))F$6$68N8NO 	rI   NrJ   rg   r,   r(   rK   r  ri   rL   c           
          | j                   d
|||||dd|}|d   }	| j                  |	      }	| j                  |	      }
d}|| j                  j                  | j
                  dk(  rd| j                  _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt               }| j
                  dk(  r& ||
j                         |j                               }n ||
|      }n| j                  j                  dk(  r=t               } ||
j                  d| j
                        |j                  d            }n,| j                  j                  dk(  rt               } ||
|      }t        ||
|j                   |j"                  	      S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Tr1  r   N
regressionsingle_label_classificationmulti_label_classificationr*   r2  r   )r   r=   r%  r#   problem_typer$  r-   r?   rD   r   r   squeezer   r   r   r   r   r   )rF   rJ   rg   r,   r(   rK   r  ri   r  r   r(  r   r  s                rH   rY   z'AlbertForSequenceClassification.forward  s   $ $++ 
))%'
 
  
]3/{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./'!//))	
 	
rI   r4  )rZ   r[   r\   r    r/   r   r   r?   r^   r_   r   r   r   r   rY   ra   rb   s   @rH   r6  r6    s    
| 
  .237260426*.;
##d*;
 ))D0;
 ((4/	;

 &&-;
 ((4/;
   4';
 +,;
 
"E	);
  ;
rI   r6  c                       e Zd Zdef fdZee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e
e   deez  fd              Z xZS )AlbertForTokenClassificationr#   c                 x   t         |   |       |j                  | _        t        |d      | _        |j
                  |j
                  n|j                  }t        j                  |      | _	        t        j                  |j                  | j                  j                        | _        | j                          y r,  )r.   r/   r$  r   r   r#  r<   r   r;   r=   r   r{   r#   r%  r   )rF   r#   r#  rG   s      rH   r/   z%AlbertForTokenClassification.__init__  s      ++!&EB --9 **++ 	 
 zz"9:))F$6$68N8NO 	rI   NrJ   rg   r,   r(   rK   r  ri   rL   c           	      H    | j                   |f||||dd|}|d   }	| j                  |	      }	| j                  |	      }
d}|<t               } ||
j	                  d| j
                        |j	                  d            }t        ||
|j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Tr  r   Nr*   r2  )	r   r=   r%  r   r   r$  r   r   r   )rF   rJ   rg   r,   r(   rK   r  ri   r  r   r(  r   r  s                rH   rY   z$AlbertForTokenClassification.forward  s      $++
))%'
 
 "!*,,71')HFKKDOO<fkk"oND$!//))	
 	
rI   r4  )rZ   r[   r\   r    r/   r   r   r?   r^   r_   r   r   r   r   rY   ra   rb   s   @rH   r?  r?    s    |    .237260426*.'
##d*'
 ))D0'
 ((4/	'

 &&-'
 ((4/'
   4''
 +,'
 
	&'
  '
rI   r?  c                   6    e Zd Zdef fdZee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  de
e   deez  fd              Z xZS )AlbertForQuestionAnsweringr#   c                     t         |   |       |j                  | _        t        |d      | _        t        j                  |j                  |j                        | _        | j                          y r,  )
r.   r/   r$  r   r   r   r   r{   
qa_outputsr   rE   s     rH   r/   z#AlbertForQuestionAnswering.__init__'  sU      ++!&EB))F$6$68I8IJ 	rI   NrJ   rg   r,   r(   rK   start_positionsend_positionsri   rL   c           
          | j                   d
|||||dd|}	|	d   }
| j                  |
      }|j                  dd      \  }}|j                  d      j	                         }|j                  d      j	                         }d }||t        |j                               dkD  r|j                  d      }t        |j                               dkD  r|j                  d      }|j                  d      }|j                  d|      }|j                  d|      }t        |      } |||      } |||      }||z   dz  }t        ||||	j                  |	j                  	      S )NTr1  r   r   r*   rm   )ignore_indexrl   )r   start_logits
end_logitsr   r   r   )r   rE  splitr=  rt   lenrC   clampr   r   r   r   )rF   rJ   rg   r,   r(   rK   rF  rG  ri   r  r   r(  rJ  rK  r  ignored_indexr  
start_lossend_losss                      rH   rY   z"AlbertForQuestionAnswering.forward1  s    $++ 
))%'
 
 "!*#?#)<<r<#: j#++B/::<''+668

&=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J+%!!//))
 	
rI   r  rZ   r[   r\   r    r/   r   r   r?   r^   r_   r   r   r   r   rY   ra   rb   s   @rH   rC  rC  %  s    |   .23726042637153
##d*3
 ))D03
 ((4/	3

 &&-3
 ((4/3
 ))D03
 ''$.3
 +,3
 
$e	+3
  3
rI   rC  c                       e Zd Zdef fdZee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e
e   deez  fd              Z xZS )AlbertForMultipleChoicer#   c                     t         |   |       t        |      | _        t	        j
                  |j                        | _        t	        j                  |j                  d      | _
        | j                          y )Nr   )r.   r/   r   r   r   r;   r#  r=   r   r{   r%  r   rE   s     rH   r/   z AlbertForMultipleChoice.__init__k  sV     !&)zz&"@"@A))F$6$6: 	rI   NrJ   rg   r,   r(   rK   r  ri   rL   c           	         ||j                   d   n|j                   d   }|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|1|j                  d|j                  d      |j                  d            nd} | j                  |f||||dd|}	|	d   }
| j	                  |
      }
| j                  |
      }|j                  d|      }d}|t               } |||      }t        |||	j                  |	j                        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
            [`PreTrainedTokenizer.encode`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
            *input_ids* above)
        Nr   r*   Tr  r2  )
rR   r   rC   r   r=   r%  r   r   r   r   )rF   rJ   rg   r,   r(   rK   r  ri   num_choicesr  r   r(  reshaped_logitsr   r  s                  rH   rY   zAlbertForMultipleChoice.forwardu  s   T -6,Aiooa(}GZGZ[\G]>G>SINN2y~~b'9:Y]	M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei ( r=#5#5b#9=;M;Mb;QR 	
 $++
))%'
 
  
]3#}= ++b+6')HOV4D("!//))	
 	
rI   r4  rR  rb   s   @rH   rT  rT  i  s    |   .237260426*.M
##d*M
 ))D0M
 ((4/	M

 &&-M
 ((4/M
   4'M
 +,M
 
$e	+M
  M
rI   rT  )r   r   r   r*  r6  r?  rC  rT  )Nr   )Er]   collections.abcr   dataclassesr   r?   r   torch.nnr   r   r    r
   r   activationsr   masking_utilsr   modeling_outputsr   r   r   r   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r   r   utils.genericr   r   utils.output_capturingr   configuration_albertr    
get_loggerrZ   loggerModuler"   r`   floatrw   ry   r   r   r   r   r   r   r   r   r   r*  r6  r?  rC  rT  __all__r   rI   rH   <module>rm     s    $ !   A A & ! 6   G & N M I 5 . 
		H	%=ryy =N !%II%<<% 
% <<	%
 LL4'% T\% % '(%8>)bii >)B#")) #Lryy "@		 @: /O / /D 
7 7 7& G
' G
 G
T \
0 \
\
~!BII !*
BII 
 _
- _
 _
D J
&; J
J
Z :
#8 :
 :
z @
!6 @
 @
F Z
3 Z
 Z
z	rI   