qitdZddlmZddlmZddlZddlmZddlmZddl m Z dd l m Z dd lmZdd lmZmZmZdd lmZmZdd lmZddlmZddlmZmZmZmZm Z ddl!m"Z"ejFe$Z%GddejLZ' dS>STzz&"<"<=   ELL)G)GHOOPWXej  N input_idstoken_type_idsr inputs_embedsreturnc||j}n|jdd}|d}||jddd|f}|:tj|tj|jj }||j |}|j|}||z}|j|} || z }|j|}|j|}|S)Nr!rdtypedevice) sizerr4zeroslongrDr)r-r+r.r2) r8r=r>rr? input_shape seq_lengthr- embeddingsr+s r;forwardzSplinterEmbeddings.forward<s  #..*K',,.s3K ^  ,,Q  ^ J"%:: "66|D)) ^^J/ \\*- r<)NNNN) __name__ __module__ __qualname____doc__r$r4 LongTensor FloatTensortuplerK __classcell__r:s@r;rr+szQ  .2260426 ##d*((4/&&-  ((4/   r<rmodulequerykeyvalueattention_maskscalingr2c tj||jdd|z}|||z}tjj |dtj j|j}tjj|||j}tj||} | jddj} | |fS)Nrr!)dimrC)ptrainingr) r4matmul transposer functionalsoftmaxfloat32torCr2r_ contiguous) rUrVrWrXrYrZr2kwargs attn_weights attn_outputs r;eager_attention_forwardrj^s<<s}}Q':;gEL!#n4 ==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K  $$r<c eZdZfdZ d dej dej dzdedzdee de ej f dZ xZ S) SplinterSelfAttentionc$t||j|jzdk7r2t |ds&t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ tj|j|j|_ tj|j|j|_ tj|j|j|_tj|j |_|j |_|jdz|_y)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()g)r#r$r'num_attention_headshasattr ValueErrorr9intattention_head_size all_head_sizerLinearrVrWrXr0attention_probs_dropout_probr2attention_dropoutrZr7s r;r$zSplinterSelfAttention.__init__vsC     : : :a ?PVXhHi#F$6$6#78 445Q8   #)#=#= #&v'9'9FS>STzz&"<"<= r<ry input_tensorr@cr|j|}|j|}|j||z}|SNrr2r.r8ryrs r;rKzSplinterSelfOutput.forward7 =1  ]3 }|'CD r<rLrMrNr$r4rrKrSrTs@r;rr1> U\\RWR^R^r<rc eZdZfdZ d dej dej dzdedzdee de ej f dZ xZ S) SplinterAttentioncbt|t||_t ||_yr)r#r$rlr8routputr7s r;r$zSplinterAttention.__init__s& )&1 (0 r<NryrYrzrgr@c n|j|f||d|}|j|d|}|f|ddz}|SN)rYrzrr)r8r)r8ryrYrzrg self_outputsattention_outputrs r;rKzSplinterAttention.forwards\!tyy  )/    ;;|A F#% QR(88r<rrrTs@r;rrsg148). ||))D0 $;  +,  u||  r<rcVeZdZfdZdej dej fdZxZS)SplinterIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r#r$rrvr'intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr7s r;r$zSplinterIntermediate.__init__s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $r<ryr@cJ|j|}|j|}|Sr)rr)r8rys r;rKzSplinterIntermediate.forwards& =1 00? r<rrTs@r;rrs#9U\\ellr<rcneZdZfdZdej dej dej fdZxZS)SplinterOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r#r$rrvrr'rr.r/r0r1r2r7s r;r$zSplinterOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= r<ryrr@cr|j|}|j|}|j||z}|Srrrs r;rKzSplinterOutput.forwardrr<rrTs@r;rrrr<rc eZdZfdZ d dej dej dzdedzdee de ej f dZ d Z xZ S) SplinterLayerct||j|_d|_t ||_t ||_t||_ y)Nr) r#r$chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr7s r;r$zSplinterLayer.__init__sI '-'E'E$*6208$V, r<NryrYrzrgr@c |j|f||d|}|d}|dd}t|j|j|j|}|f|z}|Sr)rrfeed_forward_chunkrr) r8ryrYrzrgself_attention_outputsrr layer_outputs r;rKzSplinterLayer.forwards"0 " )/"  "  2!4(,0  # #T%A%A4CSCSUe  /G+r<cL|j|}|j||}|Sr)rr)r8rintermediate_outputrs r;rz SplinterLayer.feed_forward_chunks,"//0@A{{#68HI r<r)rLrMrNr$r4rrQrrrrRrKrrSrTs@r;rrsl-48). ||))D0 $;  +,  u||  .r<rceZdZfdZe d dej dejdzdedzdedzdedzde e d e ej e zfd Z xZS) SplinterEncoderct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wr) r#r$r9r ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r8r9ir:s r;r$zSplinterEncoder.__init__sN  ]]5IaIaCb#caM&$9#cd &+#$dsA#NryrYrzoutput_hidden_states return_dictrgr@c |rdnd}|rdnd}t|jD])\} } |r||fz}| |||fi|} | d}|s!|| dfz}+|r||fz}t|||S)Nrrlast_hidden_statery attentions) enumeraterr ) r8ryrYrzrrrgall_hidden_statesall_self_attentionsr layer_module layer_outputss r;rKzSplinterEncoder.forward#s#7BD$5b4(4 POA|#$58H$H!(! M*!,M &9]1=Mrr?rzrrr@c  ||n|jj}||n|jj}||n|jj}| | t d|#|j |||j } n!||j dd} n t d| \} } | |jn |j} |tj| | f| }|&tj| tj| }|j|| }|j||||}|j||||d }|d }t||j |j" S) a token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) NzDYou cannot specify both input_ids and inputs_embeds at the same timer!z5You have to specify either input_ids or inputs_embeds)rDrB)r=rr>r?T)rYrzrrrr)r9rzruse_return_dictrr%warn_if_padding_and_no_attention_maskrErDr4onesrFrGget_extended_attention_maskrJrr ryr)r8r=rYr>rr?rzrrrgrH batch_sizerIrDextended_attention_maskembedding_outputencoder_outputssequence_outputs r;rKzSplinterModel.forwardms:2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@T  !"ZZ*j)A6RN  !"[[EJJvVN150P0PQ_al0m??%)' +  ,, 2/!5 ' *!,-)77&11  r<)NNNNNNNN)rLrMrNrOr$rrrrr4rrrRr rKrSrTs@r;rrUs /0*..2.2,0-1)-,0#'J <<$&J  t+J  t+ J llT) J ||d* J  $;J #TkJ D[J   J J r<rcXeZdZdfd Zdej dej fdZxZS)SplinterFullyConnectedLayerct|||_||_t j |j|j|_t||_t j|j|_ yr) r#r$ input_dim output_dimrrvrr act_fnr.)r8rrrr:s r;r$z$SplinterFullyConnectedLayer.__init__sV "$YYt~~t? Z( doo6r<inputsr@cl|j|}|j|}|j|}|Sr)rrr.)r8rrys r;rKz#SplinterFullyConnectedLayer.forwards2 6*  M2 }5 r<)gelurrTs@r;rrs#7ellu||r<rc(eZdZdZfdZdZxZS)QuestionAwareSpanSelectionHeadzf Implementation of Question-Aware Span Selection (QASS) head, described in Splinter's paper: ct|t|j|j|_t|j|j|_t|j|j|_t|j|j|_tj|j|jd|_ tj|j|jd|_ y)NF)bias) r#r$rr'query_start_transformquery_end_transformstart_transform end_transformrrvstart_classifierend_classifierr7s r;r$z'QuestionAwareSpanSelectionHead.__init__s %@ASASU[UgUg%h"#>v?Q?QSYSeSe#f :6;M;MvOaOab89K9KVM_M_` " &*<*P>PW\ ] ii(:(:F? ''1a0 ||M:> ++N; ##Aq!,\\-: Z''r<)rLrMrNrOr$rKrSrTs@r;rrs \(r<rc`eZdZfdZe ddej dzdej dzdej dzdej dzdej dzdejdzd ejdzd edzd edzd edzd ejdzde e zfdZ xZ S)SplinterForQuestionAnsweringct||t||_t ||_|j |_|jyrr#r$rrr splinter_qassquestion_token_idrr7s r;r$z%SplinterForQuestionAnswering.__init__C  %f- ;FC!'!9!9 r<Nr=rYr>rr?start_positions end_positionsrzrrquestion_positionsr@c | | n|jj} d} | |Dtjtj||j j d}nJtj|jdtj|j|j}|jd} d} |j||||||| | }|d}|j|| \}}| r"|jd |jd }}|d|d |z tj |j"j$zz}|d |z tj |j"j$zz}d}||t'|jd kDr|jd}t'|jd kDr|jd}|jd }|j)d||j)d|t+| }|||}|||}||zd z }| s||f|d dz}||f|zS|St-||||j.|j0 S) a token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NFr!)r]r)rClayoutrDTrYr>rr?rzrrr ignore_indexr\lossrrryr)r9rr4argmaxeqrrsrFrErGrrDrrrsqueezefinforCminlenclamp_rr ryr)r8r=rYr>rr?rrrzrrrrgquestion_positions_were_none"question_position_for_each_examplerrrr total_loss ignored_indexloss_fct start_lossend_lossrs r;rKz$SplinterForQuestionAnswering.forwardsH&1% @R@RST@U*L  %'1~+=\M_M_A`AdAd*ddL#q>'9U[[IYIY=Z=^=^&^^J  &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M  " "1m 4  M 2']CH!,@J M:H$x/14J"J/'!"+=F/9/EZMF* Q6 Q+%!!//))   r< NNNNNNNNNNN) rLrMrNr$rr4rrPrrRr rKrSrTs@r;rrs#*..2.2,0-13715)-,0#'6:b <<$&b  t+b  t+ b llT) b ||d* b ))D0b ''$.b  $;b #Tkb D[b ",,t3b  - -b b r<rzB Class for outputs of Splinter as a span selection model. ) custom_introceZdZUdZdZej dzed<dZej dzed<dZ ej dzed<dZ e ej dzed<dZ e ej dzed<y)SplinterForPreTrainingOutputa loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when start and end positions are provided): Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. start_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-start scores (before SoftMax). end_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-end scores (before SoftMax). Nr rrryr) rLrMrNrOr r4rQrrrryrRrrr<r;r2r2fs|&*D%  d ")-1L%##d*1+/J!!D(/59M5**+d2926Je''(4/6r<r2z Splinter Model for the recurring span selection task as done during the pretraining. The difference to the QA task is that we do not have a question, but multiple question tokens that replace the occurrences of recurring spans instead. ceZdZfdZe ddej dzdej dzdej dzdej dzdej dzdejdzd ejdzd edzd edzd edzd ejdzde e zfdZ dej dej fdZ xZ S)SplinterForPreTrainingct||t||_t ||_|j |_|jyrrr7s r;r$zSplinterForPreTraining.__init__rr<Nr=rYr>rr?rrrzrrrr@c `| | n|jj} | | | td| | td| |j|} |j ||||||| | } | d}|j \}}}|j || \}}| j d}||jdj|||}|d|z tj|jjzz}|d|z tj|jjzz}d}|||jdtd|dz |jdtd|dz t|jj }||j#||z||j#||z}||j#||z||j#||z}||zdz }| s||f| ddz}||f|zS|St%|||| j&| j( S) a input_ids (`torch.LongTensor` of shape `(batch_size, num_questions, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. start_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NzCquestion_positions must be specified in order to calculate the lossz?question_positions must be specified when inputs_embeds is usedrrrrr\r)r9r TypeError_prepare_question_positionsrrErrr6r4r$rCr%r'maxrr(r~r2ryr)r8r=rYr>rr?rrrzrrrrgrrrsequence_lengthr]rr num_questions attention_mask_for_each_questionr*r,r-r.rs r;rKzSplinterForPreTraining.forwardsn&1%$? @ (T[[5M5MNH!!!*}"H%x/14J"J/'!"+=F/9/EZMF* Q6 Q+%!!//))   r<c4tj||jjk(\}}tj|}tj |j d|jf|jjtj|j}t|j d|j dk(dtj|Dcgc]}tj|c}}||||f<|Scc}w)NrrBz?All samples in the batch must have at least one question token.)r4wherer9rbincountfullrEr9r(rGrDrcatr5)r8r=rowsflat_positionsr;rncolss r;r8z2SplinterForPreTraining._prepare_question_positions s${{9 8U8U+UVnt, JJ ^^A  1 1 3 4 KK $ $**##     q !Y^^A%6 6 M yy=Aa%,,q/AB . $*Bs)Dr/)rLrMrNr$rr4rrPrrRr2rKr8rSrTs@r;r4r4}s?*..2.2,0-13715)-,0#'6:y <<$&y  t+y  t+ y llT) y ||d* y ))D0y ''$.y  $;y #Tky D[y ",,t3y  - -y y vU\\ellr<r4)rr4rrr)r|):rOcollections.abcr dataclassesrr4rtorch.nnrrr activationsr modeling_layersr modeling_outputsr r r modeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrrrrconfiguration_splinterr get_loggerrLloggerModulerrfloatrjrlrrrrrrrrrrrr2r4__all__rr<r;rWs$! %&!9ZZF&63   H %//t% II% <<% % << % LL4' %  %%.5BII5r   2 299  RYY #.#N* bii* Zioiic +c c L"))$#(RYY#(Ln #:n n b  7;7  7"V4VVr r<