qiC@dZddlZddlZddlmZddlmZmZmZddlm Z ddl m Z ddl mZdd lmZmZmZmZmZdd lmZdd lmZdd lmZmZd dlmZej<eZ GddejBZ"GddejBZ#GddejBZ$GddejBZ%GddejBZ&GddejBZ'GddejBZ(GddeZ)Gdd ejBZ*Gd!d"ejBZ+eGd#d$eZ,eGd%d&e,Z-ed'(Gd)d*e,Z.eGd+d,e,Z/Gd-d.ejBZ0eGd/d0e,Z1gd1Z2y)2zPyTorch LiLT model.N)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward)auto_docstringlogging) LiltConfigc:eZdZfdZ ddZdZdZxZS)LiltTextEmbeddingsct|tj|j|j |j |_tj|j|j |_ tj|j|j |_ tj|j |j|_ tj|j|_|j#dt%j&|jj)dd|j |_tj|j|j |j*|_ y)N padding_idxeps position_idsrF) persistent)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandrselfconfig __class__s X/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/lilt/modeling_lilt.pyr"zLiltTextEmbeddings.__init__*s !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej  "..#%<<  * *F,>,>DL\L\$  c&|I|6|j||jj|j}n|j |}||j }n|j dd}|:t j|t j|jj}||j|}|j|}||z}|j|}||z }|j|}|j|}||fS)Nrdtypedevice)"create_position_ids_from_input_idsrtor>&create_position_ids_from_inputs_embedssizer2zeroslongrr'r+r)r,r0) r6 input_idstoken_type_idsr inputs_embeds input_shaper+ embeddingsr)s r9forwardzLiltTextEmbeddings.forward>s  $#FFyRVRbRbcff$$   $JJ=Y  #..*K',,.s3K  !"[[EJJtO`O`OgOghN   00;M $ : :> J"%:: "66|D)) ^^J/ \\*- <''r:c|j|j}tj|dj ||z}|j |zS)a Args: Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. x: torch.Tensor x: Returns: torch.Tensor rdim)neintr2cumsumtype_asrD)r6rErmaskincremental_indicess r9r?z5LiltTextEmbeddings.create_position_ids_from_input_idsbsP||K(,,.$||Da8@@F$N"'')K77r:c|jdd}|d}tj|jdz||jzdztj|j }|j dj|S)z Args: We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.: inputs_embeds: torch.Tensor Returns: torch.Tensor Nrrr<r)rBr2r3rrDr> unsqueezer4)r6rGrHsequence_lengthrs r9rAz9LiltTextEmbeddings.create_position_ids_from_inputs_embedsos$((*3B/ %a.||   q /D4D4D"Dq"HPUPZPZcpcwcw %%a(// <">|"L&AD[&[#&*nn5P&Q#&*ll3N&O#**3 cZ[ab b csA,D&& E/ D;;E)NN)rWrXrYr"rJrZr[s@r9r]r]s >*+r:r]c4eZdZdfd ZddZ ddZxZS)LiltSelfAttentionct||j|jzdk7r2t |ds&t d|jd|jd|j|_t |j|jz |_|j|jz|_tj|j|j|_ tj|j|j|_ tj|j|j|_ tj|j|jz|j|jz|_tj|j|jz|j|jz|_tj|j|jz|j|jz|_tj$|j&|_|j|_||_y)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ())r!r"r%num_attention_headshasattr ValueErrorrOattention_head_size all_head_sizerriquerykeyvaluerg layout_query layout_key layout_valuer.attention_probs_dropout_probr0 layer_idx)r6r7rr8s r9r"zLiltSelfAttention.__init__s     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F,,7 _kB %--aAq9DDF "/"4"4"6s";t?Q?Q>S"S* **,CD  "67  22Gr:N)rNF)rWrXrYr"rrJrZr[s@r9rwrws#>% Ar:rwcneZdZfdZdej dej dej fdZxZS)LiltSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr) r!r"rrir%denser,r-r.r/r0r5s r9r"zLiltSelfOutput.__init__"s` YYv1163E3EF f&8&8f>S>STzz&"<"<= r:r input_tensorreturncr|j|}|j|}|j||z}|Srrr0r,r6rrs r9rJzLiltSelfOutput.forward(7 =1  ]3 }|'CD r:rWrXrYr"r2TensorrJrZr[s@r9rr!1> U\\RWR^R^r:rc eZdZd fd Z d dej dej dej dzdedzdeej f dZ xZ S) LiltAttentionNct|t|||_t ||_|j }|j |jz|_t ||_||_y)Nr) r!r"rwr6routputr%rg layout_output)r6r7rori_hidden_sizer8s r9r"zLiltAttention.__init__0sa %f B $V,  ,,#//63N3NN+F3,r:rrrrrc|j||||}|j|d|}|j|d|}||f|ddz}|S)Nrrrl)r6rr) r6rrrr self_outputsattention_outputlayout_attention_outputrs r9rJzLiltAttention.forward:shyy       ;;|A F"&"4"4\!_m"T#%<= QR@PPr:rr) rWrXrYr"r2r FloatTensorbooltuplerJrZr[s@r9rr/se-48). ||||))D0   $;  u||  r:rcVeZdZfdZdej dej fdZxZS)LiltIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r!r"rrir%intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr5s r9r"zLiltIntermediate.__init__Os] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $r:rrcJ|j|}|j|}|Sr)rr)r6rs r9rJzLiltIntermediate.forwardWs& =1 00? r:rr[s@r9rrNs#9U\\ellr:rcneZdZfdZdej dej dej fdZxZS) LiltOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r!r"rrirr%rr,r-r.r/r0r5s r9r"zLiltOutput.__init___s` YYv779K9KL f&8&8f>S>STzz&"<"<= r:rrrcr|j|}|j|}|j||z}|Srrrs r9rJzLiltOutput.forwarderr:rr[s@r9rr^rr:rc eZdZd fd Z d dej dej dej dzdedzdeej f dZ d Z d Z xZ S) LiltLayerNct||j|_d|_t |||_t ||_t||_ |j}|j}|j|jz|_ |j|jz|_ t ||_ t||_||_ ||_ y)Nrr)r!r"chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr%rrglayout_intermediater)r6r7rrori_intermediate_sizer8s r9r"zLiltLayer.__init__ms '-'E'E$&vC,V4 (  ,, & 8 8#//63N3NN#)#;#;v?Z?Z#Z #3F#; '/,#8 r:rrrrrc |j||||}|d}|d}|dd}t|j|j|j|} t|j |j|j|} | | f|z}|S)N)rrrrl)rrfeed_forward_chunkrrlayout_feed_forward_chunk) r6rrrrself_attention_outputsrrr layer_outputlayout_layer_outputs r9rJzLiltLayer.forward~s"&   / "0"  2!4"8";(,0  # #T%A%A4CSCSUe 8  * *D,H,H$JZJZ\s  !45?r:cL|j|}|j||}|Sr)rrr6rintermediate_outputrs r9rzLiltLayer.feed_forward_chunks,"//0@A{{#68HI r:cL|j|}|j||}|Sr)rrrs r9rz#LiltLayer.layout_feed_forward_chunks."667GH))*=?OP r:rr) rWrXrYr"r2rrrrrJrrrZr[s@r9rrlso9*48). ||||))D0   $;  u||  : r:rceZdZfdZ d dej dej dej dzdedzdedzdedzd eej e zfd Z xZ S) LiltEncoderct|||_tjt |j Dcgc] }t|c}|_ycc}wr) r!r"r7r ModuleListrangenum_hidden_layersrlayer)r6r7_r8s r9r"zLiltEncoder.__init__sC  ]]uVE]E]?^#_!If$5#_` #_sANrrrroutput_hidden_states return_dictrc|rdnd}|rdnd}t|jD].\} } |r||fz}| ||||} | d}| d}|s&|| dfz}0|r||fz}|std|||fDSt|||S)Nrrrlc3$K|]}|| ywrr).0vs r9 z&LiltEncoder.forward..s = s)last_hidden_stater attentions) enumeraterrr ) r6rrrrrrall_hidden_statesall_self_attentionsi layer_module layer_outputss r9rJzLiltEncoder.forwards#7BD$5b4(4 POA|#$58H$H!(! M*!,M)!,M &9]1=M>> from transformers import AutoTokenizer, AutoModel >>> from datasets import load_dataset >>> tokenizer = AutoTokenizer.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> model = AutoModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = tokenizer(words, boxes=boxes, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state ```NzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embeds)r<)r>rF)rErrFrG)ror)rrrrrr)r pooler_outputrr)r7rruse_return_dictr}%warn_if_padding_and_no_attention_maskrBr>r2rCrDonesr|rIrFr4get_extended_attention_maskrrrr rr)r6rErorrFrrGrrrkwargsrH batch_size seq_lengthr>buffered_token_type_ids buffered_token_type_ids_expandedextended_attention_maskembedding_outputlayout_embedding_outputencoder_outputssequence_outputr s r9rJzLiltModel.forwardsRP2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@T <;;{T1FSD  !"ZZ*j)A6RN  !t(89*.//*H*HKZK*X'3J3Q3QR\^h3i0!A!&[ SY!Z150P0PQ_al0m)-%)' *9* &,#'"8"8dQ]"8"^,,  #2/!5# ' *!,8< 8O O4UY #]3oab6II I)-')77&11   r:)T) NNNNNNNNN)rWrXrYr"r"r$rr2rrrr rJrZr[s@r9rrs"/0*.$(.2.2,0-1)-,0#'h <<$&h llT!h  t+ h  t+ h llT) h ||d*h  $;h #Tkh D[h  u|| 9 9h h r:rz LiLT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ) custom_introcZeZdZfdZe ddej dzdejdzdejdzdej dzdej dzdejdzd ej dzd e dzd e dzd e dzd e eje zfdZ xZ S)LiltForSequenceClassificationct|||j|_||_t |d|_t ||_|jyNF)r) r!r" num_labelsr7rr LiltClassificationHead classifierrr5s r9r"z&LiltForSequenceClassification.__init__sJ   ++ f> 08 r:NrErorrFrrGlabelsrrrrc T| | n|jj} |j|||||||| |  } | d} |j| }d}||j |j }|jj |jdk(rd|j_nl|jdkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt}|jdk(r&||j|j}n|||}n|jj dk(r=t}||jd|j|jd}n,|jj dk(rt!}|||}| s|f| d dz}||f|zS|St#||| j$| j& S) a bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> from transformers import AutoTokenizer, AutoModelForSequenceClassification >>> from datasets import load_dataset >>> tokenizer = AutoTokenizer.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> model = AutoModelForSequenceClassification.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = tokenizer(words, boxes=boxes, return_tensors="pt") >>> outputs = model(**encoding) >>> predicted_class_idx = outputs.logits.argmax(-1).item() >>> predicted_class = model.config.id2label[predicted_class_idx] ```NrorrFrrGrrrrr regressionsingle_label_classificationmulti_label_classificationrrllosslogitsrr)r7r(r r=r@r> problem_typer;r=r2rDrOrsqueezerrrrrrr6rErorrFrrGr>rrrr,rr5rFrEloss_fctrs r9rJz%LiltForSequenceClassification.forwards\&1% )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ r:NrErorrFrrGr>rrrrc | | n|jj} |j|||||||| |  } | d} |j| } |j | }d}|W|j |j }t}||jd|j|jd}| s|f| ddz}||f|zS|St||| j| jS)a bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Examples: ```python >>> from transformers import AutoTokenizer, AutoModelForTokenClassification >>> from datasets import load_dataset >>> tokenizer = AutoTokenizer.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> model = AutoModelForTokenClassification.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = tokenizer(words, boxes=boxes, return_tensors="pt") >>> outputs = model(**encoding) >>> predicted_class_indices = outputs.logits.argmax(-1) ```Nr@rrrlrD) r7r(r r0r=r@r>rrr;rrrrIs r9rJz"LiltForTokenClassification.forwards V&1% ))F$6$68I8IJ r:NrErorrFrrGstart_positions end_positionsrrrrc (| | n|jj} |j||||||| | |  } | d}|j|}|j dd\}}|j dj }|j dj }d}||t|jdkDr|j d}t|jdkDr|j d}|jd}|jd|}|jd|}t|}|||}|||}||zdz }| s||f| ddz}||f|zS|St|||| j| j S) a bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. Examples: ```python >>> from transformers import AutoTokenizer, AutoModelForQuestionAnswering >>> from datasets import load_dataset >>> tokenizer = AutoTokenizer.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> model = AutoModelForQuestionAnswering.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = tokenizer(words, boxes=boxes, return_tensors="pt") >>> outputs = model(**encoding) >>> answer_start_index = outputs.start_logits.argmax() >>> answer_end_index = outputs.end_logits.argmax() >>> predict_answer_tokens = encoding.input_ids[0, answer_start_index : answer_end_index + 1] >>> predicted_answer = tokenizer.decode(predict_answer_tokens) ```Nr@rrrrL) ignore_indexrl)rE start_logits end_logitsrr)r7r(r r]splitrHrlenrBclamprr rr)r6rErorrFrrGr^r_rrrr,rr5rFrbrc total_loss ignored_indexrJ start_lossend_lossrs r9rJz LiltForQuestionAnswering.forward{s^&1%rxs AA&!9.6,*   H %S=S=l5+2995+pf fT RYY BII> ryy   7*7t4 "))4 p   i/ i iA #A A H m $7m  m `^ !4^ ^ DRYY,k 2k k \ r: