qikddlZddlmZmZddlmZmZmZmZm Z m Z m Z m Z ddl mZddlmZddlmZddlmZdd lmZdd lmZmZmZd d lmZmZej<eZ Gd deZ!GddeZ"GddeZ#Gdde Z$Gdde Z%Gdde Z&GddeZ'GddeZ(GddeZ)gdZ*y) N)InstructBlipQFormerConfigInstructBlipVisionConfig)'BaseModelOutputWithVisionQformerOutputs$InstructBlipForConditionalGeneration/InstructBlipForConditionalGenerationModelOutputInstructBlipModelInstructBlipPreTrainedModelInstructBlipQFormerModelInstructBlipVisionModelTransformersKwargs)PreTrainedConfig)FlashAttentionKwargs)BaseModelOutputWithPooling)!MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)Unpack)auto_docstringcan_return_tuplelogging)CONFIG_MAPPING AutoConfigc eZdZy)InstructBlipVideoVisionConfigN__name__ __module__ __qualname__q/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/instructblipvideo/modular_instructblipvideo.pyrr-r rc eZdZy)InstructBlipVideoQFormerConfigNrrr r!r$r$1r"r r$cFeZdZdZdZddiZeeedZ dfd Z xZ S)InstructBlipVideoConfiga [`InstructBlipVideoConfig`] is the configuration class to store the configuration of a [`InstructBlipVideoForConditionalGeneration`]. It is used to instantiate a Instructblipvideo model according to the specified arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the Instructblipvideo [Salesforce/instruct-blip-flan-t5](https://huggingface.co/Salesforce/instruct-blip-flan-t5) architecture. Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: vision_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`InstructBlipVideoVisionConfig`]. qformer_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`InstructBlipVideoQFormerConfig`]. text_config (`dict`, *optional*): Dictionary of configuration options used to initialize any [`PreTrainedConfig`]. num_query_tokens (`int`, *optional*, defaults to 32): The number of query tokens passed through the Transformer. video_token_index (`int`, *optional*): Token index of special video token. kwargs (*optional*): Dictionary of keyword arguments. Example: ```python >>> from transformers import ( ... InstructBlipVideoVisionConfig, ... InstructBlipVideoQFormerConfig, ... OPTConfig, ... InstructBlipVideoConfig, ... InstructBlipVideoForConditionalGeneration, ... ) >>> # Initializing a InstructBlipVideoConfig with Salesforce/instruct-blip-flan-t5 style configuration >>> configuration = InstructBlipVideoConfig() >>> # Initializing a InstructBlipVideoForConditionalGeneration (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration >>> model = InstructBlipVideoForConditionalGeneration(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a InstructBlipVideoConfig from a InstructBlipVideoVisionConfig, InstructBlipVideoQFormerConfig and any PreTrainedConfig >>> # Initializing Instructblipvideo vision, Instructblipvideo Q-Former and language model configurations >>> vision_config = InstructBlipVideoVisionConfig() >>> qformer_config = InstructBlipVideoQFormerConfig() >>> text_config = OPTConfig() >>> config = InstructBlipVideoConfig(vision_config=vision_config, qformer_config=qformer_config, text_config=text_config) ```instructblipvideovideo_token_idvideo_token_index) text_configqformer_config vision_configc |#td}tjdn0t|tr |j dd}t|di|}| t }tjdnt|tr t di|}| t}tjdnt|tr tdi|}||_||_ ||_ ||_ ||_ |jj|j_|jjt v|_d|_d|_t)|Tdi|y) NoptzTtext_config is None. Initializing the text config with default values (`OPTConfig`). model_typez\qformer_config is None. Initializing the InstructBlipVideoQFormerConfig with default values.z``vision_config` is `None`. initializing the `InstructBlipVideoVisionConfig` with default values.g?g{Gz?r)rloggerinfo isinstancedictgetr$rr*r,r+num_query_tokensr) hidden_sizeencoder_hidden_sizer/ruse_decoder_only_language_modelinitializer_factorinitializer_rangesuper__init__) selfr,r+r*r5r)kwargstext_model_type __class__s r!r<z InstructBlipVideoConfig.__init__ws5  (/1K KKn o  T *)oolEBO(9HKHK  !;=N KKv w  -;MnMN  9;M KKr  t ,9JMJM&*, 0!2262D2D2P2P//3/?/?/J/JNo/o,"%!% "6"r )NNN N) rrr__doc__r/ attribute_maprr$r sub_configsr< __classcell__)r@s@r!r&r&5sJ5 n%J-M"86K (#(#r r&ceZdZdZy) InstructBlipVideoPreTrainedModel)videotextNrrrinput_modalitiesrr r!rGrGs(r rGceZdZdZy)InstructBlipVideoVisionModelrHNrJrr r!rMrMsr rMc eZdZy)InstructBlipVideoQFormerModelNrrr r!rOrOr"r rOc eZdZy)4InstructBlipVideoForConditionalGenerationModelOutputNrrr r!rQrQr"r rQc PeZdZ ddejdejdej dzdejdzdej dzdej dzdej dzd ej dzd edzd edzd edzd ededzdee de e zfdZ y)InstructBlipVideoModelN pixel_valuesqformer_input_idsqformer_attention_mask input_idsattention_maskdecoder_input_idsdecoder_attention_mask inputs_embedsoutput_attentionsoutput_hidden_states return_dictinterpolate_pos_encoding use_cacher>returnc | | n|jj} |j\}}}}}|j||z|||}|j || | | | }|d}t j |jddt j|j}|jj|jddd}t j |jddt j|j}|t j|}|j|d}|j|d}t j||gd}|j|||||| | | }|dddd|jdddf}|j!|}|j||jj"|zd}|Q|j$j'|}||jj(k(}|t j|}nl||j't j*|jj(t j|jk(}|j-d}|j/dj1|j3|j}|j3|j|j4}|j7||}|jj8r|j$d ||| | | | d|}n|j$d ||||| | | | d |}t;||| S) N)rTr\r]r^r_rdtypedevicedim)rWrX query_embedsencoder_hidden_statesencoder_attention_maskr\r]r^r[rXr\r]r^r`)r[rXrYrZr\r]r^r`)vision_outputsqformer_outputslanguage_model_outputsr)configuse_return_dictshapereshape vision_modeltorchonessizelongrf query_tokensexpand ones_likerepeat_interleavecatqformerlanguage_projectionr5language_modelget_input_embeddingsr(tensorall unsqueeze expand_astoremasked_scatterr8rQ)r=rTrUrVrWrXrYrZr[r\r]r^r_r`r> batch_sizeframeschannelheightwidthrn image_embedsimage_attention_maskrzquery_attention_mask query_outputs query_outputlanguage_model_inputsspecial_image_maskoutputss r!forwardzInstructBlipVideoModel.forwards"&1%->q-A+A1(DE !% 8 8 F!6 = =j$++JfJfioJoqs t  FD//DDFyQM!*dkk.H.H!H %!&!;!.2M$2K2K2M T[[77uzzR_RfRfg3" "4!7!7!; /99"=GG VYYZgZnZno 5 8 89M9M}ObOb c%445GI^_ ;; 6 6)d))+-"3%9'# G*d)) +-"3'="3%9'#  GD))#*  r ) NNNNNNNNNFN) rrrrv FloatTensor LongTensorTensorboolrrtuplerQrrr r!rSrSs6 ;?.22659:>-1)-,0#').!%i ''i !,,i !& 0 04 7 i $$t+ i ((4/ i !++d2i !& 0 04 7i ||d*i  $;i #Tki D[i #'i $;i -.i E E!i r rSc"DeZdZee ddej dejdejdzdedzde e de e zf dZ d Zd ejd ej fd Z ddej dej dejdzd ej dzd ejdzdejdzdejdzd ej dzdedzdedzdejdzdedzdededzde e de ezf dZej$ ddej dejdzdejdzd ejdzd ejdzd ej dzdedejfdZy))InstructBlipVideoForConditionalGenerationNrTrUrVr_r>rac |j\}}}} } |j||z|| | }|jd ||dd|} t| j| j | j | j| d} | d} tj| jddtj| j} |jj| jddd}tj|jddtj| j}|tj|}|j!|d}|j!|d}tj"||gd }|j$d |||| | dd |}|| _|dddd|jd ddf}|j)|}|j||j*j,|zd}|| _| S) a pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. T)rTr_r^N)last_hidden_state pooler_output hidden_states attentionsrnrorrcrdrgri)rWrXrjrkrlr^r)rsrtrurrrrrrvrwrxryrfrzr{r|r}r~rrorrqr5)r=rTrUrVr_r>rrrrrrnrrrzrrorvideo_featuress r!get_video_featuresz>(66(66%00)  &a(  %zz,*;*;*=cr*B%**]i]p]pq((// 0B0B10Er2N $zz,*;*;*=cr*B%**]i]p]pq ! )%*__5F%G "-??A?N!7!I!I&VW!I!X!&,@BX+Y_`!a&$,, '1%".#7   *9&&q)!-C|/@/@/C-CQ*FG 11,?(// DKKInstructBlipVideoForConditionalGeneration.get_placeholder_maskls  !.2M$2K2K2M T[[77uzzR_RfRfg3" "4!7!7!; !*dkk.H.H!H /99"=GG VYYZgZnZno!!r rXrYrZr\r]labelsr^r`c <| | n|jj} |j|f||| dd|}|j}|j}|j }||j |}|tj|}|j|j|j}|j||}|j||}|jjre|jd ||| | | |d|}| r |j n|d}d}| w|j"d || |jj$j&d|}nB|jd ||||| | | | |d |}| r |j(n|d}| r |j n|d }t+||||| S) a qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. Examples: ```python >>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration >>> import torch >>> from huggingface_hub import hf_hub_download >>> import av >>> import numpy as np >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto") >>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b") >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample uniformly 4 frames from the videWhy is this video funny?o >>> total_frames = container.streams.video[0].frames >>> indices = np.arange(0, total_frames, total_frames / 4).astype(int) >>> clip = read_video_pyav(container, indices) >>> prompt = "What is happening in the video?" >>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device) >>> outputs = model.generate( ... **inputs, ... do_sample=False, ... num_beams=5, ... max_length=256, ... repetition_penalty=1.5, ... length_penalty=1.0, ... ) >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip() >>> print(generated_text) "A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front" ```NTrUrVr_r^r[rmr)logitsr vocab_size) r[rXrYrZr\r]r^rr`ri)lossrrnrorpr)rqrrrrrornrrvr|rrfrerrr8rr loss_functionr*rrrQ)r=rTrUrVrWrXrYrZr[r\r]rr^r_r`r>rrrornrrrrs r!rz1InstructBlipVideoForConditionalGeneration.forward{sb&1%.> "uO`O` "&;?.22659:>26)-,0*.#').!%P ''P !,,P !& 0 04 7 P $$t+ P ((4/ P !++d2P !& 0 04 7P ((4/P  $;P #TkP   4'P D[P #'P $;P +,!P " E E#P dU]]_6::>-12626).D''D!++d2D!& 0 04 7 D ##d* D ((4/ D((4/D#'D   DDr r)r&r$rrMrGrOrSr)+rv;transformers.models.instructblip.configuration_instructbliprr6transformers.models.instructblip.modeling_instructbliprrrrr r r r configuration_utilsrmodeling_flash_attention_utilsrmodeling_outputsrmodels.auto.modeling_autorprocessing_utilsrutilsrrrautorr get_loggerrr0rr$r&rGrMrOrQrSr__all__rr r!rs     4B:J&>>-   H % $<  %> j#.j#Z)'B)#: $<  ;j j .j Zs0Tsl r