
    qi1                         d dl ZddlmZ ddlmZmZmZ ddlm	Z	m
Z
mZmZ ddlmZmZ ddlmZ ddlmZ  G d	 d
e
d      Ze G d de             ZdgZy)    N   )BatchFeature)
ImageInputconcatenate_listmake_flat_list_of_images)MultiModalDataProcessingKwargsProcessorMixinUnpack)PreTokenizedInput	TextInput)auto_docstring)
VideoInputc                   &    e Zd ZdddddiddidZy	)
InternVLProcessorKwargsleftF)padding_sidereturn_mm_token_type_idscrop_to_patchesTreturn_tensorspt)text_kwargsimages_kwargsvideos_kwargsN)__name__
__module____qualname__	_defaults     b/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/internvl/processing_internvl.pyr   r      s.     #(-

 t
 d
Ir    r   F)totalc                       e Zd Z	 	 	 	 	 ddef fdZdee   dee   dee   dej                  dej                  d	ej                  fd
Z	e
	 	 	 ddedz  deez  ee   z  ee   z  dz  dedz  dee   def
d       ZddZed        Z xZS )InternVLProcessorNimage_seq_lengthc                 z   || _         |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _	        | j                  | j                  | j                  g| _
        t        | 0  |||fd|i| y)a  
        image_seq_length (`int`, *optional*, defaults to 256):
            The number of image token to use per image patch. it should be set so that:
            image_seq_length = (config.image_size // config.patch_size) ** 2 * (config.scale_factor**2)
        chat_templateN)r%   start_image_tokenend_image_tokenstart_image_token_idend_image_token_idcontext_image_tokenimage_tokenvideo_tokencontext_image_token_idimage_token_id	image_idssuper__init__)selfimage_processor	tokenizervideo_processorr%   r'   kwargs	__class__s          r!   r3   zInternVLProcessor.__init__+   s     !1!*!<!<(88$-$B$B!"+">">$88$00'>>--t/H/H$JaJab)_lTaleklr    textimage_num_patchesvideo_num_patchesimage_num_patches_indicesvideo_num_patches_indicesvideo_patch_indicesc	           	      0    d}	d}
g }g }g }|D ]  }|} j                   |v s j                  |v r j                   |v r؉ j                  |vs7|j                   j                         |j                   j                        k  r|	dkD  r||	dz
     nd}||	   }|j                  |||        |j	                   j                   dd      }|j                   j
                    j                    j                  z  ||	   z    j                          |	dz  }	n||
   }||
dz      }||   }||   }|j                  |||        t        |||       dj                   fdt        t                    D              }|j                  |       |j	                   j                  dd      }|
dz  }
 j                   |v r j                  |v rd|v r)|j                  d      }|j	                  d|d      }d|v r)|j                  |        |||	|
fS )z
        Processes interleaved text with <image> and <video> placeholders, replacing them with appropriate
        image and video tokens while keeping track of the patches used.
        r      z<placeholder>
c              3      K   | ]D  }d |dz    dj                    j                  j                  z  |   z   j                    F yw)FramerA   z: N)r(   r-   r%   r)   ).0inum_patchesr4   s     r!   	<genexpr>z?InternVLProcessor._insert_media_placeholders.<locals>.<genexpr>w   ss      -  Awb)?)?(@AQAQTXTiTiAilwxylzAz@{  }A  }Q  }Q  |R  S-s   A
A)r-   r.   indexappendreplacer(   r%   r)   listjoinrangelenpop)r4   r:   image_pixel_valuesvideo_pixel_valuesr;   r<   r=   r>   r?   image_indexvideo_indexprocessed_textimage_video_patchesreplace_stringsprompt
new_promptstart_index	end_indexcurrent_patch_indexend_patch_indexvideo_promptreplace_strrG   s   `                     @r!   _insert_media_placeholdersz,InternVLProcessor._insert_media_placeholdersE   s       &	.FJ""j0D4D4D
4R##z1$$J6!''(8(89J<L<LTM]M]<^^ Q\^_P_";K!O"LefK 9+ FI'../A+i/XY!+!3!3D4D4DoWX!YJ#**11243C3CdF[F[3[^op{^|3|2}  C  S  S  ~T  U  1$K
 +>k*J'&9+/&JO";<O"PK 9/ JI'../A+i/XY"&'89L_']"^K#'99 -!&s;'7!8- $L $**<8!+!3!3D4D4DoWX!YJ1$KA ""j0D4D4D
4RB "Z/-11!4'//aP
 "Z/ !!*-M&	.P 2KLLr    imagesvideosr8   returnc           
      `   |t        d       | j                  t        fd| j                  j                  i|}t        |t        t        f      s|g}g }d}t        j                  dg      }|t| j                  j                  |      }t        |      } | j                  dd|i|d   }	|	j                  d      }|	j                  d      }t        j                  |      }g }
d}t        j                  dg      }t        j                  dg      }||d	   } | j                  dd
|i|}|j                  d      }|j                   ^}}}t        j"                  ||      }t%        |      }t        j&                  |dz   t(              }d|d<   t        j                  |      |dd dg|z  }
t        j&                  |dz   t(              }d|d<   t        j                  |
      |dd |j+                  dd      }i }||`| j-                  |||||
|||      \  }}}}||t/        |      k7  rt        d      ||t/              k7  rt        d      dt1        |      i}|d   j                  dd      }|d   j                  dd      } | j                  |fi |d   }| j3                  ||dg       |rft        j                  |d         }t        j4                  |d         }d|t        j6                  || j8                        <   |j;                         |d<   t=        i |||      S )aA  
        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
        NzYou have to specify text.tokenizer_init_kwargsr   ra   r   rG   pixel_valuesr   rb   pixel_values_videosrA   zONumber of image placeholders in the prompt does not match the number of images.zONumber of video placeholders in the prompt does not match the number of videos.r   r   r   image)
modalities	input_idsmm_token_type_ids)datatensor_typer   )
ValueError_merge_kwargsr   r6   init_kwargs
isinstancerL   tuplenparrayr5   fetch_imagesr   rP   cumsumr7   shapefullsumemptyintflattenr`   rO   r   _check_special_mm_tokens
zeros_likeisinr1   tolistr   )r4   ra   r:   rb   r8   output_kwargsr;   rQ   r=   image_inputsr<   rR   r?   r>   video_kwargsvideo_inputs
batch_size
num_frames_num_frames_per_videoimage_videos_inputsrV   rS   rT   r   r   text_inputs	array_idsrk   s                                r!   __call__zInternVLProcessor.__call__   s   $ <899***#
"&.."<"<
 
 $u.6D !$&HHaSM!))66v>F-f5F/4//`v`A_`L , 0 0 ?!-!1!1.!A(*		2C(D%! hhsm$&HHaSM!(9L/4//NvNNL!-!1!12G!H);)A)A&J
Q#%77:z#B 12J"$((:>3"?%&"&(ii0D&E#!"j 0(*a(E%+,%a(,.II6G,H%ab)!3!;!;Aq!A !3BFBaBa""!!))#	C?D%{K !kS[&@ !rss!kS9M5N&N !rss $23CDW3X"Y&}599:JDQ#0#?#C#CD^`d#e $dnnTJ]=-IJ%%dKWI%N#[!9:I "k+.F GDEbggi@A/@/G/G/IK+,!GK!G3F!GUcddr    c                 R   i }|t         j                  j                  di       }|j                  |       |D cg c]   } | j                  j
                  g || " }}|D cg c]  }d| j                  |z  z    }}|j                  ||d       t        di |S c c}w c c}w )a  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        r      )num_image_tokensnum_image_patchesr   )r   r   getupdater5   get_number_of_image_patchesr%   r   )	r4   image_sizesr8   vision_datar   
image_sizer   rG   r   s	            r!   _get_num_multimodal_tokensz,InternVLProcessor._get_num_multimodal_tokens   s     "3==AA/SUVM  ( #.! A$$@@\*\m\! !
 ^ookT%:%:[%H Ioo4D[lmn,,,!
  ps   %B%B$c                 d    | j                   j                  }| j                  j                  }||z   S N)r6   model_input_namesr5   )r4   tokenizer_input_namesimage_processor_input_namess      r!   r   z#InternVLProcessor.model_input_names  s2     !% @ @&*&:&:&L&L#$'BBBr    )NNN   N)NNNr   )r   r   r   r{   r3   rL   strrs   ndarrayr`   r   r   r   r   r   r   r   r   r   r   propertyr   __classcell__)r9   s   @r!   r$   r$   )   s-     #m
 m4>M3i>M
  9>M  9>M $&::>M $&::>M  ZZ>M@  %)ae$(	]eT!]e ++d9o=EV@WWZ^^]e T!	]e
 01]e 
]e ]e~-8 C Cr    r$   )numpyrs   image_processing_utilsr   image_utilsr   r   r   processing_utilsr   r	   r
   r   tokenization_utils_baser   r   utilsr   video_utilsr   r   r$   __all__r   r    r!   <module>r      s]      2 Q Q X X C # %.e  \C \C \C~ 
r    