
    qi3                         d dl ZddlmZ ddlmZ ddlmZmZm	Z	m
Z
 ddlmZmZ ddlmZmZ ddlmZ  ej&                  e      Z G d	 d
ed      Ze G d de	             ZdgZy)    N   )BatchFeature)
ImageInput)MultiModalDataProcessingKwargsProcessorMixinUnpack)PreTokenizedInput	TextInput)auto_docstringlogging)
VideoInputc                   "    e Zd ZddddddidZy)Glm4vProcessorKwargsFT)paddingreturn_token_type_idsreturn_mm_token_type_idsreturn_metadata)text_kwargsvideos_kwargsN)__name__
__module____qualname__	_defaults     \/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/glm4v/processing_glm4v.pyr   r   "   s#     %*(,

 ,T2Ir   r   F)totalc                        e Zd Zd fd	Ze	 	 	 ddedz  deez  ee   z  ee   z  de	dz  de
e   def
d       Zdd	Z	 dd
Ze fd       Zd Z xZS )Glm4vProcessorNc                    t        |d      sdn|j                  | _        t        |d      sdn|j                  | _        t        |dd       r|j                  n|j                  | j                        | _        t        |dd       r|j                  n|j                  | j                        | _        t        | !  ||||       |j                  d      | _	        |j                  d	      | _
        y )
Nimage_tokenz	<|image|>video_tokenz	<|video|>image_token_idvideo_token_id)chat_templatez<|begin_of_video|>z<|end_of_video|>)hasattrr"   r#   getattrr$   convert_tokens_to_idsr%   super__init__video_start_idvideo_end_id)selfimage_processor	tokenizervideo_processorr&   kwargs	__class__s         r   r+   zGlm4vProcessor.__init__/   s    .5i.O;U^UjUj.5i.O;U^UjUj y"2D9 $$001A1AB 	 y"2D9 $$001A1AB 	
 	)_Tab'==>RS%;;<NOr   imagestextvideosr2   returnc                 L	    | j                   t        fd| j                  j                  i|}| | j                  dd|i|d   }|d   }ni }d}|E | j
                  dd|i|d   }|j                  d      s|j                  d	      }	n|d	   }	|d
   }
ni }d}
t        |t              s|g}|j                         }|| j                  j                  dz  }d}t        t        |            D ]  }| j                  ||   v rS||   j                         |z  }||   j!                  | j                  d|z  d      ||<   |dz  }| j                  ||   v rS||   j!                  d| j                        ||<    |
| j
                  j                  dz  }d}t        t        |            D ]  }| j"                  ||   v r|
|   d   }d}	|   }|j$                  t&        j)                  d       |j$                  dn|j$                  |_        |j*                  ddd   }g }t        dt        |            D ]  }|j-                  ||           |d| }t        |      |k  r'|j-                  |r|d   nd       t        |      |k  r't        |      D ]  }||   }| j/                  |      }||z  } ||   j!                  | j"                  |d      ||<   |
|   j                         |z  |
|   d   z  }t        |      D ]:  }| j                  ||   v s||   j!                  | j                  d|z  d      ||<   < |dz  }| j"                  ||   v r||   j!                  d| j                        ||<    |d   j                  dd      }|d   j                  dd      } | j                  |fi |d   }| j1                  ||ddg       |rt3        j4                  |d         }t3        j6                  |d         }t3        j8                  || j:                  k(  d      }t3        j8                  || j<                  k(  d      } || kD  }!d||| j>                  k(  |!z  <   d||| j>                  k(  |! z  <   |jA                         |d<   tC        i ||||      S )a  
        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
            - **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`.
            - **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`.
            - **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
        tokenizer_init_kwargsNr4   images_kwargsimage_grid_thwr6   r   r   video_metadatavideo_grid_thw   r   z<|placeholder|>    a  SmolVLM requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. Probably `video_metadata` was missing from inputs and you passed pre-sampled frames. Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results.   r   return_tensorsr   Fimagevideo)
modalities	input_ids)axismm_token_type_ids)datatensor_typer   )"_merge_kwargsr   r0   init_kwargsr/   r1   getpop
isinstancelistcopy
merge_sizerangelenr"   prodreplacer#   fpsloggerwarning_once
timestampsappendreplace_frame_token_id_check_special_mm_tokensnparray
zeros_likecumsumr,   r-   r$   tolistr   )"r.   r4   r5   r6   r2   output_kwargsimage_inputsr;   videos_inputsr<   r=   merge_lengthindexinum_image_tokensvideo_index
num_framesvideo_structuremetadatar[   unique_timestampsidxselected_timestamps	frame_idxtimestamp_secframe_structurerC   r   text_inputs	array_idsrI   startsendsis_video_modalitys"                                     r   __call__zGlm4vProcessor.__call__@   sP   * +** 
"&.."<"<
 

 /4//`v`A_`L)*:;NL!N0D00aa-P_B`aM::/0!.!2!23C!D!./?!@*+;<NM!N$%6Dyy{%//::A=LE3t9% O&&$q'1'5e'<'A'A'C|'S$"1good.>.>@QTd@dfghDGQJE &&$q'1 q'//*;T=M=MNQO %//::A=LK3t9% &O&&$q'1!/!<Q!?J&(O-k:H||+++q
 *2)=28<<HL!)!4!4SqS!9J(*%$QJ8 B)00CAB +<KZ*H'12Z?+22Na3Fr3Jghi 12Z? &+:%6 ;	(;I(F*.*E*Em*T'?:;
 #1good.>.>QRSDG&{388:lJn]hNijkNll % &+:%6 q	++tAw6&*1good6F6FHY\lHlno&pDGq  1$KG &&$q'1J q'//*;T=M=MNQM&ON '}599:JDQ#0#?#C#CD^`e#f $dnnTJ]=-IJ%%dKWgDV%W#[!9:I "k+.F G
 YYyD,?,??aHF99Y$*;*;;!DD &XYyD,?,??CTTU[\yD,?,??EVDVWX/@/G/G/IK+,!QK!Q<!Q=!Q_mnnr   c                    i }|t         j                  j                  di       }|j                  |       |j                  dd      xs | j                  j
                  }|D cg c]   } | j                  j                  g || " }}|D 	cg c]
  }	|	|dz  z   }
}	|j                  |
|d       |vt         j                  j                  di       }|j                  |       |D cg c]   } | j                  j                  g || " }}|D 	cg c]
  }	|	dz  z   }}	||d<   t        di |S c c}w c c}	w c c}w c c}	w )	aK  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.
            video_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (num_frames, height, width) per each video.
        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr:   rS   r>   )rj   num_image_patchesr   num_video_tokensr   )
r   r   rN   updater/   rS   get_number_of_image_patchesr1   get_number_of_video_patchesr   )r.   image_sizesvideo_sizesr2   vision_datar:   rS   
image_sizer|   num_patchesrj   r   
video_sizenum_video_patchesr}   s                  r   _get_num_multimodal_tokensz)Glm4vProcessor._get_num_multimodal_tokens   s    "0::>>PRSM  (&**<>a$BVBVBaBaJ #.! A$$@@\*\m\! ! Sdd;
A!=dd4D[lmn"0::>>PRSM  ( #.! A$$@@\*\m\! ! Sdd;
A!=dd.>K*+,,,#!  e!  es   $%D7D<+%EEc                 B     | j                   j                  |f||d|S )a  
        Post-process the output of the model to decode the text.

        Args:
            generated_outputs (`torch.Tensor` or `np.ndarray`):
                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
                or `(sequence_length,)`.
            skip_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
                Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
            **kwargs:
                Additional arguments to be passed to the tokenizer's `batch_decode method`.

        Returns:
            `list[str]`: The decoded text.
        )skip_special_tokensclean_up_tokenization_spaces)r0   batch_decode)r.   generated_outputsr   r   r2   s        r   post_process_image_text_to_textz.Glm4vProcessor.post_process_image_text_to_text   s5    ( +t~~**
 3)E
 	
 	
r   c                 >    t         |   }|j                  d       |S )NrI   )r*   model_input_namesr\   )r.   r   r3   s     r   r   z Glm4vProcessor.model_input_names   s#    !G5  !45  r   c                 8    d| j                    dt        |       S )Nz<|begin_of_image|>z<|end_of_image|>)r"   int)r.   rs   s     r   r]   z%Glm4vProcessor.replace_frame_token_id  s#    #D$4$4#55Ec-FXEYZZr   )NNNN)NNN)NN)TF)r   r   r   r+   r   r   r   r
   rQ   r   r	   r   r   rz   r   r   propertyr   r]   __classcell__)r3   s   @r   r    r    -   s    P"  %)Z^$(	woT!wo ++d9o=EV@WWwo T!	wo
 -.wo 
wo wor$-N Y^
6 ! !
[r   r    )numpyr_   feature_extraction_utilsr   image_utilsr   processing_utilsr   r   r   r	   tokenization_utils_baser
   r   utilsr   r   video_utilsr   
get_loggerr   rY   r   r    __all__r   r   r   <module>r      sn   *  4 % X X C , % 
		H	%+5  T[^ T[ T[n 
r   