
    qi\I                        d Z ddlmZ ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZ  ej>                  e       Z!e ed       G d de                    Z"e ed       G d de                    Z# G d dejH                        Z%e G d de             Z& ed       G d de&             Z' ed        G d! d"e&e
             Z(g d#Z)y)$zPyTorch Llava model.    )	dataclassN)nn   )ACT2FN)Cache)GenerationMixin)BaseModelOutputWithPastBaseModelOutputWithPoolingModelOutput)PreTrainedModel)Unpack)TransformersKwargsauto_docstringloggingtorch_compilable_check)can_return_tuplemerge_with_config_defaults   )	AutoModel   )LlavaConfigzJ
    Base class for Llava outputs, with hidden states and attentions.
    custom_introc                   :    e Zd ZU dZdZej                  dz  ed<   y)LlavaModelOutputWithPasta  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nimage_hidden_states)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__     Z/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/llava/modeling_llava.pyr   r   $   s    	 59**T18r%   r   zQ
    Base class for Llava causal language model (or autoregressive) outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   y)	LlavaCausalLMOutputWithPasta4  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nlosslogitspast_key_valueshidden_states
attentionsr   )r   r   r   r    r)   r!   r"   r#   r*   r+   r   r,   tupler-   r   r$   r%   r&   r(   r(   9   s     &*D%

d
")'+FE$+$(OUT\(59M5**+d2926Je''(4/648**T18r%   r(   c                   *     e Zd Zdef fdZd Z xZS )LlavaMultiModalProjectorconfigc                    t         |           t        |j                  t              rdnt        |j                        }t        j                  |j                  j                  |z  |j                  j                  |j                        | _        t        |j                     | _        t        j                  |j                  j                  |j                  j                  |j                        | _        y )Nr   bias)super__init__
isinstancevision_feature_layerintlenr   Linearvision_confighidden_sizetext_configmultimodal_projector_biaslinear_1r   projector_hidden_actactlinear_2)selfr1   num_feature_layers	__class__s      r&   r6   z!LlavaMultiModalProjector.__init__X   s    ",V-H-H#"NQTWX^XsXsTt		  ,,/AA**11

 &556		**F,>,>,J,JQWQqQq
r%   c                 l    | j                  |      }| j                  |      }| j                  |      }|S N)r@   rB   rC   )rD   image_featuresr,   s      r&   forwardz LlavaMultiModalProjector.forwardf   s2    n5/m4r%   )r   r   r   r   r6   rJ   __classcell__rF   s   @r&   r0   r0   W   s    
{ 
r%   r0   c                   <    e Zd ZU eed<   dZdZdZdZdZ	dZ
dZdZdZy)LlavaPreTrainedModelr1   model)imagetextTr+   N)r   r   r   r   r#   base_model_prefixinput_modalitiessupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph_supports_flex_attn_supports_attention_backendr$   r%   r&   rN   rN   m   s=    (&*#"3N!"&r%   rN   zu
    The Llava model which consists of a vision backbone and a language model, without a language modeling head.
    c                   \    e Zd ZddiZdef fdZd Zd Zee	 e
d      	 	 	 dd
ej                  deee   z  d	z  ded	z  ded	z  dee   deez  fd                     Zdej,                  dej                  dej                  fdZee
	 	 	 	 	 	 	 	 	 	 ddej,                  d	z  d
ej                  d	z  dej0                  d	z  dej,                  d	z  ded	z  dej                  d	z  deee   z  d	z  ded	z  dej,                  d	z  dej0                  d	z  dee   deez  fd              Z xZS )
LlavaModel^language_model.modellanguage_modelr1   c                     t         |   |       t        j                  |j                        | _        t        |      | _        t        j                  |j                        | _	        | j                          y rH   )r5   r6   r   from_configr<   vision_towerr0   multi_modal_projectorr>   r^   	post_initrD   r1   rF   s     r&   r6   zLlavaModel.__init__   sY     %11&2F2FG%=f%E"'33F4F4FGr%   c                 6    | j                   j                         S rH   )r^   get_input_embeddingsrD   s    r&   rf   zLlavaModel.get_input_embeddings   s    ""7799r%   c                 :    | j                   j                  |       y rH   )r^   set_input_embeddingsrD   values     r&   ri   zLlavaModel.set_input_embeddings   s    007r%   zWObtains image last hidden states from the vision tower and apply multimodal projection.r   Npixel_valuesr8   vision_feature_select_strategyoutput_hidden_stateskwargsreturnc                    |j                         D ci c]  \  }}|	|| }}} | j                  |fddd|}t        |t              r |j                  |   }	|dk(  r\|	d d dd f   }	nP|D 
cg c]  }
|j                  |
    }}
|dk(  r|D cg c]  }|d d dd f    }}t        j                  |d      }	| j                  |	      }|j                  d      t        j                  |d   |j                        | j                  j                  z  j                  d      j                         }t        j                  |j                  d	      |      }nt!        |      }||_        |S c c}}w c c}
w c c}w )
NT)rn   return_dictdefaultr   dimimage_sizes)devicer   )itemsra   r7   r9   r,   r!   catrb   get	as_tensorrx   
patch_sizeprodtolistsplitsqueezelistpooler_output)rD   rl   r8   rm   rn   ro   kvimage_outputsselected_image_feature	layer_idxhs_poolhsrI   split_sizess                  r&   get_image_featureszLlavaModel.get_image_features   s    $*<<>C41aQ]!Q$CC)))
!%
 	
 *C0%2%@%@AU%V"-:)?12)F&Ocd)}229=dGd-:/672ae977%*YYwB%?"334JK ::m$0!6~?T?TUY]YjYjYuYuu" 
 #[[)?)?)BKPN!.1N&4#K D  e 8s   
E2E21E8E=	input_idsinputs_embedsrI   c                 N   |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                         }|j                  d   |j                  d   z  }|j                  d      j                  |      j                  |j                        }t        ||   j                         |j                         k(  d| d|        |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )dtyperx   rt   r   r   z6Image features and image tokens do not match, tokens: z, features: )rf   r!   tensorr1   image_token_idlongrx   allsumshape	unsqueeze	expand_astor   numel)rD   r   r   rI   special_image_maskn_image_tokensn_image_featuress          r&   get_placeholder_maskzLlavaModel.get_placeholder_mask   s    !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H+//1)//2^5I5I!5LL/99"=GGVYYZgZnZno,-3359M9M9OOD^DTT`aq`rs	
 "!r%   attention_maskposition_idsr+   cache_positionrw   c           	         |d u |d uz  rt        d      | | j                         |      }|| j                  ||||
d      j                  }t	        j
                  |d      j                  |j                  |j                        }| j                  |||      }|j                  ||      } | j                  d	|||||	d|}t        |j                  |j                  |j                  |j                   |      S d       S )
Nz:You must specify exactly one of input_ids or inputs_embedsT)rl   r8   rm   rw   rr   r   ru   )r   rI   )r   r   r+   r   r   )last_hidden_stater+   r,   r-   r   r$   )
ValueErrorrf   r   r   r!   rz   r   rx   r   r   masked_scatterr^   r   r   r+   r,   r-   )rD   r   rl   r   r   r+   r   r8   rm   r   rw   ro   rI   r   outputss                  r&   rJ   zLlavaModel.forward   sK     -t";<YZZ 7D557	BM#!44)%9/M'  5  m  #YY~1=@@AUAUWdWjWjkN!%!:!:~ "; " *889K^\M%$%% 
)%+')
 
 (%77#33!//))2>2J
 	

 QU
 	
r%   )NNN)
NNNNNNNNNN)r   r   r   _checkpoint_conversion_mappingr   r6   rf   ri   r   r   r   r!   r"   r9   r   strboolr   r   r.   r
   r   
LongTensorr   Tensorr   r   rJ   rK   rL   s   @r&   r\   r\   }   s    	!"2&"{ :8 n 8<59,0-''- "DIo4- ),d
	-
 #Tk- +,- 
+	+-   
-^"))":?:K:K"]b]n]n"0  .215.204(,267;5926+/1
##d*1
 ''$.1
 t+	1

 &&-1
 1
 ((4/1
 "DIo41
 ),d
1
 ((4/1
 \\D(1
 +,1
 
)	)1
  1
r%   r\   zS
    The LLAVA model which consists of a vision backbone and a language model.
    c                    f    e Zd ZdddddZddiZdef fd	Zd
 Zd Zde	j                  fdZe	 	 ddej                  deee   z  dz  dedz  dee   deez  f
d       Zee	 	 	 	 	 	 	 	 	 	 	 	 d dej0                  dz  dej                  dz  dej2                  dz  dej0                  dz  dedz  dej                  dz  deee   z  dz  dedz  dej0                  dz  dej0                  dz  deej2                  z  dej2                  dz  dee   deez  fd              Z	 	 	 	 	 	 	 d! fd	Z xZS )"LlavaForConditionalGenerationzmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)r]   z^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightz(model.language_model.embed_tokens.weightr1   c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y )NFr3   )r5   r6   r\   rO   r   r;   r>   r=   
vocab_sizer   rc   rd   s     r&   r6   z&LlavaForConditionalGeneration.__init__%  sS     '
yy!3!3!?!?ASASA^A^ejkr%   c                 6    | j                   j                         S rH   )rO   rf   rg   s    r&   rf   z2LlavaForConditionalGeneration.get_input_embeddings+  s    zz..00r%   c                 :    | j                   j                  |       y rH   )rO   ri   rj   s     r&   ri   z2LlavaForConditionalGeneration.set_input_embeddings.  s    

''.r%   rp   c                     | j                   S rH   )r   rg   s    r&   get_output_embeddingsz3LlavaForConditionalGeneration.get_output_embeddings1  s    ||r%   Nrl   r8   rm   ro   c                 B     | j                   j                  d|||d|S )N)rl   r8   rm   r$   )rO   r   )rD   rl   r8   rm   ro   s        r&   r   z0LlavaForConditionalGeneration.get_image_features4  s5     -tzz,, 
%!5+I
 	
 	
r%   r   r   r   r+   r   labelsr   logits_to_keeprw   c                     | j                   d|||||||||
|d
|}|d   }t        |t              rt        | d      n|}| j	                  |dd|ddf         }d}|	4 | j
                  d||	| j                  j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration

        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")

        >>> prompt = "USER: <image>\nWhat's the content of the image? ASSISTANT:"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "USER:  \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed"
        ```)
r   rl   r   r   r+   r   r8   rm   r   rw   r   N)r*   r   r   )r)   r*   r+   r,   r-   r   r$   )rO   r7   r9   slicer   loss_functionr1   r>   r   r(   r+   r,   r-   r   )rD   r   rl   r   r   r+   r   r8   rm   r   r   r   rw   ro   r   r,   slice_indicesr*   r)   s                      r&   rJ   z%LlavaForConditionalGeneration.forwardC  s    ^ $** 
%)%+'!5+I)#
 
  
8B>SV8W~ot4]kmA}a,?@A%4%% f9P9P9[9[_eD +#33!//)) ' ; ;
 	
r%   c	           
      h    t        |   |f||||||d|	}
|s|	j                  dd      s||
d<   |
S )N)r+   r   r   r   r   is_first_iteration	use_cacheTrl   )r5   prepare_inputs_for_generationr{   )rD   r   r+   r   rl   r   r   r   r   ro   model_inputsrF   s              r&   r   z;LlavaForConditionalGeneration.prepare_inputs_for_generation  s\     w<	
+')))1	
 	
 VZZT%B
 ,8L(r%   )NN)NNNNNNNNNNr   N)NNNNNNF)r   r   r   r   _tied_weights_keysr   r6   rf   ri   r   Moduler   r   r!   r"   r9   r   r   r   r   r.   r
   r   r   r   r   r   r(   rJ   r   rK   rL   s   @r&   r   r     s8    #9.#@$-	&" +,VW{ 1/ryy   8<59	
''
 "DIo4
 ),d
	

 +,
 
+	+
 
  .215.204(,267;59*.26-.+/M
##d*M
 ''$.M
 t+	M

 &&-M
 M
 ((4/M
 "DIo4M
 ),d
M
   4'M
 ((4/M
 ell*M
 \\D(M
 +,M
 
,	,M
  M
d     r%   r   )r   rN   r\   )*r    dataclassesr   r!   r   activationsr   cache_utilsr   
generationr   modeling_outputsr	   r
   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   utils.genericr   r   autor   configuration_llavar   
get_loggerr   loggerr   r(   r   r0   rN   r\   r   __all__r$   r%   r&   <module>r      s/    !   !   ) ` ` - & X X I  , 
		H	% 
96 9 9 
9+ 9 90ryy , '? ' ' 
R
% R

R
j 
X$8/ X
Xv Rr%   