
    qi.                        d Z ddlmZ ddlmZ ddlmZ ddlZddlm	c m
Z ddlm	Z	 ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2  e&jf                  e4      Z5e e$d       G d de                    Z6e e$d       G d de                    Z7	 	 	 	 dBdZ8dCd Z9 G d! d"e	jt                        Z; G d# d$e	jx                        Z= G d% d&e	j|                        Z? G d' d(ej                  j|                        Z@d) ZAdDd*ZB G d+ d,e	j|                        ZC	 dEd-e	j|                  d.ej                  d/ej                  d0ej                  d1ej                  dz  d2eEd3eEfd4ZF G d5 d6e	j|                        ZG G d7 d8e      ZH G d9 d:e      ZIe$ G d; d<e             ZJe$ G d= d>eJ             ZK G d? d@eJe      ZLg dAZMy)FzPyTorch Idefics model.    )Callable)	dataclass)AnyN)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)create_causal_mask)GradientCheckpointingLayer)ModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedConfigPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)OutputRecordercapture_outputs   )IdeficsConfig)IdeficsPerceiverResampler)IdeficsVisionEmbeddingsIdeficsVisionTransformerz{
    Base class for Idefics model's outputs that may also contain a past key/values (to speed up sequential decoding).
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZe	dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)IdeficsBaseModelOutputWithPasta  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
        `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
        input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlast_hidden_statepast_key_valueshidden_states
attentionsimage_hidden_states)__name__
__module____qualname____doc__r#   torchFloatTensor__annotations__r$   r
   r%   tupler&   r'        ^/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/idefics/modeling_idefics.pyr"   r"   1   s|    & 37u((4/6$(OUT\(59M5**+d2926Je''(4/6;?u001D8?r1   r"   zS
    Base class for Idefics causal language model (or autoregressive) outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)	IdeficsCausalLMOutputWithPastae  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlosslogitsr$   r%   r&   r'   )r(   r)   r*   r+   r5   r,   r-   r.   r6   r$   r
   r%   r/   r&   r'   r0   r1   r2   r4   r4   R   s    " &*D%

d
")'+FE$+$(OUT\(59M5**+d2926Je''(4/6;?u001D8?r1   r4   c                    t        j                  | j                  d         j                  dd      j	                  d|      j                  d      j                  | j                        }| j                  d|      } |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   d|v r|d   }|j                  d|      |d<   ||j                  d|      |d	<   |d   |d   j                  d|      |d<   |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS )
Nr   r   pixel_valuesimage_encoder_embeddingsperceiver_embeddingsimage_attention_masktoken_type_idsattention_mask)	r,   arangeshapeviewrepeattodeviceindex_selectget)	input_idsexpand_sizeis_encoder_decoderr>   encoder_outputsmodel_kwargsexpanded_return_idxr=   s           r2   expand_inputs_for_generationrM   r   s    	Y__Q'(--b!4;;A{KPPQSTWWXaXhXhi  &&q*=>I#/#3#3N#CL /;/?/?@Z/[L+,+7+;+;<R+SL'(+7+;+;<R+SL'(<'%&67)7)D)DQH[)\%&!)7)D)DQH[)\%&*+7/;<R/S/`/`"0
+, N#/'3N'C'P'PQRTg'h^$ l"" 
0	1	=3?@Z3[3h3h"4
/0 l"" 
,	-	9/;<R/S/`/`"0
+, l""r1   c                 2   t         j                  t         j                  t         j                  d}|D cg c]  }||   	 }}| j	                         D ];  |r&t        fd|D              rj                  d       +j                  d       = | S c c}w )N)	LayerNormLinear	Embeddingc              3   6   K   | ]  }t        |        y wN)
isinstance).0tmodules     r2   	<genexpr>zfreeze_model.<locals>.<genexpr>   s     $]qZ%:$]s   TF)r   rO   rP   rQ   modulesanyrequires_grad_)modelmodule_exceptionsmappingmmodule_exceptions_mappedrW   s        @r2   freeze_modelra      s    \\))\\G
 5FFq
FF--/ )$]D\$]!]!!$'!!%(	)
 L  Gs   Bc                   N     e Zd ZdZ	 	 	 	 ddedz  ddf fdZd ZdefdZ xZ	S )	IdeficsDecoupledEmbeddinga  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the embeddings. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `num_additional_embeddings` > 0,
    then it will create `num_additional_embeddings` additional parameters that are always trained. If
    `num_additional_embeddings=0`, then the module defaults back to the regular behavior of `nn.Embedding`.
    Npartially_freezereturnc           	      B   |||kD  rt        d| d|       t        	|   d|||||d| || _        || _        || _        || _        |r| j                  j                  d       | j
                  dkD  r)t        j                  | j
                  |||      | _        yy)	a)  
        Args:
            num_embeddings (`int`):
                Size of the dictionary of embeddings
            num_additional_embeddings (`int`):
                Number of additional embeddings. Only useful when you `partially_freeze=True`.
            embedding_dim (`int`):
                The size of each embedding vector
            partially_freeze: (`bool`, *optional*, defaults to `False`):
                If `True`, the regular `weight` will be frozen. `additional_weight` is never frozen.
            padding_idx (`int`, *optional*):
                The padding index (needs to be less than num_embeddings)

        Note: there are a lot of other parameters to initialize a standard `nn.Embedding` such as `padding_idx`,
        `max_norm` or `norm_type`. We are not supporting these.
        Nz/padding_idx must be within num_embeddings. Got z and )num_embeddingsembedding_dimrD   dtypepadding_idxFr   )rg   rh   rD   ri   r0   )
ValueErrorsuper__init__rg   rj   num_additional_embeddingsrd   weightr[   r   rQ   additional_embedding)
selfrg   rn   rh   rd   rD   ri   rj   kwargs	__class__s
            r2   rm   z"IdeficsDecoupledEmbedding.__init__   s    6 "{^'CN{m[`ao`pqrr 	
)'#	
 	
 -&)B& 0KK&&u-))A-(*#==+	)D% .r1   c                 b   | j                   dk(  r t        j                  || j                        S |j	                         }t        j                  || j                  k\        }||   }| j                  || j                  z
        }d||<   t        j                  || j                        }|||<   |S )a  
        we have 2 embeddings, with different indices - one pretrained self.weight and another
        self.additional_embedding.weight that is being trained.

        in order to make a lookup of the input ids, we:
        1. find out the indices of the entries belonging to the 2nd embedding
        2. extract those values while subtracting the size of the first embedding (num_embeddings), since the 2nd
           embedding starts from 0 and not num_embeddings
        3. perform the 2nd embedding lookup
        4. now we handle the 1st embedding, we overwrite indices belonging to the 2nd embedding with a padding index
        5. perform the 1st embedding lookup
        6. now we overwrite the values in the 1st embedding lookup with the values of the 2nd embedding lookup

        note: for the 1st embedding lookup we could have looked up only the low indices and not do the padding, but
        then we have to create a new tensor and populate it with 2 tensors that are spread out across various indices -
        i.e. not a simple concat - I haven't benchmarked the complex case if it's any faster, given that seqlens are
        usually relatively short it's probably not faster or if faster not by much - but might be a good idea to
        measure.

        r   )	rn   F	embeddingro   cloner,   whererg   rp   )rq   rG   additional_vocab_indicesinput_ids_additional_vocabadditional_embeddingsfull_vectors         r2   forwardz!IdeficsDecoupledEmbedding.forward   s    * ))Q.;;y$++66 OO%	#(;;yD<O<O/O#P %./G%H" $ 9 9:TW[WjWj:j k /0	*+kk)T[[9 1F,-r1   c                 n    d| j                    d| j                   d| j                   d| j                   S )Nznum_embeddings=z, num_additional_embeddings=z, embedding_dim=, partially_freeze=)rg   rn   rh   rd   rq   s    r2   
extra_reprz$IdeficsDecoupledEmbedding.extra_repr  sq     !4!4 55QRVRpRpQq  rB  CG  CU  CU  BV  Vi  jn  j  j  i@  A  	Ar1   )FNNN)
r(   r)   r*   r+   boolrm   r}   strr   __classcell__rs   s   @r2   rc   rc      sH     ).3
 +3 
3j%NAC Ar1   rc   c                        e Zd ZdZ	 	 	 	 	 ddedededededdf fd	Zd
ej                  dej                  fdZ	de
fdZ xZS )IdeficsDecoupledLineara  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0,
    then it will create `out_additional_features * in_features` additional parameters that are always trained. If
    `out_additional_features=0`, then the module defaults back to the regular behavior of `nn.Linear`.
    Nin_featuresout_featuresout_additional_featuresbiasrd   re   c                 "   t         |   |||||       || _        || _        || _        || _        |r8| j                  j                  d       |r| j                  j                  d       |dkD  r t        j                  |||||      | _        yy)aG  
        out_additional_features: int. Number of additional trainable dimensions. Only makes sense when
        `partially_freeze=True`. partially_freeze: bool. If True, the regular `weight` will be frozen and extra
        parameters (if any) will be trainable. If False, default to the regular behavior of nn.Linear.
        Fr   )r   r   r   rD   ri   N)rl   rm   r   rd   r   r   ro   r[   r   r   rP   additional_fc)	rq   r   r   r   r   rd   rD   ri   rs   s	           r2   rm   zIdeficsDecoupledLinear.__init__   s     	lD&%H'>$ 0&(KK&&u-		((/"Q&!#'4"D 'r1   inputc                     t        j                  || j                  | j                        }| j                  dkD  r)| j                  |      }t        j                  ||fd      }|S )Nr   r8   )ru   linearro   r   r   r   r,   cat)rq   r   outputadditional_featuress       r2   r}   zIdeficsDecoupledLinear.forwardD  sV    %dii8''!+"&"4"4U";YY(;<bAFr1   c           
          d| j                    d| j                   d| j                   d| j                  du d| j                   
S )z=Overwriting `nn.Linear.extra_repr` to include new parameters.zin_features=z, out_features=z, out_additional_features=z, bias=Nr   r   r   r   r   rd   r   s    r2   r   z!IdeficsDecoupledLinear.extra_reprM  s    d../t?P?P>QQklp  mI  mI  lJ  JQ  RV  R[  R[  cg  Rg  Qh  h{  |@  |Q  |Q  {R  S  	Sr1   )r   TTNN)r(   r)   r*   r+   intr   rm   r,   Tensorr}   r   r   r   r   s   @r2   r   r     s     ()!%"" " "%	"
 " " 
"HU\\ ell SC Sr1   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )IdeficsRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z=
        IdeficsRMSNorm is equivalent to T5LayerNorm
        N)rl   rm   r   	Parameterr,   onesro   variance_epsilon)rq   hidden_sizeepsrs   s      r2   rm   zIdeficsRMSNorm.__init__T  s1     	ll5::k#:; #r1   c                    |j                  t        j                        j                  d      j	                  dd      }|t        j
                  || j                  z         z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                        }| j                  |z  S )N   r8   T)keepdim)rC   r,   float32powmeanrsqrtr   ro   ri   float16bfloat16)rq   r%   variances      r2   r}   zIdeficsRMSNorm.forward\  s     ##EMM266q9>>r4>P%Ht?T?T4T(UU ;; ??),,T[[->->?M{{]**r1   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r/   ro   r@   r   r   s    r2   r   zIdeficsRMSNorm.extra_reprf  s*    ))*+6$2G2G1HIIr1   )gư>)r(   r)   r*   rm   r}   r   r   r   s   @r2   r   r   S  s    $+Jr1   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )IdeficsEmbeddingc                    t         |           || _        || _        || _        d| j                  t        j                  d| j                  dt
        j                        j                  |t
        j                        | j                  z  z  z  }| j                  d|d       | j                  || j                  j                  t        j                         	       y )
N      ?r   r   ri   rD   ri   inv_freqF
persistentseq_lenrD   ri   )rl   rm   dimmax_position_embeddingsbaser,   r?   int64rC   floatregister_buffer_set_cos_sin_cacher   rD   get_default_dtype)rq   r   r   r   rD   r   rs   s         r2   rm   zIdeficsEmbedding.__init__l  s    '>$	IIQ!5;;?BB&X]XcXcBdgkgogooq
 	ZeD 	+DMM4H4HPUPgPgPi 	  	
r1   c                    || _         t        j                  | j                   |t        j                        j	                  | j
                        }t        j                  d|| j
                        }t        j                  ||fd      }| j                  d|j                         j                  |      d       | j                  d|j                         j                  |      d       y )	Nr   i,j->ijr8   r   
cos_cachedFr   
sin_cached)max_seq_len_cachedr,   r?   r   type_asr   einsumr   r   cosrC   sin)rq   r   rD   ri   rV   freqsembs          r2   r   z#IdeficsEmbedding._set_cos_sin_cache}  s    ")LL00u{{S[[\`\i\ijY4==9iiB/\3779<<+>5Q\3779<<+>5Qr1   c                    || j                   kD  r(| j                  ||j                  |j                         | j                  d | j                  |j                        | j                  d | j                  |j                        fS )Nr   r   )r   r   rD   ri   r   rC   r   )rq   xr   s      r2   r}   zIdeficsEmbedding.forward  sy    T,,,##GAHHAGG#T OOHW%((qww(7OOHW%((qww(7
 	
r1   )i   i'  NrS   )r(   r)   r*   rm   r   r}   r   r   s   @r2   r   r   k  s    
"R
r1   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr8   r   r   )r@   r,   r   )r   x1x2s      r2   rotate_halfr     sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r1   c                     ||   j                  |      }||   j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )an  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   position_idsunsqueeze_dimq_embedk_embeds           r2   apply_rotary_pos_embr     sg    * l

%
%m
4C
l

%
%m
4C3w;q>C/0G3w;q>C/0GGr1   c                   2     e Zd Zdededef fdZd Z xZS )
IdeficsMLPr   intermediate_size
hidden_actc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        t        |   | _        y )NFr   )	rl   rm   r   rP   	gate_proj	down_projup_projr	   act_fn)rq   r   r   r   rs   s       r2   rm   zIdeficsMLP.__init__  s[     	;0AN#4kNyy.?eLZ(r1   c                     | j                  | j                  | j                  |            | j                  |      z        S rS   )r   r   r   r   )rq   r   s     r2   r}   zIdeficsMLP.forward  s0    ~~dkk$..*;<t||ANOOr1   )r(   r)   r*   r   r   rm   r}   r   r   s   @r2   r   r     s*    
)
) 
) 	
)Pr1   r   rW   querykeyvaluer>   scalingdropoutc                    t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr8   )r   ri   ptrainingr   r   )r,   matmul	transposer   
functionalsoftmaxr   rC   ri   r   r   
contiguous)
rW   r   r   r   r>   r   r   rr   attn_weightsattn_outputs
             r2   eager_attention_forwardr     s     <<s}}R'<=GL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r1   c                   n    e Zd ZdZ	 	 	 	 	 ddedededededz  ded	edz  f fd
Zde	j                  dedefdZ	 	 	 	 	 dde	j                  de	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  dee   dee	j                  e	j                  f   fdZ xZS )IdeficsAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   	num_headsr   is_cross_attentionconfigqk_layer_norms	layer_idxc                    t         	|           || _        || _        || _        ||z  | _        || _        d| _        | j
                  dz  | _        || _	        |-t        j                  d| j                  j                   d       | j
                  |z  | j                  k7  rt        d| j                   d| d      || _        t!        t"        j$                  d      st        d	      | j                  rt!        |j&                  d
      s| j                  n|j&                  j(                  }t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        nt#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  || j
                  z  |d      | _        t5        | j
                        | _        || _        | j8                  rMt;        | j
                  |j<                        | _        t;        | j
                  |j<                        | _         y y )NTg      zInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).scaled_dot_product_attentionz)this model requires pytorch 2.0 or higher	embed_dimFr   r   )!rl   rm   r   r   r   head_dimr   	is_causalr   r   loggerwarning_oncers   r(   rk   r   hasattrr   r   vision_configr   rP   q_projk_projv_projo_projr   
rotary_embr   r   rms_norm_epsq_layer_normk_layer_norm)
rq   r   r   r   r   r   r   r   kv_input_dimrs   s
            r2   rm   zIdeficsAttention.__init__  s    	&"#y0}}d*" !8!8 9 :, , MMI%$*:*::QRVRbRbQc$YKr3 
 #5r}}&DEHII""(/0D0Dk(R  X^XlXlXvXv  ))  DMM)DK
 ))L)dmm2KRWXDK))DMM)DK ))  DMM)DK
 ))  DMM)DK
 ))  DMM)DK
 ii%

 +4==9, .t}}&BUBU VD .t}}&BUBU VD r1   tensorr   bszc                     |j                  ||| j                  | j                        j                  dd      j	                         S )Nr   r   )rA   r   r  r   r   )rq   r  r   r  s       r2   _shapezIdeficsAttention._shape4  s7    {{3GQQRSUVWbbddr1   r%   key_value_statesr>   r   r$   cache_positionrr   re   c                    | j                   xs |d u}|j                         \  }	}
}| j                  |      j                  |	|
| j                  | j
                        j                  dd      }|s| j                  |      j                  |	|
| j                  | j
                        j                  dd      }| j                  |      j                  |	|
| j                  | j
                        j                  dd      }n|j                         \  }}}| j                  |      j                  |	|| j                  | j
                        j                  dd      }| j                  |      j                  |	|| j                  | j
                        j                  dd      }|j                  d   }|||d   z  }|s2| j                  |t        ||
            \  }}t        |||||      \  }}|%d|i}|j                  ||| j                  |      \  }}| j                  r"| j!                  |      }| j#                  |      }t%        j&                  | j(                  j*                  t,              } || ||||f| j.                  sdn| j0                  | j2                  d|\  }}|j5                  |	|
d	      j7                         }| j9                  |      }||fS )
Nr   r   r   r   )r   r          )r   r   r8   )r   sizer  rA   r   r  r   r  r	  r@   r  maxr   updater   r   r  r  r   get_interfacer   _attn_implementationr   r   r   r   reshaper   r
  )rq   r%   r  r>   r   r$   r  rr   r   r  q_len_query_states
key_statesvalue_stateskv_len
kv_seq_lenr   r   cache_kwargsattention_interfacer   r   s                          r2   r}   zIdeficsAttention.forward7  s    "44T8HPT8T%**,UA{{=166sE4>>SWS`S`akklmopq!]388eT^^UYUbUbcmmnoqrsJ;;}5::3t~~W[WdWdeoopqstuL+002LAvq%56;;CY]YfYfgqqrsuvwJ,-223PTP]P]^hhijlmn   %%b)
&.++J!|SU=STHC';L*VY[^`l'm$L* &,n=L'6'='=j,X\XfXfht'u$J,,\:L**:6J(?(M(MKK,,.E)
 %8	%
  $}}C$,,LL	%
 	%
!\ "))#ub9DDFkk+.L((r1   )r  FNFNNNNNN)r(   r)   r*   r+   r   r   r   r   rm   r,   r   r  
LongTensorr
   r   r   r/   r}   r   r   s   @r2   r   r     sH   G #(*.$ $OWOW OW 	OW
 !OW !4'OW OW :OWbeU\\ eC ec e 15.204(,26>)||>)  ,,->) t+	>)
 &&->) >) ((4/>) +,>) 
u||U\\)	*>)r1   r   c                        e Zd Zddededz  f fdZe	 	 	 	 ddej                  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
ee   dej                  fd       Z xZS )IdeficsDecoderLayerNr   r   c                    t         |           |j                  | _        t        | j                  |j                  |j
                  ||      | _        t        | j                  |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        y )N)r   r   r   r   r   r   r   r   r   )rl   rm   r   r   num_attention_headsr   	self_attnr   r   r   mlpr   r  input_layernormpost_attention_layernormrq   r   r   rs   s      r2   rm   zIdeficsDecoderLayer.__init__z  s    !--)((00NN
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%~~r1   r%   r>   r   r$   r  rr   re   c           	         |}| j                  |      } | j                  d|||||d|\  }}t        j                  j	                  || j                  | j
                        }||z   }|}| j                  |      }| j                  |      }t        j                  j	                  || j                  | j
                        }||z   }|S )N)r%   r>   r   r$   r  r   r0   )r0  r.  r   r   r   r   r1  r/  )	rq   r%   r>   r   r$   r  rr   residualr  s	            r2   r}   zIdeficsDecoderLayer.forward  s     !,,]; *4>> 
')%+)
 
q --mt||VZVcVc-d =0 !55mD/--mt||VZVcVc-d =0r1   rS   )NNNN)r(   r)   r*   r   r   rm   r   r,   r   r(  r
   r   r   r-   r}   r   r   s   @r2   r*  r*  y  s    &} &t &&  /304(,26 ||  t+  &&-	 
   ((4/  +,  
		   r1   r*  c                   
    e Zd Zddededz  f fdZe	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  de
e   dej                  fd       Z xZS )IdeficsGatedCrossAttentionLayerNr   r   c           	      	   t         |           |j                  | _        t        | j                  |j                  d|j
                  ||j                  |      | _        t        | j                  |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        t#        j$                         | _        t#        j$                         | _        |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j2                  dd| j                              | _        t#        j.                  t1        j2                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j2                  d            | _        t#        j.                  t1        j2                  d            | _        nt9        d	|j,                   d
      |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j:                  dd| j                              | _        t#        j.                  t1        j:                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j:                  d            | _        t#        j.                  t1        j:                  d            | _        n|t9        d	|j,                   d
      |j*                  dv r;|j,                  dk(  rt#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        t#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        n|j,                  dk(  rut#        j.                  t1        j<                  d|j>                  d            | _        t#        j.                  t1        j<                  d|j>                  d            | _        n2t9        d	|j,                   d
      tA        d|j*                   d      tC        | d      rtC        | d      st9        d      y )NT)r   r   r   r   r   r   r   r,  r   zerosvectorr   r   z Unknown value for `alpha_type` ()r   >   normalrandomgaussianr  )r   stdr  zAlpha initialization scheme z not yet implemented!alpha_cross_attnalpha_densez+Alpha parameters not initialized correctly!)"rl   rm   r   r   r-  r   r   
cross_attnr   r   r   r/  r   r  r0  r1  r   r   Tanhact_cross_attn	act_densealpha_initializer
alpha_typer   r,   r8  r?  r@  rk   r   r;  alphas_initializer_rangeNotImplementedErrorr  r2  s      r2   rm   z(IdeficsGatedCrossAttentionLayer.__init__  s   !--*((00#NN!00
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%nn ggi##w.  H,(*U[[AtGWGW5X(Y%#%<<Aq$BRBR0S#T ""g-(*U[[^(D%#%<<A#?  #CFDUDUCVVW!XYY%%/  H,(*UZZ1dFVFV5W(X%#%<<

1aAQAQ0R#S ""g-(*UZZ](C%#%<<

1#>  #CFDUDUCVVW!XYY%%)II  H,(*LLcv/N/NVWYZ\`\l\lUmn)% $&<<LLcv/N/NVWYZ\`\l\lUmn$  ""g-(*LLcv/N/NVWY)% $&<<#6KjKjrs0u#v  #CFDUDUCVVW!XYY &(DVE]E]D^^s&tuu01gdM6RJKK 7Sr1   r%   r>   r'   r<   cross_attention_gater$   rr   re   c                    |t        d      |t        d      |t        d      |}| j                  |      } | j                  d	|||d|\  }}	t        j
                  j                  || j                  | j                        }|j                  |dk(  dddddf   d      }|| j                  | j                        |z  z   }|}| j                  |      }| j                  |      }t        j
                  j                  || j                  | j                        }|| j                  | j                        |z  z   }|S )
a  
        image_hidden_states (`torch.FloatTensor`):
            Input to the layer of shape `(batch, seq_len, embed_dim)`
        image_attention_mask (`torch.FloatTensor`, *optional*):
            image attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        cross_attention_gate (`torch.FloatTensor`, *optional*):
            gate of size `(batch, seq_len)` used to zero-out cross-attention output for tokens attending no images.
        Nzt`image_hidden_states` is required for Idefics cross attention module which are visual features to be conditioned on.z`cross_attention_gate` is required for Idefics cross attention module to zero-out the cross-attention hidden_states attending to no images.zMPast key value states are not implemented for Idefics cross attention module.)r%   r  r>   r   r   r  r0   )rk   rH  r0  rA  r   r   r   r   r   masked_fillrC  r?  r1  r/  rD  r@  )
rq   r%   r>   r'   r<   rI  r$   rr   r4  r  s
             r2   r}   z'IdeficsGatedCrossAttentionLayer.forward  sm   ( &# 
  ' ^  &%&uvv ,,]; +4?? 
'0/
 	
q --mt{{UYUbUb-c%113G13LaQRTXj2Y[^_ 4#6#6t7L7L#MP]#]] !55mD/--mt{{UYUbUb-c 4>>$2B2B#Cm#SSr1   rS   r'  )r(   r)   r*   r   r   rm   r   r,   r   r
   r   r   r-   r}   r   r   s   @r2   r6  r6    s    @L} @Lt @LD  /3374848(,8||8 t+8 #\\D0	8
 $llT18 $llT18 8 +,8 
		8 8r1   r6  c                        e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZe eedd	
      dZ ej$                          fd       Z xZS )IdeficsPreTrainedModelr   r\   )imagetextTr*  r6  Fr   r.  )index
layer_name)r%   r&   c                 x   t         |   |       t        |t              ryt	        j
                  |j                         t	        j                  |j                  t        j                  |j                  j                  d         j                  d             y t        |t              r6| j                  j                  dk(  r?t	        j                   |j"                         t	        j                   |j$                         y | j                  j                  dk(  r?t	        j&                  |j"                         t	        j&                  |j$                         y | j                  j                  dv rmt	        j
                  |j"                  d| j                  j(                         t	        j
                  |j$                  d| j                  j(                         y y t        |t*              r t	        j
                  |j,                         y t        |t.              rd|j0                  t        j                  d	|j2                  d
      |j2                  z  z  z  }t	        j                  |j4                  |       t        j                  |j6                        j9                  |      }t        j:                  d||      }t        j<                  ||fd      }t	        j                  |j>                  |jA                                t	        j                  |jB                  |jE                                y y )Nr8   )r   r8   r8  r   >   r;  r<  r=  r  )r   r>  r   r   r   r   r   )#rl   _init_weightsrT   r   initnormal_class_embeddingcopy_r   r,   r?   r@   expandr6  r   rE  zeros_r?  r@  ones_rG  r   latentsr   r   r   r   r   r   r   r   r   r   r   r   )rq   rW   r   rV   r   r   rs   s         r2   rS  z$IdeficsPreTrainedModel._init_weightsB  s   
 	f%f56LL//0JJv**ELL9L9L9R9RSU9V,W,^,^_f,gh ?@{{,,7F334F../..&8

6223

6--...2RRV443DKKDhDhiV//ct{{?c?cd S  9:LL( 01fkkell1fjj!.Lvzz.YZ[HJJv1V;;<DDXNALLAx8E))UEN3CJJv((#'')4JJv((#'')4 2r1   )r(   r)   r*   r   r.   base_model_prefixinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_can_compile_fullgraph_supports_attention_backendr*  r   r   _can_record_outputsr,   no_gradrS  r   r   s   @r2   rM  rM  0  st    (&*#.0QRN ""& -$%5Q;W
 U]]_5 5r1   rM  c            !           e Zd ZdZdef fdZddZddZddZe	e
e	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dedz  dej                   dz  dej                   dz  dej                   dz  dej                   dz  dej                  dz  dedz  dedz  dej                  dz  dee   deez  fd                     Z xZS )IdeficsModelz
    Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`]

    Args:
        config: IdeficsConfig
    r   c           	         t         |   |       || _        |j                  | _        |j
                  | _        t        |j
                  |j                  |j                  |j                  | j                        | _
        |j                  j                  | _        |j                  | _        |j                  | j                  _        t        |j                        | _        |j                   r]|j"                  }t%        ||j                  j&                  |j(                  |j*                  |j,                  |j.                        | _        t3        j4                  t7        |j8                        D cg c]  }t;        ||       c}      | _        |j>                  | _        |j8                  | j>                  z  }t3        j4                  t7        |      D cg c]  }tA        ||       c}      | _!        d| _"        tG        |j                  |jH                        | _%        | jM                          | jO                  |       y c c}w c c}w )N)rg   rn   rh   rd   rj   )r   Fr   )(rl   rm   r   pad_token_idrj   
vocab_sizerc   additional_vocab_sizer   freeze_text_layersembed_tokensr  
image_sizer  r   vision_modeluse_resamplerperceiver_configr   r   resampler_depthresampler_n_headsresampler_head_dimresampler_n_latentsperceiver_resamplerr   
ModuleListrangenum_hidden_layersr*  layerscross_layer_intervalr6  gated_cross_attn_layersgradient_checkpointingr   r  norm	post_initfreeze_relevant_params)rq   r   rq  inum_cross_layersrs   s        r2   rm   zIdeficsModel.__init__k  s    !.. ++5!,,&,&B&B ,,#66((
 !..99#11282M2M/4V5I5IJ %66'@$$.. 00 22 33 44(D$ mm?DVE]E]?^_! 15_
 %+$?$?!!33t7P7PP')}}KPQaKbca,VqAc(
$ ',#"6#5#56;N;NO	 	##F+ ` ds   1IINc                     || j                   }|j                  r| j                  |j                         |j                  r"t	        | j
                  |j                         y y N)r]   )r   rl  freeze_text_module_exceptionsfreeze_vision_layersra   ro  freeze_vision_module_exceptions)rq   r   s     r2   r  z#IdeficsModel.freeze_relevant_params  sQ    >[[F$$##F$H$HI&&**f>d>de 'r1   c                 X    | j                   | j                  fD ]  }t        ||        y r  )rz  r~  ra   )rq   r]   rW   s      r2   rl  zIdeficsModel.freeze_text_layers  s+    {{DII. 	FF3DE	Fr1   c                 2    t        | j                  |       y r  )ra   ro  )rq   r]   s     r2   r  z!IdeficsModel.freeze_vision_layers  s    T&&:KLr1   rG   r>   r   r$   inputs_embedsr9   r:   r;   r<   	use_cacheinterpolate_pos_encodingr  rr   re   c           	         ||j                   n|j                   }|du |duz  rt        d      || j                  |      }|
r|t        | j                        }|j
                  \  }}}||j                         nd}||z   }|2t        j                  |||j
                  d   z   |j                         }|F|D|j                         j                  d      dz
  }|j                  |dk(  d       |dd| df   }n||j                  d      }t        d |||fD              d	k7  rt        d
      |~|j                  | j                  |      }|j
                  dd	 \  }} |j!                         j"                  ||z  g|j
                  d	d  }| j%                  ||      j&                  }nJ|H|j)                         \  }}}}|j                  | j                  |      }|j#                  ||z  ||      }| j                  j*                  rN|4| j-                        }|j)                  d      |j)                  d	      }}n|j)                         \  }}}}|}n0|#j)                  d      |j)                  d	      }}nt        d      |j#                  ||z  |      }|	j)                  d      }|	j                  d      }	|	j/                  ddd|      }	|	j#                  ||||z        }	|C|j)                         \  }}}||f}|	t        j0                  ||      }	| j3                  |	      }	nd}	|	dk(  j5                  d      j                  | j                        j7                  d      j                  |      }|2t        j0                  ||ft        j8                  |j                         }t;        | j                  |||||      }|}t=        | j>                        D ]P  \  }} || j@                  z  dk(  r+| jB                  || j@                  z     }! |!|||f|	|dd|} | |f||||d|}R | jE                  |      }|j#                  ||||      }tG        |||      S )ab  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   r   )rD   r8   c              3   $   K   | ]  }|d u  
 y wrS   r0   )rU   r   s     r2   rX   z'IdeficsModel.forward.<locals>.<genexpr>  s     aQqDyas   r   z_Exactly 1 of pixel_values, image_encoder_embeddings or perceiver_embeddings has to be not-None.)ri   rD   )r9   r  zBIf `perceiver_embeddings` are passed, use_resampler should be Truer  r   r   )r   r  r>   r  r$   r   )r<   rI  r$   )r>   r   r$   r  )r#   r'   r$   )$rD   rk   rm  r   r   r@   get_seq_lengthr,   r?   longcumsummasked_fill_r   sumrC   ri   r   rA   ro  r#   r  rp  rv  rB   r   invert_attention_maskrZ   squeezer   r   	enumeraterz  r{  r|  r~  r"   )"rq   rG   r>   r   r$   r  r9   r:   r;   r<   r  r  r  rr   rD   
batch_size
seq_lengthr  past_key_values_lengthseq_length_with_past
num_imagesr'   image_seq_lenimage_hidden_sizetext_seq_lenimage_batch_sizeimage_sequence_lengthimage_hidden_shaperI  causal_maskr%   idxdecoder_layercross_attn_blocks"                                     r2   r}   zIdeficsModel.forward  s3   6 &/%:!!@T@T-t";<YZZ  --i8M0*$++>O$1$7$7!
JETE`!?!?!Afg),BB!"\\&(>ATATUVAW(W`m`t`tN %,*>)..077;a?L%%n&91='J;<8L!)33A6La<1IK_"`aaeffq  %'??F?KL%1%7%7%;"J
9<22499*z:QkT`TfTfghgiTjkL #'"3"3)D\ #4 #   &1G_GdGdGfDJ
M3D":"="=DJJW]"="^"5":"::
;RTact"u;;$$#+'+'?'?@S'T$3G3L3LQ3OQeQjQjklQm0K_KdKdKfH
J7H"6!)/B/G/G/JL_LdLdefLg,Mabb166z:P]C]_pq ,0033==bA3::1aMR388\S]`mSmn*9L9Q9Q9S63Q"24I!J#+',zz2DV'T$#'#=#=>R#S #'  $83#>"C"C"C"K!O!OVZV`V`!O!a j jop j quu 

 !"ZZ12%**]MaMaN );;'))+%
 &"+DKK"8 	CT...!3#'#?#?tG`G`@`#a  0!'! *>)=$(! ! **) /- M	. 		-0166z:}^op-+ 3+
 	
r1   rS   r0   )NNNNNNNNNNFN)r(   r)   r*   r+   r   rm   r  rl  r  r   r   r   r,   r(  r   r
   r-   r   r   r   r/   r"   r}   r   r   s   @r2   rg  rg  b  s   0,} 0,dfFM   .2.204(,2615=A9=48!%0526^
##d*^
 t+^
 &&-	^

 ^
 ((4/^
 ''$.^
 #("3"3d":^
 $//$6^
 $llT1^
 $;^
 #'+^
 ((4/^
 +,^
 
/	/^
    ^
r1   rg  c            $       0    e Zd ZddiZd fd	Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dej                  dz  deej                  z  dee   deez  f d              Z	 	 	 	 	 	 	 	 	 d fd	Z	 ddedeeef   dedeeef   f fdZ xZS )IdeficsForVisionText2Textlm_head.weightmodel.embed_tokens.weightNc                    t         |   |       t        |      | _        t	        |j
                  |j                  |j                  d|j                        | _	        |j                  dkD  r
ddd| _
        | j                          y )NFr   r   r  z.model.embed_tokens.additional_embedding.weight)r  zlm_head.additional_fc.weight)rl   rm   rg  r\   r   r   rj  rk  freeze_lm_headlm_head_tied_weights_keysr  )rq   r   ro  rs   s      r2   rm   z"IdeficsForVisionText2Text.__init__U  s{     !&)
-****$*$@$@#22
 ''!+"=0`'D# 	r1   rG   r>   r   r$   r  r9   r:   r;   r<   labelsr  r  r  logits_to_keeprr   re   c                     | j                   d|||||||||	||d|d|}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}|
* | j                  d||
| j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )aC  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoProcessor, IdeficsForVisionText2Text

        >>> model = IdeficsForVisionText2Text.from_pretrained("HuggingFaceM4/idefics-9b")
        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics-9b")

        >>> dogs_image_url_1 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image1.jpeg"
        >>> dogs_image_url_2 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image2.jpeg"

        >>> prompts = [
        ...     [
        ...         "User:",
        ...         dogs_image_url_1,
        ...         "Describe this image.\nAssistant: An image of two dogs.\n",
        ...         "User:",
        ...         dogs_image_url_2,
        ...         "Describe this image.\nAssistant:",
        ...     ]
        ... ]
        >>> inputs = processor(prompts, return_tensors="pt")
        >>> generate_ids = model.generate(**inputs, max_new_tokens=6)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)
        ```T)rG   r>   r   r$   r  r9   r:   r;   r<   r  r  return_dictr  N)r6   r  rj  )r5   r6   r$   r%   r&   r'   r0   )r\   r#   rT   r   slicer  loss_functionr   rj  r4   r$   r%   r&   r'   )rq   rG   r>   r   r$   r  r9   r:   r;   r<   r  r  r  r  r  rr   outputsr%   slice_indicesr6   r5   s                        r2   r}   z!IdeficsForVisionText2Text.forwardi  s    r 3=$** 3
)%+'%%=!5!5%=)3
 3
"  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD,#33!//)) ' ; ;
 	
r1   c                    i }|"| j                   j                  r||d<   n||d<   n||d<   |j                  dd      |d<   t        |   |f||||||
|	d||}|	#|!|d   j
                  d   }|	d d | d f   |d	<   |S )
Nr;   r:   r9   r  F)r$   r>   r  r  r   r  r<   rG   r   r<   )r   rp  poprl   prepare_inputs_for_generationr@   )rq   rG   r>   r   r  r$   r  r9   r'   r<   r  rr   images_kwargsmodel_inputsr  rs   s                  r2   r  z7IdeficsForVisionText2Text.prepare_inputs_for_generation  s      *{{((8K45<O89,8M.)4:JJ?Y[`4a01w<
+)')%!5
 
 
  +0E%k288;J3GJ;<3XL/0r1   r  rK   rI   c                     t        |   |||fi |}d|v rT|d   }|d d dd d f   j                  d      }|j                  dd      r||d<   nt	        j
                  ||gd      |d<   |j                  |d<   |S )Nr<   r8   r   r  Tr   r'   )rl   #_update_model_kwargs_for_generationr   rF   r,   r   r'   )rq   r  rK   rI   rr   r<   	last_maskrs   s          r2   r  z=IdeficsForVisionText2Text._update_model_kwargs_for_generation  s     wB
 	
 "\1#/0F#G ,QAX6@@CIT27@347<yyBVXaAbhi7j34 /6.I.I*+r1   rS   )NNNNNNNNNNNFNr   )	NNNNNNNNN)F)r(   r)   r*   r  rm   r   r   r,   r(  r   r
   r-   r   r   r   r   r/   r4   r}   r  r   dictr   r   r  r   r   s   @r2   r  r  R  s    *,GH(  .2.204(,2615=A9=48*.!%0526-.X
##d*X
 t+X
 &&-	X

 X
 ((4/X
 ''$.X
 #("3"3d":X
 $//$6X
 $llT1X
   4'X
 $;X
 #'+X
 ((4/X
 ell*X
  +,!X
" 
.	.#X
  X
z  !+b $)	 38n !	 
c3h r1   r  )r  rg  rM  )r   FNNr  )r   )r  )Nr+   collections.abcr   dataclassesr   typingr   r,   torch.nn.functionalr   r   ru    r   rT  activationsr	   cache_utilsr
   r   
generationr   masking_utilsr   modeling_layersr   modeling_outputsr   modeling_utilsr   r   r   processing_utilsr   utilsr   r   r   r   utils.genericr   utils.output_capturingr   r   configuration_ideficsr   	perceiverr   visionr   r   
get_loggerr(   r  r"   r4   rM   ra   rQ   rc   rP   r   Moduler   r   r   r   r   r   r   r   r   r*  r6  rM  rg  r  __all__r0   r1   r2   <module>r     s6  &  $ !      & ! . ) / 9 + X X & R R 7 E 0 0 E 
		H	% 
@[ @ @6 
@K @ @8 *#ZfA fAR8SRYY 8SxJRYY J0$
uxx $
N(:P P2 %II%<<% 
% <<	%
 LL4'% % %0U)ryy U)r54 5p|&@ |~ .5_ .5 .5b l
) l
 l
^x 6 xv Rr1   