
    qi8                         d dl mZmZmZmZmZmZ d dlmZ ddl	m
Z
mZ ddlmZ ddlmZ  ej                   e      Zddd	Zg d
Z G d de      ZdgZeZy)    )Regex	Tokenizerdecodersnormalizerspre_tokenizers
processors)Unigram   )
AddedTokenBatchEncoding)TokenizersBackend)loggingzsentencepiece.bpe.modelztokenizer.json)
vocab_filetokenizer_file)4ar_ARcs_CZde_DEen_XXes_XXet_EEfi_FIfr_XXgu_INhi_INit_ITja_XXkk_KZko_KRlt_LTlv_LVmy_MMne_NPnl_XXro_ROru_RUsi_LKtr_TRvi_VNzh_CNaf_ZAaz_AZbn_INfa_IRhe_ILhr_HRid_IDka_GEkm_KHmk_MKml_INmn_MNmr_INpl_PLps_AFpt_XXsv_SEsw_KEta_INte_INth_THtl_XXuk_UAur_PKxh_ZAgl_ESsl_SIc                   j    e Zd ZU dZeZddgZeZg Z	e
e   ed<   g Ze
e   ed<   	 	 	 	 	 	 	 	 	 	 ddeez  e
z  dz  dedz  f fd	Zd
 Zd Zedefd       Zej*                  deddfd       Z	 	 	 dde
e   dede
e   dz  dedef
 fdZd Zd ZdeddfdZdeddfdZdededz  dedz  fdZ xZS )MBart50Tokenizeru	  
    Construct a MBart50 tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        src_lang (`str`, *optional*):
            A string representing the source language.
        tgt_lang (`str`, *optional*):
            A string representing the target language.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.

    Examples:

    ```python
    >>> from transformers import MBart50Tokenizer

    >>> tokenizer = MBart50Tokenizer.from_pretrained("facebook/mbart-large-50", src_lang="en_XX", tgt_lang="ro_RO")
    >>> src_text = " UN Chief Says There Is No Military Solution in Syria"
    >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria"
    >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt")
    >>> # model(**model_inputs) should work
    ```	input_idsattention_maskprefix_tokenssuffix_tokensNvocab_spm_precompiled_charsmapc                    t        |
t              rt        |
dd      n|
}
t        |t              r|D cg c]  }t        |d         t	        |d         f  }}|D cg c]  }|d   	 c}t        fdt        D              }|r	|| _        nt        |      dft        |	      dft        |      dft        |      dfg}|j                  |dd         t        D ]  }|j                  t        |      df         |j                  t        |
      df       || _        nt        |      dft        |	      dft        |      dft        |      dfd	g| _        t        D ]  }| j                  j                  |df       ! | j                  j                  t        |
      df       t        t        | j                  dd
            | _        t        j                  t        d      d      g}|t        j                   |      g|z   }t        j"                  |      | j                  _        t'        j(                  ddd      | j                  _        t-        j(                  ddd      | j                  _        |j1                  dg       xs g }|j                  t               t3        | h  d|||||||	|
|d	| d| _        	 t        D cg c]  }t        |d       }}t;        | dg       xs g }|D ch c]  }t        |       }}t        |      |D cg c]  }t        |      |vs| c}z   }|| _        ||nd| _        || _         | jC                          | jD                  | j>                     | _#        | jI                  | j>                         y c c}w c c}w c c}w # t8        $ r t        t              }Y w xY wc c}w c c}w )NTF)lstriprstripr      c              3   &   K   | ]  }|v  
 y wN ).0	lang_codevocab_tokenss     b/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/mbart50/tokenization_mbart50.py	<genexpr>z,MBart50Tokenizer.__init__.<locals>.<genexpr>q   s     $g9Y,%>$gs   g        r
   )   ▁g       )unk_idbyte_fallbackz {2,} rY   always)replacementprepend_schemesplitadditional_special_tokens)	src_langtgt_lang	eos_token	sep_token	cls_token	unk_token	pad_token
mask_tokenra   )special_extra_special_tokensr   rS   )%
isinstancestrr   listfloatanyFAIRSEQ_LANGUAGE_CODES_vocab_scoresextendappendr   r	   
_tokenizerr   Replacer   PrecompiledSequence
normalizerr   	Metaspacepre_tokenizerr   decoderpopsuper__init__fairseq_offset	Exceptiongetattrrk   	_src_langrc   _build_language_code_mappingslang_code_to_idcur_lang_code_idset_src_lang_special_tokens)selfrK   rL   rb   rc   rd   re   rf   rg   rh   ri   kwargsitemhas_language_codes
vocab_listrU   normalizers_ra   codelang_tokensexisting_extratexisting_strsmerged_extrarV   	__class__s                           @rW   r   zMBart50Tokenizer.__init__U   s    KUU_adJeZ
4Fku
 eT" AFFc$q'lE$q'N3FEF056DG6L!$$gPf$g!g!%*" ^S)^S)^S)^S)	
 !!%), "8 =I%%s9~s&;<= !!3z?C"89%/" Y%Y%Y%Y%"D 4 <	""))9c*:;<%%s:&<= $""#
 $++E(OSAB$0'334MNOR^^L%0%9%9,%G"(6(@(@Ucksw(x%"*"4"4W_gk"l$*JJ/JB$O$USU!!(()?@ 	
!&?	
 	
  	7F\]d:dD9]K] !'>CIr)78AQ88N++.eQQWdIdq.ee%1"%-%9w  	**, $ 4 4T^^ D((8C G6\ ^ 	756K	7 9.esA   #N N!N+ N&$N+ ;O	O0O&N+ +OOc                    t         D ci c]  }|| j                  |       c}| _        | j                  j                         D ci c]  \  }}||
 c}}| _        ddddd| _        | j
                  j                  | j                         t        | dd      }| j                  t        |            | j
                  d<   | j
                  j                         D ci c]  \  }}||
 c}}| _	        yc c}w c c}}w c c}}w )	zFBuild language code to ID mappings and fairseq compatibility mappings.r   rP      r
   )<s><pad></s><unk>ri   <mask>N)
rq   convert_tokens_to_idsr   itemsid_to_lang_codefairseq_tokens_to_idsupdater   rm   fairseq_ids_to_tokens)r   rU   kvri   s        rW   r   z.MBart50Tokenizer._build_language_code_mappings   s     Oe 
AJIt11)<< 
 261E1E1K1K1MNA1N 	&
" 	""))$*>*>?T<:
/3/I/I#j//Z""8,7;7Q7Q7W7W7Y%Ztq!ad%Z" 
  O &[s   C6C;Dc                     | j                          t        | d      r:| j                  | j                     | _        | j                  | j                         yy)z9Called after tokenizer.json is loaded in from_pretrained.r   N)r   hasattrr   r   r   r   r   s    rW   
_post_initzMBart50Tokenizer._post_init   sH     	**,4%$($8$8$HD!,,T^^< &    returnc                     | j                   S rR   )r   r   s    rW   rb   zMBart50Tokenizer.src_lang   s    ~~r   new_src_langc                 H    || _         | j                  | j                          y rR   )r   r   )r   r   s     rW   rb   zMBart50Tokenizer.src_lang   s    %((8r   	src_textsrb   	tgt_textsrc   c                 B    || _         || _        t        |   ||fi |S rR   )rb   rc   r~   prepare_seq2seq_batch)r   r   rb   r   rc   r   r   s         rW   r   z&MBart50Tokenizer.prepare_seq2seq_batch   s*     ! w,Y	LVLLr   c                 8    | j                  | j                        S rR   )r   rb   r   s    rW   _switch_to_input_modez&MBart50Tokenizer._switch_to_input_mode  s    //>>r   c                 r    | j                   | j                  | _         | j                  | j                         S rR   )rc   r   set_tgt_lang_special_tokensr   s    rW   _switch_to_target_modez'MBart50Tokenizer._switch_to_target_mode  s,    ==  NNDM//>>r   c                    | j                  |      | _        | j                  g| _        | j                  g| _        | j                  | j                        }| j                  | j                        }t        j                  |dgz   |z   |ddgz   |z   t        t        ||z   | j                  | j                  z                     | j                  _        y)z]Reset the special tokens to the source lang setting. prefix=[src_lang_code] and suffix=[eos].$A$Bsinglepairspecial_tokensNr   r   rI   eos_token_idrJ   convert_ids_to_tokensr   TemplateProcessingrn   zipru   post_processor)r   rb   prefix_tokens_strsuffix_tokens_strs       rW   r   z,MBart50Tokenizer.set_src_lang_special_tokens
       $ : :8 D"334"//0 66t7I7IJ 66t7I7IJ)3)F)F$v-0AA"dD\14EE$58I$I4K]K]`d`r`rKr st*
&r   c                    | j                  |      | _        | j                  g| _        | j                  g| _        | j                  | j                        }| j                  | j                        }t        j                  |dgz   |z   |ddgz   |z   t        t        ||z   | j                  | j                  z                     | j                  _        y)zaReset the special tokens to the target language setting. prefix=[tgt_lang_code] and suffix=[eos].r   r   r   Nr   )r   rc   r   r   s       rW   r   z,MBart50Tokenizer.set_tgt_lang_special_tokens  r   r   return_tensorsc                 v    ||t        d      || _         | |fd|d|}| j                  |      }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)add_special_tokensr   forced_bos_token_id)
ValueErrorrb   r   )r   
raw_inputsr   rb   rc   extra_kwargsinputstgt_lang_ids           rW   _build_translation_inputsz*MBart50Tokenizer._build_translation_inputs(  sY     x/`aa jiT.i\hi00:(3$%r   )
NNNNr   r   r   r   r   r   )r   Nr$   )__name__
__module____qualname____doc__VOCAB_FILES_NAMESvocab_files_namesmodel_input_namesr	   modelrI   rn   int__annotations__rJ   rm   dictr   r   r   propertyrb   setterr   r   r   r   r   r   r   __classcell__)r   s   @rW   rF   rF   !   s   *X *$&67E!M49!!M49! +/04z9TzD 4'z9 $':z9x[&= #   __9S 9T 9 9  &*
M9
M 
M 9t#	
M
 
M 

M??

C 
D 

C 
D 

*-
9<t
ORUYz
r   rF   N)
tokenizersr   r   r   r   r   r   tokenizers.modelsr	   tokenization_pythonr   r   tokenization_utils_tokenizersr   utilsr   
get_loggerr   loggerr   rq   rF   __all__MBart50TokenizerFastrS   r   rW   <module>r      sg     [ Z % < >  
		H	% $=P`a  n Q( Qh 
 ( r   