
    qi
                     f    d Z ddlZddlmZ ddlmZ  ej                  e      Z G d d      Z	dgZ
y)	zTokenization classes for RAG.    N   )logging   )	RagConfigc                   F    e Zd Zd Zd Zed        Zd Zd Zd Z	d Z
d Zy	)
RagTokenizerc                 B    || _         || _        | j                   | _        y N)question_encoder	generatorcurrent_tokenizer)selfr   r   s      Z/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/rag/tokenization_rag.py__init__zRagTokenizer.__init__   s     0"!%!6!6    c                 z   t         j                  j                  |      rt        d| d      t        j                  |d       t         j                  j                  |d      }t         j                  j                  |d      }| j                  j                  |       | j                  j                  |       y )NzProvided path (z#) should be a directory, not a fileT)exist_okquestion_encoder_tokenizergenerator_tokenizer)	ospathisfile
ValueErrormakedirsjoinr   save_pretrainedr   )r   save_directoryquestion_encoder_pathgenerator_paths       r   r   zRagTokenizer.save_pretrained   s    77>>.)~.>>abcc
NT2 "^=Y Zn6KL--.CD&&~6r   c                     ddl m} |j                  dd       }|t        j                  |      }|j	                  ||j
                  d      }|j	                  ||j                  d      } | ||      S )N   )AutoTokenizerconfigr   )r#   	subfolderr   )r   r   )auto.tokenization_autor"   popr   from_pretrainedr   r   )clspretrained_model_name_or_pathkwargsr"   r#   r   r   s          r   r'   zRagTokenizer.from_pretrained(   s     	;Hd+>../LMF(88)&2I2IUq 9 
 "11)&2B2BNc 2 
	 $4	JJr   c                 &     | j                   |i |S r
   )r   r   argsr*   s      r   __call__zRagTokenizer.__call__:   s    %t%%t6v66r   c                 :     | j                   j                  |i |S r
   )r   batch_decoder,   s      r   r0   zRagTokenizer.batch_decode=   s    *t~~**D;F;;r   c                 :     | j                   j                  |i |S r
   )r   decoder,   s      r   r2   zRagTokenizer.decode@   s    $t~~$$d5f55r   c                 &    | j                   | _        y r
   )r   r   r   s    r   _switch_to_input_modez"RagTokenizer._switch_to_input_modeC   s    !%!6!6r   c                 &    | j                   | _        y r
   )r   r   r4   s    r   _switch_to_target_modez#RagTokenizer._switch_to_target_modeF   s    !%r   N)__name__
__module____qualname__r   r   classmethodr'   r.   r0   r2   r5   r7    r   r   r   r      s;    7
7 K K"7<670r   r   )__doc__r   utilsr   configuration_ragr   
get_loggerr8   loggerr   __all__r<   r   r   <module>rC      s<    $ 	  ( 
		H	%.0 .0b 
r   