qi5 ddlZddlmZddlmZddlZddlmZm Z m Z e rddl Z e rddl Z ddlmZmZddlmZddlmZmZmZmZdd lmZmZGd d ed ZGdded ZeGddeZdgZy)N)Path)Any)auto_docstringis_soundfile_availableis_torch_available) AudioInputmake_list_of_audio) BatchFeature) AudioKwargsProcessingKwargsProcessorMixinUnpack)PreTokenizedInput TextInputc,eZdZUdZeeefdzed<y)CsmAudioKwargsa encoded_length_kwargs (`dict[str, Any]`, *optional*): Dictionary of keyword arguments used to compute the encoded audio sequence length. This includes parameters such as `kernel_sizes`, `strides`, `dilations`, and `use_causal_conv` that define the convolutional layers used in audio encoding. The encoded length is used to determine how many audio tokens to generate for each audio input in the text sequence. Nencoded_length_kwargs)__name__ __module__ __qualname____doc__dictstrr__annotations__X/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/csm/processing_csm.pyrr$s S>D00rrF)totalcLeZdZUeed<ddddgdgdgddd d d d d idZy)CsmProcessorKwargs audio_kwargsTleftF)padding padding_sideadd_special_tokens)rrr( rr( rr(r)r(r(r(r-r(r(r(r(r(r(r)r()r(r(r(r(r(r(r(r(r(r(r(r(r(r(r() kernel_sizesstrides dilationsuse_causal_convi])r sampling_ratereturn_tensorspt) text_kwargsr" common_kwargsN)rrrrr _defaultsrrrr!r!0sG  ""' !QHJ#' & # +D1Irr!c eZdZ d fd ZeddZdedeeze eezzde e fdZ e ddeeze eze ezdzdedzd edzd edzde e f d Zed ZxZS) CsmProcessorNct|ds(d|_|j|j|_n"|j|_|j|_t|ds(d|_|j|j|_n"|j|_|j |_t ||||y)N audio_tokenz <|AUDIO|>audio_eos_tokenz <|audio_eos|>) chat_template)hasattrr>convert_tokens_to_idsaudio_token_idr?audio_eos_token_idsuper__init__)selffeature_extractor tokenizerr@ __class__s rrFzCsmProcessor.__init__Gs y-0*D "+"A"A$BRBR"SD (44D "+":":D y"34#2D &/&E&EdFZFZ&[D ##,#<#z*CsmProcessor.save_audio..s@q`aAPSUY{A[@qs "zAInvalid input path. Please provide a string, or a list of stringsz5The number of audio and saving paths must be the samer"r5)r ImportErrorr rbrrlisttupleall ValueErrorlen _merge_kwargsr!rLtorchTensorcpufloatnumpysfwrite) rGr\r]r^ output_kwargsr"r5 audio_valuerds r save_audiozCsmProcessor.save_audios&'OP P#5) kC; /&-K[4-8S@qep@q=q`a a u:[) )TU U***   %^4 $_5 !%5 4NK+u||4)oo/557==? HHQ ] 3 4rtext output_labelsdepth_decoder_labels_ratioc ^ |jtfd|jji|}|d}|d}|j dd} | dk7r"t |j jdt|tr|g}n3t|ttfrtd|Ds t d |D cgc]} | j|j} } d } |t|}t!|} t#| d kDr-| t#| k7r| t d t d | d | d||j%di} |Dcgc]"}|j&|j(dfi| $}}|j+}g}|D]}g}|j|vr]|j%d }|j|z}|j-||j/|jdd}|j|vr]d|vr'|j/d|j%d d}d|vr'|j-||}|j|fi|}i}|j1|||j%ddgg}}d }| D]} | d k(rJ|j-t3j4d |j-t7j8dgS|j-t3j:|||| zDcgc]<}t|t6j<r|j?jAn|>c}d|j-t7j8|||| zDcgc]}|j(dc}jCd|| z }!|jD|fi|}|j%dd|j1|tGd|D}|Dcgc]@}t6jHjJjM|d ||j(dz fdB}}t7jN|d |d<|r|d|jPk(jS}|j(d } |dkr-t7jT| dtW| d|z z}!||!}"n|}"t7jX|d|jPk(|d|jZk(z|dd}#d|#|"ddd f|"dddff<|#|d<t]|| Scc} wcc}wcc}wcc}wcc}w) a output_labels (bool, *optional*, default=False): Whether to return labels for training. Indices will be in `[config.audio_token_id, -100, -101]`. - `config.audio_token_id` indicates an audio frame (considering sequence length elements as frames) - `-100` will be ignored in the loss computation - `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels) depth_decoder_labels_ratio (float, *optional*, default=1.0): The ratio of audio frames to keep for the depth decoder labels. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. - **input_values** -- List of audio values to be fed to a model. Returned when `audio` is not `None`. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not `None`). - **labels** -- List of labels for the audio frames. Returned when `output_labels=True`. tokenizer_init_kwargsr8r"r6Nr7z% only supports `return_tensors='pt'`.c3<K|]}t|tywra)rbr)rcts rrez(CsmProcessor.__call__..s9[QR*Q:L9[szAInvalid input text. Please provide a string, or a list of stringsrz@No audio were provided, but there are audio tokens in the promptz)The number of audio tokens in each text (z7) should be the same as the number of provided audios (z).rz r(return_attention_mask)axis)dim padding_maskc3:K|]}|jdyw)r~N)shape)rccut_idxss rrez(CsmProcessor.__call__..sR(..,Rs)valueinput_values_cutoffs input_ids?iilabels)data tensor_type)/rlr!rI init_kwargsgetrjrJrrbrrgrhricountr>r rksumpopr[rcopyappendreplaceupdatenpzerosrmtensor concatenaternrorqcumsumrHmaxnn functionalpadstackrCnonzerorandpermintwhererDr )$rGrwr\rxryr^rtr8r"r6r}n_audio_in_textn_audior audio_arraynum_audio_tokens_listnum_audio_tokens_list_copy expanded_textsample replace_strnum_audio_tokensexpanded_audio_tokenencodingrconcatenated_audioroffsetel audio_inputsmax_lenraudio_frame_idxsn_audio_frames rand_idxsskip_frames_idxsrs$ r__call__zCsmProcessor.__call__s:+**  "&.."<"<  $M2 $^4 $)94@ T ! 7 788]^_ _ dC 6DTD%=1c9[VZ9[6[`a a>BC1774#3#34CC  &u-E%jG  ! #33G(G} !cdd ??PQ229">  $0$4$45Lb$Q !lq%]h((():):2)>XBWX% !%*?)C)C)E &M - &&&0'A'E'Ea'H$+/+;+;>N+N(&&';<#^^D,<,>$6+6 H     4d ;792 4 F* &a<&--bhhqk:(// bT0BC&--+0'9I*J$&5?r5<<4P 0VX X"$ )// U6FU\L\=]%^rbhhrl%^_ffkmfng%F# &&24112DU UL   ^T 2 KK %R=QRRG!5$##''1gr@R6R2S[]'^$ $,1;;7KQR+SD' (  $[ 1T5H5H HQQS -33A6N)S0!NN>:;sSSTWqSqAr=st #3I#> #3 [[k"d&9&99d;>OSWSjSj>jk[!F FJF#AqD)+;AqD+AA B#DN>BBGD$%L&_$s <"T'T5AT 'T%,AT*c|jj}|jj}|Dcgc] }|dk7s | }}t||zdgzScc}w)Nrr)rImodel_input_namesrHrg)rGtokenizer_input_namesfeature_extractor_input_namesnames rrzCsmProcessor.model_input_names7se $ @ @(,(>(>(P(P%;X(r$[_cq[q(r%(r),IIMcLddee)ss AAra)NNNN)NFr)rrrrF staticmethodr[r rrrgrr!rvrrrboolrprpropertyr __classcell__)rJs@rr<r<Es  T,##J 4 44Z$sTz"22 4+, 4D$(%*36 OC++d9o=EV@WWZ^^OCD OCd{ OC %*DL OC +, OCOCbffrr<)rMpathlibrtypingrrqrutilsrrrrm soundfilerr audio_utilsr r feature_extraction_utilsr processing_utilsr r rrtokenization_utils_baserrrr!r<__all__rrrrs OO94UUC 1[ 1)*yf>yfyfx  r