qi,dZddlZddlmZddlmZddlZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZmZmZdd lmZmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%eGd de!Z&GddeZ'y)zTavus video service implementation for avatar-based video generation. This module implements Tavus as a sink transport layer, providing video avatar functionality through Tavus's streaming API. N) dataclass)Optional) AudioData VideoFrame)logger)create_stream_resampler) BotStartedSpeakingFrame CancelFrameEndFrameFrameInterruptionFrameOutputAudioRawFrameOutputImageRawFrameOutputTransportReadyFrameSpeechOutputAudioRawFrame StartFrameTTSAudioRawFrameTTSStartedFrame)FrameDirectionFrameProcessorSetup) AIService)ServiceSettings)TavusCallbacks TavusParamsTavusTransportClientceZdZdZy)TavusVideoSettingsz%Settings for the Tavus video service.N)__name__ __module__ __qualname____doc__N/opt/pipecat/venv/lib/python3.12/site-packages/pipecat/services/tavus/video.pyrr*s/r#rc HeZdZUdZeZeed<ddddededed ejd e ed df fd Z d e ffd Z fdZdZdZdZdededefdZdededefdZd efdZd efdZdeffd Zdeffd Zdeffd Zded e ffd! Z!d"Z"d#Z#d$Z$d%Z%de&fd&Z'd'Z(xZ)S)(TavusVideoServiceaService that proxies audio to Tavus and receives audio and video in return. Uses the TavusTransportClient to manage sessions and handle communication. When audio is sent, Tavus responds with both audio and video streams, which are routed through Pipecat's media pipeline. In use cases with DailyTransport, this creates two distinct virtual rooms: - Tavus room: Contains the Tavus Avatar and the Pipecat Bot - User room: Contains the Pipecat Bot and the user _settingszpipecat-streamN) persona_idsettingsapi_key replica_idr(sessionr)returnc $td}||j|t| dd|i|||_||_||_||_d|_d|_ |t|_ t|_ d|_d|_d|_y)a[Initialize the Tavus video service. Args: api_key: Tavus API key used for authentication. replica_id: ID of the Tavus voice replica to use for speech synthesis. persona_id: ID of the Tavus persona. Defaults to "pipecat-stream" for Pipecat TTS voice. session: Async HTTP session used for communication with Tavus. settings: Runtime-updatable settings. Tavus has no model concept, so this is primarily used for the ``extra`` dict. **kwargs: Additional arguments passed to the parent AIService class. N)modelr)Fstreamr")r apply_updatesuper__init___api_key_session _replica_id _persona_id_other_participant_has_joined_clientr _resampler bytearray _audio_buffer _send_task_transport_destination_transport_ready) selfr*r+r(r,r)kwargsdefault_settings __class__s r$r3zTavusVideoService.__init__As*+6    ) )( 3 ="2=f=  %%-2*7;  13&[265=# %r#setupczKt||d{t|j|j|j }t d||j|j|j|jtdddd|_ |jj|d{y77w)zoSet up the Tavus video service. Args: setup: Frame processor setup configuration. N) on_joinedon_participant_joinedon_participant_leftPipecatTF)audio_in_enabledvideo_in_enabledaudio_out_enabledmicrophone_out_enabled)bot_name callbacksr*r+r(r,params) r2rDr _on_joined_on_participant_joined_on_participant_leftrr4r6r7r5rr9)r@rDrOrCs r$rDzTavusVideoService.setupls gmE""""oo"&"="= $ 9 9 ,MM''''MM!%!%"&',    ll  ''') #( (s"B;B7BB;1B92B;9B;cKt|d{|jjd{d|_y7.7w)z+Clean up the service and release resources.N)r2cleanupr9)r@rCs r$rUzTavusVideoService.cleanups=goll""$$$  $sAA!AA AAc6Ktjdyw)z!Handle bot joined the Daily room.zTavus bot joined Daily roomNrinfo)r@datas r$rQzTavusVideoService._on_joineds 12scLK|d}tjd|d|yw)z'Handle participant leaving the session.idzParticipant left z , reason: NrW)r@ participantreasonparticipant_ids r$rSz&TavusVideoService._on_participant_lefts+$T* ''7z&JKs"$c\K|d}tjd||js{d|_|jj ||j dd{|jj ||j|jjd{yy7J7w)z'Handle participant joining the session.r[zParticipant joined TN)r^callback sample_rate) rrXr8r9capture_participant_video_on_participant_video_framecapture_participant_audio_on_participant_audio_dataout_sample_rate)r@r\r^s r$rRz(TavusVideoService._on_participant_joineds$T* ).)9:;1115D .,,88 @ @"  ,,88-88 LL889   2  s%AB,B(AB,!B*"B,*B,r^ video_frame video_sourcecKt|j|j|jf|j}||_|j r|j|d{yy7w)z/Handle incoming video frames from participants.)imagesizeformatN)rbufferwidthheight color_formattransport_sourcer? push_frame)r@r^rhriframes r$rdz-TavusVideoService._on_participant_video_framese$$$##[%7%78++  ".  //%( ( ( ! (sA A+"A)#A+audio audio_sourcecKt|j|j|j}||_|j r|j |d{yy7w)z-Handle incoming audio data from participants.)rurb num_channelsN)r audio_framesrbrxrrr?rs)r@r^rurvrts r$rfz,TavusVideoService._on_participant_audio_datasZ*$$))++  ".  //%( ( ( ! (sAAAAcy)zCheck if this service can generate processing metrics. Returns: True, as Tavus service supports metrics generation. Tr"r@s r$can_generate_metricsz&TavusVideoService.can_generate_metricss r#cRK|jjd{S7w)zsGet the name of the current persona. Returns: The persona name from the Tavus client. N)r9get_persona_namer{s r$r~z"TavusVideoService.get_persona_names! \\224444s '%'rtc4Kt||d{|jj|d{|jr-|jj |jd{|j d{y7y7X7!7 w)zStart the Tavus video service. Args: frame: The start frame containing initialization parameters. N)r2startr9r>register_audio_destination_create_send_taskr@rtrCs r$rzTavusVideoService.starts} gmE"""ll  '''  & &,,99$:U:UV V V$$&&& #' V&sCBB"BB8B2B3B B BBBBcKt||d{|jd{|jd{y7577 w)zWStop the Tavus video service. Args: frame: The end frame. N)r2stop_end_conversation_cancel_send_taskrs r$rzTavusVideoService.stopsL gl5!!!$$&&&$$&&& "&&1AA AAAAAAAcKt||d{|jd{|jd{y7577 w)z\Cancel the Tavus video service. Args: frame: The cancel frame. N)r2cancelrrrs r$rzTavusVideoService.cancelsL gnU###$$&&&$$&&& $&&r directioncvKt|||d{t|tr3|j d{|j ||d{yt|t r|j|d{yt|tr"d|_ |j ||d{yt|tr|jd{yt|tr|jd{y|j ||d{y77777{7T7-7w)zProcess frames through the service. Args: frame: The frame to process. direction: The direction of frame processing. NT)r2 process_frame isinstancer _handle_interruptionsrsr_handle_audio_framerr?rstart_ttfb_metricsr stop_ttfb_metrics)r@rtrrCs r$rzTavusVideoService.process_frames g#E9555 e. /,,. . .//%3 3 3 / 0**51 1 1 8 9$(D !//%3 3 3  /))+ + + 6 7((* * *//%3 3 3% 6 / 3 1 4 + + 3sD9D('D9D+D9D-)D9D/1D95D16(D9D3(D9D5D9"D7#D9+D9-D9/D91D93D95D97D9cK|jd{|jd{|jjd{y7?7)7 w)zHHandle interruption events by resetting send tasks and notifying client.N)rrr9send_interrupt_messager{s r$rz'TavusVideoService._handle_interruptions sM$$&&&$$&&&ll11333 '&3s1AAAA!AAAAAcbK|jjd{d|_y7 w)z-End the current conversation and reset state.NF)r9rr8r{s r$rz#TavusVideoService._end_conversations)ll!!!-2* "s /- /cK|js>tj|_|j |j |_yyw)z2Create the audio sending task if it doesn't exist.N)r=asyncioQueue_queue create_task_send_task_handlerr{s r$rz#TavusVideoService._create_send_tasks:!--/DK"..t/F/F/HIDOsA Ac~K|jr+|j|jd{d|_yy7 w)z+Cancel the audio sending task if it exists.N)r= cancel_taskr{s r$rz#TavusVideoService._cancel_send_tasks4 ??""4??3 3 3"DO  3s +=;=cZK|jj}t|dzdz }|jj |j |j |d{}|jj|t|j|k\rtt|jd|||j}|j|_|jj!|d{|j|d|_t|j|k\ryy775w)z*Process audio frames for sending to Tavus.N)rbrx)r9rgintr:resamplerurbr<extendlenrbytesrxr>transport_destinationrput)r@rtrb chunk_size resampledchunks r$rz%TavusVideoService._handle_audio_frame"sll22 +/R/0 //225;;@Q@QS^__  !!),$$$%3'd((*56'"//E +/*E*EE '++//%( ( (!%!3!3JK!@D $$$%3` )s+AD+D'BD+3D)40D+%D+)D+c K |jjd{}t|tr/|jr#|jj |d{|jj |7^7!w)z0Handle sending audio frames to the Tavus client.N)rgetrrr9write_audio_frame task_done)r@rts r$rz$TavusVideoService._send_task_handler5sa++//++E%!45$,,ll44U;;; KK ! ! # +;s!BB>B B! BB)*rrr r!rSettings__annotations__straiohttp ClientSessionrr3rrDrUrQrSrRrrdrrfboolr|r~rrr rr rr rrrrrrrrr __classcell__)rCs@r$r&r&1s? "H!!+15)&)& )&  )& && )&-.)& )&V(!4(8 3L  )! )0: )JM ) )! )*3 )CF )d55 ' ''''+'44>464 3 J # A/BA&$r#r&)(r!r dataclassesrtypingrr daily.dailyrrlogururpipecat.audio.utilsrpipecat.frames.framesr r r r r rrrrrrr"pipecat.processors.frame_processorrrpipecat.services.ai_servicerpipecat.services.settingsr"pipecat.transports.tavus.transportrrrrr&r"r#r$rsp !-7    S15``     J$ J$r#