
    qig/                     >   d Z ddlZddlZddlmZ ddlZddlmZ ddl	m
Z
 ddlmZmZmZmZmZmZmZmZ ddlmZmZmZ 	 ddlmZ dd	lmZ dd
lmZmZ  G d de      Z#y# e$ r7Z  ejB                  de          ejB                  d        e"de        dZ [ ww xY w)z4Simli video service for real-time avatar generation.    N)Optional)logger)	BaseModel)CancelFrameEndFrameFrameInterruptionFrameOutputImageRawFrameTTSAudioRawFrameTTSStoppedFrameUserStartedSpeakingFrame)FrameDirectionFrameProcessor
StartFrame)
AudioFrame)AudioResampler)SimliClientSimliConfigzException: zCIn order to use Simli, you need to `pip install pipecat-ai[simli]`.zMissing module: c                        e Zd ZdZ G d de      Zdddddddddd	ee   d
ee   dee   de	de
dede	dee   f fdZd Zd Zd Zdedef fdZd Z xZS )SimliVideoServicea  Simli video service for real-time avatar generation.

    Provides real-time avatar video generation by processing audio frames
    and producing synchronized video output using the Simli API. Handles
    audio resampling, video frame processing, and connection management.
    c                   N    e Zd ZU dZdZee   ed<   dZee	   ed<   dZ
ee	   ed<   y)SimliVideoService.InputParamsa  Input parameters for Simli video configuration.

        Parameters:
            enable_logging: Whether to enable Simli logging.
            max_session_length: Absolute maximum session duration in seconds.
                Avatar will disconnect after this time even if it's speaking.
            max_idle_time: Maximum duration in seconds the avatar is not speaking
                before the avatar disconnects.
        Nenable_loggingmax_session_lengthmax_idle_time)__name__
__module____qualname____doc__r   r   bool__annotations__r   intr        N/opt/pipecat/venv/lib/python3.12/site-packages/pipecat/services/simli/video.pyInputParamsr   /   s4    	 *.-,0HSM0'+x}+r$   r&   NFr   zhttps://api.simli.ai)api_keyface_idsimli_configuse_turn_serverlatency_interval	simli_urlis_trinity_avatarparamsr'   r(   r)   r*   r+   r,   r-   r.   c                   t        |   di |	 |xs t        j                         }|.||t	        d      t        j                  dt        d       |}
n_|t	        d      |t	        d      d|i}|j                  |j                  |d	<   |j                  |j                  |d
<   t        di |}
|rt        j                  dt        d       d| _        |
xj                  dz  c_        |
xj                  dz  c_        t        ||
|d      | _        d| _        t#        j$                         | _        t)        ddd      | _        d| _        d| _        || _        || _        t5               | _        y)a  Initialize the Simli video service.

        Args:
            api_key: Simli API key for authentication.
            face_id: Simli Face ID. For Trinity avatars, specify "faceId/emotionId"
                to use a different emotion than the default.
            simli_config: Configuration object for Simli client settings.
                Use api_key and face_id instead.

                .. deprecated:: 0.0.92
                    The 'simli_config' parameter is deprecated and will be removed in a future version.
                    Please use 'api_key' and 'face_id' parameters instead.

            use_turn_server: Whether to use TURN server for connection. Defaults to False.

                .. deprecated:: 0.0.95
                    The 'use_turn_server' parameter is deprecated and will be removed in a future version.

            latency_interval: Latency interval setting for sending health checks to check
                the latency to Simli Servers. Defaults to 0.
            simli_url: URL of the simli servers. Can be changed for custom deployments
                of enterprise users.
            is_trinity_avatar: Boolean to tell simli client that this is a Trinity avatar
                which reduces latency when using Trinity.
            params: Additional input parameters for session configuration.
            **kwargs: Additional arguments passed to the parent FrameProcessor.
        NzrCannot specify both simli_config and api_key/face_id. Please use api_key and face_id (simli_config is deprecated).zThe 'simli_config' parameter is deprecated and will be removed in a future version. Please use 'api_key' and 'face_id' parameters instead, with optional 'params' for max_session_length and max_idle_time configuration.   )
stacklevelzapi_key is requiredzface_id is requiredfaceIdmaxSessionLengthmaxIdleTimezVThe 'use_turn_server' parameter is deprecated and will be removed in a future version.F   T)r'   configsimliURL	enableSFUs16monoi>  r#   )super__init__r   r&   
ValueErrorwarningswarnDeprecationWarningr   r   r   _initializedr4   r3   r   _simli_client_pipecat_resamplerasyncioEvent_pipecat_resampler_eventr   _simli_resampler_audio_task_video_task_is_trinity_avatar_previously_interrupted	bytearray_audio_buffer)selfr'   r(   r)   r*   r+   r,   r-   r.   kwargsr6   config_kwargs	__class__s               r%   r<   zSimliVideoService.__init__>   s   P 	"6":,88: #"g&9 S 
 MMF # "F  !677 !677
 'M ((44:4M4M01##//5/C/Cm, 1=1FMMh" "a1$(	
 37(/% .ufe D)-)-"3'8$&[r$   c                   K   	 | j                   s)| j                  j                          d{    d| _         | j                  j                          d{    | j	                  | j                               | _        | j	                  | j                               | _        y7 v7 O# t        $ r)}| j                  d| |       d{  7   Y d}~yd}~ww xY ww)zAStart the connection to Simli service and begin processing tasks.NTzUnable to start connection: 	error_msg	exception)rA   rB   startsendSilencecreate_task_consume_and_process_audiorH   _consume_and_process_videorI   	Exception
push_error)rN   es     r%   _start_connectionz#SimliVideoService._start_connection   s     
	]$$((..000$(! $$00222#//0O0O0QRD#//0O0O0QRD 1 3  	]//.J1#,NZ[/\\\	]s]   C)B( B$(B( B&AB( #C$B( &B( (	C1C
CCCCCc           	        K   | j                   j                          d{    | j                  j                         }|2 3 d{   }| j                  j                  |      }|D ]j  }|j                         }|j                         s$| j                  t        |j                         | j                  j                  d             d{    l 7 7 7 6 yw)z9Consume audio frames from Simli and push them downstream.N   )audiosample_ratenum_channels)rF   waitrB   getAudioStreamIteratorrC   resample
to_ndarrayany
push_framer   tobytesrate)rN   audio_iteratoraudio_frameresampled_framesresampled_frameaudio_arrays         r%   rY   z,SimliVideoService._consume_and_process_audio   s     ++00222++BBD!/ 	 	+#66??L#3 
-88:??$//("-"5"5"7(,(?(?(D(D)*  	
	 	3	 "0sM   CC CCCC	ACACCCCCCc                   K   | j                   j                          d{    | j                  j                  d      }|2 3 d{   }t	        |j                         j                         j                         |j                  |j                  fd      }|j                  |_
        | j                  |       d{    7 7 7 	6 yw)zBConsume video frames from Simli and convert them to output frames.Nrgb24)targetFormatRGB)imagesizeformat)rF   rd   rB   getVideoStreamIteratorr
   to_rgbto_imagerj   widthheightptsri   )rN   video_iteratorvideo_frameconvertedFrames       r%   rZ   z,SimliVideoService._consume_and_process_video   s     ++00222++BBPWBX!/ 	2 	2+2E!((*335==?!''););<3N
 "-N//.111 	3	2 2 "0sF   CC"CCCCA4C?C	 CC	CCframe	directionc                   K   t         	|   ||       d{    t        |t              r| j	                          d{    nFt        |t
              r8	 t        j                  t        j                  |j                  t        j                        dddf   |j                  dk(  rdnd      }|j                  |_        | j                  @t        d|j                   |j                        | _        | j"                  j%                          | j&                  j)                  |      }|D ]\  }|j+                         j-                  t        j                        j/                         }| j0                  r| j2                  j5                  |       t7        | j2                        dk\  s	 | j&                  j)                  d      D ]V  }| j2                  j5                  |j+                         j-                  t        j                        j/                                X 	 | j8                  j;                  | j2                         d{    d	| _        t=               | _        :| j8                  j?                  |       d{    _ yt        |tD              ri	 | j0                  r[t7        | j2                        dkD  rC| j8                  j;                  | j2                         d{    d	| _        t=               | _        yt        |tF        tH        f      r| jK                          d{    nUt        |tL        tN        f      r?| j0                  s"| j8                  jQ                          d{    | jR                  | _        | jU                  ||       d{    y7 7 k7 b# | j8                  j;                  | j2                         d{  7   d	| _        t=               | _        w xY w7 s# t@        $ r)}| jC                  d
| |       d{  7   Y d}~d}~ww xY w7 E# t@        $ r)}| jC                  d| |       d{  7   Y d}~yd}~ww xY w7 87 7 ̭w)zProcess incoming frames and handle Simli video generation.

        Args:
            frame: The frame to process.
            direction: The direction of frame processing.
        N)dtyper`   r:   stereo)layoutr9   i  FzError sending audio: rS   r   zError stopping TTS: )+r;   process_frame
isinstancer   r^   r   r   from_ndarraynp
frombufferra   int16rc   rb   rC   r   r   rF   setrG   rf   rg   astyperj   rK   rM   extendlenrB   playImmediaterL   sendr[   r\   r   r   r   _stopr	   r   clearBufferrJ   ri   )
rN   r   r   	old_framern   ro   
audioBytes
flushFramer]   rQ   s
            r%   r   zSimliVideoService.process_frame   s     g#E9555eZ(((***/0 Z&33MM%++RXX>tQwG%*%7%71%<6(	 ).(9(9	%**2.<y//1F1F/D+ 11557#'#8#8#A#A)#L '7 BO!0!;!;!=!D!DRXX!N!V!V!XJ33**11*=t112f<A262G2G2P2PQU2V !&J$($6$6$=$=(2(=(=(?(F(Frxx(P(X(X(Z%&!&
 '+&8&8&F&FtGYGY&Z Z Z?D <5>[ 2"0055jAAAB   /Y//C8J8J4Ka4O,,::4;M;MNNN38D0)2D& +67**, 13KLM//((44666+/+B+BD(ooeY///m 	6*8 ![d&8&8&F&FtGYGY&Z Z Z?D <5>[ 2A Zoo2Gs0KWXoYYYZ
 O  Yoo2Fqc0JVWoXXXY  7 	0s  Q0N''Q0N* Q0EO< .A4N0#(O< N-:O< O9O< Q0 AP4 ,P1-P4 *Q01Q)2AQ06Q,7*Q0!Q."Q0*Q0-O< 0)O6O
O66O< <	P.P)P!P)$Q0)P..Q01P4 4	Q&=Q!QQ!Q0!Q&&Q0,Q0.Q0c                 6  K   | j                   j                          d{    | j                  r*| j                  | j                         d{    d| _        | j                  r+| j                  | j                         d{    d| _        yy7 r7 E7 w)z2Stop the Simli client and cancel processing tasks.N)rB   stoprH   cancel_taskrI   )rN   s    r%   r   zSimliVideoService._stop  s       %%'''""4#3#3444#D""4#3#3444#D 	 	(4 5s3   BB.BB5BBBBB)r   r   r   r   r   r&   r   strr   r    r"   r<   r^   rY   rZ   r   r   r   r   __classcell__)rQ   s   @r%   r   r   '   s    ,i ,$ "&!%.2 % !/"'(,k) #k) #	k)
 {+k) k) k) k)  k) %k)Z]$2=0 =0> =0~$r$   r   )$r   rD   r>   typingr   numpyr   logurur   pydanticr   pipecat.frames.framesr   r   r   r	   r
   r   r   r   "pipecat.processors.frame_processorr   r   r   av.audio.framer   av.audio.resamplerr   simlir   r   ModuleNotFoundErrorr]   errorr[   r   r#   r$   r%   <module>r      s    ;      	 	 	 Z Y,)1.y$ y$  ,FLL;qc"#FLLVW
&qc*
++,s    A   B%2BB