
    qi                        d Z ddlZddlZddlZddlZddlmZmZ ddlmZ	 ddl
mZmZmZmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@ ddlAmBZBmCZCmDZDmEZEmFZF ddlGmHZH ddlImJZJ ddlKmLZLmMZM ddlNmOZO 	 ddlPmQZR e G d d             ZWe G d deC             ZX G d  d!e@      ZYy# eS$ r7ZT ej                  deT         ej                  d        eVdeT       dZT[Tww xY w)"zBOpenAI Realtime LLM service implementation with WebSocket support.    N)	dataclassfield)fields)AnyDictMappingOptionalType)logger)Image)ToolsSchema)OpenAIRealtimeLLMAdapter)AggregationTypeBotStoppedSpeakingFrameCancelFrameEndFrameFrameInputAudioRawFrameInputImageRawFrameInterimTranscriptionFrameInterruptionFrameLLMContextFrameLLMFullResponseEndFrameLLMFullResponseStartFrameLLMMessagesAppendFrameLLMSetToolsFrameLLMTextFrame
StartFrameTranscriptionFrameTTSAudioRawFrameTTSStartedFrameTTSStoppedFrameTTSTextFrameUserStartedSpeakingFrameUserStoppedSpeakingFrame)LLMTokenUsage)
LLMContext)LLMAssistantAggregatorParamsLLMUserAggregatorParams)LLMContextAggregatorPair)OpenAILLMContextOpenAILLMContextFrame)FrameDirection)FunctionCallFromLLM
LLMService)	NOT_GIVENLLMSettings	_NotGiven_warn_deprecated_paramis_given)Language)time_now_iso8601)traced_openai_realtime
traced_stt   )events)connectzException: zEIn order to use OpenAI, you need to `pip install pipecat-ai[openai]`.zMissing module: c                   >    e Zd ZU dZeed<   eed<   eed<   dZeed<   y)CurrentAudioResponseao  Tracks the current audio response from the assistant.

    Parameters:
        item_id: Unique identifier for the audio response item.
        content_index: Index of the audio content within the item.
        start_time_ms: Timestamp when the audio response started in milliseconds.
        total_size: Total size of audio data received in bytes. Defaults to 0.
    item_idcontent_indexstart_time_msr   
total_sizeN)__name__
__module____qualname____doc__str__annotations__intrA        V/opt/pipecat/venv/lib/python3.12/site-packages/pipecat/services/openai/realtime/llm.pyr=   r=   S   s$     LJrJ   r=   c                        e Zd ZU dZ ed       Zej                  ez  e	d<   e
dd       Zdd deeef   f fd	Zed
ed    deeef   dd fd       Z xZS )OpenAIRealtimeLLMSettingsaC  Settings for OpenAIRealtimeLLMService.

    Parameters:
        session_properties: OpenAI Realtime session properties (modalities,
            audio config, tools, etc.).  ``model`` and ``instructions`` are
            synced bidirectionally with the top-level ``model`` and
            ``system_instruction`` fields.
    c                      t         S N)r0   rI   rJ   rK   <lambda>z"OpenAIRealtimeLLMSettings.<lambda>p   s    	 rJ   )default_factorysession_propertiessettingsc                     t        | j                        sy| j                  }t        | j                        r| j                  | j                  |_        t        | j                        r| j                  |_        yy)zLPush top-level ``model``/``system_instruction`` into ``session_properties``.N)r4   rR   modelsystem_instructioninstructions)rS   sps     rK   _sync_top_level_to_spz/OpenAIRealtimeLLMSettings._sync_top_level_to_spu   s`     334((HNN#(B~~BHH//0&99BO 1rJ   deltareturnc                    t         |   |      }d|v rt        | j                        r| j                  }d|vr=|j                  1| j                  }|j                  | _        || j                  k7  r||d<   d|vr=|j
                  1| j                  }|j
                  | _        || j                  k7  r||d<   | j                  |        |S )a*  Merge a delta, keeping ``model``/``system_instruction`` in sync with SP.

        When the delta contains ``session_properties``, it **replaces** the
        stored SP wholesale (matching legacy behaviour).  Top-level field
        values always take precedence over conflicting SP values.
        rR   rU   rV   )superapply_updater4   rR   rU   rW   rV   rY   )selfrZ   changedrX   	old_modelold_si	__class__s         rK   r^   z&OpenAIRealtimeLLMSettings.apply_update   s     '&u-
  7*x8O8O/P((Bg%"((*> JJ	XX


*'0GG$#72r7R00*,//'T4444:G01
 	""4(rJ   clsc                    t        |       D ch c]  }|j                   c}dhz
  }i }i }i }t        t        j                  j
                  j                               dhz
  }|j                         D ]:  \  }}	| j                  j                  ||      }
|
|v r|	||
<   ,|
|v r|	||
<   6|	||<   < |rt        j                  di ||d<    | di |}||_
        |S c c}w )aD  Build a delta from a plain dict, routing SP keys into ``session_properties``.

        Keys that correspond to ``SessionProperties`` fields (except ``model``)
        are collected into a nested ``session_properties`` value.  ``model`` is
        always routed to the top-level field.  Unknown keys go to ``extra``.
        extrarU   rR   rI   )dataclass_fieldsnamesetr:   SessionPropertiesmodel_fieldskeysitems_aliasesgetrf   )rd   rS   fown_field_namestopsp_dictrf   sp_keyskeyvalue	canonicalinstances               rK   from_mappingz&OpenAIRealtimeLLMSettings.from_mapping   s     ,<C+@Aa166AWIM "$ " f..;;@@BCwiO"..* 	#JC((c2IO+!&Ig%%*	""c
	# (.(@(@(K7(KC$%::3 Bs   C)rS   rM   )rB   rC   rD   rE   r   rR   r:   rj   r2   rG   staticmethodrY   r   rF   r   r^   classmethodr
   r   ry   __classcell__rc   s   @rK   rM   rM   d   s     @E)@009<  : : "=  $sCx.  H #-.#:A#s(:K#	$# #rJ   rM   c                   V    e Zd ZU dZeZeed<   eZdddddddddde	d	e
e	   d
e	de
ej                     de
e   dedede	de
e   f fdZdefdZdefdZdefdZde	fdZde	defdZdee	   fdZde	fdZdef fdZdef fd Zdef fd!Zd" Zd# Zd$ Zd% Z 	 dad&e!d'e!d(e!de!fd)Z"d* Z#de$d+e%f fd,Z&d-e'fd.Z(d/ Z)d0ejT                  fd1Z+d2 Z,d3 Z-d4 Z. fd5Z/d6 Z0d7 Z1 e2d89      d:        Z3d; Z4d< Z5d= Z6d> Z7d? Z8d@ Z9e:	 dbdAe	dBedCe
e;   fdD       Z<dE Z=dFej|                  fdGZ? e2dH9      dI        Z@dJ ZAdK ZBdLe	fdMZCdN ZDdO ZEdP ZFdFej                  fdQZHdR ZIdS ZJ e2dT9      dU        ZKdVefdWZLdX ZMdeNfdYZOdZe	d[e	fd\ZP eQ        eR       d]d-eSd^eQd_eRdeTfd`ZU xZVS )cOpenAIRealtimeLLMServicea+  OpenAI Realtime LLM service providing real-time audio and text communication.

    Implements the OpenAI Realtime API with WebSocket communication for low-latency
    bidirectional audio and text interactions. Supports function calling, conversation
    management, and real-time transcription.
    	_settingsNz wss://api.openai.com/v1/realtimeFauto)rU   base_urlrR   rS   start_audio_pausedstart_video_pausedvideo_frame_detailsend_transcription_framesapi_keyrU   r   rR   rS   r   r   r   r   c       	            |	Fddl }|j                         5  |j                  d       |j                  dt        d       ddd       t        dddddddddddt        j                         	      }|t        d
t
        d
       ||_	        |Tt        dt
        d       ||_
        ||j                  |j                  |_	        |j                  |j                  |_        t
        j                  |       ||j                  |       | d|j                   }t        | @  d||d|
 || _        || _        || _        || _        || _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        i | _         i | _!        tE               | _#        | jI                  d       | jI                  d       i | _%        y# 1 sw Y   xY w)a  Initialize the OpenAI Realtime LLM service.

        Args:
            api_key: OpenAI API key for authentication.
            model: OpenAI model name.

                .. deprecated:: 0.0.105
                    Use ``settings=OpenAIRealtimeLLMSettings(model=...)`` instead.

                This is a connection-level parameter set via the WebSocket URL query
                parameter and cannot be changed during the session.
            base_url: WebSocket base URL for the realtime API.
                Defaults to "wss://api.openai.com/v1/realtime".
            session_properties: Configuration properties for the realtime session.
                If None, uses default SessionProperties.

                .. deprecated:: 0.0.105
                    Use ``settings=OpenAIRealtimeLLMSettings(session_properties=...)``
                    instead.
            settings: Runtime-updatable settings for this service.
            start_audio_paused: Whether to start with audio input paused. Defaults to False.
            start_video_paused: Whether to start with video input paused. Defaults to False.
            video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high".
                This sets the image_detail parameter in the OpenAI Realtime API.
                "auto" lets the model decide, "low" is faster and uses fewer tokens,
                "high" provides more detail. Defaults to "auto".
            send_transcription_frames: Whether to emit transcription frames.

                .. deprecated:: 0.0.92
                    This parameter is deprecated and will be removed in a future version.
                    Transcription frames are always sent.

            **kwargs: Additional arguments passed to parent LLMService.
        Nr   alwayszx`send_transcription_frames` is deprecated and will be removed in a future version. Transcription frames are always sent.   )
stacklevelzgpt-realtime-1.5F)rU   rV   temperature
max_tokenstop_ptop_kfrequency_penaltypresence_penaltyseedfilter_incomplete_user_turnsuser_turn_completion_configrR   rU   rR   z?model=)r   rS   Ton_conversation_item_createdon_conversation_item_updatedrI   )&warningscatch_warningssimplefilterwarnDeprecationWarningrM   r:   rj   r3   rU   rR   rW   rV   rY   r^   r]   __init__r   r   _audio_input_paused_video_input_paused_video_frame_detail_last_sent_time
_websocket_receive_task_context_llm_needs_conversation_setup_disconnecting_api_session_ready_run_llm_when_api_session_ready_current_assistant_response_current_audio_response_messages_added_manually_pending_function_callsri   _completed_tool_calls_register_event_handler#_retrieve_conversation_item_futures)r_   r   rU   r   rR   rS   r   r   r   r   kwargsr   default_settingsfull_urlrc   s                 rK   r   z!OpenAIRealtimeLLMService.__init__   s,   ` %0((* %%h/<& 	   5$#"!).(,%779
  "7,EwO%*")"$)$
 3E/ }!3!9!9!E);)A)A &!..:6H6U6U 3 	"778HI ))(3 Zw'7'='=&>? 	
%	
 	
  #5 #5 #5  !$(-1*#"'/4,+/('+$(*%')$%(U"$$%CD$$%CD350k s   *GG(r[   c                      y)zCheck if the service can generate usage metrics.

        Returns:
            True if metrics generation is supported.
        TrI   r_   s    rK   can_generate_metricsz-OpenAIRealtimeLLMService.can_generate_metricse  s     rJ   pausedc                     || _         y)zzSet whether audio input is paused.

        Args:
            paused: True to pause audio input, False to resume.
        N)r   r_   r   s     rK   set_audio_input_pausedz/OpenAIRealtimeLLMService.set_audio_input_pausedm       $* rJ   c                     || _         y)zzSet whether video input is paused.

        Args:
            paused: True to pause video input, False to resume.
        N)r   r   s     rK   set_video_input_pausedz/OpenAIRealtimeLLMService.set_video_input_pausedu  r   rJ   detailc                 N    |dvrt        j                  d| d       y|| _        y)zSet the detail level for video processing.

        Args:
            detail: Detail level - "auto", "low", or "high".
        )r   lowhighzInvalid video detail 'z#', must be 'auto', 'low', or 'high'N)r   warningr   )r_   r   s     rK   set_video_frame_detailz/OpenAIRealtimeLLMService.set_video_frame_detail}  s/     00NN3F8;^_`#) rJ   modalityc                 V    | j                   j                  j                  xs ddg}||v S )z;Check if a specific modality is enabled, "text" or "audio".audiotextr   rR   output_modalities)r_   r   
modalitiess      rK   _is_modality_enabledz-OpenAIRealtimeLLMService._is_modality_enabled  s-    ^^66HH]WV\L]
:%%rJ   c                 l    | j                   j                  j                  xs ddg}d|v rdgS d|v rdgS y)z#Get the list of enabled modalities.r   r   Nr   )r_   r   s     rK   _get_enabled_modalitiesz0OpenAIRealtimeLLMService._get_enabled_modalities  sF    ^^66HH]WV\L]
j 9z!8O "rJ   r>   c                 b  K   | j                         j                         }d}| j                  j                  |      sg | j                  |<   nd}| j                  |   j	                  |       |s1| j                  t        j                  |d|              d{    | d{   S 7 7 w)zRetrieve a conversation item by ID from the server.

        Args:
            item_id: The ID of the conversation item to retrieve.

        Returns:
            The retrieved conversation item.
        FTrci_)r>   event_idN)get_event_loopcreate_futurer   ro   appendsend_client_eventr:   ConversationItemRetrieveEvent)r_   r>   futureretrieval_in_flights       rK   retrieve_conversation_itemz3OpenAIRealtimeLLMService.retrieve_conversation_item  s      $$&446#77;;GD@BD44W="&009@@H"(( 44WQUV]U^O_`   | s$   BB/B+	B/&B-'B/-B/framec                 t   K   t         |   |       d{    | j                          d{    y7 7 w)zStart the service and establish WebSocket connection.

        Args:
            frame: The start frame triggering service initialization.
        N)r]   start_connectr_   r   rc   s     rK   r   zOpenAIRealtimeLLMService.start  s3      gmE"""mmo 	#   848688c                 t   K   t         |   |       d{    | j                          d{    y7 7 w)zStop the service and close WebSocket connection.

        Args:
            frame: The end frame triggering service shutdown.
        N)r]   stop_disconnectr   s     rK   r   zOpenAIRealtimeLLMService.stop  s6      gl5!!!    	" r   c                 t   K   t         |   |       d{    | j                          d{    y7 7 w)zCancel the service and close WebSocket connection.

        Args:
            frame: The cancel frame triggering service cancellation.
        N)r]   cancelr   r   s     rK   r   zOpenAIRealtimeLLMService.cancel  s6      gnU###    	$ r   c                   K   | j                   j                  j                  xrb | j                   j                  j                  j                  xr6 | j                   j                  j                  j                  j                  du }|rV| j                  t        j                                d {    | j                  t        j                                d {    | j                          d {    | j                          d {    | j                  rU| j                  t                      d {    | j                  d      r"| j                  t                      d {    y y y 7 7 7 7 l7 A7 w)NFr   )r   rR   r   inputturn_detectionr   r:   InputAudioBufferClearEventResponseCancelEvent _truncate_current_audio_responsestop_all_metricsr   
push_framer   r   r"   )r_   turn_detection_disableds     rK   _handle_interruptionz-OpenAIRealtimeLLMService._handle_interruption  s5     NN--33 V1177==V1177==LLPUU 	 
 #(()J)J)LMMM(()C)C)EFFF33555##%%%++//"9";<<<((1ooo&7888 2 ,	 NF5%< 9sl   B-E=/E10*E=E3E=2E53E=
E7,E=7E981E=)E;*E=3E=5E=7E=9E=;E=c                    K   y wrO   rI   r_   r   s     rK   _handle_user_started_speakingz6OpenAIRealtimeLLMService._handle_user_started_speaking  s	        c                   K   | j                   j                  j                  xrb | j                   j                  j                  j                  xr6 | j                   j                  j                  j                  j                  du }|rW| j                  t        j                                d {    | j                  t        j                                d {    y y 7 17 w)NF)	r   rR   r   r   r   r   r:   InputAudioBufferCommitEventResponseCreateEvent)r_   r   r   s      rK   _handle_user_stopped_speakingz6OpenAIRealtimeLLMService._handle_user_stopped_speaking  s      NN--33 V1177==V1177==LLPUU 	 
 #(()K)K)MNNN(()C)C)EFFF #NFs$   B-C%/C!0*C%C#C%#C%c                    K   d | _         y wrO   )r   r   s    rK   _handle_bot_stopped_speakingz5OpenAIRealtimeLLMService._handle_bot_stopped_speaking  s     '+$s   	total_bytessample_ratebytes_per_samplec                 2    ||z  }||z  }t        |dz        S )zGCalculate audio duration in milliseconds based on PCM audio parameters.  )rH   )r_   r   r   r   samplesduration_secondss         rK   _calculate_audio_duration_msz5OpenAIRealtimeLLMService._calculate_audio_duration_ms  s+      00"[0#d*++rJ   c           	        K   | j                   sy	 | j                   }d| _         | j                  |j                        }t        t	        j                         dz  |j
                  z
        }t        ||      }t        j                  d| d| d| d       | j                  t        j                  |j                  |j                  |             d{    y7 # t        $ r"}t        j                  d|        Y d}~yd}~ww xY ww)	a  Truncates the current audio response at the appropriate duration.

        Calculates the actual duration of the audio content and truncates at the shorter of
        either the wall clock time or the actual audio duration to prevent invalid truncation
        requests.
        Nr   zTruncating audio: duration=zms, elapsed=zms, truncate=ms)r>   r?   audio_end_msz%Audio truncation failed (non-fatal): )r   r   rA   rH   timer@   minr   tracer   r:   ConversationItemTruncateEventr>   r?   	Exceptionr   )r_   currentaudio_duration_ms
elapsed_mstruncate_mses         rK   r   z9OpenAIRealtimeLLMService._truncate_current_audio_response  s     ++	H22G+/D( !% A A'BTBT U TYY[4/'2G2GGHJj*;<KLL-.?-@ A%,mK=D
 ((44#OO")"7"7!,    	HNNB1#FGG	HsA   DCC CC DC 	D&D>DDD	directionc                 p  K   t         |   ||       d{    t        |t              rnt        |t        t
        f      rVt        |t              r|j                  nt        j                  |j                        }| j                  |       d{    ngt        |t              r(| j                  sJ| j                  |       d{    n/t        |t              r'| j                  s| j                  |       d{    nt        |t               r| j#                          d{    nt        |t$              r| j'                  |       d{    nt        |t(              r| j+                  |       d{    n{t        |t,              r| j/                          d{    nRt        |t0              r| j3                  |       d{    n(t        |t4              r| j7                          d{    | j9                  ||       d{    y7 7 7 W7 "7 7 7 7 7 \7 57 w)zProcess incoming frames from the pipeline.

        Args:
            frame: The frame to process.
            direction: The direction of frame flow in the pipeline.
        N)r]   process_frame
isinstancer   r   r,   contextr'   from_openai_context_handle_contextr   r   _send_user_audior   r   _send_user_videor   r   r$   r   r%   r   r   r   r   _handle_messages_appendr   _send_session_updater   )r_   r   r  r  rc   s       rK   r  z&OpenAIRealtimeLLMService.process_frame-  s     g#E9555e/01FGH e_5 33EMMB 
 &&w///12++++E22212++++E22201++---7844U;;;7844U;;;673355556..u555/0++---ooeY///= 	6 0 3 3-;;55-/s   H6HA;H6H7H6H"7H6H%(H6,H(-)H6H*)H6 H,(H6)H.*)H6H0(H6<H2=H6H4H6H6"H6%H6(H6*H6,H6.H60H62H64H6r  c                    K   | j                   s:|| _         | j                  d       d {    | j                          d {    y || _         | j                  d       d {    y 7 ?7 )7 	w)NFsend_new_resultsT)r   !_process_completed_function_calls_create_response)r_   r  s     rK   r  z(OpenAIRealtimeLLMService._handle_contextT  sp     }}#DM 88%8PPP''))) $DM88$8OOO Q) Ps3   )A1A+A1A-!A1%A/&A1-A1/A1c                 6   K   t        j                  d       y w)Nz%!!! NEED TO IMPLEMENT MESSAGES APPEND)r   errorr   s     rK   r  z0OpenAIRealtimeLLMService._handle_messages_appendd  s     <=s   eventc                 b   K   | j                  |j                  d             d{    y7 w)ztSend a client event to the OpenAI Realtime API.

        Args:
            event: The client event to send.
        T)exclude_noneN)_ws_send
model_dump)r_   r  s     rK   r   z*OpenAIRealtimeLLMService.send_client_eventk  s(      mmE,,$,?@@@s   %/-/c                 R  K   	 | j                   ry t        | j                  dd| j                   i       d {   | _         | j	                  | j                               | _        y 7 .# t        $ r0}| j                  d| |       d {  7   d | _         Y d }~y d }~ww xY ww)NAuthorizationzBearer )uriadditional_headerszError connecting: 	error_msg	exception)	r   websocket_connectr   r   create_task_receive_task_handlerr   r  
push_errorr_   r
  s     rK   r   z!OpenAIRealtimeLLMService._connects  s     	# $5MM#wt||n%=$% DO "&!1!1$2L2L2N!OD  	#//.@,DPQ/RRR"DOO	#sT   B'A+ B')A+ A)-A+ (B')A+ +	B$4BBBB'B$$B'c                   K   	 d| _         d| _        | j                          d {    | j                  r)| j                  j	                          d {    d | _        | j
                  r,| j                  | j
                  d       d {    d | _        t               | _        d| _         y 7 7 \7 &# t        $ r)}| j                  d| |       d {  7   Y d }~y d }~ww xY ww)NTFg      ?)timeoutzError disconnecting: r&  )r   r   r   r   closer   cancel_taskri   r   r  r,  r-  s     rK   r   z$OpenAIRealtimeLLMService._disconnect  s     	V"&D&+D#'')))oo++---"&!!&&t'9'93&GGG%)"),D&"'D *- H  	V//.CA3,GST/UUU	Vsn   C)!B4 B.-B4 B07B4 B2!B4 -C).B4 0B4 2B4 4	C&=C!CC!C)!C&&C)c                 V  K   	 | j                   sD| j                  r7| j                  j                  t        j                  |             d {    y y y 7 # t
        $ rF}| j                   s| j                  sY d }~y | j                  d| |       d {  7   Y d }~y d }~ww xY ww)NzError sending client event: r&  )r   r   sendjsondumpsr  r,  )r_   realtime_messager
  s      rK   r   z!OpenAIRealtimeLLMService._ws_send  s     	]&&4??oo**4::6F+GHHH ,;&H 	]""$//
 
 //.J1#,NZ[/\\\	]sY   B)A	A AA B)A 	B& B!8B)=B!BB!B)!B&&B)c                    K   t         |   |       d{   }ddh}|j                         |z  r| j                          d{    | j	                  |j                         |z
         |S 7 W7 *w)z=Apply a settings delta, sending a session update when needed.NrR   rV   )r]   _update_settingsrl   r   _warn_unhandled_updated_settings)r_   rZ   r`   handledrc   s       rK   r8  z)OpenAIRealtimeLLMService._update_settings  sm     077')=><<>G#++-----gllnw.FG 8 .s!   A2A..A2A0)A20A2c                   K   | j                   j                  }| j                         }| j                  r9|j	                  | j                        }|d   r
|d   |_        |d   r
|d   |_        |j
                  r:t        |j
                  t              r |j                  |j
                        |_        | j                  t        j                  |             d {    y 7 w)NtoolsrV   )session)r   rR   get_llm_adapterr   get_llm_invocation_paramsr<  rW   r  r   from_standard_toolsr   r:   SessionUpdateEvent)r_   rS   adapterllm_invocation_paramss       rK   r  z-OpenAIRealtimeLLMService._send_session_update  s     >>44,0,@,@,B==$+$E$Edmm$T! %W-!6w!? %%9:(=>R(S%
 >>jE$88HHN$$V%>%>x%PQQQs   CC$C"C$c                   K   | j                   2 3 d {   }t        j                  |      }|j                  dk(  r| j	                  |       d {    G|j                  dk(  r| j                  |       d {    p|j                  dk(  r| j                  |       d {    |j                  dk(  r| j                  |       d {    |j                  dk(  r| j                  |       d {    |j                  dk(  r| j                  |       d {    |j                  dk(  r| j                  |       d {    ?|j                  dk(  r| j                  |       d {    i|j                  d	k(  r| j                  |       d {    |j                  d
k(  r| j                  |       d {    |j                  dk(  r| j                  |       d {    |j                  dk(  r| j                  |       d {    |j                  dk(  r| j!                  |       d {    ;|j                  dk(  r| j#                  |       d {    e|j                  dk(  r| j%                  |       d {    |j                  dk(  s| j'                  |       d {   r|j(                  j*                  dv r0t-        j.                  |  d|j(                  j0                          | j3                  |       d {     y 7 7 7 7 7 n7 H7 "7 7 7 7 7 _7 87 7 7 7 7 66 y w)Nzsession.createdzsession.updatedzresponse.output_audio.deltazresponse.output_audio.donezconversation.item.addedzconversation.item.donez1conversation.item.input_audio_transcription.deltaz5conversation.item.input_audio_transcription.completedzconversation.item.retrievedzresponse.donez!input_audio_buffer.speech_startedz!input_audio_buffer.speech_stoppedzresponse.output_text.deltaz&response.output_audio_transcript.deltaz%response.function_call_arguments.doner  )response_cancel_not_active(conversation_already_has_active_response )r   r:   parse_server_eventtype_handle_evt_session_created_handle_evt_session_updated_handle_evt_audio_delta_handle_evt_audio_done#_handle_evt_conversation_item_added"_handle_evt_conversation_item_done+_handle_evt_input_audio_transcription_delta.handle_evt_input_audio_transcription_completed#_handle_conversation_item_retrieved_handle_evt_response_done_handle_evt_speech_started_handle_evt_speech_stopped_handle_evt_text_delta"_handle_evt_audio_transcript_delta(_handle_evt_function_call_arguments_done2_maybe_handle_evt_retrieve_conversation_item_errorr  coder   debugmessage_handle_evt_error)r_   r\  evts      rK   r+  z.OpenAIRealtimeLLMService._receive_task_handler  s    !__ *	 *	'++G4Cxx,,66s;;;..66s;;;::2237779911#66666>>sCCC55==cBBBPPFFsKKKTTII#NNN::>>sCCC_,44S999@@55c:::@@55c:::9911#666EE==cBBBDDCCCHHHW$!TTUXYYYyy~~ *  vQsyy/@/@.A%BC"44S999U*	 <;76CBKNC9::6BHY :Q -sd  M!ML-M9M!L0(M!9L3:(M!"L6#(M!L9(M!4L<5(M!L?)M!M)M!1M2)M!M)M!M)M!/M0)M!M)M!M)M!-M.)M!MM!/M!MM!
AM!&M'M!-M0M!3M!6M!9M!<M!?M!M!M!M!M!M!M!M!M!M!M!M!M!	llm_setup)	operationc                 @   K   | j                          d {    y 7 wrO   )r  r_   r^  s     rK   rJ  z4OpenAIRealtimeLLMService._handle_evt_session_created  s      '')))s   c                 v   K   d| _         | j                  r d| _        | j                          d {    y y 7 w)NTF)r   r   r  rb  s     rK   rK  z4OpenAIRealtimeLLMService._handle_evt_session_updated  s9     "&//38D0''))) 0)s   .979c                   K   | j                          d {    | j                  rT| j                  j                  |j                  k7  r1t        j                  d       t        j
                  d       d | _        | j                  sft        |j                  |j                  t        t        j                         dz              | _        | j                  t                      d {    t        j                  |j                        }| j                  xj                  t!        |      z  c_        t#        |dd      }| j                  |       d {    y 7 F7 v7 
w)NzpReceived a new audio delta for an already completed audio response before receiving the BotStoppedSpeakingFrame.z'Forcing previous audio response to Noner   )r>   r?   r@   ]  r9   )r   r   num_channels)stop_ttfb_metricsr   r>   r   r   r[  r=   r?   rH   r  r   r!   base64	b64decoderZ   rA   lenr    )r_   r^  r   r   s       rK   rL  z0OpenAIRealtimeLLMService._handle_evt_audio_delta  s     $$&&&''D,H,H,P,PTWT_T_,_NN C LLBC+/D(+++?!//!$))+"45,D(
 ///"3444  +$$//3u:=/ 

 ooe$$$/ 	' 5 	%s5   E#ECE#(E)A-E#E!E#E#!E#c                 l   K   | j                   r"| j                  t                      d {    y y 7 wrO   )r   r   r"   rb  s     rK   rM  z/OpenAIRealtimeLLMService._handle_evt_audio_done"  s,     ''///"3444 (4s   )424c                   K   |j                   j                  dk(  r}|j                   j                  | j                  vr.|j                   | j                  |j                   j                  <   n-t	        j
                  d|j                   j                   d       | j                  d|j                   j                  |j                          d{    | j                  j                  |j                   j                        r"| j                  |j                   j                  = y|j                   j                  dk(  r3|j                   | _        | j                  t                      d{    yy7 7 w)zQHandle conversation.item.added event - item is added but may still be processing.function_callzFunction call z already tracked, skippingr   N	assistant)itemrI  call_idr   r   r   _call_event_handleridr   ro   roler   r   r   rb  s     rK   rN  z<OpenAIRealtimeLLMService._handle_evt_conversation_item_added(  s    88==O+ xxt'C'CCAD,,SXX-=-=>0@0@/AA[\]&&'Esxx{{TWT\T\]]]
 ((,,SXX[[9--chhkk:88==K'/2xxD,//";"=>>> ( 	^ ?s%   CE4E0BE4)E2*E42E4c                    K   | j                  d|j                  j                  |j                         d{    y7 w)z>Handle conversation.item.done event - item is fully completed.r   N)rq  ro  rr  rb  s     rK   rO  z;OpenAIRealtimeLLMService._handle_evt_conversation_item_done?  s.     &&'Esxx{{TWT\T\]]]s   5?=?c                    K   | j                  t        |j                  dt               |      t        j
                         d {    y 7 w)N result)r  )r   r   rZ   r6   r-   UPSTREAMrb  s     rK   rP  zDOpenAIRealtimeLLMService._handle_evt_input_audio_transcription_deltaE  sA     oo%cii5E5GPST$--  
 	
 	
s   AAAA
transcriptis_finallanguagec                    K   yw)z+Handle a transcription result with tracing.NrI   )r_   rz  r{  r|  s       rK   _handle_user_transcriptionz3OpenAIRealtimeLLMService._handle_user_transcriptionL  s     
 	r   c                 V  K   | j                  d|j                  d       d{    | j                  t        |j                  dt               |      t        j                         d{    | j                  |j                  dt        j                         d{    y7 7 :7 	w)z{Handle completion of input audio transcription.

        Args:
            evt: The transcription completed event.
        r   Nrv  rw  T)rq  r>   r   r   rz  r6   r-   ry  r~  r5   ENrb  s     rK   rQ  zGOpenAIRealtimeLLMService.handle_evt_input_audio_transcription_completedS  s      &&'Es{{TXYYYoos~~r3C3EcR##
 	
 	

 --cnndHKKPPP 	Z	

 	Qs4   !B)B#AB)*B%+2B)B'B)%B)'B)r^  c                    K   | j                   j                  |j                  j                  d       }|r#|D ]  }|j	                  |j                          y y wrO   )r   popro  rr  
set_result)r_   r^  futuresr   s       rK   rR  z<OpenAIRealtimeLLMService._handle_conversation_item_retrievedb  sO     ::>>sxx{{DQ! ,!!#((+, s   AAllm_responsec                   K   t        |j                  j                  d      rJ|j                  j                  j                  r*|j                  j                  j                  j                  nd }t        |j                  j                  j                  |j                  j                  j                  |j                  j                  j                  |      }| j                  |       d {    | j                          d {    | j                  t                      d {    d | _        |j                  j                  dk(  r5| j                  |j                  j                   d   d          d {    y |j                  j"                  D ]'  }| j%                  d|j&                  |       d {    ) y 7 7 7 7 L7 w)Ninput_token_details)prompt_tokenscompletion_tokenstotal_tokenscache_read_input_tokensfailedr  r\  r'  r   )hasattrresponseusager  cached_tokensr&   input_tokensoutput_tokensr  start_llm_usage_metricsstop_processing_metricsr   r   r   statusr,  status_detailsoutputrq  rr  )r_   r^  r  tokensro  s        rK   rS  z2OpenAIRealtimeLLMService._handle_evt_response_doneh  su     s||))+@A""66 LL22@@ 	 	 ,,,,99!ll00>>++88$1	
 **6222**,,,oo57888+/(<<(*//CLL,G,G,PQZ,[/\\\LL'' 	ZD**+I477TXYYY	Z 	3,8 ] Zs[   C*G
,G -G
G G
%G&AG
9G:>G
8G9G
G
G
G
G
c                    K   |j                   r/t        |j                         }| j                  |       d {    y y 7 wrO   )rZ   r   r   )r_   r^  r   s      rK   rV  z/OpenAIRealtimeLLMService._handle_evt_text_delta  s7      99 +E//%((( (s   6A?Ac                 p   K   |j                   r$| j                  |j                          d {    y y 7 wrO   )rZ   #_push_output_transcript_text_framesrb  s     rK   rW  z;OpenAIRealtimeLLMService._handle_evt_audio_transcript_delta  s/      99::399EEE Es   +646r   c                    K   t        |      }d|_        | j                  |       d {    t        |t        j
                        }d|_        | j                  |       d {    y 7 @7 w)NF)aggregated_byT)r   append_to_contextr   r#   r   SENTENCEincludes_inter_frame_spaces)r_   r   llm_text_frametts_text_frames       rK   r  z<OpenAIRealtimeLLMService._push_output_transcript_text_frames  se      &d++0(oon--- &d/:R:RS592oon--- 	.
 	.s!   'A.A*:A.$A,%A.,A.c                   K   	 t        j                  |j                        }| j                  j	                  |j
                        }|r| j                  |j
                  = t        | j                  |j
                  |j                  |      g}| j                  |       d{    t        j                  d|j                          yt        j                  d|j
                          t        j                  dt        | j                  j                                       y7 # t        $ r"}t        j                   d|        Y d}~yd}~ww xY ww)zHandle completion of function call arguments.

        Args:
            evt: The response.function_call_arguments.done event.
        )r  tool_call_idfunction_name	argumentsNzProcessed function call: z,No tracked function call found for call_id: zAvailable pending calls: z+Failed to process function call arguments: )r4  loadsr  r   ro   rp  r.   r   rh   run_function_callsr   r[  r   listrl   r  r  )r_   r^  argsfunction_call_itemfunction_callsr
  s         rK   rX  zAOpenAIRealtimeLLMService._handle_evt_function_call_arguments_done  s    	L::cmm,D "&!=!=!A!A#++!N!00= ( $%([[&8&=&="&	" --n===89K9P9P8QRS!Mckk][\/T5Q5Q5V5V5X0Y/Z[	 >  	LLLFqcJKK	LsN   EBD) #D'$&D) 
EAD) &E'D) )	E2E
EEEc                    K   | j                          d {    | j                  t               d {    | j                          d {    y 7 :7 7 	wrO   )r   broadcast_framer$   broadcast_interruptionrb  s     rK   rT  z3OpenAIRealtimeLLMService._handle_evt_speech_started  sL     33555""#;<<<))+++ 	6<+s1   AAAAAAAAAc                    K   | j                          d {    | j                          d {    | j                  t               d {    y 7 :7 $7 	wrO   )start_ttfb_metricsstart_processing_metricsr  r%   rb  s     rK   rU  z3OpenAIRealtimeLLMService._handle_evt_speech_stopped  sL     %%'''++---""#;<<< 	(-<s1   AAAAAAAAAc                 8  K   |j                   j                  dk(  r}|j                   j                  j                  dd      d   }| j                  j                  |d      }|r5|D ]0  }|j                  t        |j                   j                               2 yyw)a7  Maybe handle an error event related to retrieving a conversation item.

        If the given error event is an error retrieving a conversation item:

        - set an exception on the future that retrieve_conversation_item() is waiting on
        - return true
        Otherwise:
        - return false
        item_retrieve_invalid_item_id_r9   NTF)	r  rZ  r   splitr   r  set_exceptionr  r\  )r_   r^  r>   r  r   s        rK   rY  zKOpenAIRealtimeLLMService._maybe_handle_evt_retrieve_conversation_item_error  s      99>><<ii((..sA6q9G>>BB7DQG% GF((3993D3D)EFGs   BBc                 J   K   | j                  d|        d {    y 7 w)NzError: r  )r,  rb  s     rK   r]  z*OpenAIRealtimeLLMService._handle_evt_error  s      oo'#o888s   #!#c                    K   t        j                  d       | j                          d{    d| _        | j	                  d       d{    | j                          d{    y7 >7 7 	w)zReset the conversation by disconnecting and reconnecting.

        This is the safest way to start a new conversation. Note that this will
        fail if called from the receive task.
        zResetting conversationNTFr  )r   r[  r   r   r  r   r   s    rK   reset_conversationz+OpenAIRealtimeLLMService.reset_conversation  sg      	-.    .2*44e4LLLmmo 	! 	Ms3   )A0A* A0A,A0$A.%A0,A0.A0llm_requestc                   K   | j                   sd| _        y | j                         }| j                  rt	        j
                  d|j                  | j                                |j                  | j                        }|d   }|D ]T  }t        j                  |      }d| j                  |j                  j                  <   | j                  |       d {    V | j                          d {    d| _        t	        j
                  d       | j!                  t#                      d {    | j%                          d {    | j'                          d {    | j                  t        j(                  t        j*                  | j-                                            d {    y 7 7 7 7 r7 \7 w)	NTzNSetting up conversation on OpenAI Realtime LLM service with initial messages: messagesro  FzCreating response)r   )r  )r   r   r>  r   r   r[  get_messages_for_loggingr   r?  r:   ConversationItemCreateEventr   ro  rr  r   r  r   r   r  r  r   ResponsePropertiesr   )r_   rB  rC  r  ro  r^  s         rK   r  z)OpenAIRealtimeLLMService._create_response   s    &&37D0,0,@,@,B --LL`ah  bB  bB  CG  CP  CP  bQ  aR  S
 %,$E$Edmm$T!,Z8H  288dC=A--chhkk:,,S1112 ++--- 27D.()oo79:::++---%%'''$$&&22TEaEaEcd
 	
 	
 2 . 	;-'	
sm   CF>F2F>0F41<F>-F6.F>F8F>F:AF>,F<-F>4F>6F>8F>:F><F>r  c                   K   d}| j                   j                         D ]  }|j                  d      s|j                  d      dk7  s*|j                  d      }|s>|| j                  vsM|r+d}| j	                  ||j                  d             d {    | j                  j                  |        |r| j                          d {    y y 7 =7 w)NFrs  contentIN_PROGRESSr  T)r   get_messagesro   r   _send_tool_resultaddr  )r_   r  sent_new_resultr\  r  s        rK   r  z:OpenAIRealtimeLLMService._process_completed_function_calls'  s     }}113 	AG{{6"w{{9'='N&{{>:L8R8R$R'*."44\7;;yCYZZZ..22<@	A '')))  [ *s:   1CC	CC,(CC6CCCCc                    K   t        j                  |j                        j                  d      }| j	                  t        j                  |             d {    y 7 w)Nutf-8)r   )rh  	b64encoder   decoder   r:   InputAudioBufferAppendEvent)r_   r   payloads      rK   r  z)OpenAIRealtimeLLMService._send_user_audio9  sD     ""5;;/66w?$$V%G%Gg%VWWWs   AA!AA!c           	      >  K   | j                   s| j                  s| j                  syt        j                         }|| j                  z
  dk  ry|| _        t        j                  d|        t        j                         }t        j                  |j                  |j                  |j                        j                  |d       t        j                   |j#                               j%                  d      }d| }t'        j(                  dd	t'        j*                  d
|| j,                        g      }	 | j/                  t'        j0                  |             d{    y7 # t2        $ r(}| j5                  d|        d{  7   Y d}~yd}~ww xY ww)zxSend user video frame to OpenAI Realtime API.

        Args:
            frame: The InputImageRawFrame to send.
        Nr9   z(Sending video frame to OpenAI Realtime: JPEG)formatr  zdata:image/jpeg;base64,r\  userinput_image)rI  	image_urlr   )rI  rs  r  r  zSend error: r  )r   r   r   r  r   r   r  ioBytesIOr   	frombytesr  sizeimagesaverh  r  getvaluer  r:   ConversationItemItemContentr   r   r  r  r,  )r_   r   nowbufferdatadata_uriro  r
  s           rK   r  z)OpenAIRealtimeLLMService._send_user_video=  sU     ##t':':$//iik%%%)"?wGH ejj%++>CCFSYCZ 1299'B -TF3 &&""&&33

	@(()K)KQU)VWWW 	@//l1#,>/???	@sN   D6F9(E) !E'"E) &F'E) )	F2F
FFFFFr  rx  c                    K   t        j                  d|t        j                  |            }| j	                  t        j
                  |             d {    y 7 w)Nfunction_call_output)rI  rp  r  r  )r:   r  r4  r5  r   r  )r_   r  rx  ro  s       rK   r  z*OpenAIRealtimeLLMService._send_tool_resulth  sI     &&' ::f%

 $$V%G%GT%RSSSs   AAAAuser_paramsassistant_paramsr  r  c                    t        j                  d       t        j                  |      }d|_        t        |||      S )a  Create an instance of OpenAIContextAggregatorPair from an OpenAILLMContext.

        NOTE: this method exists only for backward compatibility. New code
        should instead do::

            context = LLMContext(...)
            context_aggregator = LLMContextAggregatorPair(context)

        Constructor keyword arguments for both the user and assistant aggregators can be provided.

        Args:
            context: The LLM context.
            user_params: User aggregator parameters.
            assistant_params: Assistant aggregator parameters.

        Returns:
            OpenAIContextAggregatorPair: A pair of context aggregators, one for
            the user and one for the assistant, encapsulated in an
            OpenAIContextAggregatorPair.

        .. deprecated:: 0.0.99
            `create_context_aggregator()` is deprecated and will be removed in a future version.
            Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
            See `OpenAILLMContext` docstring for migration guide.
        aW  As of version 0.0.92, TranscriptionFrames and InterimTranscriptionFrames now go upstream from OpenAIRealtimeLLMService, so if you're using TranscriptProcessor, say, you'll want to adjust accordingly:

pipeline = Pipeline(
  [
    transport.input(),
    context_aggregator.user(),

    # BEFORE
    llm,
    transcript.user(),

    # AFTER
    transcript.user(),
    llm,

    transport.output(),
    transcript.assistant(),
    context_aggregator.assistant(),
  ]
)

Also, LLMTextFrames are no longer pushed from OpenAIRealtimeLLMService when it's configured with output_modalities=['audio']. Listen for TTSTextFrames instead.

Once you've made the appropriate changes (if needed), you can dismiss this warning by updating to the new context-setup pattern:

  context = LLMContext(messages, tools)
  context_aggregator = LLMContextAggregatorPair(context)
Fr  )r   r   r'   r  expect_stripped_wordsr*   )r_   r  r  r  s       rK   create_context_aggregatorz2OpenAIRealtimeLLMService.create_context_aggregatorp  sG    J 	I	
8 00916.'?O
 	
rJ   )re  r   rO   )WrB   rC   rD   rE   rM   SettingsrG   r   adapter_classrF   r	   r:   rj   boolr   r   r   r   r   r   r  r   r   r   r   r   r   r   r   r   r   r   r   rH   r   r   r   r-   r  r'   r  r  ClientEventr   r   r   r   r8  r  r+  r7   rJ  rK  rL  rM  rN  rO  rP  r8   r5   r~  rQ  ConversationItemRetrievedrR  rS  rV  rW  r  rX  rT  rU  
ErrorEventrY  r]  r  r  r  r  r   r  r  r)   r(   r+   r*   r  r|   r}   s   @rK   r   r      sU    )H(( -M  $:AE8<#(#("(48H6 H6 }	H6
 H6 %V%=%=>H6 45H6 !H6 !H6  H6 $,D>H6Td *T **T *	*S 	*&S &T &
c  > ! !!+ !9&
G, ST,,-0,LO,	,$HX%0 %0> %0NPZ P >AV-?-? A#"V ]"R:+Z k2* 3*
*%85?.^
 NR)-9A(9K Q,V=]=] , n5Z 6Z6)F.c .&%LN,
=
FL]L] &9 m4$
 5$
L* *$X)@,> )@VTC T T 0G/H9U9WE
!E
 -	E

 7E
 
"E
rJ   r   )ZrE   rh  r  r4  r  dataclassesr   r   r   rg   typingr   r   r   r	   r
   logurur   PILr   %pipecat.adapters.schemas.tools_schemar   2pipecat.adapters.services.open_ai_realtime_adapterr   pipecat.frames.framesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   pipecat.metrics.metricsr&   *pipecat.processors.aggregators.llm_contextr'   +pipecat.processors.aggregators.llm_responser(   r)   5pipecat.processors.aggregators.llm_response_universalr*   1pipecat.processors.aggregators.openai_llm_contextr+   r,   "pipecat.processors.frame_processorr-   pipecat.services.llm_servicer.   r/   pipecat.services.settingsr0   r1   r2   r3   r4   pipecat.transcriptions.languager5   pipecat.utils.timer6   (pipecat.utils.tracing.service_decoratorsr7   r8   rv  r:   websockets.asyncio.clientr;   r)  ModuleNotFoundErrorr
  r  r  r=   rM   r   rI   rJ   rK   <module>r     s$   I  	   ( 2 5 5   =      2 2 A [ > H  5 / W ,F     e e ePh
z h
A  ,FLL;qc"#FLLXY
&qc*
++,s   D D?2D::D?