qiLddlZddlmZmZddlmZddlmZmZddl m Z ddl m Z m Z mZmZGd d eZeGd d eZeGd deZeedZy)N)ABCabstractmethod)deque) attach_tracertraced)PagedAttentionCache)FutureRequestState RequestState RequestStatusloggerceZdZdZddedefdZedefdZ e de de d e e d zfd Zed efd Zedd eded d fdZed ed e e fdZed efdZedZed ed efdZedede d efdZdedeed e e fdZdede e de deed d f dZ d de ede de deeded ee e eff dZdeed d fdZy )! ScheduleraA Abstract base class for scheduling requests in the continuous batch processor. Schedulers manage the lifecycle of requests from when they are added to the waiting queue to when they are scheduled for processing. Different schedulers implement different strategies for prioritizing and batching requests. cacheretain_cache_on_finishc$i|_i|_t|_||_||_t j|_t|_ g|_ d|_ |jrd|_y|jj|_y)NF)active_requestswaiting_requestsrwaiting_requests_orderrr threadingLock_cancellation_lockset_requests_to_cancel_requests_to_forkblock_new_requestsnum_full_attention_groupsconfigsliding_windowcache_budget_module)selfrrs g/opt/pipecat/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/scheduler.py__init__zScheduler.__init__sx8:9;27'# &<#"+.."2-0U 57"'+0+J+J4 PUP\P\PkPk statec|jr|j|jvrn|jj|j}|jt |j d|_|j|_|j|_||j|j<|jj|jy)z#Adds a request to the waiting list.N) r request_idrpoptokens_to_processleninitial_tokensallocated_blocksposition_offsetrrappend)r"r& old_states r#add_waiting_requestzScheduler.add_waiting_request,s  & &5+;+;t?S?S+S,,001A1ABI&+&=&=I,,-/'E #&/%?%?E "$-$=$=E !27e../ ##**5+;+;$$U%5%56266u7G7GH,77 &&.DJJ$$U%5%56266u7G7GH||}777*/*H*H'13. -55 8||}4449>$$U%5%56,77 266u7G7GH-;LM-JE *&4]l&CE #r% candidates safety_margincg}d}||jjz}|D] } |jj} | |k} | r?|r=| jtj k7r t jd| d|||fS| j} |j| n| |jz} || kr|j| |} tt| |}|j| |}|sId}| dk(r@| j|jvr(t jd| jd||fS|j!| | ||t| j"}||z}|| z}|jj$rF| j|jj&z}||z}||jj&z}nd}| j( }|j+t-| ||| j}|jj/|ddu}|r|j1||dk(s|dk(s||fS||fS) a>Schedules candidate requests for the current batch. This method contains the common logic shared by all schedulers: it checks token and cache budgets, allocates cache blocks if needed, updates request states, and tracks which waiting requests should be removed from the waiting queue. FzJOutside safety margin, breaking out of scheduling loop. num_free_blocks = z safety_margins = NTrzBreaking mid-loop for request z because the cache is full)r num_blocksget_num_free_blocksrWr r]rinforMr!r`r\r+rSr(rrcr*allow_block_sharingrNrZr/r r)rD)r"rdr2r3rTrescheduled_requestsone_allocation_failedsafety_marginsr&num_free_blocksoutside_safety_margin cache_neededr_ request_lenallocation_successfultokens_in_current_blocktokens_after_forwardcomplete_blocks has_new_tokenreq_id was_waitings r#_process_candidateszScheduler._process_candidatess  %&)>)>>? E"jj<<>O$3n$D !$); P]PfPf@f aNbbudrcvwr"#888m!,,.L $ 8 8 @ lUYUmUmFm l*"77?abNc.1<@K$($C$CE;$W !)(,%#a'E,<,<@U@U,UKK"@AQAQ@RRl mnB"#888A  " "5.,Hj ke556K K 'L L (Lzz--*/*;*;*= @U@U*U''>'L$"6$**:O:O"O"#!& > >>M  % %&8 &_ `%%F//33FDAMK266v>q LA$5!#888C? B"#888r%cht|jDcgc] }||vs| c}|_ycc}w)z8Removes processed requests from the waiting queue order.N)rr)r"rTrws r#_cleanup_waiting_queuez Scheduler._cleanup_waiting_queues.&+"&"="= rOqAqV r' # rs //)F)T))__name__ __module__ __qualname____doc__r r9r$rr r1rintlistr r7r:strr>rBrErHrJrSrr`rcfloattupleryr{r6r%r#rrsE l1 l4 l  = =  =R3RcRdK]F^aeFeRR  MdM M 777PT7 7 CDI   535 5   -  -  s t     sW[ $<]`ad]ejnorjs2$D$DS $D $D -0H $D  $DX # S9&S9S9 S9 -0H S9  S9 t&'- .S9j S VZ r%rc\eZdZdZd dededeffd Zede de de e d zfd Z xZ S) FIFOScheduleraKThis scheduler processes requests in the order they arrive, meaning decoding requests has priority over prefilling requests. Additionally, it includes a safety margin mechanism to prevent cache exhaustion. By default, when 80% of the cache is full, new requests will not be scheduled to prioritize decoding active requests.rrrec4t|||||_y)a[Initializes the FIFO scheduler. The safety margin is the percentage of free blocks under which we stop scheduling new prefill requests, so safety_margin = 0.1 means that when there is less than 10% of free blocks, or equivalently when more than 90% of blocks are already allocated, we stop scheduling new prefill requests. N)superr$re)r"rrre __class__s r#r$zFIFOScheduler.__init__#s  67*r%r2r3r4Ncg}g}|jjD]`}|jtjk(r|j |2|jtj k(sP|j |b|js/|jD] }|j |j|"||z}t}|j|||||j\} } |j|| s| ry| S)Nre)rvaluesrWr r]r/r[rrrrryrer{ r"r2r3priority_statessecond_priority_statesr&rwrdrTrkrls r#r7zFIFOScheduler.schedule_batch+s.057))002 5E||}555&&u-!9!99&--e4  5&&55 M&--d.C.CF.KL M%'== -0U*484L4L    .,, 5M5 11 ##$FG"&;!!r%)Fg?)r}r~rrr r9rr$rrrr r7 __classcell__)rs@r#rrsZq+1+4+hm+  "3 "c "dK]F^aeFe "  "r%rc:eZdZdZedededeedzfdZy)PrefillFirstSchedulerzScheduler that prioritizes split prefill requests over decoding requests. This scheduler ensures that split prefill requests (which are continuations of partially processed prompts) are completed before processing new decoding requests.r2r3r4Ncg}g}|jjD]`}|jtjk(r|j |2|jtj k(sP|j |b|js/|jD] }|j |j|"||z}t}|j||||d\} } |j|| s| ry| S)Nr|r) rrrWr r[r/r]rrrrryr{rs r#r7z$PrefillFirstScheduler.schedule_batchWs .057))002 5E||}777&&u-!7!77&--e4  5&&55 M&--d.C.CF.KL M%'== -0U*484L4L    . 5M5 11 ##$FG"&;!!r%) r}r~rrrrrr r7r6r%r#rrQs< !"3!"c!"dK]F^aeFe!" !"r%r)fifo prefill_first)rabcrr collectionsr utils.metricsrrrr requestsr r r rrrrSCHEDULER_MAPPINGr6r%r#rsw#2&MMB B L."I."."f'"I'"'"V *r%