
    qiϨ              '          d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
m
Z
mZ ddlmZ ddlmZmZmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlm Z  erddl!m"Z"  ejF                  e$      Z%dZ&dZ'de(de)e(e(f   fdZ*e G d d             Z+e G d d             Z,e G d d             Z- ed       G d d             Z.e G d d             Z/e G d d             Z0e G d  d!             Z1de(de)e(e(f   fd"Z2de(de3fd#Z4e G d$ d%             Z5e G d& d'             Z6 G d( d)      Z7d*e(de8e)e(e(f      fd+Z9d,e(dee)e(e:e;f      fd-Z<d.d/d0e(d1e(dee)e(e:e;ef      fd2Z=d3e;de(fd4Z>dd5de(d6ed7   d8e:d9e;d:e:d;e;d<e(d=e(d>e3d?e3d@e3dAee   de5fdBZ?	 	 	 	 	 	 	 d\dDe(dEe(d.d/dFe3d?e3d>e3dGe3d@e3dHee7   dIee   de6fdJZ@dKe6ddfdLZAdKe6dMe(ddfdNZBdMe(de6fdOZCd]dKe6d.d/dPe3dIee   ddf
dQZDdKe6ddfdRZE	 	 d^dCdCdCdCdCddddddCdCdCddSdDee(   dEee(   d.d/dFe3d?e3d>e3dGe3d@e3dTee8e(      dUee8e(      dVee(   dKee(   dWee(   dXe3dPe3dYe3dZee3e(df   de6f$d[ZFy)_zzShared logic for bucket operations.

This module contains the core buckets logic used by both the CLI and the Python API.
    N)	dataclassfield)datetimetimezone)Path)TYPE_CHECKINGAnyIteratorLiteralOptionalUnion   )	constantslogging)BucketNotFoundError)XetFileDatadisable_progress_barsenable_progress_barsparse_datetime)
StatusLineHfApihf://buckets/  pathreturnc                     | j                  dd      }t        |      dk  s
|d   r|d   st        d|  d      |d    d|d    }t        |      dkD  r|d   nd}||fS )zSplit 'namespace/name(/optional/prefix)' into ('namespace/name', 'prefix').

    Returns (bucket_id, prefix) where prefix may be empty string.
    Raises ValueError if path doesn't contain at least namespace/name.
    /   r   r   zInvalid bucket path: 'z)'. Expected format: namespace/bucket_name )splitlen
ValueError)r   parts	bucket_idprefixs       J/opt/pipecat/venv/lib/python3.12/site-packages/huggingface_hub/_buckets.py_split_bucket_id_and_prefixr(   5   sy     JJsAE
5zA~U1XU1X1$7`abb8*AeAhZ(IUaU1XRFf    c                   J    e Zd ZU dZeed<   eed<   eed<   eed<   eed<   d Z	y)	
BucketInfoa  
    Contains information about a bucket on the Hub. This object is returned by [`bucket_info`] and [`list_buckets`].

    Attributes:
        id (`str`):
            ID of the bucket.
        private (`bool`):
            Is the bucket private.
        created_at (`datetime`):
            Date of creation of the bucket on the Hub.
        size (`int`):
            Size of the bucket in bytes.
        total_files (`int`):
            Total number of files in the bucket.
    idprivate
created_atsizetotal_filesc                 *   |j                  d      | _        |j                  d      | _        t        |j                  d            | _        |j                  d      | _        |j                  d      | _         | j                  j                  di | y )Nr,   r-   	createdAtr/   
totalFiles )	popr,   r-   r   r.   r/   r0   __dict__update)selfkwargss     r'   __init__zBucketInfo.__init__[   so    **T"zz),(K)@AJJv&	!::l3&v&r)   N)
__name__
__module____qualname____doc__str__annotations__boolr   intr:   r4   r)   r'   r+   r+   C   s)      	GM
I'r)   r+   c                       e Zd ZU eeeef   ed<   eed<    ed      Z	e
e   ed<    ed      Ze
e   ed<    ed      Zeed	<    ed      Ze
e   ed
<   ddZy)_BucketAddFilesourcedestinationN)defaultxet_hashr/   Finitmtimecontent_typec                    d | _         t        | j                  t        t        f      r't        j                  | j                        d   | _         | j                   't        j                  | j                        d   | _         t        t        | j                  t              s6t        j                  j                  | j                        dz        | _        y t        j                         dz        | _        y )Nr   r   )rL   
isinstancerE   r?   r   	mimetypes
guess_typerF   rB   bytesosr   getmtimetimerK   )r8   s    r'   __post_init__z_BucketAddFile.__post_init__n   s     dkkC;/ ) 4 4T[[ A! DD$ ) 4 4T5E5E Fq ID8B4;;PU8VBGGT[[)D0

\`\e\e\gjn\n

r)   r   N)r;   r<   r=   r   r?   r   rQ   r@   r   rH   r   r/   rB   rK   rL   rU   r4   r)   r'   rD   rD   d   sg    #tU"###D1Hhsm1-D(3--E"E3""'U"3L(3-3	
r)   rD   c                       e Zd ZU eed<   y)_BucketDeleteFiler   N)r;   r<   r=   r?   r@   r4   r)   r'   rX   rX   z   s    
Ir)   rX   T)frozenc                   &    e Zd ZU dZeed<   eed<   y)BucketFileMetadataa%  Data structure containing information about a file in a bucket.

    Returned by [`get_bucket_file_metadata`].

    Args:
        size (`int`):
            Size of the file in bytes.
        xet_file_data (`XetFileData`):
            Xet information for the file (hash and refresh route).
    r/   xet_file_dataN)r;   r<   r=   r>   rB   r@   r   r4   r)   r'   r[   r[      s    	 Ir)   r[   c                       e Zd ZU dZeed<   dZeed<    ed      Zeed<    ed      Z	eed<    ed      Z
eed	<   ddZy
)	BucketUrla  Describes a bucket URL on the Hub.

    `BucketUrl` is returned by [`create_bucket`]. At initialization, the URL is parsed to populate properties:
    - endpoint (`str`)
    - namespace (`str`)
    - bucket_id (`str`)
    - url (`str`)
    - handle (`str`)

    Args:
        url (`str`):
            String value of the bucket url.
        endpoint (`str`, *optional*):
            Endpoint of the Hub. Defaults to <https://huggingface.co>.
    urlr    endpointFrI   	namespacer%   handleNc                    | j                   xs t        j                  | _         | j                  j	                  | j                   d      j                  d      }|j                  d      r|t        d      d  }t        |      \  }}|rt        d| j                         |j                  d      d   | _        || _        d| j                   | _        y )Nr    r   zbuckets/zUnable to parse bucket URL: r   r   )r`   r   ENDPOINTr_   replacestrip
startswithr"   r(   r#   r!   ra   r%   rb   )r8   url_pathr%   r&   s       r'   rU   zBucketUrl.__post_init__   s    ;);); 88##DMM26<<SAz*J 12H7A	6;DHH:FGG"-a0"%dnn%56r)   rV   )r;   r<   r=   r>   r?   r@   r`   r   ra   r%   rb   rU   r4   r)   r'   r^   r^      sJ      
HHc&Is&&Is&U#FC#7r)   r^   c                   f    e Zd ZU dZed   ed<   eed<   eed<   eed<   ee	   ed<   ee	   ed<   d	 Z
y
)
BucketFilez
    Contains information about a file in a bucket on the Hub. This object is returned by [`list_bucket_tree`].

    Similar to [`RepoFile`] but for files in buckets.
    filetyper   r/   rH   rK   uploaded_atc                 X   |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  dd       }|rt        |      nd | _        |j                  dd       }|rt        |      | _        y d | _        y )Nrl   r   r/   xetHashrK   
uploadedAt)r5   rl   r   r/   rH   r   rK   rm   )r8   r9   rK   rm   s       r'   r:   zBucketFile.__init__   s    JJv&	JJv&	JJv&	

9-

7D).3^E*
jjt4:E>+64r)   N)r;   r<   r=   r>   r   r@   r?   rB   r   r   r:   r4   r)   r'   rj   rj      s>     &/
I
IMH(##Pr)   rj   c                   B    e Zd ZU dZed   ed<   eed<   ee   ed<   d Z	y)BucketFolderz
    Contains information about a directory in a bucket on the Hub. This object is returned by [`list_bucket_tree`].

    Similar to [`RepoFolder`] but for directories in buckets.
    	directoryrl   r   rm   c                    |j                  d      | _        |j                  d      | _        |j                  dd       xs |j                  dd       }|r)t        |t              r|| _        y t        |      | _        y d | _        y )Nrl   r   rp   rm   )r5   rl   r   rN   r   r   rm   )r8   r9   rm   s      r'   r:   zBucketFolder.__init__   s|    JJv&	JJv&	jjt4W

=RV8W  '{H=[ 	CQR]C^ 	  	r)   N)
r;   r<   r=   r>   r   r@   r?   r   r   r:   r4   r)   r'   rr   rr      s)     +

I(##
r)   rr   c                     | j                  t              st        d|  dt               t        | j	                  t                    S )zParse a bucket path like hf://buckets/namespace/bucket_name/prefix into (bucket_id, prefix).

    Returns:
        tuple: (bucket_id, prefix) where bucket_id is "namespace/bucket_name" and prefix may be empty string.
    zInvalid bucket path: z. Must start with )rg   BUCKET_PREFIXr#   r(   removeprefixr   s    r'   _parse_bucket_pathry      s@     ??=)06HXYY&t'8'8'GHHr)   c                 ,    | j                  t              S )z!Check if a path is a bucket path.)rg   rv   rx   s    r'   _is_bucket_pathr{      s    ??=))r)   c                       e Zd ZU dZed   ed<   eed<   dZee	   ed<   dZ
eed<   dZee   ed	<   dZee   ed
<   dZee   ed<   y)SyncOperationz,Represents a sync operation to be performed.)uploaddownloaddeleteskipactionr   Nr/   r    reasonlocal_mtimeremote_mtimebucket_file)r;   r<   r=   r>   r   r@   r?   r/   r   rB   r   r   r   r   rj   r4   r)   r'   r}   r}     sY    6:;;
ID(3-FC!%K#%"&L(3-&(,K*%,r)   r}   c                   r    e Zd ZU dZeed<   eed<   eed<    ee      Zee	   ed<   de
eeeef   f   fdZy	)
SyncPlanz Represents a complete sync plan.rE   dest	timestamp)default_factory
operationsr   c                 *   t        d | j                  D              }t        d | j                  D              }t        d | j                  D              }t        d | j                  D              }t        d | j                  D              }|||||dS )Nc              3   @   K   | ]  }|j                   d k(  sd  yw)r~   r   Nr   .0ops     r'   	<genexpr>z#SyncPlan.summary.<locals>.<genexpr>       KBRYY(5JaK   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  s     ObryyJ7NOr   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  r   r   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  s     G"2993FAGr   c              3   Z   K   | ]#  }|j                   d v s|j                  xs d % yw)r~   r   r   N)r   r/   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  s&     g"299PfCfAgs   ++)uploads	downloadsdeletesskips
total_size)sumr   )r8   r   r   r   r   r   s         r'   summaryzSyncPlan.summary  s    K$//KKODOOOO	K$//KKGGGggg
"$
 	
r)   N)r;   r<   r=   r>   r?   r@   r   listr   r}   dictr   rB   r   r4   r)   r'   r   r     sF    *K
IN&+D&AJ]#A
c5c?23 
r)   r   c                   j    e Zd ZdZ	 	 	 d
deee      deee      deeeeef         fdZdede	fd	Z
y)FilterMatcherz4Matches file paths against include/exclude patterns.Ninclude_patternsexclude_patternsfilter_rulesc                 F    |xs g | _         |xs g | _        |xs g | _        y)a  Initialize the filter matcher.

        Args:
            include_patterns: Patterns to include (from --include)
            exclude_patterns: Patterns to exclude (from --exclude)
            filter_rules: Rules from filter file as list of ("+"/"-", pattern) tuples
        Nr   r   r   )r8   r   r   r   s       r'   r:   zFilterMatcher.__init__1  s+     !1 6B 0 6B(.Br)   r   r   c                 $   | j                   D ]"  \  }}t        j                  ||      s|dk(  c S  | j                  D ]  }t        j                  ||      s y | j                  D ]  }t        j                  ||      s y | j                  ryy)zCheck if a path should be included based on the filter rules.

        Filtering rules:
        - Filters are evaluated in order, first matching rule decides
        - If no rules match, include by default (unless include patterns are specified)
        +FT)r   fnmatchr   r   )r8   r   signpatterns       r'   matcheszFilterMatcher.matchesB  s     ".. 	#MD'tW-s{"	#
 ,, 	GtW-	 ,, 	GtW-	
    r)   )NNN)r;   r<   r=   r>   r   r   r?   tupler:   rA   r   r4   r)   r'   r   r   .  sh    > 15048<	/"49-/ #49-/ tE#s(O45	/"C D r)   r   filter_filec                    g }t        |       5 }|D ]  }|j                         }|r|j                  d      r'|j                  d      r%|j                  d|dd j                         f       ]|j                  d      r%|j                  d|dd j                         f       |j                  d|f        	 ddd       |S # 1 sw Y   |S xY w)zParse a filter file and return a list of (sign, pattern) tuples.

    Filter file format:
    - Lines starting with "+" are include patterns
    - Lines starting with "-" are exclude patterns
    - Empty lines and lines starting with "#" are ignored
    #r   r   N-)openrf   rg   append)r   rulesflines       r'   _parse_filter_filer   _  s     E	k	 *a 
	*D::<D4??3/s#c48>>#345%c48>>#345 c4[)
	** L* Ls   B+CC
local_pathc              #   >  K   t         j                  j                  |       } t         j                  j                  |       st	        d|        t        j
                  |       D ]  \  }}}|D ]  }t         j                  j                  ||      }t         j                  j                  ||       }|j                  t         j                  d      }t         j                  j                  |      }t         j                  j                  |      dz  }|||f   yw)zsList all files in a local directory.

    Yields:
        tuple: (relative_path, size, mtime_ms) for each file
    z Local path must be a directory: r   r   N)rR   r   abspathisdirr#   walkjoinrelpathre   sepgetsizerS   )	r   root_filesfilename	full_pathrel_pathr/   mtime_mss	            r'   _list_local_filesr   |  s      ,J77==$;J<HII''*- +a 	+HT84Iwwy*=H''4H77??9-Dww''	2T9HD(**	++s   DDapir   r%   r&   c              #     K   | j                  ||xs dd      D ]  }t        |t              r|j                  }|rH|j	                  |dz         r|t        |      dz   d }n$||k(  rd|v r|j                  dd      d   n|}nj|}|j                  r|j                  j                         dz  nd}||j                  ||f  yw)	zList all files in a bucket with a given prefix.

    Yields:
        tuple: (relative_path, size, mtime_ms, bucket_file) for each file.
            bucket_file is the BucketFile object from list_bucket_tree.
    NT)r&   	recursiver   r   r   r   )
list_bucket_treerN   rr   r   rg   r"   rsplitrK   r   r/   )r   r%   r&   itemr   r   r   s          r'   _list_remote_filesr     s      $$Yv~QU$V 2dL)yy v|,Fa 1269Tk4;;sA.r2t H48JJ4::'')D0A		8T11'2s   CCr   c                 n    t        j                  | dz  t        j                        j	                         S )z3Convert mtime in milliseconds to ISO format string.r   )tz)r   fromtimestampr   utc	isoformat)r   s    r'   _mtime_to_isor     s&    !!(T/hllCMMOOr)   )r   r   r   source_sizesource_mtime	dest_size
dest_mtimesource_newer_labeldest_newer_labelignore_sizesignore_timesignore_existingr   c                    t        |dk(  r|n|      }t        |dk(  r|n|      }| |||d}|
rt        dddd|S ||k7  }||z
  t        kD  }|r1|rt        d|||d|S ||z
  t        kD  }|r|nd}t        dd|d|S |	r|rt        d|d|d|S t        ddd	d|S |s|r|rdn|}t        d|||d|S t        ddd
d|S )a  Compare source and dest files and return the appropriate sync operation.

    This is a unified helper for both upload and download directions.

    Args:
        path: Relative file path
        action: "upload" or "download"
        source_size: Size of the source file (bytes)
        source_mtime: Mtime of the source file (milliseconds)
        dest_size: Size of the destination file (bytes)
        dest_mtime: Mtime of the destination file (milliseconds)
        source_newer_label: Label when source is newer (e.g., "local newer" or "remote newer")
        dest_newer_label: Label when dest is newer (e.g., "remote newer" or "local newer")
        ignore_sizes: Only compare mtime
        ignore_times: Only compare size
        ignore_existing: Skip files that exist on receiver
        bucket_file: BucketFile object (for downloads only)

    Returns:
        SyncOperation describing the action to take
    r~   )r   r/   r   r   r   z&exists on receiver (--ignore-existing))r   r   )r   r   r   z
same mtimezsize differsz	same size	identicalr4   )r   r}   _SYNC_TIME_WINDOW_MS)r   r   r   r   r   r   r   r   r   r   r   r   local_mtime_isoremote_mtime_isobase_kwargssize_differssource_newer
dest_newerskip_reasonr   s                       r'   _compare_files_for_syncr     s6   H $Fh4FLJWO$6X3EZ<X &(	#K kF3[k_jkk)+L :-1EEL r7IWbrfqrr$|37KKJ.8*lK R{RkRR	 n~S^nbmnn R{RkRR<'3^9KF fv;fZeff R{RkRRr)   FrE   r   r   existingfilter_matcherstatusc
                    |xs
 t               }t        |        xr t        |      }
t        |       xr t        |       }|
s|st        d      t        | |t	        j
                  t        j                        j                               }d}|
rt        j                  j                  |       }t        |      \  }}t        j                  j                  |      st        d|       i }t        |      D ]?  \  }}}|j                  |      r||f||<   |	s"|	j!                  dt#        |       d       A |	r|	j%                  dt#        |       d       i }|	r	 |j'                  |      j(                  }	 t-        |||      D ]K  \  }}}}|j                  |      r||f||<   |	s#|d| nd}|	j!                  d	t#        |       | d       M 	 |	r|	j%                  d	t#        |       d       t5        |j7                               t5        |j7                               z  }|	r|	j%                  dt#        |       d       t9        |      D ]  }|j;                  |      }|j;                  |      }|rv|st|r9|j<                  j?                  tA        d||d   dtC        |d                      e|j<                  j?                  tA        d||d   dtC        |d                      |r<|r:|\  }}|\  }}|j<                  j?                  tE        |d||||dd|||             |r|s|s|j<                  j?                  tA        d||d   dtC        |d                       |S t        |       \  }}t        j                  j                  |      }i }i } |	r	 |j'                  |      j(                  }t-        |||      D ]P  \  }}}}!|j                  |      r||f||<   |!| |<   |	s(|d| nd}|	j!                  d	t#        |       | d       R |	r|	j%                  d	t#        |       d       i }t        j                  j                  |      rMt        |      D ]?  \  }}}|j                  |      r||f||<   |	s"|	j!                  dt#        |       d       A |	r|	j%                  dt#        |       d       t5        |j7                               t5        |j7                               z  }|	r|	j%                  dt#        |       d       t9        |      D ]>  }|j;                  |      }|j;                  |      }|r|s|r9|j<                  j?                  tA        d||d   dtC        |d                      e|j<                  j?                  tA        d||d   dtC        |d         | j;                  |                   |rL|rJ|\  }}|\  }}|j<                  j?                  tE        |d||||dd|||| j;                  |                   |r|s|s|j<                  j?                  tA        d||d   dtC        |d                      A |S # t*        $ r Y w xY w# t.        $ r t0        j3                  d
| d       Y w xY w# t*        $ r Y w xY w)zCompute the sync plan by comparing source and destination.

    Returns:
        SyncPlan with all operations to be performed
    z[One of source or dest must be a bucket path (hf://buckets/...) and the other must be local.rE   r   r   NzSource must be a directory: zScanning local directory (z files)r   r    zScanning remote bucket (zBucket 'z' not found, treating as empty.zComparing files (z paths)r   r   znew file (--existing)r   )r   r   r/   r   r   r~   znew filezlocal newerzremote newer)r   r   r   r   r   r   r   r   r   r   r   r   znot in source (--delete))r   r   r/   r   r   r   )r   r   r/   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )#r   r{   r#   r   r   nowr   r   r   rR   r   r   ry   r   r   r   r7   r"   donebucket_infor0   	Exceptionr   r   loggerdebugsetkeyssortedgetr   r   r}   r   r   )"rE   r   r   r   r   r   r   r   r   r   	is_uploadis_downloadplanremote_totalr   r%   r&   local_filesr   r/   r   remote_filesr   	total_str	all_pathsr   
local_inforemote_info
local_sizer   remote_sizer   bucket_file_mapr   s"                                     r'   _compute_sync_planr     s   " $6}N#F++E0EI!&)G/$2G.GK[vww,,x||,668D #'LWW__V,
.t4	6ww}}Z(;J<HII (9*(E 	V$HdH%%h/)-x(8H% :3{;K:LGTU		V
 KK4S5E4FgNO"y9EE		P/A#yRX/Y d+$!!))(3.2H-=L*6B6N!L> 2TVIMM$<S=N<OPY{Za"bcd KK23|3D2EWMN ((*+c,2C2C2E.FF	KK+C	N+;7CD9% 8	D$.J&**40K+OO**%#)!%!+A#:(5jm(D OO**%#+!%!+A#-(5jm(D *4'
K,7)\&&+!'$.%0"-#/+8)7%1%1(7  KF&&!'!(^9%2;q>%Ba8	v K /v6	6WW__T*
 *,"y9EE 6HYX^5_ 	`1HdHk%%h/*.)9X&,7)2>2Ja~.PR	 8\9J8KI;V]^_	` KK23|3D2EWMN77==$,=j,I Z($!))(3-18,<K)MM$>s;?O>PPW"XY	Z
 KK4S5E4FgNO ))+,s;3C3C3E/FF	KK+C	N+;7CD9% :	D&**40K$.J:OO**%#)!%!,Q#:)6{1~)F OO**%#-!%!,Q#-)6{1~)F(7(;(;D(A	 ,7)\*4'
K&&+!)$/%1",#.+9)6%1%1(7$3$7$7$=  !ZF&&!'!']9$1*Q-$@e:	x K]   # 	PLL8I;.MNO	P^  s<    Y7 </Z ,+Z ,Z0 7	ZZ"Z-,Z-0	Z=<Z=r   c                    d| j                   | j                  | j                  | j                         d}|j	                  t        j                  |      dz          | j                  D ]  }d|j                  |j                  |j                  d}|j                  |j                  |d<   |j                  |j                  |d<   |j                  |j                  |d	<   |j	                  t        j                  |      dz           y)
z1Write a sync plan as JSONL to a file-like object.header)rl   rE   r   r   r   
	operation)rl   r   r   r   Nr/   r   r   )rE   r   r   r   writejsondumpsr   r   r   r   r/   r   r   )r   r   r  r   op_dicts        r'   _write_planr    s     ++		^^<<>F GGDJJv%& oo ,iiGGii	#
 77 ggGFO>>%%'^^GM"??&&(ooGN#	

7#d*+,r)   	plan_filec                 ^    t        |d      5 }t        | |       ddd       y# 1 sw Y   yxY w)z!Save a sync plan to a JSONL file.wN)r   r  )r   r  r   s      r'   
_save_planr    s-    	i	 D!  s   #,c                 f   t        |       5 }|j                         }ddd       st        d|        t        j                  |d         }|j                  d      dk7  rt        d      t        |d   |d   |d	   
      }|dd D ]  }t        j                  |      }|j                  d      dk7  r-|j                  j                  t        |d   |d   |j                  d      |j                  dd      |j                  d      |j                  d                    |S # 1 sw Y   xY w)z#Load a sync plan from a JSONL file.NzEmpty plan file: r   rl   r  z0Invalid plan file: expected header as first linerE   r   r   r   r   r  r   r   r/   r   r    r   r   )r   r   r/   r   r   r   )
r   	readlinesr#   r  loadsr   r   r   r   r}   )r  r   linesr  r   r   r  s          r'   
_load_planr    s1   	i A ,YK899 ZZa!Fzz&X%KLLhF^%D ab	 
**T";;v+-x(V_[[({{8R0#KK6$[[8		
	
 KC s   D&&D0verbosec                    t        | j                         xr t        | j                        }t        | j                        xr t        | j                         }|r5t        j                  j                  | j                        }t        | j                        \  }}|j                  d      }g }	g }
| j                  D ];  }|j                  dk(  rt        j                  j                  ||j                        }|r| d|j                   n|j                  }|r&t        d|j                   d|j                   d       |	j                  ||f       |j                  dk(  rZ|r| d|j                   n|j                  }|r&t        d|j                   d|j                   d       |
j                  |       |j                  dk(  s|st        d	|j                   d|j                   d       > |	s|
r|rpg }|	r|j                  d
t        |	       d       |
r|j                  dt        |
       d       |j                  dj                  |      j!                                |j#                  ||	xs d|
xs d       yy|r t        | j                        \  }}|j                  d      }t        j                  j                  | j                        }t        j$                  |d       g }g }| j                  D ]  }|j                  dk(  rt        j                  j                  ||j                        }t        j$                  t        j                  j'                  |      d       |r&t        d|j                   d|j                   d       |j(                  |j                  |j(                  |f       |r| d|j                   n|j                  }|j                  ||f       |j                  dk(  ret        j                  j                  ||j                        }|r&t        d|j                   d|j                   d       |j                  |       j|j                  dk(  s{|st        d	|j                   d|j                   d        t        |      dkD  r2|r|j                  dt        |       d       |j+                  ||       |r |r|j                  dt        |       d       |D ]  }t        j                  j-                  |      s#t        j.                  |       t        j                  j'                  |      }||k7  s]	 t        j0                  |       t        j                  j'                  |      }||k7  r; yy# t2        $ r Y w xY w)zExecute a sync plan.r   r~   z  Uploading: z ()r   z  Deleting: r   z  Skipping: z
uploading z filesz	deleting z, N)addr   T)exist_okr   z  Downloading: r   zDownloading z	Deleting z local files)r{   rE   r   rR   r   r   ry   rstripr   r   r   printr   r   r"   r   
capitalizebatch_bucket_filesmakedirsdirnamer   download_bucket_filesexistsremovermdirOSError)r   r   r  r   r   r   r   r%   r&   	add_filesdelete_pathsr   
local_fileremote_pathr$   download_filesdelete_files	file_pathparents                      r'   _execute_planr6  9  s   #DKK00O_TYY5OI!$++.Qtyy7Q3QKWW__T[[1
.tyy9	6s# @B	"$// 	>ByyH$WW\\*bgg>
7="''3277M"''"RYYKqAB  *k!:;h&7="''3277L	BII;a@A##K0f$RWWIR		{!<=	>  LL:c)n-=V!DELL9S->,?v!FGDIIe,779:""%#+t #  % 
.t{{;	6s#WW__TYY/
 	J. QS"$// 	>ByyJ&WW\\*bgg>
BGGOOJ7$GOBGG9BryykCD>>-"))2>>:*FG;AVHAbggY"7rwwK"));
*CDh&WW\\*bgg>
L	BII;a@A##J/f$RWWIR		{!<='	>, ~"l3~+>*?vFG%%i@ lKK)C$5#6lCD% 
	Iww~~i(		)$3
*(!#!8 
*
	U 
f # s   94V77	WWc                     | j                         }t        d| j                   d| j                          t        d|d           t        d|d           t        d|d           t        d	|d
           y)z!Print a summary of the sync plan.zSync plan: z -> z  Uploads: r   z  Downloads: r   z  Deletes: r   z	  Skips: r   N)r   r$  rE   r   )r   r   s     r'   _print_plan_summaryr8    s~    llnG	K}D
45	K	*+
,-	M'+./
01	K	*+
,-	Igg&'
()r)   )r   r   r   r   r   includeexcludefilter_fromr   applydry_runr  quiettokenr9  r:  r;  r<  r=  r>  r?  c                d   |ddl m}  ||      }|r| s|rt        d      |t        d      |rt        d      |rt        d      |rt        d	      |rt        d
      |	rt        d      |
rt        d      |rt        d      |rt        d      |rt        d      t        |      }t	        |       }|st        |       t        d       |r
t                	 t        ||||       |rt                	 |st        d       |S | r|st        d      t        |       }t        |      }|r|rt        d      |s|st        d      |r|rt        d      |r|rt        d      |r|rt        d      |rLt        j                  j                  |      rZt        j                  j                  |      s;t        d|       t        j                  j                  |       st        d|        d}|
rt        |
      }t!        ||	|      }t	        | xr |       }t#        | |||||||||
      }|rt%        |t&        j(                         |S |r)t+        ||       |st        |       t        d|        |S |st        |       |j-                         }|d   d k(  r|d!   d k(  r|d"   d k(  r|st        d#       |S |st        d$       |r
t                	 t        ||||       |rt                	 |st        d       |S # |rt                w w xY w# |rt                w w xY w)%a  Sync files between a local directory and a bucket.

    This is equivalent to the ``hf buckets sync`` CLI command. One of ``source`` or ``dest`` must be a bucket path
    (``hf://buckets/...``) and the other must be a local directory path.

    Args:
        source (`str`, *optional*):
            Source path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``.
            Required unless using ``apply``.
        dest (`str`, *optional*):
            Destination path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``.
            Required unless using ``apply``.
        api ([`HfApi`]):
            The HfApi instance to use for API calls.
        delete (`bool`, *optional*, defaults to `False`):
            Delete destination files not present in source.
        ignore_times (`bool`, *optional*, defaults to `False`):
            Skip files only based on size, ignoring modification times.
        ignore_sizes (`bool`, *optional*, defaults to `False`):
            Skip files only based on modification times, ignoring sizes.
        existing (`bool`, *optional*, defaults to `False`):
            Skip creating new files on receiver (only update existing files).
        ignore_existing (`bool`, *optional*, defaults to `False`):
            Skip updating files that exist on receiver (only create new files).
        include (`list[str]`, *optional*):
            Include files matching patterns (fnmatch-style).
        exclude (`list[str]`, *optional*):
            Exclude files matching patterns (fnmatch-style).
        filter_from (`str`, *optional*):
            Path to a filter file with include/exclude rules.
        plan (`str`, *optional*):
            Save sync plan to this JSONL file instead of executing.
        apply (`str`, *optional*):
            Apply a previously saved plan file. When set, ``source`` and ``dest`` are not needed.
        dry_run (`bool`, *optional*, defaults to `False`):
            Print sync plan to stdout as JSONL without executing.
        verbose (`bool`, *optional*, defaults to `False`):
            Show detailed per-file operations.
        quiet (`bool`, *optional*, defaults to `False`):
            Suppress all output and progress bars.
        token (Union[bool, str, None], optional):
            A valid user access token. If not provided, the locally saved token will be used.

    Returns:
        [`SyncPlan`]: The computed (or loaded) sync plan.

    Raises:
        `ValueError`: If arguments are invalid (e.g., both paths are remote, conflicting options).

    Example:
        ```python
        >>> from huggingface_hub import HfApi
        >>> api = HfApi()

        # Upload local directory to bucket
        >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket")

        # Download bucket to local directory
        >>> api.sync_bucket("hf://buckets/username/my-bucket", "./data")

        # Sync with delete and filtering
        >>> api.sync_bucket(
        ...     "./data",
        ...     "hf://buckets/username/my-bucket",
        ...     delete=True,
        ...     include=["*.safetensors"],
        ... )

        # Dry run: preview what would be synced
        >>> plan = api.sync_bucket("./data", "hf://buckets/username/my-bucket", dry_run=True)
        >>> plan.summary()
        {'uploads': 3, 'downloads': 0, 'deletes': 0, 'skips': 1, 'total_size': 4096}

        # Save plan for review, then apply
        >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket", plan="sync-plan.jsonl")
        >>> api.sync_bucket(apply="sync-plan.jsonl")
        ```
    Nr   r   )r?  z,Cannot specify source/dest when using apply.z#Cannot specify both plan and apply.z'Cannot specify delete when using apply.z-Cannot specify ignore_times when using apply.z-Cannot specify ignore_sizes when using apply.z(Cannot specify include when using apply.z(Cannot specify exclude when using apply.z,Cannot specify filter_from when using apply.z)Cannot specify existing when using apply.z0Cannot specify ignore_existing when using apply.z(Cannot specify dry_run when using apply.)enabledzExecuting plan...)r  r   zSync completed.z7Both source and dest are required (unless using apply).z?Remote to remote sync is not supported. One path must be local.z?One of source or dest must be a bucket path (hf://buckets/...).z2Cannot specify both ignore_times and ignore_sizes.z1Cannot specify both existing and ignore_existing.z%Cannot specify both dry_run and plan.z!Destination must be a directory: z&Source must be an existing directory: r   )
rE   r   r   r   r   r   r   r   r   r   zPlan saved to: r   r   r   r   zNothing to sync.z
Syncing...)hf_apir   r#   r  r   r8  r$  r   r6  r   r{   rR   r   r*  r   r   r   r  r  sysstdoutr  r   )rE   r   r   r   r   r   r   r   r9  r:  r;  r   r<  r=  r  r>  r?  r   	sync_planr   source_is_bucketdest_is_bucketr   r   r   s                            r'   sync_bucket_internalrH    sa   F !% TKLLBCCFGGLMMLMMGHHGHHKLLHIIOPPGHHu%	I.	*%&!#	')S'&I$&#$ RSS&v.$T*NNZ[[NZ[[MNNOLMM4@AA 77>>$d(;@GHHww}}V$EfXNOO L)+6"  !N E	 9'k:F"!!'%I Iszz*9d#	*OD6*+ I&!GyQ7;#71#<ASWXAX$%l#igfE " G $& z  " s   "L #L  L L/)FFFFFNN)FN)NN)Gr>   r   r  rO   rR   rC  rT   dataclassesr   r   r   r   pathlibr   typingr   r	   r
   r   r   r   r    r   r   errorsr   utilsr   r   r   r   utils._terminalr   rB  r   
get_loggerr;   r   rv   r   r?   r   r(   r+   rD   rX   r[   r^   rj   rr   ry   rA   r{   r}   r   r   r   r   rB   floatr   r   r   r   r  r  r  r  r6  r8  rH  r4   r)   r'   <module>rQ     sB  
    	 
  ( '  I I   ' [ [ '  
		H	%   c eCHo  ' ' '@ 
 
 
*    $    %7 %7 %7P P P P2 
 
 
6IS IU38_ I*# *$ * 	- 	- 	- 
 
 
8. .bC DsCx,A :+# +(5c53I*J +*2G 2 2S 2XeTWY\^cehThNiEj 2DPE Pc P$ "&ES
ES ()ES 	ES
 ES ES ES ES ES ES ES ES #ES ESX !.2 cc
c 
c 	c
 c c c c ]+c SMc cV,h ,d ,:X # $ ## #( #V` `w ` `xX[} `hl `F*h *4 *  !n
 !#'#'!%$(%nSMn
3-n 
	n
 n n n n n d3i n d3i n #n 3-n C=n n  !n" #n$ sD!%n& 'nr)   