
     jx              &          d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZmZ ddlmZmZ ddlmZ ddlmZmZmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% erddl&m'Z'  ejP                  e)      Z*dZ+dZ,e G d d             Z-e G d d             Z.e G d d             Z/e G d d             Z0 ed       G d d             Z1e G d d             Z2e G d d              Z3e G d! d"             Z4d#e5d$efd%Z6d#e5d$e7fd&Z8e G d' d(             Z9e G d) d*             Z: G d+ d,      Z;d-e5d$e<e=e5e5f      fd.Z>d#e5d$e=e?e@f   dz  fd/ZAd0e5d$e	e=e5e?e@f      fd1ZBd2d3d4e5d5e5d$e	e=e5e?e@ef      fd6ZCd7e@d$e5fd8ZDdd9d#e5d:ed;   d<e?d=e@d>e?d?e@d@e5dAe5dBe7dCe7dDe7dEedz  d$e9fdFZE	 	 	 	 	 	 	 d`dHe5dIe5d2d3dJe7dCe7dBe7dKe7dDe7dLe;dz  dMedz  d$e:fdNZFdOe:d$dfdPZGdOe:dQe5d$dfdRZHdQe5d$e:fdSZIdadOe:d2d3dTe7dMedz  d$df
dUZJdOe:d$dfdVZK	 	 dbdGdGdGdGdGddddddGdGdGddWdHe5dz  dIe5dz  d2d3dJe7dCe7dBe7dKe7dDe7dXe<e5   dz  dYe<e5   dz  dZe5dz  dOe5dz  d[e5dz  d\e7dTe7d]e7d^e7e5z  dz  d$e:f$d_ZLy)czzShared logic for bucket operations.

This module contains the core buckets logic used by both the CLI and the Python API.
    N)Iterator)	dataclassfield)datetimetimezone)Path)TYPE_CHECKINGAnyLiteral   )	constantslogging)_validate_relative_filename)BucketNotFoundError)HfUri
StatusLineXetFileDatadisable_progress_barsenable_progress_barsparse_datetimeparse_hf_uri)_looks_like_hf_urlHfApizhf://buckets/  c                   J    e Zd ZU dZeed<   eed<   eed<   eed<   eed<   d Z	y)	
BucketInfoa  
    Contains information about a bucket on the Hub. This object is returned by [`bucket_info`] and [`list_buckets`].

    Attributes:
        id (`str`):
            ID of the bucket.
        private (`bool`):
            Is the bucket private.
        created_at (`datetime`):
            Date of creation of the bucket on the Hub.
        size (`int`):
            Size of the bucket in bytes.
        total_files (`int`):
            Total number of files in the bucket.
    idprivate
created_atsizetotal_filesc                 *   |j                  d      | _        |j                  d      | _        t        |j                  d            | _        |j                  d      | _        |j                  d      | _         | j                  j                  di | y )Nr   r   	createdAtr!   
totalFiles )	popr   r   r   r    r!   r"   __dict__update)selfkwargss     d/home/min/hermes_workspace/demo_review/venv/lib/python3.12/site-packages/huggingface_hub/_buckets.py__init__zBucketInfo.__init__W   so    **T"zz),(K)@AJJv&	!::l3&v&    N)
__name__
__module____qualname____doc__str__annotations__boolr   intr-   r&   r.   r,   r   r   ?   s)      	GM
I'r.   r   c                       e Zd ZU eez  ez  ed<   eed<    ed      Zedz  ed<    ed      Z	e
dz  ed<    ed      Ze
ed	<    ed      Zedz  ed
<   ddZy)_BucketAddFilesourcedestinationNdefaultxet_hashr!   Finitmtimecontent_typec                    d | _         t        | j                  t        t        f      r't        j                  | j                        d   | _         | j                   't        j                  | j                        d   | _         t        t        | j                  t              s6t        j                  j                  | j                        dz        | _        y t        j                         dz        | _        y Nr   r   )rA   
isinstancer9   r3   r   	mimetypes
guess_typer:   r6   bytesospathgetmtimetimer@   r*   s    r,   __post_init__z_BucketAddFile.__post_init__j   s     dkkC;/ ) 4 4T[[ A! DD$ ) 4 4T5E5E Fq ID8B4;;PU8VBGGT[[)D0

\`\e\e\gjn\n

r.   returnN)r/   r0   r1   r3   r   rG   r4   r   r=   r!   r6   r@   rA   rM   r&   r.   r,   r8   r8   `   sd    $J .HcDj.T*D#**E"E3"$%0L#*0	
r.   r8   c                       e Zd ZU eed<   eed<   eed<   eed<    ed      Zedz  ed<    ed	      Zeed
<    ed	      Z	edz  ed<   ddZ
y)_BucketCopyFiler:   r=   source_repo_typesource_repo_idNr;   r!   Fr>   r@   rA   c                     t        j                  | j                        d   | _        t	        t        j
                         dz        | _        y rC   )rE   rF   r:   rA   r6   rK   r@   rL   s    r,   rM   z_BucketCopyFile.__post_init__   s7    %001A1AB1Et+,
r.   rN   )r/   r0   r1   r3   r4   r   r!   r6   r@   rA   rM   r&   r.   r,   rQ   rQ   v   sS    MT*D#**E"E3"$%0L#*0-r.   rQ   c                       e Zd ZU eed<   y)_BucketDeleteFilerI   N)r/   r0   r1   r3   r4   r&   r.   r,   rV   rV      s    
Ir.   rV   T)frozenc                   &    e Zd ZU dZeed<   eed<   y)BucketFileMetadataa%  Data structure containing information about a file in a bucket.

    Returned by [`get_bucket_file_metadata`].

    Args:
        size (`int`):
            Size of the file in bytes.
        xet_file_data (`XetFileData`):
            Xet information for the file (hash and refresh route).
    r!   xet_file_dataN)r/   r0   r1   r2   r6   r4   r   r&   r.   r,   rY   rY      s    	 Ir.   rY   c                       e Zd ZU dZeed<   dZeed<    ed      Zeed<    ed      Z	eed<    ed      Z
eed	<   ddZy
)	BucketUrla  Describes a bucket URL on the Hub.

    `BucketUrl` is returned by [`create_bucket`]. At initialization, the URL is parsed to populate properties:
    - endpoint (`str`)
    - namespace (`str`)
    - bucket_id (`str`)
    - url (`str`)
    - uri (`HfUri`)

    Args:
        url (`str`):
            String value of the bucket url.
        endpoint (`str`, *optional*):
            Endpoint of the Hub. Defaults to <https://huggingface.co>.
    url endpointFr>   	namespace	bucket_iduriNc                    | j                   xs t        j                  | _         | j                  j	                  | j                   d      j                  d      }|j                  d      r|t        d      d  }t        |      }|j                  rt        d| j                         |j                  j                  d      d   | _        |j                  | _        || _        y )Nr^   /zbuckets/zUnable to parse bucket URL: r   )r_   r   ENDPOINTr]   replacestrip
startswithlen_parse_bucket_uripath_in_repo
ValueErrorr   splitr`   ra   rb   )r*   url_pathparseds      r,   rM   zBucketUrl.__post_init__   s    ;);); 88##DMM26<<SAz*J 12H"8,;DHH:FGG-a0r.   rN   )r/   r0   r1   r2   r3   r4   r_   r   r`   ra   rb   r   rM   r&   r.   r,   r\   r\      sJ      
HHc&Is&&Is&E"C"r.   r\   c                   f    e Zd ZU dZed   ed<   eed<   eed<   eed<   edz  ed<   edz  ed	<   d
 Z	y)
BucketFilez
    Contains information about a file in a bucket on the Hub. This object is returned by [`list_bucket_tree`].

    Similar to [`RepoFile`] but for files in buckets.
    filetyperI   r!   r=   Nr@   uploaded_atc                 X   |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  dd       }|rt        |      nd | _        |j                  dd       }|rt        |      | _        y d | _        y )Nrs   rI   r!   xetHashr@   
uploadedAt)r'   rs   rI   r!   r=   r   r@   rt   )r*   r+   r@   rt   s       r,   r-   zBucketFile.__init__   s    JJv&	JJv&	JJv&	

9-

7D).3^E*
jjt4:E>+64r.   )
r/   r0   r1   r2   r   r4   r3   r6   r   r-   r&   r.   r,   rq   rq      s<     &/
I
IMd?D Pr.   rq   c                   B    e Zd ZU dZed   ed<   eed<   edz  ed<   d Zy)BucketFolderz
    Contains information about a directory in a bucket on the Hub. This object is returned by [`list_bucket_tree`].

    Similar to [`RepoFolder`] but for directories in buckets.
    	directoryrs   rI   Nrt   c                    |j                  d      | _        |j                  d      | _        |j                  dd       xs |j                  dd       }|r)t        |t              r|| _        y t        |      | _        y d | _        y )Nrs   rI   rw   rt   )r'   rs   rI   rD   r   r   rt   )r*   r+   rt   s      r,   r-   zBucketFolder.__init__   s|    JJv&	JJv&	jjt4W

=RV8W  '{H=[ 	CQR]C^ 	  	r.   )	r/   r0   r1   r2   r   r4   r3   r   r-   r&   r.   r,   ry   ry      s(     +

ID 
r.   ry   rI   rO   c                 v   | j                  t        j                        st        |       r(t	        |       }|j
                  st        d|  d      |S | j                  dd      }t        |      dk  s
|d   r|d   st        d|  d      |d    d|d    }dj                  |dd	       }t        d
||      S )a  Parse a bucket path into a HfUri.

    Accepts:
    - `hf://buckets/namespace/name(/path/in/repo)` URIs,
    - Hugging Face web URLs such as `https://huggingface.co/buckets/namespace/name(/tree/path)`,
    - plain `namespace/name(/path/in/repo)` paths.
    zInvalid bucket path: z*. Must be a bucket URI (hf://buckets/...).rd      r   r   zInvalid bucket path: 'z)'. Expected format: namespace/bucket_nameNbucket)rs   r   rk   )rh   r   HF_PROTOCOLr   r   	is_bucketrl   rm   ri   joinr   )rI   ro   partsra   prefixs        r,   rj   rj      s     y,,-1CD1Id#4TF:deffJJsAE
5zA~U1XU1X1$7`abb8*AeAhZ(IXXeABi Fh96BBr.   c                 l    | j                  t        j                        syt        |       j                  S )zCheck if a path is a bucket path.

    Do not raise if the path is not a hf:// URI.
    Raise if the path is a hf:// URI but with an incorrect format.
    F)rh   r   r   r   r   )rI   s    r,   _is_bucket_pathr     s*     ??9001'''r.   c                       e Zd ZU dZed   ed<   eed<   dZedz  ed<   dZ	eed<   dZ
edz  ed	<   dZedz  ed
<   dZedz  ed<   y)SyncOperationz,Represents a sync operation to be performed.)uploaddownloaddeleteskipactionrI   Nr!   r^   reasonlocal_mtimeremote_mtimebucket_file)r/   r0   r1   r2   r   r4   r3   r!   r6   r   r   r   r   rq   r&   r.   r,   r   r     sY    6:;;
ID#*FC"Kt"#L#*#%)Kd")r.   r   c                   n    e Zd ZU dZeed<   eed<   eed<    ee      Zee	   ed<   de
eeez  f   fdZy	)
SyncPlanz Represents a complete sync plan.r9   dest	timestamp)default_factory
operationsrO   c                 *   t        d | j                  D              }t        d | j                  D              }t        d | j                  D              }t        d | j                  D              }t        d | j                  D              }|||||dS )Nc              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   .0ops     r,   	<genexpr>z#SyncPlan.summary.<locals>.<genexpr>6       K/BRYY(5Ja/   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   r   s     r,   r   z#SyncPlan.summary.<locals>.<genexpr>7  s     OObryyJ7NOr   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   r   s     r,   r   z#SyncPlan.summary.<locals>.<genexpr>8  r   r   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr   r   s     r,   r   z#SyncPlan.summary.<locals>.<genexpr>9  s     G"2993FAr   c              3   Z   K   | ]#  }|j                   d v s|j                  xs d % yw)r   r   r   N)r   r!   r   s     r,   r   z#SyncPlan.summary.<locals>.<genexpr>:  s'     g"299PfCfAs   ++)uploads	downloadsdeletesskips
total_size)sumr   )r*   r   r   r   r   r   s         r,   summaryzSyncPlan.summary5  s    K$//KKODOOOO	K$//KKGGGggg
"$
 	
r.   N)r/   r0   r1   r2   r3   r4   r   listr   r   dictr6   r   r&   r.   r,   r   r   ,  sA    *K
IN&+D&AJ]#A
c39n- 
r.   r   c            
       j    e Zd ZdZ	 	 	 d
dee   dz  dee   dz  deeeef      dz  fdZdedefd	Z	y)FilterMatcherz4Matches file paths against include/exclude patterns.Ninclude_patternsexclude_patternsfilter_rulesc                 F    |xs g | _         |xs g | _        |xs g | _        y)a  Initialize the filter matcher.

        Args:
            include_patterns: Patterns to include (from --include)
            exclude_patterns: Patterns to exclude (from --exclude)
            filter_rules: Rules from filter file as list of ("+"/"-", pattern) tuples
        Nr   r   r   )r*   r   r   r   s       r,   r-   zFilterMatcher.__init__L  s+     !1 6B 0 6B(.Br.   rI   rO   c                 $   | j                   D ]"  \  }}t        j                  ||      s|dk(  c S  | j                  D ]  }t        j                  ||      s y | j                  D ]  }t        j                  ||      s y | j                  ryy)zCheck if a path should be included based on the filter rules.

        Filtering rules:
        - Filters are evaluated in order, first matching rule decides
        - If no rules match, include by default (unless include patterns are specified)
        +FT)r   fnmatchr   r   )r*   rI   signpatterns       r,   matcheszFilterMatcher.matches]  s     "..MD'tW-s{" /
 ,,GtW- - ,,GtW- -
    r.   )NNN)
r/   r0   r1   r2   r   r3   tupler-   r5   r   r&   r.   r,   r   r   I  sh    > .2-159	/s)d*/ s)d*/ 5c?+d2	/"C D r.   r   filter_filec                    g }t        |       5 }|D ]  }|j                         }|r|j                  d      r'|j                  d      r%|j                  d|dd j                         f       ]|j                  d      r%|j                  d|dd j                         f       |j                  d|f        	 ddd       |S # 1 sw Y   |S xY w)zParse a filter file and return a list of (sign, pattern) tuples.

    Filter file format:
    - Lines starting with "+" are include patterns
    - Lines starting with "-" are exclude patterns
    - Empty lines and lines starting with "#" are ignored
    #r   r   N-)openrg   rh   append)r   rulesflines       r,   _parse_filter_filer   z  s     E	k	aD::<D4??3/s#c48>>#345%c48>>#345 c4[)  
 L 
 Ls   B+CCc                     	 t        j                  |       }t        j                  |j                        ry|j
                  |j                  dz  fS # t        $ r Y yw xY w)zStat a local file and return (size, mtime_ms).

    Returns None if the path is missing or is a directory. Uses a single
    ``os.stat`` call so callers don't pay for multiple syscalls per file.
    Nr   )rH   statOSErrorS_ISDIRst_modest_sizest_mtime)rI   sts     r,   _stat_localr     sU    WWT] ||BJJ::r{{T)))	  s   A 	AA
local_pathc              #     K   t         j                  j                  |       } t         j                  j                  |       st	        d|        t        j
                  |       D ]  \  }}}|D ]}  }t         j                  j                  ||      }t        |      }|1t         j                  j                  ||       }|j                  t         j                  d      }||d   |d   f   yw)zsList all files in a local directory.

    Yields:
        tuple: (relative_path, size, mtime_ms) for each file
    z Local path must be a directory: Nrd   r   r   )rH   rI   abspathisdirrl   walkr   r   relpathrf   sep)r   root_filesfilename	full_path	stat_inforel_paths           r,   _list_local_filesr     s      ,J77==$;J<HII''*-aHT84I#I.I wwy*=H''4HIaL)A,66  .s   C.C0apir   ra   r   c              #     K   | j                  ||xs dd      D ]  }t        |t              r|j                  }|rH|j	                  |dz         r|t        |      dz   d }n$||k(  rd|v r|j                  dd      d   n|}nj|}t        |       |j                  r|j                  j                         dz  nd}||j                  ||f  yw)	zList all files in a bucket with a given prefix.

    Yields:
        tuple: (relative_path, size, mtime_ms, bucket_file) for each file.
            bucket_file is the BucketFile object from list_bucket_tree.
    NT)r   	recursiverd   r   r   r   )list_bucket_treerD   ry   rI   rh   ri   rsplitr   r@   r   r!   )r   ra   r   itemrI   r   mtime_mss          r,   _list_remote_filesr     s      $$Yv~QU$VdL)yy v|,Fa 1269Tk4;;sA.r2t H#H-48JJ4::'')D0A		8T11+ Ws   CCr   c                 n    t        j                  | dz  t        j                        j	                         S )z3Convert mtime in milliseconds to ISO format string.r   )tz)r   fromtimestampr   utc	isoformat)r   s    r,   _mtime_to_isor     s&    !!(T/hllCMMOOr.   )r   r   r   source_sizesource_mtime	dest_size
dest_mtimesource_newer_labeldest_newer_labelignore_sizesignore_timesignore_existingr   c                    t        |dk(  r|n|      }t        |dk(  r|n|      }| |||d}|
rt        dddd|S ||k7  }||z
  t        kD  }|r1|rt        d|||d|S ||z
  t        kD  }|r|nd}t        dd|d|S |	r|rt        d|d|d|S t        ddd	d|S |s|r|rdn|}t        d|||d|S t        ddd
d|S )a  Compare source and dest files and return the appropriate sync operation.

    This is a unified helper for both upload and download directions.

    Args:
        path: Relative file path
        action: "upload" or "download"
        source_size: Size of the source file (bytes)
        source_mtime: Mtime of the source file (milliseconds)
        dest_size: Size of the destination file (bytes)
        dest_mtime: Mtime of the destination file (milliseconds)
        source_newer_label: Label when source is newer (e.g., "local newer" or "remote newer")
        dest_newer_label: Label when dest is newer (e.g., "remote newer" or "local newer")
        ignore_sizes: Only compare mtime
        ignore_times: Only compare size
        ignore_existing: Skip files that exist on receiver
        bucket_file: BucketFile object (for downloads only)

    Returns:
        SyncOperation describing the action to take
    r   )rI   r!   r   r   r   z&exists on receiver (--ignore-existing))r   r   )r   r   r   z
same mtimezsize differsz	same size	identicalr&   )r   r   _SYNC_TIME_WINDOW_MS)rI   r   r   r   r   r   r   r   r   r   r   r   local_mtime_isoremote_mtime_isobase_kwargssize_differssource_newer
dest_newerskip_reasonr   s                       r,   _compare_files_for_syncr     s6   H $Fh4FLJWO$6X3EZ<X &(	#K kF3[k_jkk)+L :-1EEL r7IWbrfqrr$|37KKJ.8*lK R{RkRR	 n~S^nbmnn R{RkRR<'3^9KF fv;fZeff R{RkRRr.   Fr9   r   r   existingfilter_matcherstatusc
                    |xs
 t               }t        |        xr t        |      }
t        |       xr t        |       }|
s|st        d      t        | |t	        j
                  t        j                        j                               }d}|
rt        j                  j                  |       }t        |      }|j                  |j                  }}t        j                  j                  |      st        d|       i }t!        |      D ]?  \  }}}|j#                  |      r||f||<   |	s"|	j%                  dt'        |       d       A |	r|	j)                  dt'        |       d       i }|	r	 |j+                  |      j,                  }	 t1        |||      D ]K  \  }}}}|j#                  |      r||f||<   |	s#|d| nd}|	j%                  d	t'        |       | d       M 	 |	r|	j)                  d	t'        |       d       t9        |j;                               t9        |j;                               z  }|	r|	j)                  dt'        |       d       t=        |      D ]  }|j?                  |      }|j?                  |      }|rv|st|r9|j@                  jC                  tE        d||d   dtG        |d                      e|j@                  jC                  tE        d||d   dtG        |d                      |r<|r:|\  }}|\  }} |j@                  jC                  tI        |d|||| dd|||             |r|s|s|j@                  jC                  tE        d||d   dtG        |d                       |S t        |       }|j                  |j                  }}t        j                  j                  |      }i }i }!|	r	 |j+                  |      j,                  }t1        |||      D ]P  \  }}}}"|j#                  |      r||f||<   |"|!|<   |	s(|d| nd}|	j%                  d	t'        |       | d       R |	r|	j)                  d	t'        |       d       i }t        j                  j                  |      r|rNt!        |      D ]?  \  }}}|j#                  |      r||f||<   |	s"|	j%                  dt'        |       d       A n[|D ]V  }t        j                  jK                  ||      }#tM        |#      }$|$1|$||<   |	s9|	j%                  dt'        |       d       X |	r|	j)                  dt'        |       d       t9        |j;                               t9        |j;                               z  }|	r|	j)                  dt'        |       d       t=        |      D ]>  }|j?                  |      }|j?                  |      }|r|s|r9|j@                  jC                  tE        d||d   dtG        |d                      e|j@                  jC                  tE        d||d   dtG        |d         |!j?                  |                   |rL|rJ|\  }} |\  }}|j@                  jC                  tI        |d|| ||dd||||!j?                  |                   |r|s|s|j@                  jC                  tE        d||d   dtG        |d                      A |S # t.        $ r Y {w xY w# t2        $ r t4        j7                  d
| d       Y Gw xY w# t.        $ r Y Sw xY w)zCompute the sync plan by comparing source and destination.

    Returns:
        SyncPlan with all operations to be performed
    z[One of source or dest must be a bucket path (hf://buckets/...) and the other must be local.r9   r   r   NzSource must be a directory: zScanning local directory (z files)rd   r^   zScanning remote bucket (zBucket 'z' not found, treating as empty.zComparing files (z paths)r   r   znew file (--existing)r   )r   rI   r!   r   r   r   znew filezlocal newerzremote newer)rI   r   r   r   r   r   r   r   r   r   r   r   znot in source (--delete))r   rI   r!   r   r   r   )r   rI   r!   r   r   r   )rI   r   r   r   r   r   r   r   r   r   r   r   )'r   r   rl   r   r   nowr   r   r   rH   rI   r   rj   r   rk   r   r   r   r)   ri   donebucket_infor"   	Exceptionr   r   loggerdebugsetkeyssortedgetr   r   r   r   r   r   r   )%r9   r   r   r   r   r   r  r   r  r  	is_uploadis_downloadplanremote_totalr   ro   ra   r   local_filesr   r!   r   remote_filesr   	total_str	all_pathsrI   
local_inforemote_info
local_sizer   remote_sizer   bucket_file_mapr   
local_filer   s%                                        r,   _compute_sync_planr  -  sk   " $6}N#F++E0EI!&)G/$2G.GK[vww,,x||,668D  $LWW__V,
"4("IIv':':6	ww}}Z(;J<HII (9*(E$HdH%%h/)-x(8H% :3{;K:LGTU	 )F
 KK4S5E4FgNO"y9EE		P/A#yRX/Y+$!!))(3.2H-=L*6B6N!L> 2TVIMM$<S=N<OPY{Za"bc 0Z KK23|3D2EWMN ((*+c,2C2C2E.FF	KK+C	N+;7CD9%D$.J&**40K+OO**%#)!%!+A#:(5jm(D OO**%#+!%!+A#-(5jm(D *4'
K,7)\&&+!'$.%0"-#/+8)7%1%1(7  KF&&!'!(^9%2;q>%Ba &V K_ #6*"IIv':':6	WW__T*
 *,"y9EE 6HYX^5_1HdHk%%h/*.)9X&,7)2>2Ja~.PR	 8\9J8KI;V]^_ 6` KK23|3D2EWMN77==$0A*0M,HdH%--h715x0@H-(B3{CSBTT[&\]	 1N !-H!#j(!CJ +J 7I ( ,5K)(B3{CSBTT[&\] !- KK4S5E4FgNO ))+,s;3C3C3E/FF	KK+C	N+;7CD9%D&**40K$.J:OO**%#)!%!,Q#:)6{1~)F OO**%#-!%!,Q#-)6{1~)F(7(;(;D(A	 ,7)\*4'
K&&+!)$/%1",#.+9)6%1%1(7$3$7$7$=  !ZF&&!'!']9$1*Q-$@e &x K}   # 	PLL8I;.MNO	P`  s<   5[? /\ +\ \8 ?	\\"\54\58	]]r  c                    d| j                   | j                  | j                  | j                         d}|j	                  t        j                  |      dz          | j                  D ]  }d|j                  |j                  |j                  d}|j                  |j                  |d<   |j                  |j                  |d<   |j                  |j                  |d	<   |j	                  t        j                  |      dz           y)
z1Write a sync plan as JSONL to a file-like object.header)rs   r9   r   r   r   
	operation)rs   r   rI   r   Nr!   r   r   )r9   r   r   r   writejsondumpsr   r   rI   r   r!   r   r   )r  r   r   r   op_dicts        r,   _write_planr'  )  s     ++		^^<<>F GGDJJv%& ooiiGGii	#
 77 ggGFO>>%%'^^GM"??&&(ooGN#	

7#d*+ r.   	plan_filec                 ^    t        |d      5 }t        | |       ddd       y# 1 sw Y   yxY w)z!Save a sync plan to a JSONL file.wN)r   r'  )r  r(  r   s      r,   
_save_planr+  F  s$    	i	D! 
		s   #,c                 f   t        |       5 }|j                         }ddd       st        d|        t        j                  |d         }|j                  d      dk7  rt        d      t        |d   |d   |d	   
      }|dd D ]  }t        j                  |      }|j                  d      dk7  r-|j                  j                  t        |d   |d   |j                  d      |j                  dd      |j                  d      |j                  d                    |S # 1 sw Y   xY w)z#Load a sync plan from a JSONL file.NzEmpty plan file: r   rs   r   z0Invalid plan file: expected header as first liner9   r   r   r  r   r"  r   rI   r!   r   r^   r   r   )r   rI   r!   r   r   r   )
r   	readlinesrl   r$  loadsr  r   r   r   r   )r(  r   linesr   r  r   r&  s          r,   
_load_planr0  L  s$   	iA 
 ,YK899 ZZa!Fzz&X%KLLhF^%D ab	**T";;v+-x(V_[[({{8R0#KK6$[[8		
	  KC 
s   D&&D0verbosec                    t        | j                         xr t        | j                        }t        | j                        xr t        | j                         }| j                  D ]  }t	        |j
                          |r%t        j
                  j                  | j                        }t        | j                        }|j                  |j                  }
}	g }g }| j                  D ]'  }|j                  xdk(  r t        j
                  j                  ||j
                        }|
r|
 d|j
                   n|j
                  }|r&t        d|j
                   d|j                   d       |j                  ||f       xdk(  rZ |
r|
 d|j
                   n|j
                  }|r&t        d|j
                   d|j                   d       |j                  |       dk(  s|st        d	|j
                   d|j                   d       * |s|r|rpg }|r|j                  d
t!        |       d       |r|j                  dt!        |       d       |j#                  dj                  |      j%                                |j'                  |	|xs d|xs d       yy|r$t        | j                        }|j                  |j                  }
}	t        j
                  j                  | j                        }t        j(                  |d       g }g }| j                  D ]  }|j                  dk(  rt        j
                  j                  ||j
                        }t        j(                  t        j
                  j+                  |      d       |r&t        d|j
                   d|j                   d       |j,                  |j                  |j,                  |f       |
r|
 d|j
                   n|j
                  }|j                  ||f       |j                  dk(  ret        j
                  j                  ||j
                        }|r&t        d|j
                   d|j                   d       |j                  |       j|j                  dk(  s{|st        d	|j
                   d|j                   d        t!        |      dkD  r2|r|j#                  dt!        |       d       |j/                  |	|       |r |r|j#                  dt!        |       d       |D ]  }t        j
                  j1                  |      s#t        j2                  |       t        j
                  j+                  |      }||k7  s]	 t        j4                  |       t        j
                  j+                  |      }||k7  r; yy# t6        $ r Y w xY w)zExecute a sync plan.r   rd   z  Uploading: z ()r   z  Deleting: r   z  Skipping: z
uploading z filesz	deleting z, N)addr   T)exist_okr   z  Downloading: r   zDownloading z	Deleting z local files)r   r9   r   r   r   rI   rH   r   rj   r   rk   r   r   printr   r   ri   r  
capitalizebatch_bucket_filesmakedirsdirnamer   download_bucket_filesexistsremovermdirr   )r  r   r1  r  r  r  r   r   ro   ra   r   	add_filesdelete_pathsr  remote_pathr   download_filesdelete_files	file_pathparents                       r,   _execute_planrF  w  s   #DKK00O_TYY5OI!$++.Qtyy7Q3QK oo#BGG,  WW__T[[1
"499-"IIv':':6	 ;=	"$//B))!#j"''!BJ;AVHAbggY"7rwwKbggYb1EF$$j+%>?;AVHAbggY"7rwwKRWWIR		{!DE ''4wL	BII;a@A "" LL:c)n-=V!DELL9S->,?v!FGDIIe,779:""%#+t #  % 
"4;;/"IIv':':6	WW__TYY/
 	J. EG"$//ByyJ&WW\\*bgg>
BGGOOJ7$GOBGG9BryykCD>>-"))2>>:*FG;AVHAbggY"7rwwK"));
*CDh&WW\\*bgg>
L	BII;a@A##J/f$RWWIR		{!<=' ", ~"l3~+>*?vFG%%i@ lKK)C$5#6lCD%Iww~~i(		)$3
*(!#!8 
* &U 
f # s   4W	WWc                     | j                         }t        d| j                   d| j                          t        d|d           t        d|d           t        d|d           t        d	|d
           y)z!Print a summary of the sync plan.zSync plan: z -> z  Uploads: r   z  Downloads: r   z  Deletes: r   z	  Skips: r   N)r   r6  r9   r   )r  r   s     r,   _print_plan_summaryrH    s~    llnG	K}D
45	K	*+
,-	M'+./
01	K	*+
,-	Igg&'
()r.   )r   r   r   r  r   includeexcludefilter_fromr  applydry_runr1  quiettokenrI  rJ  rK  rL  rM  rN  rO  c                   |Fddl m}  ||j                  ||j                  |j                  |j
                  |j                        }|r| s|rt        d      |t        d      |rt        d      |rt        d      |rt        d	      |rt        d
      |	rt        d      |
rt        d      |rt        d      |rt        d      |rt        d      t        |      }t        |       }|st        |       t        d       |r
t                	 t        ||||       |rt                	 |st        d       |S | r|st        d      t        |       }t        |      }|r|rt        d      |s|st        d      |r|rt        d      |r|rt        d      |r|rt        d      |rLt         j"                  j%                  |      rZt         j"                  j'                  |      s;t        d|       t         j"                  j'                  |       st        d|        d}|
rt)        |
      }t+        ||	|      }t        | xr |       }t-        | |||||||||
      }|rt/        |t0        j2                         |S |r)t5        ||       |st        |       t        d|        |S |st        |       |j7                         }|d   d k(  r|d!   d k(  r|d"   d k(  r|st        d#       |S |st        d$       |r
t                	 t        ||||       |rt                	 |st        d       |S # |rt                w w xY w# |rt                w w xY w)%a  Sync files between a local directory and a bucket.

    This is equivalent to the ``hf buckets sync`` CLI command. One of ``source`` or ``dest`` must be a bucket path
    (``hf://buckets/...``) and the other must be a local directory path.

    Args:
        source (`str`, *optional*):
            Source path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``.
            Required unless using ``apply``.
        dest (`str`, *optional*):
            Destination path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``.
            Required unless using ``apply``.
        api ([`HfApi`]):
            The HfApi instance to use for API calls.
        delete (`bool`, *optional*, defaults to `False`):
            Delete destination files not present in source.
        ignore_times (`bool`, *optional*, defaults to `False`):
            Skip files only based on size, ignoring modification times.
        ignore_sizes (`bool`, *optional*, defaults to `False`):
            Skip files only based on modification times, ignoring sizes.
        existing (`bool`, *optional*, defaults to `False`):
            Skip creating new files on receiver (only update existing files).
        ignore_existing (`bool`, *optional*, defaults to `False`):
            Skip updating files that exist on receiver (only create new files).
        include (`list[str]`, *optional*):
            Include files matching patterns (fnmatch-style).
        exclude (`list[str]`, *optional*):
            Exclude files matching patterns (fnmatch-style).
        filter_from (`str`, *optional*):
            Path to a filter file with include/exclude rules.
        plan (`str`, *optional*):
            Save sync plan to this JSONL file instead of executing.
        apply (`str`, *optional*):
            Apply a previously saved plan file. When set, ``source`` and ``dest`` are not needed.
        dry_run (`bool`, *optional*, defaults to `False`):
            Print sync plan to stdout as JSONL without executing.
        verbose (`bool`, *optional*, defaults to `False`):
            Show detailed per-file operations.
        quiet (`bool`, *optional*, defaults to `False`):
            Suppress all output and progress bars.
        token (Union[bool, str, None], optional):
            A valid user access token. If not provided, the locally saved token will be used.

    Returns:
        [`SyncPlan`]: The computed (or loaded) sync plan.

    Raises:
        `ValueError`: If arguments are invalid (e.g., both paths are remote, conflicting options).

    Example:
        ```python
        >>> from huggingface_hub import HfApi
        >>> api = HfApi()

        # Upload local directory to bucket
        >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket")

        # Download bucket to local directory
        >>> api.sync_bucket("hf://buckets/username/my-bucket", "./data")

        # Sync with delete and filtering
        >>> api.sync_bucket(
        ...     "./data",
        ...     "hf://buckets/username/my-bucket",
        ...     delete=True,
        ...     include=["*.safetensors"],
        ... )

        # Dry run: preview what would be synced
        >>> plan = api.sync_bucket("./data", "hf://buckets/username/my-bucket", dry_run=True)
        >>> plan.summary()
        {'uploads': 3, 'downloads': 0, 'deletes': 0, 'skips': 1, 'total_size': 4096}

        # Save plan for review, then apply
        >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket", plan="sync-plan.jsonl")
        >>> api.sync_bucket(apply="sync-plan.jsonl")
        ```
    Nr   r   )r_   rO  library_namelibrary_version
user_agentheadersz,Cannot specify source/dest when using apply.z#Cannot specify both plan and apply.z'Cannot specify delete when using apply.z-Cannot specify ignore_times when using apply.z-Cannot specify ignore_sizes when using apply.z(Cannot specify include when using apply.z(Cannot specify exclude when using apply.z,Cannot specify filter_from when using apply.z)Cannot specify existing when using apply.z0Cannot specify ignore_existing when using apply.z(Cannot specify dry_run when using apply.)enabledzExecuting plan...)r1  r  zSync completed.z7Both source and dest are required (unless using apply).z?Remote to remote sync is not supported. One path must be local.z?One of source or dest must be a bucket path (hf://buckets/...).z2Cannot specify both ignore_times and ignore_sizes.z1Cannot specify both existing and ignore_existing.z%Cannot specify both dry_run and plan.z!Destination must be a directory: z&Source must be an existing directory: r   )
r9   r   r   r   r   r   r  r   r  r  zPlan saved to: r   r   r   r   zNothing to sync.z
Syncing...)hf_apir   r_   rQ  rR  rS  rT  rl   r0  r   rH  r6  r   rF  r   r   rH   rI   r<  r   r   r   r  r'  sysstdoutr+  r   )r9   r   r   r   r   r   r  r   rI  rJ  rK  r  rL  rM  r1  rN  rO  r   	sync_planr  source_is_bucketdest_is_bucketr   r  r   s                            r,   sync_bucket_internalr\    s   F !\\))//~~KK
 TKLLBCCFGGLMMLMMGHHGHHKLLHIIOPPGHHu%	I.	*%&!#	')S'&I$&#$ RSS&v.$T*NNZ[[NZ[[MNNOLMM4@AA 77>>$d(;@GHHww}}V$EfXNOO L)+6"  !N E	 9'k:F"!!'%I Iszz*9d#	*OD6*+ I&!GyQ7;#71#<ASWXAX$%l#igfE " G $& z  " s   M M MM&)FFFFFNN)FN)NN)Mr2   r   r$  rE   rH   r   rW  rK   collections.abcr   dataclassesr   r   r   r   pathlibr   typingr	   r
   r   r^   r   r   _local_folderr   errorsr   utilsr   r   r   r   r   r   r   utils._hf_urisr   rV  r   
get_loggerr/   r
  BUCKET_PREFIXr   r   r8   rQ   rV   rY   r\   rq   ry   r3   rj   r5   r   r   r   r   r   r   r   r6   floatr   r   r   r   r   r  r'  r+  r0  rF  rH  r\  r&   r.   r,   <module>rh     sk  
    	  
  $ ( '  . .   6 '   /  
		H	%    ' ' '@ 
 
 
* - - -    $    $ $ $N P P P2 
 
 
6CC CE C,(# ($ (  	* 	* 	* 
 
 
8. .bC DsCx,A :*c *eCJ/$6 *7# 7(5c53I*J 7,2G 2 2S 2XeTWY\^cehThNiEj 2HPE Pc P$ #ES
ES ()ES 	ES
 ES ES ES ES ES ES ES ES tES ESX !+/tt
t 
t 	t
 t t t t "D(t $Jt tx,h ,d ,:X # $ ## #( #Ve ew e esUYz eei eP*h *4 *  u
 ! $ $"#%u$Ju
*u 
	u
 u u u u u #Yu #Yu tu *u :u u  !u" #u$ #:%u& 'ur.   