
    "h                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlZ	 d dlZddlmZ ddlmZ ddlmZmZmZ ddlmZmZmZ  ej8                  d	      Z G d
 de      Zd Z d Z!d Z" G d dejF                  jH                        Z% G d dejF                  jL                        Z'd Z( G d dejF                  jR                        Z* G d de      Z+y# e$ r	 e
sd dlZY w xY w)    N)	lru_cache)TYPE_CHECKING   )AsyncFileSystem)_DEFAULT_CALLBACK)
filesystemopensplit_protocol)
isfilelikemerge_offset_rangesother_pathszfsspec.referencec                   $     e Zd Z fdZd Z xZS )ReferenceNotReachablec                 :    t        |   |  || _        || _        y N)super__init__	referencetarget)selfr   r   args	__class__s       ]/var/www/html/sandstorm/venv/lib/python3.12/site-packages/fsspec/implementations/reference.pyr   zReferenceNotReachable.__init__   s    $"    c                 :    d| j                    d| j                   S )NzReference "z" failed to fetch target )r   r   r   s    r   __str__zReferenceNotReachable.__str__!   s    T^^,,Edkk]SSr   )__name__
__module____qualname__r   r   __classcell__r   s   @r   r   r      s    
Tr   r   c                 :    t        | j                               d   S Nr   )listvalues)ds    r   _firstr(   %   s    
Ar   c                     |j                  |       }t        |t        t        f      r|d   rt	        |d         d   S |d   S y r$   )get
isinstancer%   tupler
   )path
referencesrefs      r   _prot_in_referencesr0   )   sF    
..
C#e}%,/F~c!f%a(>A> &r   c                     t        | t              rt        | |      | giS i }| D ]/  }t        ||      }|j                  |g       j	                  |       1 |S r   )r+   strr0   
setdefaultappend)pathsr.   outr-   protocols        r   _protocol_groupsr8   /   s`    %#E:6@@
C 2&tZ8x$++D12 Jr   c                       e Zd Zd Zy)RefsValuesViewc              #     K   | j                   j                  j                         D ]'  }t        j                  |      j                          ) | j                   j                  j                         E d {    | j                   j                         D ]c  }| j                   j                  |      }t        |      dk(  r| j                   |dz       A| j                   j                  |      E d {    e y 7 7 	w)Nr   /0)_mapping	zmetadatar&   jsondumpsencode_itemslistdir_get_chunk_sizeslen_generate_all_records)r   valfieldchunk_sizess       r   __iter__zRefsValuesView.__iter__:   s     ==**113 	+C**S/((**	+==''..000]]**, 	BE--88?K;1$mmEDL11}}::5AAA	B 	1 Bs%   A6D8C>9A=D6D 7D DNr   r   r    rJ    r   r   r:   r:   9   s    	Br   r:   c                       e Zd Zd Zy)RefsItemsViewc                 z    t        | j                  j                         | j                  j                               S r   )zipr=   keysr&   r   s    r   rJ   zRefsItemsView.__iter__G   s)    4==%%')=)=)?@@r   NrK   rL   r   r   rN   rN   F   s    Ar   rN   c                 h    d}d}t        | d d d   |d d d         D ]  \  }}|||z  z  }||z  } |S )Nr      )rP   )idxsizesrG   multiss         r   ravel_multi_indexrZ   K   sS    
CDC"IuTrT{+ 1q4x	 Jr   c                       e Zd ZdZed        Zed        Z	 ddZed        Z	ddZ
ddZd	 Z ed
      d        Zd Zd Zd Zd Zd Zd Z ed      d        Zd Zed        Zd Zd dZd dZd Zd Zd Zd Zy)!LazyReferenceMapperao  This interface can be used to read/write references from Parquet stores.
    It is not intended for other types of references.
    It can be used with Kerchunk's MultiZarrToZarr method to combine
    references into a parquet store.
    Examples of this use-case can be found here:
    https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storagec                     dd l }|S r$   )numpy)r   nps     r   r_   zLazyReferenceMapper.np]   s
    	r   c                     dd l }|S r$   )pandas)r   pds     r   rb   zLazyReferenceMapper.pdc   s
    	r   Nc                     | _         i  _        i  _        d _        |t	        j
                  d      n| _         j                  j                  dj                   j                   dg             j                  d<   t        j                   j                  d         }|d    _        |d    _         j                   dz    _        |xs  j                    _        | _        t!        |       fd	       }| _        y)
a  
        Parameters
        ----------
        root : str
            Root of parquet store
        fs : fsspec.AbstractFileSystem
            fsspec filesystem object, default is local filesystem.
        cache_size : int, default=128
            Maximum size of LRU cache, where cache_size*record_size denotes
            the total number of references that can be loaded in memory at once.
        categorical_threshold : int
            Encode urls as pandas.Categorical to reduce memory footprint if the ratio
            of the number of unique urls to total number of refs for each variable
            is greater than or equal to this number. (default 10)


        Nfile/
.zmetadatarecord_sizemetadataz/{field}/refs.{record}.parq)maxsizec                 0   j                   j                  | |      }t        j                  j                  j                  |            }j                  j                  |d      }|j                  D ci c]  }|||   j                   }}|S c c}w )zcached parquet file loader)rH   recordfastparquet)engine)
urlformatioBytesIOfscat_filerb   read_parquetcolumnsr&   )rH   rk   r-   datadfcrefsr   s          r   	open_refsz/LazyReferenceMapper.__init__.<locals>.open_refs   s~     88??v?>D::dgg..t45D%%d=%AB-/ZZ8Ar!u||O8D8K 9s   8B)rootrI   rB   dirsfsspecr   rr   rs   joinr?   loadsrg   r>   rn   out_root
cat_threshr   rz   )r   r{   rr   r   
cache_sizecategorical_thresholdmetrz   s   `       r   r   zLazyReferenceMapper.__init__i   s    ( 		/1z&##F+r$(GG$4$4HHdii./%
L! jj\23}-Z99<< -DII/ 
:	&	 
'	 #r   c                     i | d}|j                  dj                  |dg      t        j                  |      j	                                t        ||fi |S )Nrh   rg   re   rf   )piper~   r?   r@   rA   r\   )rg   r{   rr   kwargsr   s        r   createzLazyReferenceMapper.create   sK    k:
$-.

30F0F0HI"46v66r   c                 \   | j                   V| j                  D cg c]  }|j                  dd      d    }}|D ch c]  }|s|j                  d      r| c}| _         | j                   }|r,|D cg c]!  }t        j
                  j                  |      # }}|S c c}w c c}w c c}w )zList top-level directoriesre   rS   r   .)r|   r>   split
startswithosr-   basename)r   r   pr|   listingr-   s         r   rC   zLazyReferenceMapper.listdir   s    9904?1AGGCOA&?D?$(HqAall36GHDI)):AB$rww''-BGB @H Cs   BB$B$B$5&B)c                    |s| j                         }t        dg| j                  D cg c]	  }d|vs| c}z   | j                  D cg c]	  }d|vs| c}z         }|du r|j	                  |       t        |      S |D cg c]  }|ddd
 }}|D cg c]N  }|dt        || j                  v r"t        j                  | j                  |         n| j                  |         dP }}t        ||z   d 	      S |j                  dd
      }t        |      d
kD  rt        d      |d   }	t        | j                  D cg c]  }|j                  | d      s| c}| j                  D cg c]  }|j                  | d      s| c}z         }|D cg c]N  }|dt        || j                  v r"t        j                  | j                  |         n| j                  |         dP }}| j                  |	      }
|du rt        |      t        |
      z   S | j                  |	      }t        |
|      D cg c]  \  }}|d   r	|d|d   d }}}||z   S c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}}w )zShortcut file listingsrf   re   F	directoryr   nametypesizerd   c                     | d   S )Nr   rL   )rY   s    r   <lambda>z(LazyReferenceMapper.ls.<locals>.<lambda>   s
    AfI r   keyrS   z(Cannot list within directories right nowrT   )rC   setr>   rB   updatesortedrE   r?   r@   r   FileNotFoundErrorr   _keys_in_fieldr%   rF   rP   )r   r-   detaildirnamesr   othersdirinfofileinfopartsrH   rQ   recsrecrecinfos                 r   lszLazyReferenceMapper.ls   s   ||~H$(NNFDco4FG$(KKCD3d?4CDF
 h'f~%KSCG{A>G  #  !"4>>1 

4>>$#78![[.H  'H,2EFF

3"u:>#$NOOa"nnLd4&
0KTL $Kt$qz/JtKL
 
  t~~- JJt~~d34T*
 
 ""5)U?<$t*,,))%0 !t_
c1v 63r7;
 

 '!!i GC
$ MK
"
sH   	I
I
	I
I
I#AI(?I-I--I2
I2
AI79I<c                    || j                   v r| j                   |   S || j                  v r0t        j                  | j                  |         j	                         S d|vs| j                  |      rt        |      |j                  d      \  }}| j                  |      \  }}}| j                   j                  ||fi       j                  |d      }|t        |r|S 	 | j                  |      \  }}}|dk(  ry| j                  ||      }	g d}
|
D cg c]  }||	v r|	|   |   nd }}|d   }||S |d   t        d      |d	d
 ddgk(  r|dd	 S |dd
 S # t        t        t        f$ r t        |      w xY wc c}w )zfGet the reference for one key

        Returns bytes, one-element list or three-element list.
        re   FNr   r   r-   offsetr   rawrT   zThis reference has been deletedrS      )rB   r>   r?   r@   rA   _is_metaKeyErrorr   _key_to_recordr*   rz   
ValueError	TypeErrorr   )r   r   rH   sub_keyrk   _mayberi
chunk_sizery   ru   rx   	selectionr   s                 r   _load_one_keyz!LazyReferenceMapper._load_one_key   s   
 $++;;s##DNN"::dnnS1299;;^t}}S13-3w**3/1b155guE=NL	 %)%8%8%="FB
Q>>%0D 4AHIAAIT!WR[47I	Im?JQ<<==Qq>aV#Ra= !} I'89 	 3-	  Js    E ;E E< E9i   c                 2   |j                  d      \  }}| j                  |      }t        |      dk(  ry|j                  d      D cg c]  }t        |       }}t	        ||      }|| j
                  z  }|| j
                  z  }	||	t        |      fS c c}w )z3Details needed to construct a reference for one keyre   r   )r   r   r   r   )r   rD   rE   intrZ   rg   )
r   r   rH   chunkrI   rx   	chunk_idxchunk_numberrk   r   s
             r   r   z"LazyReferenceMapper._key_to_record  s     yy~u++E2{q %*[[%56SV6	6(K@!1!11D,,,r3{+++	 7s   Bc                     || j                   vr[| j                  | d   }t        |d   |d         D cg c]  \  }}t        j                  ||z         }}}|| j                   |<   | j                   |   S c c}}w )z6The number of chunks along each axis for a given fieldz/.zarrayshapechunks)rI   r>   rP   mathceil)r   rH   zarrayrY   rx   
size_ratios         r   rD   z$LazyReferenceMapper._get_chunk_sizes  s    (((^^ugX$67F-0&BR-S%)Q		!a% J  '1DU#&&	s   "A9c                     | j                  ||      }t        t        |j                                     }t	        |      dk(  r	d |D        S t	        |      dk(  r|d   S d |D        S )z8The references for a given parquet file of a given fieldr   c              3   2   K   | ]  }t        |        y wr   r%   .0ts     r   	<genexpr>z7LazyReferenceMapper._generate_record.<locals>.<genexpr>*  s     (DG(   rS   r   c              3   L   K   | ]  }|d    st        |dd        n|d      yw)r   Nr   r   s     r   r   z7LazyReferenceMapper._generate_record.<locals>.<genexpr>0  s)     B!qtD2AK15Bs   "$)rz   iterrP   r&   rE   )r   rH   rk   ry   its        r   _generate_recordz$LazyReferenceMapper._generate_record$  s`    ~~eV,#dkkm$%t9>(R((Y!^; CrBBr   c              #      K   d}| j                  |      D ]  }||z  }	 t        j                  || j                  z        }t	        |      D ]  }| j                  ||      E d{     y7 w)zJLoad all the references within a field by iterating over the parquet filesrS   N)rD   r   r   rg   ranger   )r   rH   nrecchrk   s        r   rF   z)LazyReferenceMapper._generate_all_records2  ss     ''. 	BBJD	yy 0 001Dk 	<F,,UF;;;	<;s   A#A/%A-&A/c                     t        |       S r   )r:   r   s    r   r&   zLazyReferenceMapper.values;  s    d##r   c                     t        |       S r   )rN   r   s    r   itemszLazyReferenceMapper.items>  s    T""r   c                     t        |       S r   )idr   s    r   __hash__zLazyReferenceMapper.__hash__A  s    $xr      c                 $    | j                  |      S r   )r   )r   r   s     r   __getitem__zLazyReferenceMapper.__getitem__D  s    !!#&&r   c                    d|v r| j                  |      sx|j                  d      \  }}| j                  |      \  }}}| j                  j	                  ||fi       }|||<   t        |      | j                  k(  r| j                  ||       y y || j                  |<   t        j                  t        |t              r|j                         n|      | j                  |<   y Nre   )r   r   r   rB   r3   rE   rg   writer?   r   r+   bytesdecoder>   )	r   r   valuerH   r   rk   rX   r   subdicts	            r   __setitem__zLazyReferenceMapper.__setitem__H  s    #:dmmC099S>LE5..s3LFAqkk,,eV_bAGGAJ7|t///

5&) 0  %DKK"&**",UE":#DNN3r   c                 0    | j                  d      xs d| v S )N.zz/.zr   r   s    r   r   zLazyReferenceMapper._is_metaW  s    ~~d#3u|3r   c                    || j                   v r| j                   |= y || j                  v r| j                  |= y d|v r| j                  |      sx|j                  d      \  }}| j	                  |      \  }}}| j                   j                  ||fi       }d ||<   t        |      | j                  k(  r| j                  ||       y y d | j                   |<   y r   )	rB   r>   r   r   r   r3   rE   rg   r   )r   r   rH   r   rk   r   r   s          r   __delitem__zLazyReferenceMapper.__delitem__[  s    $++C DNN"s#cz$--"4"yy~u#22371++00%"E!%w<4#3#33JJuf- 4 $(C r   c           
         dd l }dd l}dd l}| j                  ||f   }|xs | j                   d| d| d}	|j                  | j                  |j                  d      }
|j                  | j                  d      }|j                  | j                  d      }|j                  | j                  |j                  d      }d}d}|j                         D ]e  \  }}t        |t              r-|dz  }|d   |
|<   t        |      dkD  s2|d   ||<   |d	   ||<   C|dz  }|j                  j                  |      ||<   g |j                  |
|||d
d      }|j                   j#                         |j                   j%                         xs dz  | j&                  kD  r|d   j)                  d      |d<   ddd}ddg}| j*                  j-                  |xs | j                   d| d       |j/                  |	d|xs t1        | j*                  dd       ddd||       |j3                          | j                  j5                  ||f       y )Nr   re   z/refs.z.parqO)dtypeint64rS   r   r   F)copyr-   categoryr   utf8)r   r-   r   Texist_okrl   storage_optionszstd)rm   r   compressionindexstatsobject_encoding	has_nulls)kerchunk.dfr^   ra   rB   r   fullrg   nanzerosr   r+   r%   rE   rw   	_proc_raw	DataFramer-   countnuniquer   astyperr   mkdirs
to_parquetgetattrclearpop)r   rH   rk   base_urlr   kerchunkr_   rb   	partitionfnr5   offsetsrV   rawsnrawnpathjrv   rw   r   r   s                        r   r   zLazyReferenceMapper.writel  sD    KK0	)DMM*!E7&F (("&&<((4++7(;))9wwt''sw; ( 		6GAt$%
7at9q=!%aGAJ#AwE!H	"++//5Q		6 \\!	   
 77==?bggoo/415GF**:6BvJ")6:UO	(3dmm4AeW=M
 + 9tww 148+ 	 	
 	(r   c                    t        | j                        D ]-  }t        |t              s|\  }}| j	                  ||||       / t        | j                        D ]H  }|dk7  s	d|v st        j                  | j                  j                  |            | j                  |<   J | j                  | j                  d}t        j                  |      j                         | j                  d<   | j                  j                  dj                  |xs | j                  dg      | j                  d          | j                   j#                          y)zOutput any modified or deleted keys

        Parameters
        ----------
        base_url: str
            Location of the output
        )r  r   rf   r   r   re   N)r%   rB   r+   r,   r   r?   r   r  r>   rg   r@   rA   rr   r   r~   r   rz   cache_clear)r   r  r   thingrH   rk   kr   s           r   flushzLazyReferenceMapper.flush  s    $++& 	E%' %v

%$3	  	 dkk" 	CAL TQY$(JJt{{q/A$Bq!	C  >>$:J:JK$(JJsO$:$:$<L!HHh/$-->?KK%	
 	""$r   c                 $   d}| j                         D ]J  }|j                  d      r|dz  }| j                  |      }| j                  j	                  |      }||z  }L |t        | j                        z  }|t        | j                        z  }|S )Nr   r   rS   )rC   r   rD   r_   productrE   r>   rB   )r   r  rH   rI   nchunkss        r   __len__zLazyReferenceMapper.__len__  s    \\^ 	!E$
"33E:''//+6 	! 	T^^$$T[[!!r   c              #     K   t        | j                        }|j                  | j                         |D ]  }t	        |t
              s|  | j                         D ]  }| j                  |      E d {     y 7 wr   )r   r>   r   rB   r+   r2   rC   r   )r   metasbitrH   s       r   rJ   zLazyReferenceMapper.__iter__  sr      DNN#T[[! 	C#s#		 \\^ 	2E**5111	21s   AB 	-B 6A>7B c                 F    	 | j                  |       y# t        $ r Y yw xY w)NTF)r   r   )r   items     r   __contains__z LazyReferenceMapper.__contains__  s*    	t$ 		s    	  c           
   #     K   | j                  |      }t        |      dk(  r|dz    yt        j                  d |D         }|D ]2  }|dz   dj	                  |D cg c]  }t        |       c}      z    4 yc c}w w)z|List key names in given field

        Produces strings like "field/x.y" appropriate from the chunking of the array
        r   r<   Nc              3   2   K   | ]  }t        |        y wr   )r   )r   rX   s     r   r   z5LazyReferenceMapper._keys_in_field.<locals>.<genexpr>  s     "A58"Ar   re   r   )rD   rE   	itertoolsr  r~   r2   )r   rH   rI   indsindrx   s         r   r   z"LazyReferenceMapper._keys_in_field  s     
 ++E2{q $,  "A["AB 	@C#+#)>Q#a&)> ???	@)>s   AB A;-B )NN   
   T) TNN)r   r   r    __doc__propertyr_   rb   r   staticmethodr   rC   r   r   r   r   rD   r   rF   r&   r   r   r   r   r   r   r   r  r  rJ   r#  r   rL   r   r   r\   r\   T   s    Y  
   SU-#^ 7 7
:"x'R t_
, 
,'C<$# r]' ' 4 4("8)t%B	2@r   r\   c                        e Zd ZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 d fd	Zd dZd dZd dZd Z	d Z
efd	Zd!d
Zd"dZd#dZd Zd#dZd Zd Zd Zd$dZd%dZd Zd Zd Zd%dZd& fd	Zd Zd Zd Zd Zd Z d Z! xZ"S )'ReferenceFileSystema  View byte ranges of some other file as a file system
    Initial version: single file system target, which must support
    async, and must allow start and end args in _cat_file. Later versions
    may allow multiple arbitrary URLs for the targets.
    This FileSystem is read-only. It is designed to be used with async
    targets (for now). This FileSystem only allows whole-file access, no
    ``open``. We do not get original file details from the target FS.
    Configuration is by passing a dict of references at init, or a URL to
    a JSON file containing the same; this dict
    can also contain concrete data for some set of paths.
    Reference dict format:
    {path0: bytes_data, path1: (target_url, offset, size)}
    https://github.com/fsspec/kerchunk/blob/main/README.md
    r   c                 @   t        |   di | || _        |	| _        |
| _        i | _        i | _        i | _        || _        || _	        t        |t              rt        di |xs |xs i d|i}t        j                  j                  |fi |\  }}|j!                  |      r^t        j"                  |dfi |5 }t$        j'                  d|       t)        j*                  |      }ddd       | j-                  |	       n<t$        j'                  d|       t/        |||      | _        n| j-                  ||	       t        |t              r|j3                         D ci c]A  \  }}|t        |t              r)t        j4                  |j7                  dd      d	   fi |n|C c}}| _        d| j                  vrt5        d
      | j                  d<   y|Dt        |j8                  t:              r|j8                  d	   n|j8                  }|| j                  |<   || j
                  j=                         D ]g  }t?        |      r |       }t        j                  jA                  |      \  }}|s:|| j                  vsIt5        |fi |xs i }|| j                  |<   i || j0                  j=                         D ]  }t?        |      r |       }t        |tB              s&|d	   s,t        j                  jA                  |d	         \  }}|| j                  vs`t5        |fi |xs i }|| j                  |<    n |r-|| j                  vrt5        |fi |xs i }|| j                  |<   |xs t5        d
      | j                  d<   y# 1 sw Y   xY wc c}}w )a  
        Parameters
        ----------
        fo : dict or str
            The set of references to use for this instance, with a structure as above.
            If str referencing a JSON file, will use fsspec.open, in conjunction
            with target_options and target_protocol to open and parse JSON at this
            location. If a directory, then assume references are a set of parquet
            files to be loaded lazily.
        target : str
            For any references having target_url as None, this is the default file
            target to use
        ref_storage_args : dict
            If references is a str, use these kwargs for loading the JSON file.
            Deprecated: use target_options instead.
        target_protocol : str
            Used for loading the reference file, if it is a path. If None, protocol
            will be derived from the given path
        target_options : dict
            Extra FS options for loading the reference file ``fo``, if given as a path
        remote_protocol : str
            The protocol of the filesystem on which the references will be evaluated
            (unless fs is provided). If not given, will be derived from the first
            URL that has a protocol in the templates or in the references, in that
            order.
        remote_options : dict
            kwargs to go with remote_protocol
        fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict))
            Directly provide a file system(s):
                - a single filesystem instance
                - a dict of protocol:filesystem, where each value is either a filesystem
                  instance, or a dict of kwargs that can be used to create in
                  instance for the given protocol

            If this is given, remote_options and remote_protocol are ignored.
        template_overrides : dict
            Swap out any templates in the references file with these - useful for
            testing.
        simple_templates: bool
            Whether templates can be processed with simple replace (True) or if
            jinja  is needed (False, much slower). All reference sets produced by
            ``kerchunk`` are simple in this sense, but the spec allows for complex.
        max_gap, max_block: int
            For merging multiple concurrent requests to the same remote file.
            Neighboring byte ranges will only be merged when their
            inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0
            to only merge when it requires no extra bytes. Pass a negative
            number to disable merging, appropriate for local target files.
            Neighboring byte ranges will only be merged when the size of
            the aggregated range is <= ``max_block``. Default is 256MB.
        cache_size : int
            Maximum size of LRU cache, where cache_size*record_size denotes
            the total number of references that can be loaded in memory at once.
            Only used for lazily loaded references.
        kwargs : passed to parent class
        r7   rbzRead reference from URL %sNz$Open lazy reference dict from URL %s)rr   r   :rS   r   rd   rL   )"r   r   r   template_overridessimple_templates	templatesfss	_dircachemax_gap	max_blockr+   r2   dictr}   core	url_to_fsisfiler	   loggerinfor?   load_process_referencesr\   r.   r   r   r   r7   r,   r&   callabler
   r%   )r   for   ref_storage_argstarget_protocoltarget_optionsremote_protocolremote_optionsrr   r6  r7  r;  r<  r   r   dicref_fsfo2ftextr  optsr/   r7   r   r   s                            r   r   zReferenceFileSystem.__init__  sU   R 	"6""4 0"b# #;~;GVC !++//:c:KFC}}R [[T1S1 (QKK <bA99Q<D( ((/AB BBG"5)# $$R);<b$  "xxz At !$- %%aggc1oa&8ADADH 488#!+F!3> #-R[[%"@Abkk  )+DHH_%"~~,,. ,C=%C$kk88=! 8#HG1E2GB)+DHHX&, "--/ 	C=%Cc4(SV"(++"<"<SV"DKHatxx/'K^5IrK-/*	 dhh>OF0D"FB(*DHH_%1z&1y( ( s   ;,N!ANNc           
         | j                  |      }t        j                  d|        	 | j                  |   }t        |t              r|j                         }t        |t              rGt        j                  d| d       |j                  d      rt        j                  |dd        }|d d fS t        |      dk(  r%t        j                  d| d|        |d   }||}}nV|\  }}}	t        j                  d| d	| d
| d|	        ||	z   }
||dk\  r||z   }n|
|z   }n|}||dk\  r||z   }n|
|z   }n|
}|| j                  }|||fS # t        $ r t        |      w xY w)Nzcat: zReference: z, type bytes   base64:   rS   z, whole file => r   z => z	, offset z, size )_strip_protocolrA  debugr.   r   r   r+   r2   rA   r   r   base64	b64decoderE   r   )r   r-   startendpartrn   start1end1start0r   end0s              r   _cat_commonzReferenceFileSystem._cat_common  s   ##D)uTF^$	*??4(D dC ;;=DdE"LL;tfL9:z*''QR1t##t9>LL;tf,<TFCDq'C #DF $CLL;tfDYvhgdVTUD=D A:#e^F!E\F!8!C<D#:D;++CFD  G  	*#D))	*s   E E"c                   K   | j                  |||      \  }}}t        |t              r||| S t        |      \  }}		 | j                  |   j                  |||       d {    y 7 # t        $ r}
t        ||      |
d }
~
ww xY wwN)rY  rZ  )r`  r+   r   r
   r9  	_cat_file	Exceptionr   r   r-   rY  rZ  r   part_or_urlr^  r_  r7   r   es              r   rc  zReferenceFileSystem._cat_file  s     $($4$4TC$4$P!VTk5)uS))$[1!	B((8$..{%S.QQQ 	B'k:A	Bs:   <B	$A+ #A)$A+ (B	)A+ +	B4BBB	c                     | j                  |||      \  }}}t        |t              r||| S t        |      \  }}		 | j                  |   j                  |||      S # t        $ r}
t        ||      |
d }
~
ww xY wrb  )r`  r+   r   r
   r9  rs   rd  r   re  s              r   rs   zReferenceFileSystem.cat_file  s    $($4$4TC$4$P!VTk5)uS))$[1!	B88H%..{&d.SS 	B'k:A	Bs    A 	A9'A44A9c                 "    || j                   |<   y)z2Temporarily add binary data or reference as a fileNr.   )r   r-   r   r   s       r   	pipe_filezReferenceFileSystem.pipe_file  s     %r   c                    K   | j                  |      rt        j                  |d      S | j                  |       d {   }t	        |d      5 }|j                  |       d d d        y 7 +# 1 sw Y   y xY wwNTr   wb)isdirr   makedirsrc  r	   r   )r   rpathlpathr   rv   rO  s         r   	_get_filezReferenceFileSystem._get_file  sj     ::e;;ut44^^E**% 	!GGDM	 	 +	 	s(   =A9A+ A9A-"
A9-A62A9c                    | j                  |      rt        j                  |d      S  | j                  |fi |}|j	                  t        |             t        |      r|j                  |       n&t        |d      5 }|j                  |       d d d        |j                  t        |             y # 1 sw Y   $xY wrm  )
ro  r   rp  rs   set_sizerE   r   r   r	   absolute_update)r   rq  rr  callbackr   rv   rO  s          r   get_filezReferenceFileSystem.get_file  s    ::e;;ut44t}}U-f-#d)$eKKeT" a  T+ s   ?B44B=c                    |r| j                  d       | j                  ||      }t        j                  dd      }t	        ||      }|r2| j                  |D cg c]  }| j                  |      r| c}      }n| j                  |      }t        ||      D ]  \  }	}
|	|v s|j                  |
||	          ! y c c}w )Nr,  )	recursiverd   T)
auto_mkdir)	r   expand_pathr}   r   r   catro  rP   rk  )r   rq  rr  rz  r   rr   targetsrrv   remotelocals              r   r*   zReferenceFileSystem.get  s    GGBK  ) <v$7eU+88C1TZZ]QCDD88E?D 0 	2MFE~UDL1	2 Ds   B>2B>c           	      .   t        |t              r|rt        t        |t              r|st	        d |D              rt        t        || j                        }i }|j                         D ]U  \  }}| j                  |   }	g g g }}}
|D ]K  }	 | j                  |      \  }}}|
j                  |       |j                  |       |j                  |       M g }g }g }g }t               }t        |
|||      D ]u  \  }}}}t        |t              r|||<   |!|j                  |       |j                  |       |j                  |       |j                  |       |j                  |       w t        |
|||      D ]S  \  }}}}|||vs|j                  |       |j                  |       |j                  |       |j                  |       U t!        t        |      t        |      t        |      d| j"                  | j$                        \  }}}|	j'                  |||      }t        |
|||      D ]  \  }}}}||v rt        ||||      D ]i  \  }}}}||k(  r#||t        |t(              r|||<   '||| ||<   0||k(  s6||k\  s<||k  sBt        |t(              r|||<   X|||z
  ||z
  xs d  ||<   k  X |j+                         j                         D ]\  \  }} t        | t(              s|| j                  v s&||   }!t-        || j                  |         }"|!|"_        |dk(  r|"|dk7  sX|"||<   ^ t1        |      dk(  rt        |t              rd|vrt3        |      S |S # t        $ r}|dk(  r |dk7  r|||<   Y d }~d }~ww xY w)Nc              3   $   K   | ]  }d |v  
 yw)*NrL   )r   r   s     r   r   z*ReferenceFileSystem.cat.<locals>.<genexpr>  s     7OQq7Os   raiseomitT)sortr;  r<  rS   r  )r+   r2   NotImplementedErrorr%   anyr8   r.   r   r9  r`  r4   r   r   rP   r   addr   r;  r<  
cat_rangesrd  r   r   	__cause__rE   r(   )#r   r-   rz  on_errorr   
proto_dictr6   protor5   rr   urlsstartsendsr   urY   rg  errurls2starts2ends2paths2whole_files	new_paths
new_startsnew_ends	bytes_outr_   nsnebr  vexnew_exs#                                      r   r}  zReferenceFileSystem.cat  s   dC Y%%dD!yC7O$7O4O%%%dDOO<
&,,. D	BLE5%B!#R$&D %	%"..q1GAq!KKNMM!$KKN% EGEF%K!$e< %
1aa'CFY  OOA&LLONN1%LLOMM!$% "$e< %
1a=Qk%9LLONN1%LLOMM!$% /BUWU../+Iz8 iXFI "$e< B
1a8%(J)%T 
BMBBQwBJ"*%a3%&CF%&qVCFqQ"Wb%a3%&CF%&q2vR0@D%ACF
BBoD	BL HHJ$$& 		$DAq!Y'A,@V.q$//!2DE#% w& L'#CF		$ s8q=Zc2s$#;
Q ) %7*6)!$A	%s   AM00	N9NNc                     |j                  dd       }|| j                  |       y |dk(  r| j                  ||       y t        d|       )NversionrS   )r6  z Unknown reference spec version: )r*   _process_references0_process_references1r   )r   r.   r6  verss       r   rD  z'ReferenceFileSystem._process_references^  sS    ~~i.<%%j1QY%%jEW%X?vFGGr   c                     || _         y)z&Make reference dict for Spec Version 0Nrj  )r   r.   s     r   r  z(ReferenceFileSystem._process_references0i  s	    $r   c                 f     j                   r j                  rdd li  _         j	                  |j                  di              t        d       fd       }|j                  di       j                         D ]  \  }}t        |t              rF|j                  d      r%t        j                  |dd         j                  |<   | j                  |<   \ j                  r|d   }d|v rQ j                   r= |j                  dd	      j                  d
d      j                  di  j                  }n ||      }t        |      dk(  r|gn
||d   |d   g j                  |<   | j                  |<     j                  j!                   j#                  |j                  dg                    y )Nr   r8  i  c                 Z     j                  |       j                  di j                  S NrL   )Templaterenderr8  )r  jinja2r   s    r   _render_jinjaz?ReferenceFileSystem._process_references1.<locals>._render_jinjas  s&    ,6??1%,,>t~~>>r   ry   zbase64:rT  {{{z}}}rS   r   genrL   )r7  r8  r  r.   _process_templatesr*   r   r   r+   r2   r   rW  rX  replacero   rE   r   _process_gen)r   r.   r6  r  r  r  r  r  s   `      @r   r  z(ReferenceFileSystem._process_references1m  s~   $$
{B ?@	4	? 
	? NN62.446 	'DAq!S!<<	*)/)9)9!AB%)@DOOA&%&"aD19,,$AIIdC0$WT3/#V6&*nn6  *!,,/FaKaSa1qt_"%&"%	'& 	t00r1JKLr   c                     i | _         | j                  |j                  | j                         |j                         D ]1  \  }}d|v rdd l|ffd	| j                   |<   #|| j                   |<   3 y )Nr  r   c                 F     j                  |       j                  di |S r  )r  r  )tempr   r  s     r   r   z8ReferenceFileSystem._process_templates.<locals>.<lambda>  s*     >V__>&>#!># r   )r8  r6  r   r   r  )r   tmpr  r  r  s       @r   r  z&ReferenceFileSystem._process_templates  sn    "".JJt../IIK 	&DAqqy01 %#q! %&q!	&r   c                 ^   i }|D ]  }|d   j                         D ci c]H  \  }}|t        |t              r|n/t        |j	                  dd      |d   |j	                  dd            J c}}fdt        j                  j                          D        }|D ]  }dd l} |j                  |d         j                  di || j                  }	 |j                  |d	         j                  di || j                  }
d
|v rd|v r{t         |j                  |d
         j                  di || j                        }t         |j                  |d         j                  di || j                        }|
||g||	<   d
|v d|v z  rt        d      |
g||	<     |S c c}}w )N
dimensionsrY  r   stopsteprS   c              3   d   K   | ]'  }t        t        j                         |             ) y wr   )r=  rP   rQ   )r   r&   	dimensions     r   r   z3ReferenceFileSystem._process_gen.<locals>.<genexpr>  s,       S)623s   -0r   rn   r   lengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.rL   )r   r+   r%   r   r*   r&  r  r&   r  r  r  r8  r   r   )r   gensr6   r  r  r  productsprr  r   rn   r   r  r  s                @r   r  z ReferenceFileSystem._process_gen  s    	%C
  -335	 Aq a& 155!,aivq9IJKI'//1A1A1CDH  %8fooc%j188P2PP8fooc%j188P2PPO(c/ =H6==UUdnnUF !=H6==UUdnnUF !$VV4CH#o(c/:$K 
 !$uCH'%	%> 
=s   AF)c                 d   dg i| _         | j                  j                         }|D ]  \  }}t        |t        t
        f      rt        |      }nt        |      dk(  rd }n|\  }}}d|v r|j                  dd      d   nd}|}|r|| j                   vryg | j                   |<   | j                   j                  d|v r|j                  dd      d   ndg       j                  |ddd       | j                  |      }|r|| j                   vry| j                   |   j                  |d|d       
 y )Nr,  rS   re   r   r   r   rd   )dircacher.   r   r+   r   r2   rE   rsplitr3   r4   _parent)	r   r   r-   r[  r   r   rY  parpar0s	            r   _dircache_from_itemsz(ReferenceFileSystem._dircache_from_items  s"   R__""$ 	TJD$$-4yTa!%5$,/4K$++c1%a(RCD4t}}4&(d#((.1TkDKKQ'*r2&$QGH||D) 4t}}4 MM#%%tVT&RS#	Tr   c                 N    | j                  |      }t        j                  |      S r   )rs   rp   rq   )r   r-   mode
block_sizecache_optionsr   rv   s          r   _openzReferenceFileSystem._open  s    }}T"zz$r   c                 z   | j                  |      }t        | j                  t              r	 | j                  j	                  ||      S | j                  s| j                          | j                  |      }|t        |      |r|S |D cg c]  }|d   	 c}S # t
        $ r Y nw xY wt        d| d      c c}w )N'z' is not a known keyr   )
rU  r+   r.   r\   r   r   r   r  r  _ls_from_cache)r   r-   r   r   r6   os         r   r   zReferenceFileSystem.ls  s    ##D)doo':;))$77 }}%%'!!$';#D))J#&'a&	''  #av-A$BCC (s   B B8	B&%B&c                 J    | j                  |      xs | j                  |      S r   )ro  r@  r   r-   r   s      r   existszReferenceFileSystem.exists  s    zz$44;;t#44r   c                     | j                   r| j                   v S t        | j                  t              r| j                  j	                  d      v S t        fd| j                  D              S )Nr,  c              3   F   K   | ]  }|j                   d         yw)re   Nr   )r   r   r-   s     r   r   z,ReferenceFileSystem.isdir.<locals>.<genexpr>  s      IAq||tfAJ/Is   !)r  r+   r.   r\   rC   r  r   r-   s    `r   ro  zReferenceFileSystem.isdir  sY    ==4==(()<=4??222666 IIIIr   c                     || j                   v S r   rj  r  s     r   r@  zReferenceFileSystem.isfile  s    t&&r   c                 2   K    | j                   ||fi |S wr   )r   )r   r-   r   r   s       r   _lszReferenceFileSystem._ls  s     twwtV.v..r   c                 P   |rt        |   f|||d|S r0| j                        t        fd| j                  D              }nt        | j                        }|r?| j
                  s| j                          |D ci c]  }|| j                  |      d    c}S |S c c}w )N)maxdepthwithdirsr   c              3   F   K   | ]  }|j                        s|  y wr   r   )r   r  r-   s     r   r   z+ReferenceFileSystem.find.<locals>.<genexpr>  s     HQQ\\$5GqHs   !!r   )r   findrU  r   r.   r  r  r  )	r   r-   r  r  r   r   r  r  r   s	    `      r   r  zReferenceFileSystem.find  s    7<'(6MS  ''-DH$//HHAt'A==))+:;<QAt**1-a00<<H =s   B#c                    | j                   j                  |      }|Dt        |t        t        f      r|dt        |      dS t        |      dkD  r	|d|d   dS |dd dg}n2| j                  |d      }|D cg c]  }|d   |k(  s| }}|s|dddS |d   d	   Rt        | j                   |   d         \  }}| j                  |   j                  | j                   |   d         |d   d	<   |d   S c c}w )
Nrd   r   rS   r   Tr   r   r   r   )
r.   r*   r+   r2   r   rE   r   r
   r9  r   )r   r-   r   r6   out0r  protr   s           r   rB  zReferenceFileSystem.info  s	   oo!!$'?#U|, $fc#hGGSA $fc!fEE!%vtDE''$%C"8!ai4&7A8D8 $k1EE76?"$T__T%:1%=>GD!"hhtn11$//$2G2JKDGFOAw 9s   8C3C3c                 ,   K   | j                  |      S wr   )rB  r  s      r   _infozReferenceFileSystem._info!  s     yys   c                 x   K   | j                   j                  |d        | j                  j                          y wr   )r.   r  r  r
  r  s      r   _rm_filezReferenceFileSystem._rm_file$  s/     $	
 	s   8:c                 ^   K   || j                   |<   | j                  j                          y wr   )r.   r  r
  )r   r-   rv   s      r   
_pipe_filezReferenceFileSystem._pipe_file*  s$      $s   +-c                    K   t        |d      5 }|j                         | j                  |<   d d d        | j                  j	                          y # 1 sw Y   $xY ww)Nr4  )r	   readr.   r  r
  )r   rr  rq  rO  s       r   	_put_filezReferenceFileSystem._put_file/  sJ     % 	.!%&VVXDOOE"	.	. 	.s   AA#AAAc                    i }| j                   j                         D ]0  \  }}t        |t              r	 |j	                  d      ||<   ,|||<   2 t        j                  |dfi |5 }|j                  t        j                  d|d      j                                ddd       y# t
        $ r, dt        j                  |      z   j	                         ||<   Y w xY w# 1 sw Y   yxY w)z+Write modified references into new locationasciirS  rn  rS   )r  ry   N)r.   r   r+   r   r   UnicodeDecodeErrorrW  	b64encoder}   r	   r   r?   r@   rA   )r   rn   r   r6   r  r  rO  s          r   	save_jsonzReferenceFileSystem.save_json5  s    OO))+ 	DAq!U#IXXg.CF A	 [[d6o6 	F!GGDJJ1c:;BBDE	F 	F	 * I(6+;+;A+>>FFHCFI	F 	Fs   B&'6C&2CCC')NNNNNNNNTi   i @Br)  r-  )F)Fr  r   )r4  NNr+  )NFF)#r   r   r    r.  r7   r   r`  rc  rs   rk  rs  r   rx  r*   r}  rD  r  r  r  r  r  r  r   r  ro  r@  r  r  rB  r  r  r  r  r  r!   r"   s   @r   r2  r2    s     H
 U2n(!TBB& /@ 
,2ZxH%M>&!FT, ("5J'/"*
Fr   r2  ),rW  collectionsrp   r&  loggingr   r   	functoolsr   typingr   fsspec.corer}   ujsonr?   ImportErrorasynr   	callbacksr   r>  r   r	   r
   utilsr   r   r   	getLoggerrA  RuntimeErrorr   r(   r0   r8   abc
ValuesViewr:   	ItemsViewrN   rZ   MutableMappingr\   r2  rL   r   r   <module>r     s      	    	    
 # ) 3 3 @ @			-	.TL T?
B[__// 
BAKOO-- A
^@+//88 ^@BL	F/ L	FM  s   C$ $C21C2