"hmp 2UddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZmZmZmZe rddlZddlmZedZnedZedZej4dZeeegefZGd d ZGd d eZ Gd deZ!GddeZ"GddeZ#GddeZ$GddeZ%GddeZ&GddeeefZ'GddeZ(deiZ)de*d<d d!dZ+ee e$e!e#e"e%e&e(f D] Z,e+e, y)") annotationsN)FutureThreadPoolExecutor) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTuple OrderedDictTypeVar) ParamSpecPTfsspecc0eZdZUdZdZded<ddZd dZy) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]namec.||_||_||_yN) blocksizefetchersize)selfrrrs K/var/www/html/sandstorm/venv/lib/python3.12/site-packages/fsspec/caching.py__init__zBaseCache.__init__9s"  ct|d}| |j}||jk\s||k\ry|j||SNrr)rrrstartstops r_fetchzBaseCache._fetch>s@ =E <99D DII $||E4((rNrintrFetcherrr'returnNoner# int | Noner$r,r)bytes)__name__ __module__ __qualname____doc__r__annotations__rr%rrrr's !D-  )rrcdeZdZdZdZ d d fd Zd dZd dZd dZd dZ xZ S) MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapct||||| tn||_||_|j |_yr)superrsetblockslocation _makefilecache)rrrrr;r: __class__s rrzMMapCache.__init__Ss: GT2%~ce6   ^^% rc4ddl}ddl}|jdk(r tS|j)t j j|js|j |j}t|_ nt|jd}|j|jdz |jd|jnt|jd}|j|j|jS)Nrzwb+1zrb+)r6tempfiler bytearrayr;ospathexists TemporaryFiler9r:openseekwriteflushfileno)rr6rBfds rr<zMMapCache._makefile`s 99>;  == t}}(E}}$++-!e $--/ GGDIIM " HHTN HHJdmmU+Btyydii00rctjd|d||d}| |j}||jk\s||k\ry||jz}||jz}t ||dzDcgc]}||j vs|}}|r|j d}||jz}t||jz|j}tjd|d|d||j|||j|||j j||r|j||Scc}w)NzMMap cache fetching -rrr@zMMap get block #z () loggerdebugrrranger:popminrr=add) rr#end start_block end_blockineedsstartsends rr%zMMapCache._fetchvs0 +E7!C59: =E ;))C DII #t~~- 4>>)  i!m<Ua@TUU A'Fv. :D LL+A3b$@ A&*ll64&@DJJvd # KKOOA zz%$$Vs 2D;D;c@|jj}|d=|S)Nr=)__dict__copyrstates r __getstate__zMMapCache.__getstate__s  ""$ 'N rcd|jj||j|_yr)r^updater<r=r`s r __setstate__zMMapCache.__setstate__s" U#^^% r)NN) rr'rr(rr'r;z str | Noner:zset[int] | Noner)r*)r)zmmap.mmap | bytearrayr#r,rVr,r)r-r)dict[str, Any]rarhr)r*) r.r/r0r1rrr<r%rbre __classcell__r>s@rr5r5Hsj D $"& & & & &  &  &  &1,%. &rr5c0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadcRt||||d|_d|_d|_y)Nrr)r8rr=r#rVrrrrr>s rrzReadAheadCache.__init__s) GT2  rc|d}|||jkDr |j}||jk\s||k\ry||z }||jk\r8||jkr)|j||jz ||jz S|j|cxkr|jkr:nn7|j||jz d}|t |z}|j}nd}t |j||j z}|j|||_||_|jt |jz|_||jd|zSr!)rr#rVr=lenrTrr)rr#rVlparts rr%zReadAheadCache._fetchs& =E ;# /))C DII # %K DJJ 3$((?::edjj033CD D ZZ5 +488 +::edjj023D TNAHHED$))S4>>12\\%-  ::DJJ/djj!n$$rr&rfr.r/r0r1rrr%rjrks@rrmrms D %rrmc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstc6t||||d|_yr)r8rr=rps rrzFirstChunkCache.__init__s GT2#' rc|xsd}|xs |j}||jkr|j[||jkDr+|jd|}|d|j|_||dS|jd|j|_|j||}||jkDr||j|j|z }|S|j||SNr)rrr=r)rr#rVdatarts rr%zFirstChunkCache._fetchs TYY 4>> !zz!'<<3/D!%&6!7DJ<'!\\!T^^< ::eC(DT^^# T^^S99K<<s+ +rr&rfrurks@rrwrws D(,rrwceZdZdZdZ d d fd Zd dZdZddZddZ ddZ dfd Z dd Z xZ S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcachect||||tj||z |_||_t j||j|_ yr) r8rmathceilnblocks maxblocks functools lru_cache _fetch_block_fetch_block_cachedrrrrrr>s rrzBlockCache.__init__sR GT2yy !12 "#A9#6#6y#A$BSBS#T rcVd|jd|jd|jdS)Nzrrrrs r__repr__zBlockCache.__repr__s2$T^^$45II;ja 9 rc6|jjSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infors rrzBlockCache.cache_info ''2244rc$|j}|d=|S)Nrr^r`s rrbzBlockCache.__getstate__s  ' ( rc|jj|tj|d|j|_y)Nr)r^rdrrrrr`s rrezBlockCache.__setstate__s< U##J9#6#6u[7I#J   $  rc|d}| |j}||jk\s||k\ry||jz}||jz}t||dzD]}|j||j ||||S)Nrrr@start_block_numberend_block_number)rrrRr _read_cache)rr#rVrr block_numbers rr%zBlockCache._fetchs =E ;))C DII ##dnn4$..0""46F6JK 3L  $ $\ 2 3  1-    rc||jkDrtd|d|jd||jz}||jz}tj d|t |||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks ()zBlockCache fetching block %dr ValueErrorrrPinfor8r%)rrr#rVblock_contentsr>s rrzBlockCache._fetch_block6s} $,, & /))-a9  t~~-dnn$ 2LAs3rc||jz}||jz}||k(r|j|}|||Sg}|j|j||dt|dz|D]"} |j|j| $|j|j|d|dj |Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. Nr@rrrappendrRjoin rr#rVrr start_posend_posblockoutrs rrzBlockCache._read_cacheFsDNN* & !1 1334FGE7+ +C JJt//0BCIJO P !&&81&<>N O C  433LAB C JJt//0@A(7K L88C= r rr'rr(rr'rr'r)r*r)strrgrirf)rr'r)r- r#r'rVr'rr'rr'r)r-)r.r/r0r1rrrrrbrer%rrrjrks@rr~r~s, DMOUU'.U69UFIU U 5   . "!"!""!8;"!OR"! "!rr~cZeZdZUdZdZded< d d fd Zd dZd dZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r-rrc`t||||d|_d|_d|_||_y)Nr)r8rr=r#rVtrim)rrrrrr>s rrzBytesCache.__init__zs2 GT2 !% # rcf|d}| |j}||jk\s||k\ry|jN||jk\r?|j3||jkr$||jz }|j|||z|z S|jr$t |j||jz}n|}||k(s||jkDry|j||jkr;|j||jkDr |j |||_||_nJ|jJ|jJ||jkr|j|j|z |jkDr|j |||_||_n|j ||j}||_||jz|_n|j||jkDr|j|jkDrnk||jz |jkDr|j |||_||_n0|j |j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|jrq|j|jz |jdzz}|dkDrC|xj|j|zz c_|j|j|zd|_|S)Nrrr@) rr#rVr=rrTrrrr)rr#rVoffsetbendnewrnums rr%zBytesCache._fetchs =E ;))C DII # JJ "#$dhhTZZ'F::fv|e';< < >>tyy#"67DD 5=EDII- JJ %$**"4 HH dhheT2DJDJ::) ))88' ''tzz!88#txx#~'F!%eT!:DJ!&DJ,,udjj9C!&DJ!$tzz!1DJ%$/88dii'488^dnn4!%eT!:DJ!&DJ,,txx6C!%c!1DJ::DJJ/#jj&3,"67 9988djj(dnnq.@ACQw dnns22 !ZZ(<(>?  rc,t|jSr)rrr=rs r__len__zBytesCache.__len__s4::r)T) rr'rr(rr'rboolr)r*rf)r)r') r.r/r0r1rr2rr%rrjrks@rrrksS "D-!IM'.69AE ?BrrcXeZdZUdZdZded< d dfd Zd dZxZS) AllBytesz!Cache entire contents of the fileallrrcrt||||||jd|j}||_yr{)r8rrrr|)rrrrr|r>s rrzAllBytes.__init__s6 GT2 <<<499-D rc |j||Sr)r|r"s rr%zAllBytes._fetchsyyt$$r)NNNN) rr,rzFetcher | Nonerr,r|z bytes | Noner)r*r+ r.r/r0r1rr2rr%rjrks@rrrsX+D-!%"&!         %rrc\eZdZUdZdZded<idf dfd Zd fd ZxZS) KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrTc t||||||_|rt|j }|dg}|j |dg} |ddD]m\} } |d\} } | | k(r&| | f|d<| dxx|j | | fz cc<9|j | | f| j |j | | fott|| |_ y||_ y)Nrr@) r8rstrictsortedkeysrSrdictzipr|)rrrrr|r_ old_offsetsoffsetsr:r#r$start0stop0r>s rrzKnownPartsOfAFile.__init__s GT2   -K"1~&Ghh{1~./F*12 ; t ' E>#)4.GBK2J$((E4="99JNNE4=1MM$((E4="9: ;S&12DIDIrc|d}| |j}d}|jjD]_\\}}}||cxkr|ksn||z }||||z|z }|jr||cxkr|krnn|d||z t |z zz }|cS|}n|j t d||fdtjd||fdtjd|d||t|1||zS) Nrrz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching rO) rr|itemsrrrrrwarningswarnrPrQr8r%) rr#r$rloc0loc1r|offr>s rr%zKnownPartsOfAFile._fetchs, =E <99D"&))//"3  LT4$u#t#dl3te!34{{dd&:d&: 7dUlSX&=>>CJ !E% . << Eudm_TVWX X  4eT]OD2 3   8qGHUW^E4000r) rr'rr(rr'r|zdict[tuple[int, int], bytes]rrrrr+rrks@rrrse,"D-!.0   +   <(1(1rrcPeZdZdZGddeZd d dZd dZd dZddZ ddZ y ) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c6eZdZUded<ded<ded<ded<y)UpdatableLRU.CacheInfor'hitsmissesmaxsizecurrsizeN)r.r/r0r2r3rr CacheInforFs    rrctj|_||_||_d|_d|_tj|_ yr{) collectionsr _cache_func _max_size_hits_misses threadingLock_lock)rfuncmax_sizes rrzUpdatableLRU.__init__Ls<+6+B+B+D  !  ^^% rc^|rtd|j|j5||jvrH|jj ||xj dz c_|j|cdddS ddd|j |i|}|j5||j|<|xjdz c_t|j|jkDr|jjdddd|S#1swYxYw#1swY|SxYw)Nz Got unexpected keyword argument r@Flast) TypeErrorrrr move_to_endrrrrrrpopitem)rargskwargsresults r__call__zUpdatableLRU.__call__Ts >v{{}oNO O ZZ )t{{" ''- a {{4(  ) )" ) T,V, ZZ 0 &DKK  LLA L4;;$..0 ###/  0   ) ) 0  sA D)A#D"D"D,cb|j5||jvcdddS#1swYyxYwr)rr)rrs r is_key_cachedzUpdatableLRU.is_key_cachedgs* ZZ '4;;& ' ' 's%.c|j5||j|<t|j|jkDr|jj ddddy#1swYyxYw)NFr)rrrrrr)rrrs radd_keyzUpdatableLRU.add_keyksV ZZ 0 &DKK 4;;$..0 ###/ 0 0 0s AA$$A-c|j5|j|jt|j|j |j cdddS#1swYyxYw)N)rrrr)rrrrrrrrrs rrzUpdatableLRU.cache_infoqsP ZZ >>T[[)ZZ|| "   s AAA&N))rzCallable[P, T]rr'r)r*)rzP.argsrzP.kwargsr)r)rrr)r)rrrrr)r*r)r) r.r/r0r1r rrrrrrr3rrrr?s, J &&'0 rrceZdZUdZdZded< d dfd ZddZddZddZ dd Z dd Z ddfd Z dd Z xZS)BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrct||||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_y)Nr@ max_workers)r8rrrrrrrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrs rrzBackgroundBlockCache.__init__ss GT2yy !12 "#/0A0A9#M 2q A6:'37"+.."2rcVd|jd|jd|jdS)Nz ?II;ja 9 rc6|jjSrrrs rrzBackgroundBlockCache.cache_inforrc<|j}|d=|d=|d=|d=|d=|S)Nrr rrrrr`s rrbz!BackgroundBlockCache.__getstate__s<  ' ( $ % . / / " & ' rc|jj|t|j|d|_t d|_d|_d|_tj|_ y)Nrr@r ) r^rdrrrrr rrrrrr`s rrez!BackgroundBlockCache.__setstate__sZ U##/0A0A5CU#V 2q A*.'!"+.."2rc|d}| |j}||jk\s||k\ry||jz}||jz}d}d}|j5|j|jJ|jj rbt jd|jj|jj|jd|_d|_nKt||jcxkxr|knc}|r&|j}|j}d|_d|_ddd|?t jd|jj|j|t||dzD]}|j||dz} |j5|j]| |jkrN|jj| s3| |_|jj!|j"| d|_ddd|j%||||S#1swYxYw#1swY+xYw)Nrrz3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.r@asyncr)rrrrrdonerPrrrrrrRrrr submitrr) rr#rVrrfetch_future_block_number fetch_future must_joinrend_block_plus_1s rr%zBackgroundBlockCache._fetchs^ =E ;))C DII ##dnn4$..0$(!  $ $ 2!!-66BBB%%**,KK UV,,44**113T5T5T7;D3)-D&!%*::,+,!I !594S4S1'+'9'9 ;?7-1*7 2<  # KKB C  $ $ , ,##%'@  ""46F6JK 3L  $ $\ 2 3 ,a/  $ $ ""*$ 400>>?OP2B/%)%:%:%A%A%%'7&"   1-    o 2 2X  sC"H>7A*I >I Ic||jkDrtd|d|jd||jz}||jz}tj d||t |||}|S)rrrrz!BlockCache fetching block (%s) %dr)rrlog_infor#rVrr>s rrz!BackgroundBlockCache._fetch_blocks $,, & /))-a9  t~~-dnn$ 7<Ps3rc||jz}||jz}||k(r|j|}|||Sg}|j|j||dt|dz|D]"} |j|j| $|j|j|d|dj |Srrrs rrz BackgroundBlockCache._read_cache"sDNN* & !1 1,,-?@E7+ +C JJt//0BCIJO P !&&81&<>N O C  433LAB C JJt//0@A(7K L88C= rrrrrrg)r)r*rf)sync)rr'rrr)r-r)r.r/r0r1rr2rrrrbrer%rrrjrks@rrr{s2'D-&MO 3 3'. 369 3FI 3  3 53J X "!"!""!8;"!OR"! "!rrz!dict[str | None, type[BaseCache]]cachescr|j}|s |tvrtd|dt||t|<y)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rr!r)clsclobberrs rregister_cacher%Ms= 88D tv~+D83Fvd|nUVVF4Lr)F)r#ztype[BaseCache]r$rr)r*)- __future__rrrloggingrrDrrconcurrent.futuresrrtypingrrrr r r r r r6typing_extensionsrrr getLoggerrPr'r-r(rr5rmrwr~rrrrrr!r2r%cr3rrr-s_" 9   +#A A CL   8 $ C:u$ %))BN& N&b'%Y'%T,i,>F!F!RZZz%y%*_1 _1D971a4=9xI!9I!\ )-) (    A1 r