
    +#h                         d dl Z d dlZd dlZd dlmZ d dlZdddddZddZd Zd	 Z	d
e
e   de
e   fdZddZd dlZ G d de      Z G d de      Zd ZddZd Zd Zd Zd ZddZd Z G d d      Zd Zy)    N)defaultdictTF)	conditionpadsepc                    dj                  |D cg c]  }t        |       c}      }dj                  t        j                  j	                         j                  d      |      }| r|s|nd| d}t        |d|       |S c c}w )N z[{}] {}z%b %d, %H:%M:%S
T)flushr   )joinstrformatdatetimenowstrftimeprint)r   r   r   sxmsgs         L/var/www/html/sandstorm/venv/lib/python3.12/site-packages/dsp/utils/utils.pyprint_messager   	   sw    !$Q#a&$%A


8,,002;;<MNPQ
RCcBse2,c3'J %s   Bc                     d| rdnd d| rdnd d}t         j                   j                         j                  |      }|S )Nz%Y-%m/-z%d_z%H.%M.%S)r   r   r   )daydir
format_strresults      r   	timestampr      sD    C0&3c2J(SJ""$--j9FM    c              #   d  K   t        d| j                          t        j                  t        j                  j                  | j                        dz  dz  d      5 }| D ]&  }| |j                  t        |      dz  dz         ( |j                          d d d        y # 1 sw Y   y xY ww)Nz#> Reading g      @MiB)totalunit)	r   nametqdmospathgetsizeupdatelenclose)filepbarlines      r   	file_tqdmr/      s     	K		{
#$	ggoodii(61F:
 	 	5DJKKD	F*V34	5 	

  s   AB0<B$	B0$B-)B0c                     t         j                  j                  |       rt        d       t	        d| d       y t        d       t	        d| d       t        j
                  |        y )Nr	   z#> Note: Output directoryzalready exists

z#> Creating directoryz

)r&   r'   existsr   r   makedirs)r'   s    r   create_directoryr3   '   sG    	ww~~dd149MNd-tV<
Dr   seqreturnc                 n    t               }| D cg c]  }||v r|j                  |      r| c}S c c}w )z<
    Source: https://stackoverflow.com/a/480227/1493011
    )setadd)r4   seenr   s      r   deduplicater:   1   s/    
 5D=!19A===s   	222c              #      K   d}|t        |       k  r/| |||z    }|r||fn| |t        |      z  }|t        |       k  r/y wNr   )r*   )groupbsizeprovide_offsetoffsetLs        r   batchrB   :   sV     F
3u:
&6E>*,{!4#a& 3u:
 s   ?AAc                   8     e Zd Z fdZ fdZ fdZd Z xZS )dotdictc                     |j                  d      r |j                  d      rt        |   |      S 	 | |   S # t        $ r& t        dt        |       j                   d| d      w xY w)N__'z' object has no attribute ')
startswithendswithsuper__getattr__KeyErrorAttributeErrortype__name__selfkey	__class__s     r   rK   zdotdict.__getattr__R   st    >>$CLL$67&s++	]9 	] 1T$Z%8%8$99TUXTYYZ![\\	]s	   9 /A(c                 v    |j                  d      r"|j                  d      rt        |   ||       y || |<   y NrF   )rH   rI   rJ   __setattr__)rQ   rR   valuerS   s      r   rV   zdotdict.__setattr__Z   s2    >>$CLL$6GU+DIr   c                 p    |j                  d      r!|j                  d      rt        |   |       y | |= y rU   )rH   rI   rJ   __delattr__rP   s     r   rY   zdotdict.__delattr__`   s.    >>$CLL$6G$S	r   c                 R    t        t        j                  t        |       |            S N)rD   copydeepcopydict)rQ   memos     r   __deepcopy__zdotdict.__deepcopy__f   s    t}}T$Z677r   )rO   
__module____qualname__rK   rV   rY   r`   __classcell__)rS   s   @r   rD   rD   Q   s    ]8r   rD   c                   T    e Zd Zej                  Zej                  Zej                  Z	y)dotdict_laxN)
rO   ra   rb   r^   getrK   __setitem__rV   __delitem__rY    r   r   re   re   k   s!    ((K""K""Kr   re   c                 "    g }| D ]  }||z  }	 |S r[   ri   )rA   r   _lists      r   flattenrl   q   s(     F % Mr   c           	          t        |       dk(  r| S t        | d         }|dk  r,t        |      D cg c]  }| D cg c]  }||   	 c} c}}S t        |  } |r| S t        |       S c c}w c c}}w )zc
    A much faster A, B, C = zip(*[(a, b, c), (a, b, c), ...])
    May return lists or tuples.
    r   d   )r*   rangeziplist)rA   lazywidthidxelems        r   zipstarrv   {   sn     1v{!IEs{5:5\Bcq)tc)BBQA1!$q'!	 *Bs   	A,A'A,'A,c                     t        |       t        t        fv rt        |       nd }t        t	        | |            }|d t        |      fv sJ d       |S )Nz$zip_first() failure: length differs!)rN   tuplerq   r*   rp   )L1L2lengthL3s       r   	zip_firstr}      sM    RUDM1SWtF	c"bk	BdCG_$L&LL$Ir   c                 6    d| v rt        |       S t        |       S )N.)floatint)vals    r   int_or_floatr      s    
czSzs8Or   c                     t        t              }| D ]-  ^}}t        |      dk(  r|d   n|}||   j                  |       / |S )N   r   r   rq   r*   append)lstgroupsfirstrests       r   groupby_first_itemr      sM    F #d)q.tAwduT"# Mr   c              #      K   t        t              }d}d}| D ]O  ^}}t        |      dk(  r|d   n|}|r||k7  r|||   f ||vs
J | d       ||   j                  |       |}d}Q |S w)zE
    Requires items in list to already be grouped by first item.
    FNr   r   z( seen earlier --- violates precondition.Tr   )r   r   started
last_groupr   r   s         r   process_grouped_by_first_itemr      s     
 FGJ d)q.tAwdu
*vj122V#B@AB# 	uT"
 Ms   A*A,c                 J    t        |       g|z  }t        j                  |d|iS )z
    Collect data into fixed-length chunks or blocks
        Example: grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
        Source: https://docs.python.org/3/library/itertools.html#itertools-recipes
    	fillvalue)iter	itertoolszip_longest)iterablenr   argss       r   grouperr      s+     NaD  $<)<<r   c              #   :   K   d}| D ]  }|||z   f ||z  } y wr<   ri   )lengthsr@   r{   s      r   lengths2offsetsr      s8     F v''& s   c                        e Zd ZddZd Zd Zy)NullContextManagerNc                     || _         y r[   dummy_resource)rQ   r   s     r   __init__zNullContextManager.__init__   s
    ,r   c                     | j                   S r[   r   )rQ   s    r   	__enter__zNullContextManager.__enter__   s    """r   c                      y r[   ri   )rQ   r   s     r   __exit__zNullContextManager.__exit__   s    r   r[   )rO   ra   rb   r   r   r   ri   r   r   r   r      s    -#r   r   c                 t   | j                   y g }|D ]  }| j                   |   }t        |      r2t        |d         t        k(  r|D cg c]  }| j                  |    }}n)|D cg c]  }| j
                  j                  |d        }}dj                  |      }|j                  |        |S c c}w c c}w )Nr    z [SEP] )	qid2backgroundsr*   rN   r   
collectioncollectionXrf   r   r   )r   qidsqbackgroundsqidbackpidr   s          r   load_batch_backgroundsr      s    #L 	##C(t9d1g#-156#%6A6:>?3!!%%c2.?A?NN1A	  7?s   B0%#B5)Fr[   )r   r   r&   collectionsr   r%   r   r   r/   r3   rq   r   r:   rB   r\   r^   rD   re   rl   rv   r}   r   r   r   r   r   r   r   ri   r   r   <module>r      s      	 #  !%%T 
>T#Y >49 >( 8d 84#$ #"&6= r   