
    ##h
                     4   d dl mZmZ d dlZdej                  dedej                  fdZ	 	 	 ddej                  j                  dej                  d	ej                  d
ej                  deej                     de	dee	   dee
   deej                  df   fdZy)    )OptionalTupleNhidden_statesn_repreturnc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
       N)shapeexpandreshape)r   r   batchnum_key_value_headsslenhead_dims         e/var/www/html/sandstorm/venv/lib/python3.12/site-packages/transformers/integrations/sdpa_attention.py	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TT    modulequerykeyvalueattention_maskdropoutscaling	is_causalc           	         t        | d      r,t        || j                        }t        || j                        }|}	|-|	j                  dk(  r|	d d d d d d d |j                  d   f   }	|j                         }|j                         }|j                         }||j                  d   dkD  xr |	d u }t        j                  j                         r*t        |t        j                        r|j                         }t        j                  j                  j                  ||||	|||      }
|
j                  dd      j                         }
|
d fS )Nnum_key_value_groups      r	   )	attn_mask	dropout_pscaler   )hasattrr   r   ndimr
   
contiguoustorchjit
is_tracing
isinstanceTensoritemnn
functionalscaled_dot_product_attention	transpose)r   r   r   r   r   r   r   r   kwargscausal_maskattn_outputs              r   sdpa_attention_forwardr4      s@    v-.V889%!<!<= K!k&6&6!&;!!Q?SYYr]?":; E
..
CE
 KKNQ&>;$+>	 yy*Y"ENN$	((%%BB C K ''1-88:Kr   )g        NN)typingr   r   r'   r+   intr   r-   Modulefloatboolr4    r   r   <module>r;      s    " 	UU\\ 	U# 	U%,, 	U$ # $/HHOO/<</ 
/ <<	/
 U\\*/ / e_/ ~/ 5<</r   