
    #hI             !       s   U d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl mZmZ d dlm	Z	m
Z
 d dlmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmc mZ d dlm Z m!Z!m"Z" d dl#m$Z$ d dl%m&Z& d d	lm'Z'm(Z(m)Z)m*Z*m+Z+ d d
l,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z3 d dl4m5Z5 ejl                  jn                  Z7g Z8ee9   e:d<   ejv                  jx                  jz                  Z= G d de      Z>	 ddedej~                  de@fdZA eeAej~                  j                  d      ZC eeAej~                  j                        ZD eeAej~                  j                        ZFde"deGde"fdZH e$e=j                         e0d      eDde"de"fd                     ZI e$e=j                         e0d      eDde"de"fd                      ZJ e$e=j                         e0d      eDde"de"d!eLd"eLfd#                     ZK e$e=j                         e0d      eDd$e"d%eLd&eLd'eLd(e@d)e"fd*                     ZM e$e=j                  j                  g      d+        ZP e$e=j                  jD                  g      d,e"fd-       ZQ e$e=j                         e0       eDd.e"de"fd/                     ZR e$e=j                         e0d      eDd$e"d.e"fd0                     ZS e$e=j                         e0d      d$e"d.e"d1eLd2eLfd3              ZT e$e=j                         e0       eDd.e"de"fd4                     ZU e$e=j                         e0       eDd$e"d.e"de"fd5                     ZV e$e=j                         e0d      d$e"d.e"d"eLfd6              ZW e$e=j                         e0d      eDd$e"d.e"d7eLd8e@fd9                     ZX e$e=j                         e0d      eDdd:e"d.e"d;e9fd<                     ZY e$e=j                        eDd$e"d=e"fd>              ZZ e$e=j                         e0       eDd.e"de"fd?                     Z[ e$e=j                         e0d      eDd$e"d.e"de"fd@                     Z\ e$e=j                        d.e"dAe"de"fdB       Z] e$e=j                        d$e"d.e"dAe"dee"e"f   fdC       Z^ e$e=j                         e0       eDd$e"d.e"dDe"dEeLdFeLdGe@d8e@de"fdH                     Z_ e$e=j                         e0d      eDd$e"d.e"dIe"de"fdJ                     Z`dKe"dLeGfdMZadNej                  fdOZc e$e=j                         e0       eDe>j                  j                  fd.e"dPe"dLeGde"fdQ                     Zd e$e=j                         e0d      eDd$e"d=e"dPe"dLeGfdR                     Zg e$e=j                        ddS       Zi e$e=j                         e0       eDe>j                  j                  dTfd.e"dPe"dLeGd!eLfdU                     Zj e$e=j                  j                        eDd$e"d.e"dPe"dLeGd!eLf
dV              Zk e$e=j                  j                        eDd$e"d.e"dPe"dLeGd!eLde"fdW              Zn e$e=j                  j                        eDd$e"d.e"dPe"dLeGdXeLf
dY              Zo e$e=j                  j                        eDd$e"d.e"dPe"dLeGdXeLde"fdZ              Zqd$e"d.e"dPe"dAee"   dLeGd[eGd\e"de"fd]Zr e$e=j                         e0d      eDd$e"d.e"deGde"fd^                     Zs e$e=j                         e0d      d$e"d.e"dPe"dAee"   dLeGd[eGd\e"de"fd_              Zt e$e=j                         e0d      d$e"d.e"dPe"dAee"   dLeGd[eGd\e"de"fd`              Zu e$e=j                         e0       eDde>j                  j                  fd.e"dPe"dAee"   dLeGde"f
da                     Zv e$e=j                         e0d      eDde>j                  j                  fd$e"d.e"dPe"dAee"   dLeGde"fdb                     Zw e$e=j                         e0       eDe>j                  j                  fd=e"dPe"dLeGde"fdc                     Zx e$e=j                         e0d      eDe>j                  j                  fd$e"d.e"dPe"dLeGde"f
dd                     Zy e$e=j                         e0       dd=e"dee"dfeLfdg              Zz e$e=j                         e0       dhe"die"de"fdj              Z{ e$e=j                         e0       d$e"dkeeG   deGdleGdmeGdneGfdo              Z| e$e=j                  jD                        	 	 	 	 dd.e"deGdleeG   dmeeG   dneGf
dq       Z~de"deGdleeG   dmeeG   deeGeGf   f
drZ e$e=j                          e0       	 	 	 	 dd=e"dse"deGdleeG   dmeeG   dneGfdt              Z e$e=j                         e0       d$e"dkeeG   deGdueGfdv              Z e$e=j                         e0       d$e"dkeeG   dweGdxeGdyeGf
dz              Zd$e"de"d{ej                  fd|Z e$e=j                         e0d      eCd$e"d}e"deGd{ej                  fd~                     Z e$e=j
                         e0       eCd$e"d}e"deGd{ej                  fd                     Zd Z e$e=j                         e0       d=e"deeG   deeG   deeG   deeG   de"fd              Z e$e=j                         e0       eDd=e"deeG   deeG   deeG   deeG   deeG   de"fd                     Z e$e=j                         e0       d$e"de"d&eLfd              Z e$e=j                         e0       d:e"deeG   deGdeGdneGde"fd              Z e$e=j                  j                        eD	 dd$e"d.e"deeL   de"fd              Z e$e=j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d=e"dfeLdee@   fd                     Z e$e=j                          e0dd      d=e"dfeLdee@   fd              Z e$e=j"                         e0       de"deGde@fd              Z e$e=j$                         e0d      de"deGde@fd              Z e$e=j&                         e0       	 	 	 ddAe"de"deGde@de@de"fd              Z e$e=j(                         e0       d$e"de"deGdeGde@f
d              ZdeeG   fdZdee"   deGdeGdee"   fdZdee"   fdZdee"   deGfdZdee"   deGdeGfdZ e$e=j4                  j                  e=j4                  j                  g      	 ddee"   deGdeGdee"   de"f
d       Z e$e=j6                  j                  e=j6                  j                  g      	 	 dd.e"deeG   deGdeee"      deee"      f
d       Z e$e=j8                  jD                        dd=e"deGdeGdee"df   fd       Z e$e=j:                  j                        	 dd=e"deeG   deGdee"df   fd       Z e$e=j<                  jD                        dd.e"deGdeGdee"df   fd       Ze=j>                  j@                  j                  e7j                        	 dd.e"de"deGdee"df   fd       Z e$e=jD                         e0d      eDdd.e"de"de"d!eGd%eGf
d                     Z e$e=jF                         e0       eD	 	 	 dd.e"de"de"d!eGd%eGde@fd                     Z e$e=jH                         e0       eDdd.e"de"de"d!eGd%eGf
d                     Z e$e=jJ                  j                        eDd$e"d=e"de"de"dee"   deGdeGdeGdeGdee@   deee"   ee"   ee"   f   fd              Z e$e=jJ                  j                        d$e"d=e"de"de"dee"   deGdeGdeGdeGdee@   dejD                  dejD                  dejD                  deee"   ee"   ee"   f   fdÄ       Zdee"   dee"   fdĄZ e$e=jP                  j                        de"d=e"deeG   de"de"dAee"   dee"   dee@   deee"   ee"   ee"   f   fdȄ       Z e$e=jP                  j                        de"d=e"deeG   de"de"dAee"   dee"   dee@   dejD                  dejD                  dejD                  deee"   ee"   ee"   f   fdɄ       Zd=e"dAee"   dee"   dee"   dee"   dGe@deLdeLde@dee"e"e"ee"   ee"   f   fd΄Z e$e=jV                         e0dddЫ      d=e"dAee"   dee"   dee"   dee"   dGe@deLdeLdee"e"e"f   fdф              Ze=jV                  j                  j                  e7j                        e=jV                  j                  j                  e7j                        d=e"dAee"   dee"   dee"   dee"   dGe@deLdeLdee"e"e"f   fd҄              Ze=jZ                  j                  j                  e7j                        ddee"   fdӄ       Z e$e=j^                  j                        d=e"dAee"   dee"   de"de"deLdeLdee"e"e"f   fdԄ       Z e$e=j`                  j                        d=e"dAee"   dee"   de"de"dGe@deLdeLdee"e"e"f   fdՄ       Z e$e=j`                  jb                        d=e"dAee"   dee"   dGe@deLdeLdee"e"e"f   fdք       Z e$e=jf                  j                        d=e"dAee"   dee"   de"de"dGe@deLdeLdee"e"e"e"e"f   fdׄ       Zd=e"dAee"   dee"   de"de"deLdGe@de"fd؄Z e$e=jj                  j                        d=e"dAee"   dee"   de"de"deLdeLdee"e"e"e"f   fdل       Z e$e=jl                  j                        d=e"dAee"   dee"   de"de"deLdeLdee"e"e"e"e"e"f   fdڄ       Z e$e=jn                  j                        d=e"dAee"   dee"   de"de"deLdeLdee"e"e"e"f   fdۄ       Z e$e=jp                         e0dd      eDdd܄                     Z e$e=jt                         e0       dddddddݜdee"e(f   dNeej                     deejv                     de@de@deejx                     fd              Z e$e=jz                  e=j|                  e=j~                  g       e0       d               Ze=j                  j                  j                  e7j                         e$e=j                         e0dddd      d=e"dAe"dee"   dee"   dee"   dGe@deLdeLfd                     Zd Z e$e=j                  j                        de"d=e"dAee"   dee"   dee"   dee"   dee"   de@deLdee@   de"dee"ee"   ee"   f   fd       Z e$e=j                  j                        de"d=e"dAee"   dee"   dee"   dee"   dee"   de@deLdee@   dee"ee"   ee"   f   fd       Z e$e=j                  j                        de"d=e"dAee"   dee"   dee"   dee"   dee"   de@deLdee@   dejD                  dejD                  dejD                  dee"ee"   ee"   f   fd       Z e$e=j                         e0ddd«      d=e"d$e"dAe"dee"   dee"   dee"   dee"   deLfd              Z e$e=j                         e0ddd«      d=e"d$e"dAe"dee"   dee"   dee"   dee"   deLde"fd              Z e$e=j                         e0       eDd=e"deeGeGf   fd                     Zd.e*de*deeG   deGfdZ e$e=j                         e0       d.e*de*deeG   fd              Z e$e=j                         e0       d=e*de*deeG   deeG   deeG   f
d              Z e$e=j                        dpdde*deGdue*de*d%e(f
d       Z e$e=j                         e0       dpdde*deGdue*de*d%e(f
d              Zdpdde*deGdue*de*de@d%e(fdZ e$e=j                  j                        e=j                  j                  j                  e7j                        dd              Z e$e=j                        de*deGdue*de*fd       Z e$e=j                         e0       de*deGdue*de*fd              Zde*deGdue*de*de@f
dZ e$e=j                         e0d}dI      eDd.e"dee"e"f   fd                     Z e$e=j                         e0       	 	 	 dde"d ee@eGeLf   dee@eGeLf   deej                     fd              Z e$e=j                        dd       Zאd Zؐd Z e$e=j                  j                         e$e=j                  j                         e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d=e"deeeG      deeeL      de"fd                                                               Z e$e=j                  j                         e$e=j                  j                         e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d=e"deeeG      deeeL      de"fd	                                                               Zdd
Z e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                         e0dd      	 dd=e"deeG   deeL   de"fd                            Z e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                         e0dd      	 dd=e"deeG   deeL   de"fd                            Z e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                         e0dd      	 	 dd=e"deeG   deeL   deeL   de"f
d                            Z e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                         e0dd      	 	 dd=e"deeG   deeL   deeL   de"f
d                            Z e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                         e0dd      	 	 	 dd=e"deeG   deeL   deeL   deeL   de"fd                            Z e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                         e0dd      	 	 	 dd=e"deeG   deeL   deeL   deeL   de"fd                            ZeD	 dd=e"deeG   deeeL      de@de"f
d       Zd Zd Zd Zd Z	 ddZd Zd ZddZdd Zd! Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d"                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d#                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d$                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d%                      Zd& Zdd'Zdd(Zd) Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d*                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d+                      Zd, Zd- Z  e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d.                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d/                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d0                      Z e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d1                      Z e$e=j                  j                         e$e=j                  j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        e=j                  j                  j                  e7j                        d2                                                         Z e$e=j                  j                  e=j                  j                  g       e0       	 dd=e"deeG   d3e@deeL   de"f
d4              Z
 e$e=j                  j                  e=j                  j                  g      e=j                  j                  j                  e7j                         e0       	 	 dd=e"deeG   d3e@deeL   deeL   de"fd5                     Z e$e=j                  j                  e=j                  j                  g       e0       	 	 	 dd=e"deeG   d3e@deeL   deeL   deeL   de"fd6              Z	dd7Zd8 Zdsee"   d9ee"   d:e"de"fd;Zd9e+de"fd<ZeDd=e"deeG   d3e@deeeL      de"f
d=       Z e$e=j"                  j                        d>e"d?e"de@fd@       Z e$e=j$                  e=j&                  g       e0       dA               Z e$e=j(                  g      dB        Z e$e=j*                  g      ddC       Z e$e=j,                  g      dD        Z e$e=j.                  g      dE        Zd.e"dPe"dAee"   dLeGd[eGdee"e"f   fdFZ e$e=j2                         e0d}d\      d.e"dPe"dAee"   dLeGd[eGdee"e"f   fdG              Z e$e=j4                         e0d}d\      d.e"dPe"dAee"   dLeGd[eGdee"e"f   fdH              Zde"dIeLde"fdJZde"dIeLde"fdKZdLe"de+fdMZdNe+dOe"de"fdPZdOee"   de"fdQZdReGd3e@dNej                  dejv                  fdSZ dTe"dUeGdVeGd3e@fdWZ!dTe"dXeGdUeGdVeGd3e@f
dYZ"dTe"deeG   d3e@fdZZ#dTe"deeG   d3e@fd[Z$ e$e=jJ                         e0       eDdTe"deeG   d3e@fd\                     Z%	 	 	 	 dd>e"d]e"d^eGd_eGd3e@d`e@de"fdaZ& e$e=jN                         e0       eD	 	 	 dd>e"d]e"d^eGd_eGd3e@de"fdb                     Z' e$e=jP                         e0       eDdc                      Z( e$e=jR                         e0       dde>j                  j                  fdd              Z)deejD                  dfejD                  dge@de@fdhZ*e=jV                  j                  j                  e7j                        e=jV                  j                  j                  e7j                         e0di      ddjdk                     Z+ e$e=jX                  j                  e=jX                  j                  g      e=jX                  j                  j                  e7j                         e0       eD	 	 dd=e"deeGeGf   d3e@dleeL   dmeeL   de"fdn                            Z- e$e=jX                  j                        e=jX                  j                  j                  e7j                        e=jX                  j                  j                  e7j                         e0       eD	 dd>e"deeeGeGf      d3e@deeeLeLf      de"f
do                                   Z. e$e=j^                         e$e=j`                         e$e=jb                        eD e0       d>e"deeGdf   de"fdp                                   Z2 e$e=jf                         e$e=jh                         e$e=jj                        eD e0       d>e"deeGdf   de"fdq                                   Z6d>e"deeGdf   dreeGeGeGge"f   de"fdsZ7 e$e=jp                         e$e=jr                         e$e=jt                         e0d      dt                             Z; e$e=jx                         e0dudv      dddwdx              Z< e$e=jz                         e0       dddydz              Z= e$e=j|                  j                  e=j|                  j                  g       e0       dej~                  ddd{dme(dNeej                     d|ej                  deejv                     de@f
d}              ZA e$e=j|                  j                  g      dej~                  ddd{dle(dme(dNeej                     d|ej                  deejv                     de@fd~       ZC e$e&      d        ZD e$e=j                        e=j                  j                  j                  e7j                         e0       dpdpde>j                  j                  fd=e"dPe"dfe(de(dAee"   dLeGde"fd                     ZE e$e=j                        e=j                  j                  j                  e7j                         e0d}d      d=e"dPe"dLeGdee"e"f   fd                     ZF e$e=j                  j                        	 	 ddddde"de"d,e"deLde@dee"   d&eeL   dee"e"f   fd       ZHd ZI e$e=j                  g       e0       eDdd                     ZJ e$e=j                         e0       d               ZK e$e=j                        d        ZL e$e=j                  j                  e=j                  j                  g      dddd.e"dNeej                     dee"   de"fd       ZN e$e=j                  j                  e=j                  j                  g      dd.e"deeG   fd       ZQ e$ejx                  jz                  j                        dd       ZR e$e=j                         e0       dddd              ZS e$e=j                  j                        ddd.ejD                  deej                     dejD                  fd       ZT e$e=j                  j                        dddeej                     fd       ZVdddZWddddZX e$e=j                         e0       d               ZY e$e=j                        dd       ZZ eIe=j                  e=j                          eIe=j                  e=jD                          eIe=j                  e=jH                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                          eIe=j                  e=j                           eIe=j                  e=j                          eIe=j                  e=j                          eIe=j
                  e=j                         y(      N)Enum)partialreduce)chainproduct)AnyCallablecastIterableListOptionalTupleUnion)	sym_floatsym_intTensorregister_decomposition)	out_dtype)IntLike
NumberTypesuggest_memory_format
TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_outout_wrapper)_pytree)tree_map__all__c                       e Zd ZdZdZdZy)	Reductionr         N)__name__
__module____qualname__NONEMEANSUM     Y/var/www/html/sandstorm/venv/lib/python3.12/site-packages/torch/_decomp/decompositions.pyr#   r#   /   s    DD
Cr-   r#   Fftype_promotioncompute_dtype_onlyc                 J     t        j                          fd       }|S )Nc                     t        j                  | i |D cg c]  }t        |t              s| }}t	        j
                  |di\  fd}fd} 
t        ||       i t        ||      }	r|S t        ||      S c c}w )Ntype_promotion_kindc                 J    t        | t              r| j                        S | S N
isinstancer   to)xcomputation_dtypes    r.   increase_precz0type_casts.<locals>.inner.<locals>.increase_precG   s"    !V$tt-..r-   c                 J    t        | t              r| j                        S | S r6   r7   )r:   result_dtypes    r.   decrease_precz0type_casts.<locals>.inner.<locals>.decrease_precM   s!    !V$ttL))r-   )pytreearg_tree_leavesr8   r   utilselementwise_dtypesr    )argskwargsr:   	flat_argsr<   r?   rr;   r>   r1   r/   r0   s          @@r.   innerztype_casts.<locals>.inner=   s     --t>v>
*QPVBWA
	 
 +0*B*B+
,:+
'<
		 xt,P0OPHM1--1
s
   BB)	functoolswraps)r/   r0   r1   rH   s   ``` r.   
type_castsrK   8   s'    
 __Q. .6 Lr-   T)r0   r1   )r0   r:   dimreturnc                 j    t        || j                         z
        D ]  }| j                  d      }  | S N)rangerL   	unsqueeze)r:   rL   _s      r.   _unsqueeze_to_dimrT   j   s2    3=! KKOHr-   
grad_inputout_gradyc                 4    | d||z  z
  j                         z  S Nr$   conj_physicalrV   rW   s     r.   tanh_backwardr]   p   s      q1q5y//111r-   c                 4    | |d|z
  z  j                         z  S rY   rZ   r\   s     r.   sigmoid_backwardr_   w   s      qAE{11333r-   beta	thresholdc                 t    ||z  j                         }t        j                  ||z  |kD  | | |z  |dz   z        S N      ?)exptorchwhere)rV   r:   r`   ra   zs        r.   softplus_backwardri   ~   s=     
TA;;DI-xAS9QRRr-   grad_outputalphascaleinput_scale	is_resultself_or_resultc                     ||z  }|}|}|r&t        j                  |dk  | |z  ||z   z  | |z        S t        j                  |dk  | |z  |z  t        j                  ||z        z  | |z        S Nr   )rf   rg   re   )	rj   rk   rl   rm   rn   ro   negcoefposcoef
negiptcoefs	            r.   elu_backwardru      s     emGGJ{{a*$(@A'!
 	
 {{a*$w.>J;V1WW'!
 	
r-   c                 .    t        j                  | |      S r6   )rf   	full_likeselfvalues     r.   fill_scalarr{      s    ??4''r-   rz   c                     t        j                  j                         dk(  fd       t        j	                  |       S )Nr   c                  ,    d j                          dS )Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrL   )rz   s   r.   <lambda>zfill_tensor.<locals>.<lambda>   s    RSXS\S\S^R__jk r-   )rf   _checkrL   atencopyrx   s    `r.   fill_tensorr      s3    	LL		qk 99T5!!r-   ry   c                 f    t        j                  t        j                  | dz   d      d      dz  S N   r   min   maxrf   clampry   s    r.   hardsigmoidr      s)     ;;u{{4!83;a??r-   c                 H    t        j                  |dkD  |dk  z  | dz  d      S )Ng      g      @gUUUUUU?        rf   rg   rj   ry   s     r.   hardsigmoid_backwardr      s0     ;;	$y! r-   min_valmax_valc                 B    t        j                  ||k  ||k\  z  d|       S )Nr   r   )rj   ry   r   r   s       r.   hardtanh_backwardr      s$    
 ;;DGO<c;OOr-   c                 l    | t        j                  t        j                  | dz   d      d      z  dz  S r   r   r   s    r.   	hardswishr      s.     %++ekk$(:BBQFFr-   c           
      x    t        j                  |dk  dt        j                  |dk  | |dz  dz   z  |             S )Nr   r         ?r   r   s     r.   hardswish_backwardr      sA     ;;r	DAI{tax3.>?M r-   c                 6    t        j                  ||k  d|       S rq   r   )rj   ry   ra   s      r.   threshold_backwardr      s     ;;ty(![99r-   negative_slopeself_is_resultc                 <    t        j                  |dkD  | | |z        S rq   r   )rj   ry   r   r   s       r.   leaky_relu_backwardr      s      ;;taxkN.JKKr-   gradapproximatec                    d}d}d}|dk(  ri||z  dz  }d}||z  }||z  }	||||	z  z   z  }
t        j                  |
      }d|z  }d|z   }d|z  }d||z  z
  }|dd|z  |z  z   z  }||z  |z  }| ||z   z  S |}||z  dz  }ddt        j                  ||z        z   z  }|t        j                  ||z  d	z        z  }| |||z  z   z  S )
Ng;f?g;f?gmBP?tanhr   gHm?r$   r   g      )rf   r   erfre   )r   ry   r   M_SQRT2	M_SQRT1_2
M_2_SQRTPIkBetakKappax_sqx_cuberH   
tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs                        r.   gelu_backwardr      s'    %G&I'Jf*$s*d{/0ZZ&
TzJ+j:55 AF
T(9$9:/14DD)99::Y&,Q4&=112eiitd 233sTCZ'((r-   inputc                     t        j                  t        j                  |            }t        j                  |      }||z  d||z  z
  z  }| ||z   z  S rY   )rf   r   Fsoftplussigmoid)rj   r   input_tanh_softplusinput_sigmoidouts        r.   mish_backwardr     sV      **QZZ%67MM%(M
-
1':=P'P#P
QC-344r-   c                 2    | t        j                  |       z  S r6   )rf   r   r   s    r.   silur     s     %--%%%r-   c                 \    ddt        j                  |       z   z  }| |z  d|d|z
  z  z   z  S rY   )rf   re   )rj   ry   r   s      r.   silu_backwardr     s<     1uyy$''(G AG(<$<==r-   weightc                 <    t        j                  | dkD  | || z        S rq   r   )ry   r   s     r.   _prelu_kernelr   $  s    ;;taxv}55r-   c                 ~    t        j                  |dkD  | || z        }t        j                  |dkD  d|| z        }||fS )Nr   r   r   )rj   ry   r   
input_gradweight_grads        r.   _prelu_kernel_backwardr   )  sE     TAX{F[4HIJ++dQhTK-?@K$$r-   noiseloweruppertrainingc                 x    |r||z
  dkD  r| j                  |      S ||z   dz  }t        j                  | |||      S )Ngư>r%   )mulr   r   )rj   ry   r   r   r   r   r   r   s           r.   rrelu_with_noise_backwardr   4  sK     EEMD(u%%%-1,''~~
 	
r-   bufferc                     |dk  }t        j                  |dd      }t        j                  |dd      }t        j                  t        j                  |             }| |||d|z   z  z  z
  z  S )Nr   r$   rP   )rf   rg   re   abs)rj   ry   r   in_negative	max_derivsignrh   s          r.   log_sigmoid_backwardr   I  sg     (KKA.I;;{Ar*D		599T?"#A)da1q5k&::;;r-   loss	reductionc                     |t         j                  j                  k(  rt        j                  |       S |t         j
                  j                  k(  rt        j                  |       S | S r6   )r#   r*   rz   rf   meanr+   sum)r   r   s     r.   apply_loss_reductionr   V  sH    INN(((zz$	imm))	)yyr-   dtypec                     | t         j                  k(  rt         j                  S | t         j                  k(  rt         j                  S | t         j
                  k(  rt         j                  S y r6   )rf   	complex32float16	complex64float32
complex128float64r   s    r.   to_real_dtyper   _  sK    }}	%//	!}}	%""	"}} 
#r-   targetc                 *    | |z
  dz  }t        ||      S )Nr%   )r   )ry   r   r   r   s       r.   mse_lossr   n  s     6MaDi00r-   c                 |    |t         j                  j                  k(  rd|j                         z  nd}|||z
  z  | z  S )N       @)r#   r*   rz   numel)rj   r   r   r   norms        r.   mse_loss_backwardr   x  s;     #,y~~/C/C"C3D56>"[00r-   c                     t        j                  | ||      }| j                  t        d            }t        j                  ||d      }t        j
                  |      }t        j                  |||      S )N)rL   r   z-infTrL   keepdim)rf   softmaxeqfloatall
zeros_likerg   )ry   rL   r   r   maskedmasked_rowszeross          r.   safe_softmaxr    s[    
--#U
3CWWU6]#F))FT:KS!E;;{E3//r-   rd   c                     | |z
  j                         }t        j                  ||k  d|dz  z  |z  |d|z  z
        }t        ||      S )Nr   r%   )r   rf   rg   r   )ry   r   r   r`   r   s        r.   smooth_l1_lossr    sO     6M D;;td{C$'MD$8$t:KLDi00r-   c                    |t         j                  j                  k(  rd|j                         z  nd}||z
  }t	        j
                  |      }|| z  }t	        j                  ||k  ||z  |z  |t	        j                  |      z        S rc   )r#   r*   rz   r   rf   r   rg   r   )	rj   ry   r   r   r`   r   r:   abs_x	norm_grads	            r.   smooth_l1_loss_backwardr    s{    
 "+inn.B.B!B3DvAIIaLE{"I;;AEJJqM! r-   c                 h    t        | ||||      }t        ||j                         t        ||d      S NT	copy_fromcopy_toexact_dtype)r  r   shaper   )rj   ry   r   r   r`   rU   results          r.   smooth_l1_loss_backward_outr    s3     %[$	4PFj&,,/FJDQQr-   deltac           
          |t         j                  j                  k(  rd|j                         z  nd}||z
  }t	        j
                  || k  | | z  |z  t	        j
                  ||kD  || z  |z  ||z  | z              S rc   )r#   r*   rz   r   rf   rg   )rj   ry   r   r   r  r   r:   s          r.   huber_loss_backwardr    s    
 "+inn.B.B!B3DvA;;	UF
	e#AItk1E94!8k;QR r-   c                 h    t        | ||||      }t        ||j                         t        ||d      S r	  )r  r   r  r   )rj   ry   r   r   r  rU   r  s          r.   huber_loss_backward_outr    s3     !dFIuMFj&,,/FJDQQr-   ignore_indextotal_weightc                    |j                         dk  rdnd}|t        j                  j                  k(  r| |z  } |j	                  |      }t        j                  ||k7  |d      }t        j                  |      }	t        j                  |	||d      }	|	j                         | j                         cxkD  rdkD  rn n| j	                  |      } |Nt        |j                               D 
cg c]  }
d }}
|j                  d   ||<   |j                  |      }| |z  } t        j                  ||k7  | d      } |	| z  S c c}
w )Nr%   r   r$   g      )rL   r#   r*   rz   rR   rf   rg   r   scatterrQ   r  reshape)rj   ry   r   r   r   r  r  channel_dimsafe_targetrU   rS   	new_shapes               r.   _nll_loss_backwardr    s    xxzA~!1KINN(((!L0k*F++f4fa@K!!$'Jz;TJJ~~+//+/a/!++K8 %dhhj 121Q2	2!'a	+	*!F*++f4k1EK## 3s   *	D<c                    |j                         dkD  sJ d       t        j                  |j                         |      }|j                  |      }|dz  dk(  sJ d| d|        |dz  }|j	                  |d|      }|j	                  |||      }t        j                  |      }d|z
  |z  |z  | z  }	|| z  }t        j                  ||	g|      S )Nr   z*glu does not support 0-dimensional tensorsr%   z.Halving dimension must be even, but dimension z	 is size rd   r~   )rL   rB   canonicalize_dimsizenarrowrf   r   cat)
rj   ry   rL   wrap_dimnIn	inputSize	firstHalf
secondHalfgradInputFirstHalfgradInputSecondHalfs
             r.   glu_backwardr+    s     88:>GGG>%%dhhj#6H
))H
Ca1Q	7z3%PQqIHa3IXy)<Jz2	!	!%77)CkQ  ,k999(*=>HMMr-   c           	         d|j                         cxk  rdk  sJ d        J d       |j                         dk  sJ d       |j                         dk(  xr |j                         dk(  }|sA|j                  d   |j                  d   k(  s"J d|j                   d|j                   d       |j                         dk(  s'J d	|j                   d
|j                          df       |'|j                         |j                  d   k(  sJ d       |t        j                  j
                  k(  r}|j                         dk(  rj| j                         dk(  r| j                  d   |j                  d   k(  srJ d|j                  d    d| j                          d| j                  d           | j                         dk  r| j                         dk(  sJ d| j                          t        | ||||||      S )Nr   r%   input tensor should be 1D or 2Dr$   ;0D or 1D target tensor expected, multi-target not supportedsize mismatch (got input: 
, target: ):expected total_weight to be a single element tensor, got: z (z
 elements)rP   z<weight tensor should be defined either for all or no classesz7Expected a tensor of dimension 1 and tensor.size[0] == z but got: dimension z and tensor.size[0] == z7Expected a single element grad_output tensor, but got: )rL   r  r   r#   r)   rz   r  )rj   ry   r   r   r   r  r  no_batch_dims           r.   nll_loss_backwardr4    s    
aB!BBB!BB

EDE 88:?8vzz|q'8L

1a(J	#DJJ<z&,,qIJ  1$ D
b!3!3!5 6jA' $ 	&,,.DJJrN:FEF: INN(((TXXZ1_ A%+*;*;A*>$**Q-*O 	
EdjjQRm_ U)oo/00GHYHYZ[H\G]_	
O OO"{'8'8':a'?	YD[EVEVDWX	Y? T669lL r-   c           	      >   |j                         dk(  sJ d|j                                 |j                         dk(  sJ d|j                                 |j                  d   |j                  d   k(  r>|j                  d   |j                  d   k(  r|j                  d   |j                  d   k(  s!J d|j                   d	|j                          |j                         dk(  s&J d
|j                   d|j                          d       t        | ||||||      S )N   zSonly batches of spatial inputs supported (4D tensors), but got input of dimension: r   zUonly batches of spatial targets supported (3D tensors) but got targets of dimension: r   r%   r$   r/  r0  r2  z ( z, elements))rL   r  r   r  )rj   ry   r   r   r   r  r  s          r.   nll_loss2d_backwardr7  7  sE    	
aj	\]a]e]e]g\hij 	

n	^_e_i_i_k^lmn 	

1a(JJqMV\\!_,JJqMV\\!_,I 
$DJJ<z&,,H	I	-
 1$ 	""#3|'9'9';&<K	I$
 T669lL r-   c           	      "   |dz
  t        j                  t        j                  |        | j                  dd            z  |t        j                  t        j                  |       | j                  dd            z  z
  }|||z  }t        ||      S )Nr$   r,   i)rf   maximumlog1pnew_fulllogr   )ry   r   r   r   r   s        r.   binary_cross_entropyr=  Z  s     QJ%--TEDMM"d3 uyyb$0GHHID f}i00r-   c                     d}| ||z
  z  t        j                  |d|z
  z  |      z  }|||z  }|t        j                  j                  k(  r||j                         z  }|S )Ng-q=r$   r   )rf   r   r#   r*   rz   r   )rj   ry   r   r   r   EPSILONr  s          r.   binary_cross_entropy_backwardr@  p  sg     GD6M*U[[T9JPW-XXF&INN((($**,&Mr-   c                 r    t        j                  t        j                  |  |z              }t        ||      S r6   )rf   r:  re   r   )r   r   r   r   s       r.   soft_margin_lossrB    s.     ;;uyy%&12Di00r-   c                     || z  t        j                  ||z        dz
  z  }|t        j                  j                  k(  r||j                         z  }|S rY   )rf   r   r#   r*   rz   r   )rj   ry   r   r   rU   s        r.   soft_margin_loss_backwardrD    sM     +%v})E)IJJINN((($**,.
r-   otherpc                 6    t         j                  | |z
  |      S )N)rF  )r   r   )r   rE  rF  s      r.   distrH    s     99UU]a9((r-   x1x2c                    | j                  d      j                  dd      }t        j                  |t        j                        }|j                  d      j                  dd      }t        j                  |t        j                        }t        j
                  | j                  d      ||gd      }t        j
                  |||gd      }|j                  |j                        }|j                  d      j                         S )Nr%   rP   Tmemory_formatr   )powr   rf   	ones_likecontiguous_formatr#  r   matmulmT	clamp_minsqrt)	rI  rJ  x1_normx1_padx2_normx2_padx1_x2_r  s	            r.   _euclidean_distr\    s     ffQimmB%G__WE4K4KLFffQimmB%G__WE4K4KLF
))RVVBZ&12
6C
))R)2
.CZZFA##%%r-   input_sizesstartendstepc                 X    | j                  |      }t        j                  || ||||      S r6   )	new_zerosrf   slice_scatter)rj   r]  rL   r^  r_  r`  rU   s          r.   slice_backwardrd    s/     &&{3Jz;UCNNr-   r$   c                    ddl m}m} | j                         }|dk(  rt	        d      t        j                  | j                         |      }t        | j                               }t        | j                               }	|dk  rt	        d      ||nd}
||nt        j                  } ||
dk        r|
||   z  }
 ||dk        r|||   z  } ||
dk        rd}
n ||
||   kD        r||   }
 |||
k        r|
}n, ||t        j                  k(        s ||||   kD        r||   }| j                         |
|	|   z  z   }||
z
  }||z   dz
  |z  ||<   |	|xx   |z  cc<   | j                  rt        d      | j                  ||	|      S )Nr   )guard_size_obliviousstatically_known_truez,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver$   z<Slice decomposition for quantized tensors aren't implemented)%torch.fx.experimental.symbolic_shapesrf  rg  rL   RuntimeErrorrB   r   listr!  stridesysmaxsizestorage_offsetis_quantizedNotImplementedError
as_strided)ry   rL   r^  r_  r`  rf  rg  ndimsizesstrides	start_valend_valrn  lens                 r.   slice_forwardrx    s   
 88:DqyIJJ

 
 S
1CE4;;=!Gqy899*I_c#++GIM*U3Z	GaK(5:IM*		i%*4	5#J	Gi/0	w#++5	6:N%*; *((*Y-EEN
I
C*q.T)E#JCLDL!J
 	
 ug~>>r-   c                 n    | j                   |   dt        ffd} ||dd      } |||      }||fS )zn
    Normalize start and end such that both are in the range
    [0, x.get_size()[dim]] and start <= end.
    rM   c                 L    | |S | dk  r| z   } t        t        | |      |      S rq   r   r   )valr   r   defaultdim_sizes       r.   
clamp_wrapz(_normalize_start_end.<locals>.clamp_wrap  s0    ;N7.C3sE?E**r-   r   )r  int)r:   rL   r^  r_  r  r~  s        @r.   _normalize_start_endr    sJ     wws|H+# + ua1-E
S%8
4C#:r-   srcc           	      |   t        j                  | j                  |      }| j                  |   }t	        | |||      \  }}t        | j                        }||z
  |dz
  z   |z  ||<   |j                  |      }|dk(  r||k(  r|dk(  r|j                         S d g| j                         z  }t        j                  || j                        }	|	|z
  |z  ||<   t        j                  || j                  t        j                        }
|dk7  rt        j                  |
|	|k\        }
||k7  rt        j                  |
|	|k        }
|dk7  rt        j                  |
|	|z
  |z  dk(        }
dg| j                         z  }d||<   |
j                  |      }
t         j#                  |
t         j%                  ||
|d      |       S )Nr$   r   devicer  r   rP   )rB   r   rr  r  r  rj  expandclonerL   rf   aranger  onesboollogical_andviewr   rg   _unsafe_masked_index)r   r  rL   r^  r_  r`  r~  src_sizeindicesidxmask
mask_shapes               r.   rc  rc    s    
 
 S
1C{{3H%eS%=JE3EKK H5[D1H-$6HSM
**X
CzcXo$!)yy{fuyy{"G
,,x
5C%KD(GCL::hu||5::FDz  se|4
h  sSy1qy  et';q'@Auyy{"JJsO99Z D::dD55c4!LeTTr-   indexc                 T    | j                  |      }t        j                  || ||      S r6   )rb  rf   select_scatter)rj   r]  rL   r  rU   s        r.   select_backwardr  9  s+     &&{3J
KeDDr-   offsetdim1dim2c                 V    | j                  |      }t        j                  || |||      S r6   )rb  rf   diagonal_scatter)rj   r]  r  r  r  rU   s         r.   diagonal_backwardr  @  s-    
 &&{3J!!*k64NNr-   input_dtypec                 F    | j                   |k7  r|j                  |      }|S r6   )r   r9   )rj   rU   r  s      r.   _cast_grad_to_input_dtyper  I  s&     K']];/
r-   outputc                 ~    | |z  }||t        j                  ||d      z  z
  }t        | ||      j                         S NTr   )rf   r   r  
contiguous)rj   r  rL   r  new_grad_outputrU   s         r.   _softmax_backward_datar  Q  sK     "F*O 6EIIS$- $ J %[*kJUUWWr-   c                 ~    | t        j                  |      t        j                  | |d      z  z
  }t        | ||      S r  )rf   re   r   r  )rj   r  rL   r  rU   s        r.   _log_softmax_backward_datar  c  sA     uyy0599d4   J %[*kJJr-   c                     | |dz  z   ||dz
  z  z
  }t        t        j                  t        j                  |      } |d||      j	                  d      } |d||z  |      j	                  d      }	||	z   S )z/Utility function to implement im2col and col2imr%   r$   r   r  r   rP   )r   rf   r  int64rR   )
input_dkernel_d
dilation_d	padding_dstride_dr  blocks_d	arange_kwblocks_d_indiceskernel_grids
             r.    _im2col_col2im_indices_along_dimr  o  s     Q&x!|)DDHEKKGI !Hh7AA!D Ax*4jAKKBOK k))r-   kernel_sizedilationpaddingrk  c           
         t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        dd} |d        |d        |d	d
        |d       | j                  t              }t        j                  |dv xr t	        d dd  D              fd       t        d t        dd        D              t        j                  t	        d D              fd       |dk(  }|s| j                  d      } | j                  \  }}	}
}\  }}\  }}\  }}\  }}t        |
||||| j                        }t        |||||| j                        }t        j                  | ||||f      }|j                  d      j                  d      }|d d d d ||f   }|j                  dddddd      }|j                  d      }|j                  d      }|j                  ||	|z  |z  ||z        }|s|j                  d      }|S )Nr%   c                       y)Nz"im2col(): only 2D kernel supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>      r-   c                       y)Nz$im2col(): only 2D dilation supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  r  r-   c                       y)Nz#im2col(): only 2D padding supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  r  r-   c                       y)Nz"im2col(): only 2D stride supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  r  r-   c                 ~    |rt        d | D              nt        d | D              }t        j                  |d        y )Nc              3   &   K   | ]	  }|d kD    ywr   Nr,   .0rF  s     r.   	<genexpr>z1im2col.<locals>.check_positive.<locals>.<genexpr>       (Q1q5(   c              3   &   K   | ]	  }|d k\    ywr  r,   r  s     r.   r  z1im2col.<locals>.check_positive.<locals>.<genexpr>       ;RqAF;Rr  c                       y)Nz<{param_name} should be greater {'than' zero, but got {param}r,   r,   r-   r.   r   z0im2col.<locals>.check_positive.<locals>.<lambda>  r  r-   r   rf   r   param
param_namestrictconds       r.   check_positivezim2col.<locals>.check_positive  s3    ,2s(%((;RE;R8RX	
r-   r  r  r  Fr  rk  r   r6  c              3   &   K   | ]	  }|d k7    ywr  r,   r  ds     r.   r  zim2col.<locals>.<genexpr>       :!qAv:r  r   c                       dt                S )NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tupler  s   r.   r   zim2col.<locals>.<lambda>       --25\N< r-   c              3   \   K   | ]$  \  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z    & yw)r$   r%   Nr,   )r  r   paddilkersts         r.   r  zim2col.<locals>.<genexpr>  sF      "Cc3 	
S1s7]SC!G_,q0R77s   *,rN  c              3   &   K   | ]	  }|d kD    ywr  r,   )r  cs     r.   r  zim2col.<locals>.<genexpr>  s     'aAE'r  c                  F    dt        dd         d d  d d d dS )	Nz!Given an input with spacial size rN  , kernel_size=, dilation=
, padding=	, stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r  )r  r  output_sizer  r  rk  s   r.   r   zim2col.<locals>.<lambda>  sL    3E%*4E3F G"m;xj 9)9VH -]D	F r-   r6  r   rP   r$   r      T)rf   r   rw  r  r   r  ziprR   r  r  r   r  permuter!  r  squeeze)r   r  r  r  rk  r  rr  batched_input	batch_dimr  input_hinput_wstride_hstride_w	padding_h	padding_w
dilation_h
dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indicespadded_inputr  num_blocks_rownum_blocks_colr  r  s    ````                     @@r.   im2colr    s~    
LL[!Q&(TU	LLX!#%ST	LLW"$QR	LLV!#OP
 ;.8Z(8Yu568$KKEu:D	LL:3:uRSz::	<
  &)"#J;'
 K 
LL';''	F 	F AIM"/4{{,I{GWHh"Iy%J
$Hh9:y(ELL ::y(ELL 55Iy) LML+55b9CCBG!Q 24FFGF^^Aq!Q1-F',,Q/N',,Q/N^^;)H4n~6UF "Mr-   r  c                   !" t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        dd} |d	        |d
        |dd        |d        |d       | j                  "t        "      }t        j                  |dv xr t	        d "dd  D              "fd       d   d   z  }t        j                  "d   |z  dk(  "fd       t              D 	
cg c]"  \  }	}
}}}d|	d|
z  z   ||dz
  z  z
  dz
  |z  z   $ }}}}
}	}|d   |d   z  !t        j                  "d   !k(  !"fd       t        j                  !dkD  !"fd       |dk(  }|s| j                  d      } | j                  "\  }}\  }}\  }}\  }}\  }}| j                  "d   "d   |z  gz   |z         } | j                  dddddd      } t        |||||| j                        }t        |d      }t        |||||| j                        }t              D cg c]  \  }}|d|z  z    }}}| j                  "d   "d   t              z  g|z         }d d ||f} t        j                  || | d      }t!        j"                  || | | | f      }|s|j%                  d      }|S c c}}}}
}	w c c}}w )Nr%   c                       y)Nzonly 2D output_size supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  r  r-   c                       y)Nzonly 2D kernel supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  r  r-   c                       y)Nzonly 2D dilation supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  r  r-   c                       y)Nzonly 2D padding supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  r  r-   c                       y)Nzonly 2D stride supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  r  r-   Tc                 ~    |rt        d | D              nt        d | D              }t        j                  |d        y )Nc              3   &   K   | ]	  }|d kD    ywr  r,   r  s     r.   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r  c              3   &   K   | ]	  }|d k\    ywr  r,   r  s     r.   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r  c                       y)Nz9{param_name} should be greater than zero, but got {param}r,   r,   r-   r.   r   z0col2im.<locals>.check_positive.<locals>.<lambda>  r  r-   r  r  s       r.   r  zcol2im.<locals>.check_positive  s3    ,2s(%((;RE;R8RU	
r-   r  r  r  Fr  rk  r  )r%   r   c              3   &   K   | ]	  }|d k7    ywr  r,   r  s     r.   r  zcol2im.<locals>.<genexpr>  r  r  rN  c                       dt                S )NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: r  r  s   r.   r   zcol2im.<locals>.<lambda>  r  r-   r   r$   c                      dd    d  S )Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rN  z and kernel_size=r,   )r  r  s   r.   r   zcol2im.<locals>.<lambda>  s#     ==B2YK H"m% r-   rP   c                  :    d d d d d d  dd    d	S 
NzGiven output_size=r  r  r  r  z , expected input.size(-1) to be 	 but got rP   .r,   Lr  r  r  r  r  rk  s   r.   r   zcol2im.<locals>.<lambda>  F    $[M} M:Zy	& B))*9U2YKqB r-   c                  :    d d d d d d  dd    d	S r  r,   r  s   r.   r   zcol2im.<locals>.<lambda>  r  r-   r   r6  r  
accumulater  )rf   r   rw  r  r   r  rR   r  r  r  r  rT   rb  prodr   _unsafe_index_putr   r  r  )#r   r  r  r  r  rk  r  rr  prod_kernel_sizer   r  r  r  r  colr  out_hout_wr  r  r  r  r  r  r  r  indices_rowindices_colorF  output_padded_sizer  r  r  r  s#    `````                           @@r.   col2imr!    s    
LL[!Q&(OP	LL[!Q&(JK	LLX!#%IJ	LLW"$GH	LLV!#EF
 ;.8Z(7Ie468$;.KKEu:D	LL:3:uRSz::	<
 #1~A6	LLb	$$)	% '*(K'
 "Cc3 	
S1s7]SC!G_,q0R77C  	AQA	LLb	Q	B 	B 
LL	A	B 	B AIM"KKELE5Hh"Iy%J
$Hh MM58U1X1A%AB[PSVVWEMM!Q1a+E2xY%,,K $K3K2xY%,,K 14K0IJ1!a!e)JJ__	q58tK0014FFF {K
0C##FC4#HFUU6YJ
YJ
KLF"MkV Ks   6'M9Mr  c                 z    | |j                  |       |z  z  j                  t        j                  |             }|S NrL  )type_asr  rB   r   )rj   r  rl   rG   s       r.   native_dropout_backwardr%  7  sB     
[1E9	:AA11+> 	B 	A Hr-   
input_size	dimensionr!  c                    t        |      dk(  rt        j                  | d      S t        j                  t        |      |      }t        j
                  ||   | j                  t        j                        }|j                  d||      j                         }| j                  d|dz         j                  ||dz         } | j                  |      }d|z  |fz   }t        j                  ||| d      j                         S )Nr   r  rP   r$   r6   Tr  )rw  rf   squeeze_copyrB   r   r  r  int32unfoldflattenmovedimrb  r   r  r  )	r   r&  r'  r!  r`  rL   r  rU   r  s	            r.   unfold_backwardr.  F  s    
 :!!!$**

 
 Z)
<C
,,z#t{{%++
NC
**Qd
#
+
+
-C<<C!G$,,S#':D 
+JcMSF"E!!*eTd!KVVXXr-   epsc           
      .   |A|}d|z
  }t        j                  t        j                  ||k\  ||k        | |d|z
  z  z  d      S t        j                  t        j                  |dk\  |dk        | |d|z
  z  z  |j                  dt	        d                  S )Nrd   r   r,   nan)rf   rg   r  r;  r   )rj   ry   r/  lohis        r.   logit_backwardr4  Y  s    
 2X{{dbj$"*543:./
 	
 {{dck43;743:./MM"eEl+
 	
r-   trainc                 d    |r|dk7  rt         j                  | ||      d   S | j                         S rq   )r   native_dropoutr  )r   rF  r5  s      r.   dropoutr8  n  s3     a""5!U3A66{{}r-   out0out1c                    |r|dk7  r|dk(  r:t        j                  |       t        j                  | t         j                        fS | j                  j                  st        d      t        j                  |       |kD  }|| z  t        dd|z
  z        z  }||fS | t        j                  | t         j                        fS )Nr   r$   r   z?result type Float can't be cast to the desired output type Longrd   )	rf   r   r  r   is_floating_pointri  	rand_liker   rP  )r   rF  r5  	bool_maskress        r.   r7  r7  x  s     a6$$U+U-=-=e5::-VWW{{,,Q  OOE*Q.	%%sQw"88YuuEJJ?@@r-   half_to_floatc                    | j                         } |r| j                  t        j                  k(  sJ t	        j
                  | t        j                  j                        \  }}| j                  |      } | j                         dk(  rt        j                  |       }n0t        j                  | |d      }t        j                  | |z
        }|t        j                  ||d      z  }|s|j                  |      }|S Nr4   r   T)r   )r  r   rf   halfrB   rC   ELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTr9   r   re   amaxr   )r:   rL   r@  r;   r>   unnormalizedx_maxr  s           r.   _softmaxrJ    s    
 	
Aww%**$$$&+&>&>	uDDLL'#| 	
AwwyA~yy|

1c40yyU+EIIlCFFF<(Mr-   )r  c                     | j                         } |r| j                  t        j                  k(  sJ t	        j
                  | t        j                  j                        \  }}| j                  |      } | j                         dk(  r| }nt        j                  | |d      }| |z
  }t        j                  t        j                  t        j                  |      |d            }||z
  }|s|j                  |      }|S rB  )r  r   rf   rD  rB   rC   rE  rF  r9   r   rG  r<  r   re   )	r:   rL   r@  r;   r>   shiftedrI  shifted_logsumexpr  s	            r.   _log_softmaxrN    s    
 	
Aww%**$$$&+&>&>	uDDLL'#| 	
AwwyA~

1c40e)		%))EIIg,>T"RS((F<(Mr-   r  padding_idxscale_grad_by_freqsparsec                     | j                         dk(  sJ d       |j                  dk  r4| j                  d|      }|j                  dk(  r|j                  d      }|S | |   S )Nr%   z'weight' must be 2-Dr$   r   )rL   rr  index_selectr  )r   r  rO  rP  rQ  r   s         r.   	embeddingrT    sd     ::<1444||q!!!W-<<1++a.C
gr-   num_weightsc                 t   t        j                  | t         j                  j                        \  }}| j	                  |      } t        |t        j                        }|rZ|j                  |f      }t        j                  |      }t        j                  ||g|d      }||   }	| |	j                  d      z  } t        ||k(  | j                        }
| j                  |
d      }| j                  |f| j                   |j                  d  z         }t        j                  ||g|d      j	                  |      S )NrC  Tr  rP   r   )rB   rC   rE  rF  r9   r   rf   longrb  rP  r   r  rR   rT   rr  masked_fillr  )rj   r  rU  rO  rP  r;   r>   countsr  grad_weights_scaler  r   grad_weights                r.   embedding_dense_backwardr\    s,    ',&>&>)N)N)V)V'#| ..!23K%guzz:G""K>2w'''	4D'Q#G_!$6$@$@$DDW3[5E5EFD""4+D''	**7<<>::K !!+y$4!PSS r-   c                 "    d}| D ]  }||z  }	 |S rY   r,   )r:   rG   is      r.   r  r    s$    	A 	QHr-   tensors
num_chunksc                 Z   g }| D ]  }|j                         }||   |z   dz
  |z  |z  }|||   k7  r;dgdz  |j                  |z
  dz
  z  d|||   z
  gz   }t        j                  ||d      }|d | t	        j
                  |dg      z   }|j                  |j                  |              |S )Nr$   r   r%   rP   )r!  rr  r   constant_pad_ndrf   Sizeappendr  )	r_  rL   r`  padded_tensorstensortensor_sizepad_along_dimr  	view_sizes	            r.   
_pad_chunkrj    s    
 N 6kkm$S)J6:zIJVK,,#'V[[3.23C 007 C ))&#q9F%

J3C(DD	fkk)456 r-   c                 R    | d   j                   }| D ]  }|j                   |k7  s y y)Nr   FTrr  )r_  rr  rf  s      r.   have_same_ndimsrm    s2    1:??D ;;$ r-   c                     | d   j                         d | }| D ]-  }t        j                  |j                         d | |k(  d        / y )Nr   c                       y)NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr,   r,   r-   r.   r   z+leading_dimension_matches.<locals>.<lambda>  r  r-   )r!  rf   r   )r_  rL   leading_dim_sizesrf  s       r.   leading_dimension_matchesrq    sN    
)$3/ 
KKM$3#44]	

r-   c                    t        j                  |dk\  d        t        j                  t        |       dkD  d        | d   j                  }| d   j                  }| D ]r  }t        j                  |j                         dkD  d        t        j                  |j                  |k(  d        t        j                  |j                  |k(  d        t t        |       r(t        j                  | d   j                         |      }nEt        j                  |dk\  d        | D ]&  }t        j                  ||j                  k  d	        ( t        | |       |S )
Nr$   c                       y)Nz&_chunk_cat expects positive num_chunksr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r-   r   c                       y)Nz0_chunk_cat expects a non-empty input tensor listr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>   r  r-   c                       y)Nz#_chunk_cat expects non-empty tensorr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>%  r  r-   c                       y)Nz8_chunk_cat expects all input tensors with the same dtyper,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>(  r  r-   c                       y)Nz8_chunk_cat expects all inputs tensors on the same devicer,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>,  r  r-   c                       y)NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>3  r  r-   c                       y)Nz3_chunk_cat expects dim < ndim for all input tensorsr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>8  r  r-   )rf   r   rw  r   r  r   rm  rB   r   rL   rr  rq  )r_  rL   r`  expected_dtypeexpected_devicerf  s         r.   _preprocess_chunk_cat_inputsr|    s-   
 
LLq"RS	LLGqT QZ%%Naj''O 	
V\\^a')VWLLN*N	
 	MM_,N	
	
 w$$WQZ^^%5s;1Ha	
  	FLLfkk!M	
 gs+Jr-   r   c                     t        | ||      }t        | ||      }|t        j                  ||dz         S t        j                  ||dz   |       |S )Nr$   )r   )r|  rj  rf   r#  )r_  rL   r`  r   re  s        r.   
_chunk_catr~  >  sS     'wZ
@Cj9N
{yyq11		.#'s3
r-   split_sizesc                    t         j                  | ||      }|.|D cg c]"  }|j                  t        j                        $ c}S t        ||      D ])  \  }}t        ||j                         t        ||d       + y c c}w )Nr~   rL  Tr
  )	r   split_with_sizesr  rf   rQ  r  r   r  r   )ry   r  rL   r   splitssr  splits           r.   split_with_sizes_copyr  O  s     ""4#">F
{HNO1e&=&=>OO f- 	NMFEfekk2UFM	N  Ps   'B
split_size.c                 D    t         j                  j                  | ||      S r6   )r   r  r   )r   r  rL   s      r.   unsafe_splitr  b  s    ::UJ44r-   c                 D    t         j                  j                  | ||      S r6   )r   r  r}  )r   r  rL   s      r.   unsafe_split_with_sizesr  g  s       ((SAAr-   c                    | j                   }||   }|dk(  r|dk(  sJ | j                         fS ||z   dz
  |z  }ddlm}  ||      }t	        |      D cg c]  }| }}|||z  |z
  z
  |d<   t        j                  | ||      S c c}w )Nr   r$   )	guard_intrP   )r  detachrh  r  rQ   rf   r  )	ry   r  rL   r]  r~  chunksr  r^  r  s	            r.   r  r  n  s    **K3HQ1}}#a'J6F @vF',V}5!:5K5 J$7($BCKO;;t[#.. 6s   	Btensor_indices_or_sectionsc                    |j                   j                  dk(  sJ |j                  t        j                  k(  sJ |j                         t        j                  dk(  xs dk(  fd       dk(  r4|j                         }t        |t              sJ | j                  ||      S |D cg c]  }|j                          }}| j                  ||      S c c}w )Ncpur$   r   c                      d  dS )Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr,   )	split_dims   r.   r   zAtensor_split_tensor_indices_or_sections_py_impl.<locals>.<lambda>  s     <<E;eM r-   )r  typer   rf   r  rL   r   itemr8   r   tensor_split)ry   r  rL   sectionsr^  r  r  s         @r.   /tensor_split_tensor_indices_or_sections_py_implr    s     &,,11U:::%++u{{:::*..0I	LLQ()q.	M
 A~-224(G,,,  3//%?@1668@@   #.. As   ,Cmat1mat2c                     | j                         s&| j                         st        |      }t        |      }|t        j                  ||      z  }|dk(  r|S ||| z  z   S rq   )r<  
is_complexr  rf   mm)ry   r  r  r`   rk   r   s         r.   addmmr    s]     !!#DOO,=4yE

%((4&
&Cqy
 r-   use_geluc                     t        | ||||      }|r8| j                  rt        j                  |d      S t        j                  |      S t        j	                  |      S )Nr   )r   )r  is_cudar   gelurelu)ry   r  r  r`   rk   r  r   s          r.   _addmm_activationr    sO     dD$
.C<<99Sf95599S>!99S>r-   vecc                     | j                         s&| j                         st        |      }t        |      }|t        j                  ||      z  }|dk(  r|S ||| z  z   S rq   )r<  r  r  rf   mv)ry   r  r  r`   rk   r   s         r.   addmvr    s[     !!#DOO,=4yE

%((4%
%Cqy
r-   r   rstdgammaNCHxWgroupoutput_maskc
           	      R   t        j                  | ||d       t        j                  || d       t        j                  |d       t        j                  |j                         z  z  k(  fd       t        j                  j                  fk(  fd       t        j                  d u xs j                         k(  fd       t              \  }
}t        j                  |dk(  fd       t        j                  | |      j                        j                  dg	      }| j                        j                  dg	      }d }d }d }|	d   r*d
|
z  z  }t        j                  |j                  d            j                  |
      j                  d      }t        j                  |j                  d            j                  |
      j                  d      }t        j                  |j                  d      j                  d|
            }n|j                  |
      j                  d      }|j                  |
      j                  d      }t        j                  |j                  d      t        j                  d|
f|j                              }|z  |z
  |z  |z  |z  |z  }| z  ||z  |z  z
  }|j                  d      }t        |d      }t        |d      }t        j                  | j                  |
      |      t        j                  |j                  |
      |      z   |z   }|j                  |j                        j!                  |j"                        }|	d   rk|j                  |
      |j                  |
      j                  d      z  z
  |j                  d      z  j                  dg	      j                        }|	d   r|j                  dg	      }|||fS )NF)allow_cpu_scalar_tensorsc                      d z  z   dS )NzExpect input to have z	 elementsr,   )r  r  r  s   r.   r   z,native_group_norm_backward.<locals>.<lambda>  s    'A}I> r-   c                  .    d  d dj                    S )NzExpect mean to have shape (, z
, but got r  )r  r  r   s   r.   r   z,native_group_norm_backward.<locals>.<lambda>  s    -aS5'DJJ<P r-   c                  <    d  dj                          S d S )NzExpect gamma to have z elements but got rP   )r   )r  r  s   r.   r   z,native_group_norm_backward.<locals>.<lambda>  s-    's*<eN_U[[]<hi eg<hi r-   r   c                      d  d S )NzExpect number of channels z, to be evenly-divisible by number of groups r,   )r  r  s   r.   r   z,native_group_norm_backward.<locals>.<lambda>  s    ,QC/[\a[bc r-   r%   r~   rd   rP   r$   r  r6  )rB   check_same_devicecheck_same_shaperf   r   r   r  divmodr   r  r   rR   r  r  r  rT   r9   r   )rj   r   r   r  r  r  r  r  r  r  cpg_remdsdbd_inputd_gammad_biasr  ds_valdb_valc1c2c3s     ` `````              r.   native_group_norm_backwardr    s    
UD$ 
5+N	4F	LLQ$> 
LL

q%j P 
LL+!+i
 q% IC	LL	c 
;	&	+	+Aq#	6	:	:s	:	CB			!Q	$	(	(aS	(	1B $G $G#F1~39YYr5??1#56>>q%MQQRSTFYYr5??1#56>>q%MQQRSTFr"a,B
 ZZ5#.2215FZZ5#.2215Fr"

Auc?4;;?B tmf$,t3d:Q>S4Z&4-!++\\"r1%r1%IIk))!UC=rBiiaS92>? 	
 //%++.11%++>1~ E3'"''!UC*@4>>RTCU*UU..$% SaSS\WQZ 	 1~QCWf%%r-   out2c
                    t        | |||||||||	
      }|
||f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r	  )r  	enumerater   r  r   )rj   r   r   r  r  r  r  r  r  r  r9  r:  r  r  rU   r^  rG   s                    r.   native_group_norm_backward_outr  1  sz    " (UD$q!S%F d#J&! Q1=jmQWW5Q
14PQ
 r-   c                 ,    | | j                  |      S | S r6   r9   )r:   r   s     r.   _maybe_castr  N  s    }ttE{Hr-   grad_outnormalized_shapebiasc                 8  " |j                   }|j                         }	t        j                  |j                        ""fd| |||fD        \  }
}}}|
J |	t        |      z
  }||d  }|d | }g }g }t        |	      D ]*  }||k\  r|j                  |       |j                  |       , t        |      }t        |      }ddl	m
}  ||dk        s ||dk        rN|d   r|j                  |      nd |d   r|j                  ||d        nd |d   r|j                  ||d        fS d fS t        ||j                               }t        ||j                               }||z
  |z  }||
|z  }n|
}||z  }t        j                  ||d      }t        j                  ||      }t        j                  ||d      }t        j                  ||      }||z
  |z
  }d }d } d }!|d   r||z  |z  }|d   r0|.t        |      dkD  rt        j                  |
|z  |d      } n|
|z  } |d   r8|6t        |      dkD  rt        j                  |
|d      }!n|
j!                         }!t#        ||j                        t#        | |j                        t#        |!|j                        fS )Nc              3   d   K   | ]'  }||j                        j                         n| ) y wr6   )r9   r  r  r:   r;   s     r.   r  z-native_layer_norm_backward.<locals>.<genexpr>c  s5      9 12**,1D9s   -0r   rf  r$   r%   TF)r  rL   rB   get_computation_dtyper   rw  rQ   rd  r  rh  rf  rb  rT   rf   r   r   r  r  )#r  r   r  r   r  r   r  r  input_shape
input_ndimgrad_out_cast
input_castweight_cast	bias_castaxis
inner_dims
outer_dimsinner_dim_indicesouter_dim_indicesr^  r  Mrf  x_hat
grad_x_hatabr  r  r  rH   r  d_weightr  r;   s#                                     @r.   native_layer_norm_backwardr  U  s    ++KJ33EKK@9E64095M:{I $$$,--DTU#JUd#J#%#%: (9$$Q'$$Q'	( 	ZAZAJAF#';AF'C,7NEOOK(3>q>EOOK./t3>q>EOOK./
 	
 HL
 	

 T:>>#34DT:>>#34D$$&E"[0
"
QA		*/6A	:u	%B	2($	/B	5"	BEBJE $G!%H#F1~!8u$1~+1 !A%yy!68I5QH$u,H1~)/ !A%YY}.?GF"((*F 	GU[[)Hekk*FEKK( r-   c          
          t        | |||||||      }||	|
f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r	  )r  r  r   r  r   )r  r   r  r   r  r   r  r  r9  r:  r  r  rU   r^  rG   s                  r.   native_layer_norm_backward_outr    sw     (%)4vt[F d#J&! Q1=jmQWW5Q
14PQ
 r-   running_meanrunning_varmomentum
functionalc	                 2   dgt        t        d| j                                     z   }	t        j                  | j
                        }
|}|}|r"t        j                  | j
                        }
| j                  |
      }t        j                  ||	dd      \  }}t        j                  ||z         }| |z
  |z  }t        j                  ||	      }t        j                  ||	      }|!||z  d|z
  |z  z   }|s|j                  |       |2| j                         | j                  d   z  }t        j                  ||	      }|||dz
  z  z  }||z  d|z
  |z  z   }|s|j                  |       n||J |j                  |
d      }|}|j                  |
d      }|}|}dt        j                  ||z         z  }| j                  j                   dk7  r|}|}n"| j#                  d	      }| j#                  d	      }t%        || j                         dz
        }t%        || j                         dz
        }| |z
  |z  }|2|j'                         }t%        || j                         dz
        }||z  }|2|j'                         }t%        || j                         dz
        }||z   }| j                  j                   dk(  r8|j                  | j
                        }|j                  | j
                        }|j                  | j
                        ||||fS )
Nr   r%   r   T)rL   
correctionr   r$   )r   r   r  r   )rj  rQ   rL   rB   r  r   r9   rf   var_meanrsqrtr  copy_r   r  rU  r  r  rb  rT   r,  )r   r   r  r  r  r   r  r/  r  reduction_dimsr;   new_running_meannew_running_var	input_acc
biased_varr   r  r  	save_mean	save_rstdnsqueezed_varunbiased_varinvstds                           r.   native_batch_norm_helperr    s    S4a 566N33EKK@#!O!77DHH#4H5	 >>>a

D {{:+,$,$&MM$7	MM$7	#')3q8||6SS""#34"A.A !==^DL'1A;7L&5X8TTO!!/2'K,CCC#->TJ'!nn+<4nH%ejjs!234<<%$II-I-I uyy{Q7"6599;?;$,&(!"6599;?;&||~ uyy{Q7$||E!LLu{{L3	LLu{{L3				$ r-   r  save_invstdc                 >    t        | |||||||d	      \  }}	}
}}||	|
fS NFr  r   r   r  r  r  r   r  r/  r  r  r  rS   s               r.   native_batch_normr    s=     *Bvt\;(CQV*&FIy!Q 9i''r-   c           
          ||t         j                  | |||||      S |t        d      |t        d      |rt         j                  | |||||||      S t         j                  | ||||||      S )Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r   _native_batch_norm_legitri  $_native_batch_norm_legit_no_training)r   r   r  r  r  r   r  r/  s           r.   native_batch_norm_decompositionr	  /  s      3,,648S
 	
 <
 	
 <
 	
 ,,64{HhPS
 	
 8864{Hc
 	
r-   c                 l   | j                  |      }||z   dz
  |z  }|dk(  rZ|dk(  rU|D cg c]  }| }}|||z  |z
  z
  ||dz
  <   t        j                  j                  j                  j                  | ||      S t        j                  j                  j                  j                  | ||      S c c}w Nr$   r   )r!  rf   opsr   r  r}  r  r   )rf  r  rL   r~  r  rS   r  s          r.   unsafe_chunk_py_implr  T  s    {{3HV#a'F2JQ8q=+12az22",
V0Ch0N"OFQJyy~~55==fkSVWW99>>&&--fj#FF 3s   	B1c           
      N    t         j                  j                  | ||||d||      S r  )r   r  r}  )r   r   r  r  r  r  r/  s          r.   r  r  `  s5     ((00	 	r-   c                 >    t        | |||||||d	      \  }}	}
}}||	|
fS r  r  r  s               r.   r  r  v  s=     *Bvt\;(CQV*&FIy!Q 9i''r-   c                 >    t        | ||d d |||d	      \  }}}}	}	|||fS r  r  )
r   r   r  r   r  r/  r  r  r  rS   s
             r.   !_native_batch_norm_legit_no_statsr    s<     *BvtT48S%*&FIy!Q 9i''r-   c                 f    t        | |||||||d	      \  }}	}
}}|J d       |J d       ||	|
||fS )NT#new_running_mean should not be None"new_running_var should not be Noner  )r   r   r  r  r  r   r  r/  r  r  r  r  r  s                r.   #_native_batch_norm_legit_functionalr    sl    " 	!vt\;(CQU	 'N)NN'&L(LL&9i)9?JJr-   c           	      T   t         j                  j                  | ||||d|      }d}|t         j                  j                  j                  k(  r t         j                  j                  | |      }t        j                  |t         j                  | j                  | j                        S )a  
    Return a reserve tensor for batch norm, used only by cudnn to pass forward state to the
    backward pass. This is needed for `_batch_norm_with_update` and `_batch_norm_no_update`,
    which support a variety of backends including cudnn. We create this tensor here to get
    the correct shape in the traced graph if we detect that will call the cudnn kernel,
    and rely on DCE to avoid materializing this tensor.
    Tr   )r   layoutr  )
rf   _C_select_batch_norm_backend_BatchNormBackendCudnn(_get_cudnn_batch_norm_reserve_space_sizeemptyuint8r  r  )	r   r   r  r  r  r/  r   backendreserve_sizes	            r.   _get_batch_norm_reserve_tensorr!    s      hh11vt\;cG L%((,,222xxHHPXY;;EKKU\\ r-   c                 d    t        | ||||d||d	      \  }}}	}
}
t        | |||||d      }|||	|fS )NTFr   r  r!  r   r   r  r  r  r  r/  r  r  r  rS   reserves               r.   _batch_norm_with_updater'    sa     *B
*&FIy!Q -vt\;dG 9i00r-   c                     t        | ||||d||d	      \  }}}	}
}t        | |||||d      }|
J d       |J d       |||	||
|fS )NTr#  r  r  r$  )r   r   r  r  r  r  r/  r  r  r  new_rmnew_rvr&  s                r.   "_batch_norm_with_update_functionalr+    s      	!vt\;hT	 -vt\;dG DDDCCCIy'66BBr-   c                 d    t        | ||||d||d	      \  }}}	}
}
t        | |||||d      }|||	|fS )NFr#  r$  r%  s               r.   _batch_norm_no_updater-     sa     *B
*&FIy!Q -vt\;eG 9i00r-   c                     |J t        j                  |       |k  j                  t         j                        }|j	                  |       | z  d|z  z  }||fS )Nr   rd   )rf   r=  r9   r  r$  )r   rF  	generatorr  r?  s        r.   _fused_dropout_decompositionr0    s[     OOE"Q&***=D
,,u

%q
1C;r-   )r   r  r  
pin_memorynon_blockingrM  r  r1  r2  rM  c                   |r|t         j                  k(  sJ d       |rJ d       t        | t         j                  t        t
        t        t        f      sJ |0|.|,t        | t         j                        r| j                         S | S d}t        | t         j                        r| }nt        j                  |       }|c||j                  k7  rT|1|j                  dk(  r"t         j                  j                  ||      }d}t         j                  j                  |||      }|$|s"t         j                  j                  ||      }d}|t        j                  ||      S |S )NTODOFr  TrL  )rf   stridedr8   r   r  r   r  complexr  scalar_tensorr  r  _primsconvert_element_type
device_put)	r:   r   r  r  r1  r2  rM  dtype_convertedx_tensors	            r.   _to_copyr=  %  s2    5==08&80!6!>a%,,UD'BCCC~%-M,Aa&779HO!U\\"&&q)f7!5||885IH"O<<**8V\J<<44XuE {{8=AAOr-   c                 ,    t         j                  |       S r6   )r   alias)r:   s    r.   nop_decompositionr@  S  s     ::a=r-   out3exponential_average_factorepsilonc           
         t         j                  | |||||||      \  }}	}
|r%||	|
| j                  dt        j                        fS ||j                  d      |j                  d      | j                  dt        j                        fS )Nr  r   )r   r  rb  rf   r  )r   r   r  r  r  r   rB  rC  r  r  r  s              r.   cudnn_batch_normrE  [  s     $$"	GAq! 1aU[[ABB	EKK0	 r-   c                     t        |      D ]>  \  }}|dk(  s|| j                  k  r| j                  |   |k(  r.| j                  |      } @ | S rY   )r  rr  r  rR   )r:   broadcast_maskr  r  s       r.   _broadcast_batch_norm_backwardrH  }  sO    / "
d19dQVVm0ED!A" Hr-   r&  c                 *    t        | |||||||||	
      S r6   )native_batch_norm_backward)r  r   r   r  r  r  r   r5  r/  r  r&  s              r.   batch_norm_backwardrK    s/     & r-   c
                   & |j                   }
||j                   }n|
}t        j                  |j                         &&fd| ||||||fD        \  }}}}}}}|j                  }|j	                         }|dk\  sJ d       d}t        t        |            ||   z  }|}|}|r||"J ||J |}t        j                  ||z         }dg|z  }||   ||<   g }t        |      D ]  }||k7  s	|j                  |        t        ||      }d|z  }t        j                  ||      }t        j                  |||z
  z  |      }t        ||z  |      }t        t        j                  ||z  ||z        |      } |t        ||      dz  }!nt        ||z  |      }!|r||z
  | z  }"||"z
  |z
  |!z  }#n||!z  }#|	d   r||z  }$nd }$|	d   r|}%nd }%|#j                  |
      t        |$|      t        |%|      fS )Nc              3   H   K   | ]  }||j                        n|  y wr6   r  r  s     r.   r  z-native_batch_norm_backward.<locals>.<genexpr>  s,      	 $%=a7	s   "r%   z$rank of the input must be at least 2r$   rd   )r   rB   r  r  rL   r  rj  rf   r  rQ   rd  rH  r   r   r9   r  )'r  r   r   r  r  r  r   r5  r/  r  r  weight_dtyper  r  r  running_mean_castrunning_var_castsave_mean_castsave_invstd_castr  
input_rankr  num_featuresr   r  rG  reduction_axesr^  r   grad_output_sumdot_p	grad_mean
proj_scale
grad_scaleprojrU   r[  	grad_biasr;   s'                                         @r.   rJ  rJ    st    ++K||"33EKK@	 
	 ++KJ?BBB?D[)*[->>LDFF$666 !,1A1MMM -34!"j 0N&t,N4 "N: %9!!!$% *$?DDii~>OIImzD'89>JE./E~VI/		%$,QW0XZhiJ3FNKcQ
3[ .

 T!Z/$t+y8JF
"Z/
1~fn1~#		 	k"K.I|, r-   c
                    t        | |||||||||	
      }|
||f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r	  )rJ  r  r   r  r   )r  r   r   r  r  r  r   r5  r/  r  r9  r:  r  r  rU   r^  rG   s                    r.   native_batch_norm_backward_outr^  	  s    " (F d#J&! Q1=jmQWW5Q
14PQ
 r-   save_varc                 B    t         j                  || |||||d|g d
      S NT)TTTr   rJ  )r   rj   r   r  r  r  r_  rC  s           r.   miopen_batch_norm_backwardrc  ,	  s5     ** r-   reserveSpacec	                 B    t         j                  || |||||d|g d
      S ra  rb  )	r   rj   r   r  r  r  r_  rC  rd  s	            r.   cudnn_batch_norm_backwardrf  F	  s5     ** r-   c                    | j                   | j                  t              t        j                  dv fd       | j                  dd  D ]  }t        j                  |dk7  fd         d   |d   z  dk(  rxd   |d   z  dk(  rjt        d t        dd  |      D              }t        d t        dd  ||      D              }t        j                  j                  j                  | ||      S d	 d
 fd} |d   |d         \  }}}}	 |d   |d         \  }
}}}| dt        |d      |
f   }|	s|st        j                  |d      S d } |||||	d      \  }} |||||d      \  }}d }t        t        |j                  d         t        |j                  d               D ]!  \  }}||d|d d |f   }||d|d d |f   z   }# |||z  z  S )Nr  c                      d  S )Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r,   rl  s   r.   r   z%adaptive_avg_pool2d.<locals>.<lambda>k	  s    KD6R r-   rN  r   c                  "    dt                dS )Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape r  r  r  s   r.   r   z%adaptive_avg_pool2d.<locals>.<lambda>p	  s     99>uaI r-   rP   c              3   ,   K   | ]  \  }}||z    y wr6   r,   )r  r^  r  s      r.   r  z&adaptive_avg_pool2d.<locals>.<genexpr>v	  s     G$!QqAvG   c              3   :   K   | ]  \  }}}||d z
  |z  z
    ywr$   Nr,   )r  r^  r  r  s       r.   r  z&adaptive_avg_pool2d.<locals>.<genexpr>w	  s'      
 '1aAQ!O
s   c                 8    t        j                  | |z  |d      S )Ntruncrounding_moderf   divr  r  r  s      r.   start_indexz(adaptive_avg_pool2d.<locals>.start_index|	  s    yyQ99r-   c                 J    t        j                  | dz   |z  |z   dz
  |d      S )Nr$   ro  rp  rr  rt  s      r.   	end_indexz&adaptive_avg_pool2d.<locals>.end_index	  s&    yy!a%1q1,awGGr-   c                    t        j                  |t         j                        } |||       }| |z  dz   }| |z  }|dk(  xs ||z  dk(   }|r|dz  }n
|dk(  r|dz  }t        j                  |t         j                        }|j                  d      |z   }|rUt        j                  | dz
  |j
                  |j                        }	t        j                  ||	      } |||       }
|
|z
  }n|}||||fS )Nr  r$   r   rP   r  )rf   r  r  rR   r7  r   r  minimum)in_sizeout_sizeorangei0	maxlengthin_size_modadaptive	range_maxr  maxvali1lengthr  rw  ru  s               r.   compute_idxz(adaptive_avg_pool2d.<locals>.compute_idx	  s   hvU[[I73 x'!+	(#q(GH{,Ba,GHNIANILL6M	ll2* ((!399SZZF --V,C 68W5B"WFFFIx//r-   .r6  )r   rP   r~   c                     t        |t              r| |fS |dk  sJ ||j                  d      k\  }|dk(  rt        |d      }t	        j
                  | |d      } t        ||       }| |fS )Nr   rP   rN  r6  r   )r8   r   rR   rT   rf   rX  )valsr  r  r  rL   r  s         r.   
maybe_maskz'adaptive_avg_pool2d.<locals>.maybe_mask	  sw    fg&< 7N7 0 0 44Dby(q1$$T45D&vt4F<r-   )r  rL   r   )r  r  rw  rf   r   r  r  nnr  
avg_pool2drT   r   r   rQ   )r   r  r  rk  kernelr  idxhlength_hrange_max_h
adaptive_hidxwlength_wrange_max_w
adaptive_wr  r  retr^  jr  rw  rr  r  ru  s                      @@@@@r.   adaptive_avg_pool2dr  a	  s.   
 \\FKKEu:D	LLR [[ 
FI	

 Ry;r?"a'E"IB,G1,LG#eBCj+*FGG 
+.uRSz;+O
 
 xx""--eVVDD:H0@ /:%)[QS_.U+D(K.9%)[QS_.U+D(K'a0$67Djzz$H--   hjbND(  hjbND(
 Cdjjn-uTZZ^/DE +1;sAq!|$CS!Q\**C	+
 (X%&&r-   c                    t        j                  d| d       t        t        j                  | j
                  d |        }t        t        j                  |      }dg| j                  z  }| j
                  d |  |d |  |t        j                  || j                        j                  |      |z  z   j                  d      }| j                  t        | j
                  d |        t        |      z         }t        j                  |j                  d      |g| j                  d      d      j                  |j
                        S )Nmax_unpoolingd_forward_outr$   r  rP   Fr  )rB   alert_not_deterministicr   operatorr   r  rr  r   r  r  r  r  rb  rj  r  )	ry   r  r  rL   nchwindices_nc_shapeindices_flatr  s	            r.   _max_unpoolndr  	  s    
!!M#m"DE	djj3$/	0B	k	*BsTYY"jj3$/Usd$++b+5::;KLrQQgbk  ^^DEcT!23d;6GGHF!!r\NDLL,< " 
d6<<r-   c                     t        j                  j                  t         j                  k(  fd       t        j                  t	              dk(  fd       t        j                   j
                  dv  fd       t        j                   j                  j                  k(   fd       t        d j
                        D ].  t        j                   j                        dkD   fd	       0 t         d      S )
Nc                  "    d j                    S )Nz2elements in indices should be type int64 but got: r   )r  s   r.   r   zmax_unpool2d.<locals>.<lambda>	  s    DW]]OT r-   r%   c                  "    dt                dS )NzMThere should be exactly two elements (height, width) in output_size, but got 
 elements.rw  r  s   r.   r   zmax_unpool2d.<locals>.<lambda>	      ;'(
4 r-   r  c                  $    d j                    dS )NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with  dimensions.rl  r   s   r.   r   zmax_unpool2d.<locals>.<lambda>	  s    %%)YYK|= r-   c                  <    dj                    d j                    S NzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: r  )r  ry   s   r.   r   zmax_unpool2d.<locals>.<lambda>	  s,    PQUQ[Q[P\ ]229--B r-   r$   r   c                  *    dj                    d  dS )NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got  with dimension  being empty.r  )r^  ry   s   r.   r   zmax_unpool2d.<locals>.<lambda>
  s%    ::,&6qcH r-   )
rf   r   r   r  rw  rr  r  rQ   r!  r  )ry   r  r  r^  s   ```@r.   max_unpool2dr  	  s     
LL$T 
LLKA	
 
LL		V	
 
LL

gmm#	
 1dii  
IIaL1	

 wQ77r-   c                     t        j                  j                  t         j                  k(  d        t        j                   j                  dv  fd       t        j                  t              dk(  fd       t        j                  t              dk(  fd       t        j                  t              dk(  fd       t        j                   j                  j                  k(   fd       t        d	 j                        D ].  t        j                   j                        d
kD   fd       0 t        j                  d
   d
kD  xr d	   d
kD  xr d   d
kD  fd       t         d      S )Nc                       y)Nz(elements in indices should be type int64r,   r,   r-   r.   r   zmax_unpool3d.<locals>.<lambda>
  r  r-   r6  r  c                  $    d j                    dS )NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with r  rl  r   s   r.   r   zmax_unpool3d.<locals>.<lambda>
  s    ^_d_i_i^jjvw r-   r   c                  "    dt                dS )NzVThere should be exactly three elements (depth, height, width) in output_size, but got r  r  r  s   r.   r   zmax_unpool3d.<locals>.<lambda>#
  r  r-   c                  "    dt                dS )NzRThere should be exactly three elements (depth, height, width) in stride, but got: r  r  rk  s   r.   r   zmax_unpool3d.<locals>.<lambda>*
  s    dehioepdqq{| r-   c                  "    dt                dS )NzSThere should be exactly three elements (depth, height, width) in padding, but got: r  r  )r  s   r.   r   zmax_unpool3d.<locals>.<lambda>.
  s    efijqfress}~ r-   c                  <    dj                    d j                    S r  r  )r  r   s   r.   r   zmax_unpool3d.<locals>.<lambda>2
  s,    PQVQ\Q\P] ^229--B r-   r$   r   c                  *    dj                    d  dS )NzZmax_unpooling3d(): Expected input to have non-zero size for non-batch dimensions, but got r  r  r  )r^  r   s   r.   r   zmax_unpool3d.<locals>.<lambda>;
  s%     ;;-'7s-I r-   r%   c                      d  S )Nz5strides should be greater than zero, but got stride: r,   r  s   r.   r   zmax_unpool3d.<locals>.<lambda>D
  s    GxP r-   )
rf   r   r   r  rr  rw  r  rQ   r!  r  )r   r  r  rk  r  r^  s   `````@r.   max_unpool3dr  
  sD    
LL$&X 
LL

fw 
LLKA	
 
LLFq| 
LLG~ 
LLw}}$	
 1ejj! 
JJqMA	

 
LLq	A9&)a-9F1IMP
 a88r-   )rk   rf  c                $    t        | |||d|      S )NTinplacerk   
_index_addr:   rL   r  rf  rk   s        r.   
index_add_r  J
  s     aeVTGGr-   c                $    t        | |||d|      S )NFr  r  r  s        r.   	index_addr  V
  s     aeVU%HHr-   r  c                8   t        j                  | j                        t        j                  j                  dk  fd       j                  dk(  rj                  d      nd|j                  dkD  r|j                        ndt        j                  k(  fd       dk7  rft        j                  | j                        t        j                  t        k(  xs t        j                  t                    fd       |z  }| j                  dk(  }|r| j                  d      n| }dz  fz   }|rt        j                  nt        j                  }	 |	|||d      }
|r| S |r|
j                  d      S |
j!                         S )	Nr$   c                  $    d j                    dS Nz(Index should have dimension 1 or 0 (got r1  rl  r  s   r.   r   z_index_add.<locals>.<lambda>o
      :5::,aH r-   r   c                      d d d S )NzNumber of indices (z') should be equal to tensor.size(dim) (z), for dim=r,   )rL   
index_sizerg  s   r.   r   z_index_add.<locals>.<lambda>u
  s!    %j\1XYdXeeqmplrs r-   c                  (    dt                d dS )Nzalpha argument of type z cannot be safely cast to type !)r  )rk   python_types   r.   r   z_index_add.<locals>.<lambda>|
  s    -d5k]:YZeYffgh r-   r6   Tr  )rB   canonicalize_dimsrr  rf   r   r!  dtype_to_typer   r  is_weakly_lesser_typer  rR   r   
index_put_	index_putr  r  )r:   rL   r  rf  r  rk   zero_dimrI  r  r  r   r  r  rg  s    ``  `     @@@r.   r  r  c
  sM    
!
!!&&#
.C	LL

aH #(**/AqJ&,kkAo&++c"1K	LLz!s z))!''24 E**4;Dh	

 %vv{H#QB
C-5(
"C#*I
BV
5C!)s{{1~?s~~/??r-   c           
         t        j                  t        |       dkD  d        t        |       }| d   j                         }|dd  }t	        d | D              }|r||f}n||f}||z   }| d   j                  ||      }dt        |      z  }	t        |      D ]j  }
| |
   }t        j                  ||	d||j                  d      z
  fz   |      }|rt        j                  ||d|
      }Rt        j                  ||d|
      }l |S )Nr   c                       y)Nz#received an empty list of sequencesr,   r,   r-   r.   r   zpad_sequence.<locals>.<lambda>
  r  r-   r$   c              3   >   K   | ]  }|j                  d         ywr  r!  )r  r:   s     r.   r  zpad_sequence.<locals>.<genexpr>
  s     /!&&)/s   )r   r   rL   r  )
rf   r   rw  r!  r   r;  rQ   r   rb  r  )	sequencesbatch_firstpadding_valuesequences_sizemax_sizetrailing_dimsmax_lenout_dimsr   dim_paddingsr^  currseqrows                r.   pad_sequencer  
  s#    
LLY!#%RS^N|  "HQRLM/Y//G"G,^,-'H
A,

-
8CC..L>" @A,""\Q',,q/(A$BBM
 %%c3AQ%?C%%c3AQ%?C@ Jr-   c                 "    t        | |||d      S )NTr  _index_copyr:   rL   r  rf  s       r.   index_copy_r  
  s    q#ufd;;r-   c                 "    t        | |||d      S )NFr  r  r  s       r.   
index_copyr  
  s     q#ufe<<r-   c                   t        j                  | j                  |      }t        j                  j                  dk  fd       | j                  dk(  }|r| j                  d      n| }j                  dk(  rj                  d      nd|z  fz   }|rt        j                  nt        j                  } ||||      }	|r| S |r|	j                  d      S |	j                         S )Nr$   c                  $    d j                    dS r  rl  r  s   r.   r   z_index_copy.<locals>.<lambda>
  r  r-   r   r6   )rB   r  rr  rf   r   rR   r   r  r  r  r  )
r:   rL   r  rf  r  r  rI  r  r  r   s
     `       r.   r  r  
  s     
!
!!&&#
.C	LL

aH
 vv{H#QB"'**/EOOAuE
C-5(
"C#*I
BV
$C!)s{{1~?s~~/??r-   c                 *   t        j                  | j                  d      |       }t        j                  t        j                  |              }| j
                  s| j                  r| j                  d      }n|}|t        j                  |      z
  |fS )Nr,   r  )rf   ry  rb  re   r   r  is_xpur:  )ry   r   rh   r   s       r.   log_sigmoid_forwardr  
  sn     --r*D
1C		599T?"#A||t{{%Q''r-   lowhighr/  c                     t        j                  | j                  t        |      t        |      | j                  | j
                  |      S )N)r  r  r   r  r/  )prims_uniform_helperr  r   r   r  )r:   r  r  r/  s       r.   uniformr  
  s=       	cNt_ggxx r-   c                 <    | j                  t        | |||            S r6   )r  r  )ry   r  r  r/  s       r.   uniform_r  
  s    ::gdCy9::r-   c                 "   t        |       dz
  }|>t        j                  |d u d        t        j                  t        |      |k(  d        |S |t        j                  |d u d        t        j                  t        |      |k(  d        g }t        |      D ]Z  \  }}t	        |      |k(  r$|j                  | |dz      t	        |      z         8|j                  t        | |dz      |z               \ |S t        j                  dd        y )Nr%   c                       yNz9Must specify exactly one of output_size and scale_factorsr,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  r  r-   c                       yN r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  r  r-   c                       yr  r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  r  r-   c                       yr   r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  r  r-   Fc                       yr  r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>  r  r-   )rw  rf   r   r  r  rd  r   )r&  r  scale_factorsspatial_dimensionsr^  r  s         r.   upsample_compute_output_sizer  
  s   Z1,T!O	
 	S%);;ZH 4O	
 	S'+==zJm, 	CDAq1v{"":a!e#4s1v#=>""7:a!e+<q+@#AB		C
 	LLRr-   c                     | y | |   S r6   r,   )scalesr  s     r.   get_scale_valuer
    s    ~#;r-   r  c                 x    t        | j                         ||      }|r|nd gt        |      z  }t        | ||      S r6   r  r!  rw  _upsample_nearestr   r  r  osizer	  s        r.   _upsample_nearest_vecr    s@     ){MRE&TFSZ,?  UE622r-   c                 |    t        | j                         ||      }|r|nd gt        |      z  }t        | ||d      S NTexactr  r  s        r.   _upsample_nearest_exact_vecr  #  sB     ){MRE&TFSZ,?  UE6>>r-   c                    g }t        |      }|rdnd}t        |      D ]  }||   }| j                  | |z      }	||   |	|	||   z  z  n|	|z  }
t        j                  |t        j
                  | j                        }||z   |
z  j                  t        j                        }t        |dz
  |z
        D ]  }|j                  d      } |j                  |        |S )Nr   r   r  r$   rP   )rw  rQ   r  rf   r  r   r  r9   r  rR   rd  )r   r  r	  r  r  num_spatial_dimsr  r  r  isizerl   output_indicesinput_indicesrS   s                 r.   !_compute_upsample_nearest_indicesr  8  s     G;'SsF#$ & A--12/5ay/D*+%RW-e5==V(61U:>>u{{K'!+a/0 	8A)33B7M	8}%+&, Nr-   )preserve_memory_formatr  r	  c                     t        | ||g      S r6   r  r   r  r	  s      r.   upsample_nearest1dr   X  s     UK&::r-   c                 "    t        | ||gd      S r  r  r  s      r.   upsample_nearest_exact1dr"  d  s     UK&FFr-   scales_hscales_wc                      t        | |||g      S r6   r  r   r  r#  r$  s       r.   upsample_nearest2dr'  r  s     UK(H1EFFr-   c                 $    t        | |||gd      S r  r  r&  s       r.   _upsample_nearest_exact2dr)    s     UK(H1ETRRr-   scales_dc                 "    t        | ||||g      S r6   r  r   r  r*  r#  r$  s        r.   upsample_nearest3dr-    s     UK(Hh1OPPr-   c                 &    t        | ||||gd      S r  r  r,  s        r.   _upsample_nearest_exact3dr/    s!     {Xx:$ r-   r  c                 D   t        | |||      }d d g|z   }t        j                  | |      }|j                  dk(  rdt	        j
                  |       }| j                  d   }| j                  j                  dk(  r|dk  rt        j                  }|j                  |      }|S )Nr  r6  r$   cudarL  )r  r   _unsafe_indexrr  rB   r   r  r  r  rf   rQ  r  )	r   r  r	  r  spatial_indicesr  r  rM  
n_channelss	            r.   r  r    s     8{F%O Tl_,Gw/F{{a33E: [[^
<<&:>!33M"""?Mr-   c           	          |r|rd}n|rd}n|rd}nd}t        |       |z  dk(  sJ t        |              t        dt        |       |      D cg c]  }t        | |||z           c}S c c}w )Nr  r6  r   r%   r   )rw  rQ   r  )params
has_biaseshas_projections
group_sizer^  s        r.   gather_paramsr:    s{    o
	
	

v;#q(5#f+5(38CK3T./fQZ()  s   A(c                 ~    |r'| d|z     |d|z     }}| d|z  dz      |d|z  dz      }}n| |   ||   }}d\  }}||||fS )Nr%   r$   NNr,   )r6  hiddensr^  bidirectional
cur_params
cur_hiddenbidir_paramsbidir_hiddens           r.   params_hiddensrC    sk    !'AAJ
%+AEAI%6A	8Jl!'GAJJ
%/"lz<==r-   c                     ||kD  sJ |j                  | j                  d|||z
               | j                  dd|      S rq   )rd  r"  )r@  last_batch_size
batch_sizer=  s       r.   update_hidden_for_packedrG    sE    Z'''NN:$$Q
Oj4PQRQ:..r-   c           	      t    ||k(  r| S ||k  sJ t        j                  | |j                  d|||z
        f      S rq   )rf   concatr"  )r@  rE  rF  
inp_hiddens       r.    update_hidden_for_packed_reverserK    sP     *$Z'''<<a*2NO	
 r-   c           	      X   |d   }|d   }|r|d   nd }	|r|d   nd }
g }g }|r|d   n|d   }|j                  dd|      }t        j                  | t        |            }|r|d d d   }|D ]V  } | j                  d   }||k(  rn|rt        ||||      }nt        ||||      } || |||	||
      }|}|j                  |       X |r|j                          n!|j                  |       |j                          t        j                  |d      }|st        j                  |d      n|}||fS )Nr   r$   r%   r   rP   )
r"  rf   r  rj  r  rK  rG  rd  reverser#  )inphiddenr6  r7  	hidden_fnbatch_sizesrM  	ih_weight	hh_weightih_biashh_biasstep_outputr=  rE  r@  	split_inpr^  r   
hidden_outs                      r.   one_layer_rnn_datarY    sR    q	Iq	I%fQi4G%fQi4GK"$G)0k"ok!nOq!_5JCk!23IddO	 'IIaLa9OQJ 2OQJ sJ	7IwW
:&#'& z"
))K
#C.57A&:J
?r-   c                       fd}|S )Nc                 D     t        j                  |||      | z         S r6   r   linearr^  r@  rR  rT  rS  rU  nonlinearitys         r.   rH   zrnn_cell.<locals>.inner'  s     AHHZGDqHIIr-   r,   r_  rH   s   ` r.   rnn_cellra  &  s    J Lr-   c                       fd}|S )Nc                 r    t        j                  | ||      }  t        j                  |||      | z         S r6   r\  r^  s         r.   rH   zrnn_cell_data.<locals>.inner.  s2    HHQ	7+AHHZGDqHIIr-   r,   r`  s   ` r.   rnn_cell_datard  -  s    J Lr-   c           	      x   |d   }|d   }|r|d   nd }|r|d   nd }	t        j                  | ||      }
|r|
j                  d      n|
}
|j                  d      }g }|
D ]   } |||||||	      }|j	                  |       " |r|j                          t        j                  |d      }||j                  d      fS )Nr   r$   r%   r   )	r   r]  fliprR   rd  rM  rf   r#  r  )rN  rO  r6  r7  rP  rM  rR  rS  rT  rU  precomputed_inputr@  rV  r^  r   s                  r.   one_layer_rnnrh  5  s    q	Iq	I%fQi4G%fQi4Gi95<)..q1BS!!!$JK 'q*i)WU
:&' 
))K
#C
""1%%%r-   c                    |d   }|d   }|r|d   }|d   }nFt        j                  |j                               }t        j                  |j                               }|d   j                  d      }	|d   j                  d      }
g }d}|	j                  d      }d}d}d}d}| j	                         } |	j	                         }	|
j	                         }
t         j
                  j                  j                  j                  | |||||	|
|||||||||      }|d   |d   |d   }}}||j                  d      |j                  d      ffS )Nr   r$   r%   r   F)
rf   r   r!  rR   r  r  r   mkldnn_rnn_layerr}  r  )rN  rO  r6  r7  rM  w0w1w2w3hxcxrQ  modehidden_size
num_layersr>  r  r5  outputsrW   hycys                         r.   mkldnn_one_layer_lstmrw  K  sS   	B	BAYAY[[#[[#			Q	B			Q	BKD''!*KJ MKE ..
C	B	Biinn--55





!G$ 
GAJ
2rArzz!}bjjm,,,r-   c
                    |r| j                  dd      n| } g }
t        |      D ]  }t        ||||      \  }}}}|r
||dz
  k  r|nd} |	| |||      \  }}|
j                  |       |r! |	| |||d      \  }}|
j                  |       |r*t	        j
                  |g|j                         dz
        } n|} |dk7  s|s||dz
  k  st	        j                  | |d      }  |r| j                  dd      n| } | |
fS )Nr   r$   r   T)rM  )r5  )	transposerQ   rC  rd  rf   r#  rL   r8  )r   rO  r6  r7  rs  r8  r5  r>  r  layer_fnfinal_hiddensr^  r?  r@  rA  rB  fwd_inp
fwd_hiddenbwd_inp
bwd_hiddens                       r.   _rnn_helperr  }  s"    &1EOOAq!eEM: >=KFA}>
:
Jl $
QU(:'&uj*jQZ("*|\:t#GZ   ,IIw0'++-!2CDEEa<Ea*q.&8MM%=E)>, &1EOOAq!eE-r-   c	                     |j                  d      }	t        ||d      }t        | |	|||||||t        t        t        t        j                              
      \  }
}|
t        j                  |d      fS Nr   FrP  )	unbindr:  r  r   rh  ra  rf   r   stackr   ro  r6  r7  rs  r8  r5  r>  r  rO  r   r{  s               r.   rnn_tanh_inputr    t     YYq\F6:u5F$%**)=>C M1---r-   c	                     |j                  d      }	t        ||d      }t        | |	|||||||t        t        t        t        j                              
      \  }
}|
t        j                  |d      fS r  )	r  r:  r  r   rh  ra  rf   r  r  r  s               r.   rnn_relu_inputr    r  r-   c	                     |j                  d      }	t        ||d      }t        | |	||||||dt        t        |t        t        j                              
      \  }
}|
t        j                  |d      fS Nr   FrQ  rP  )	r  r:  r  r   rY  rd  rf   r  r  datarQ  ro  r6  r7  rs  r8  r5  r>  rO  r   r{  s               r.   rnn_relu_datar    {     YYq\F6:u5F$##EJJ/	
C  M1---r-   c	                     |j                  d      }	t        ||d      }t        | |	||||||dt        t        |t        t        j                              
      \  }
}|
t        j                  |d      fS r  )	r  r:  r  r   rY  rd  rf   r   r  r  s               r.   rnn_tanh_datar    r  r-   c                 l   t        j                  |||      | z   }|j                  d|      }|d   j                         }	|d   j                         }
|d   j	                         }|d   j                         }|
|z  |	|z  z   }||j	                         z  }||nt        j                  ||d       }||fS )Nr6  r   r$   r%   r   r   r]  chunkr   r   )rN  ro  rp  rS  rU  	hr_weight	chunk_dimgateschunked_gatesin_gateforget_gate	cell_gateout_gaterv  ru  s                  r.   	lstm_cellr  *  s    HHRG,s2EKK9-MA&&(G"**,Ka %%'IQ'')H	r	Wy0	1B	BGGI	B ahhr9d&CBr6Mr-   c           
      (   |d   }|d   }|r|d   nd }|r|d   nd }t        |      dk(  r|d   nt        |      dk(  r|d   nd }	|d   j                  d      }
|d   j                  d      }t        j                  | ||      }|r|j	                  d      n|}g }|D ](  } t        | |
||||	d      \  }
}|j                  |
       * |r|j                          t        j                  |d      }||
j                  d      |j                  d      ffS )Nr   r$   r%   r   r  r6  r  )rw  rR   r   r]  rf  r  rd  rM  rf   r#  r  )rN  rO  r6  r7  rM  rR  rS  rT  rU  r  ro  rp  rg  rV  r   s                  r.   one_layer_lstmr  8  s*   q	Iq	I%fQi4G%fQi4G[A%q	Fq8H6!9d  
		Q	B			Q	Bi95<)..q1BSK  3B	7IQRSB2 
))K
#CA

1...r-   c           
         |d   }|d   }|r|d   nd }|r|d   nd }	t        |      dk(  r|d   nt        |      dk(  r|d   nd }
g }g }|r|d   n|d   }t        j                  | t        |            }|r|d d d   }|d   }|d   }|j	                  dd|      |j	                  dd|      }}|D ]  } | j
                  d   }t        j                  | ||      } ||k  ra|j                  |j	                  d|||z
        |j	                  d|||z
        f       |j	                  dd|      |j	                  dd|      }}||kD  rXt        j                  ||j	                  d|||z
        fd      }t        j                  ||j	                  d|||z
        fd      }t        | ||||	|
d      \  }}|}|j                  |        |r|j                          ||f}nZ|j                  ||f       |j                          t        | \  }}t        j                  |d      t        j                  |d      f}t        j                  |d      }||fS )	Nr   r$   r%   r   r  r6  rP   r  )rw  rf   r  rj  r"  r  r   r]  rd  rI  r  rM  r  r#  )rN  rO  r6  r7  rQ  rM  rR  rS  rT  rU  r  rV  r=  rE  rW  orig_hxorig_cxro  rp  r^  rX  hidden0hidden1r   s                           r.   one_layer_lstm_datar  S  sx   q	Iq	I%fQi4G%fQi4G[A%q	Fq8H6!9d  KG)0k"ok!nOCk!23IddO	QiGQiG^^Aq/2GNN	1o5B  IIaLhhsIw/ NNIIaOa$78IIaOa$78 YYq!Q'1a);B W^^AO8KLMqB W^^AO8KLMqB 3B	7IQRSB236 "X
Bx =YYw*EIIgq,AA

))K
#C
?r-   c                 4    d } || ||      rt         S t        S )a*  Check whether we could use decompose lstm with mkldnn_rnn_layer.
    All the below conditions need to be met:
        * ``torch._C._get_mkldnn_enabled()`` returns ``True``.
        * All the input args are on CPU.
        * The dtypes of args are either torch.float or torch.bfloat16.
        * Inference.
        * ``has_projections`` returns ``False``.

    Args:
        * input: the input sequence to LSTM
        * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM
        * params: the weight and bias tensors of LSTM
    c                 N   t         j                  j                         sy| gt        |      z   t        t	        j
                  |            z   }|D ch c]  }|j                   }}t        |      dk7  ry|j                         }|t        j                  d      k7  ry|D ch c]  }|j                   }}|D ]&  }|t         j                  t         j                  fvs& y | j                  ry|d   j                  d      |d   j                  d      k7  }	|	ryyc c}w c c}w )NFr$   r  r   r%   T)rf   r  _get_mkldnn_enabledrj  r   from_iterabler  rw  popr   r   bfloat16requires_gradr!  )
r   ro  r6  r_  tdevicesr  dtypesr   r8  s
             r.   
use_mkldnnz2select_one_layer_lstm_function.<locals>.use_mkldnn  s    xx++-'DH$tE,?,?,G'HH%,-188--w<1U\\%((#*+a!''++ 	EU[[%..99	 Q%**Q-2a5::a=8) . ,s   D#D")rw  r  )r   ro  r6  r  s       r.   select_one_layer_lstm_functionr    s!    : %V$$$r-   c	                    t        |      dk(  sJ d       t        |||d   j                  d      |d   j                  d      k7        }t        t	        |d   |d               }	t        | ||      }
t        | |	||||||||

      \  }}t        t	        |       }|t        j                  |d   d      t        j                  |d   d      fS )Nr%   lstm expects two hidden statesr   r$   )	rw  r:  r!  rj  r  r  r  rf   r  )r   ro  r6  r7  rs  r8  r5  r>  r  rO  rz  r   r{  s                r.   	lstm_implr    s     r7a<999<6:r!uzz!}1

1/MNF#beRU#$F-eR@H$C m,-MM!,a0%++mA>NPQ2RRRr-   c	                    t        |      dk(  sJ d       t        |||d   j                  d      |d   j                  d      k7        }t        t	        |d   |d               }	t        | |	||||||dt        t        |      
      \  }
}t        t	        |       }|
t        j                  |d   d      t        j                  |d   d      fS )Nr%   r  r   r$   F)rQ  )
rw  r:  r!  rj  r  r  r   r  rf   r  r  s               r.   lstm_data_implr    s     r7a<999<6:r!uzz!}1

1/MNF#beRU#$F$#=C m,-MM!,a0%++mA>NPQ2RRRr-   c                 &   | j                  dd      }t        j                  |||      j                  dd      }|d   |d   z   j                         }|d   |d   z   j                         }	|d   |d   |z  z   j	                         }
||
z
  |	z  |
z   S )Nr   r$   r%   r   )r  r   r]  r   r   rN  r@  rR  rT  rS  rU  chunked_igateschunked_hgates
reset_gate
input_gatenew_gates              r.   gru_cellr    s    YYq!_NXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH!Z/(::r-   c                 P   t        j                  | ||      j                  dd      }t        j                  |||      j                  dd      }|d   |d   z   j                         }|d   |d   z   j                         }	|d   |d   |z  z   j	                         }
||
z
  |	z  |
z   S )Nr   r$   r   r%   r  r  s              r.   gru_cell_datar    s    XXc9g6<<QBNXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH!Z/(::r-   c	                     t        ||d      }t        | |j                  d      ||||||dt        t        |t
              
      \  }	}
|	t        j                  |
d      fS )NFr   r  )r:  r  r  r   rY  r  rf   r  )r  rQ  ro  r6  r7  rs  r8  r5  r>  r   r{  s              r.   gru_impl_datar    si     6:u5F$
		!"}UC M1---r-   c	                     t        ||d      }t        | |j                  d      |||||||t        t        t
              
      \  }	}
|	t        j                  |
d      fS )NFr   r  )r:  r  r  r   rh  r  rf   r  )r   ro  r6  r7  rs  r8  r5  r>  r  r   r{  s              r.   gru_implr  7  sf     6:u5F$
		!2C M1---r-   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j
                  j                  | ||||      S Nr   r$   )r  r!  r
  rf   r  r   _upsample_bilinear2d_aar   r  align_cornersr  r  scale_hscale_ws          r.   upsample_bilinear2d_aa_vecr  U  sV     ){MREmQ/GmQ/G99>>11umWg r-   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j
                  j                  | ||||      S r  )r  r!  r
  rf   r  r   _upsample_bicubic2d_aar  s          r.   upsample_bicubic2d_aa_vecr  a  sV     ){MREmQ/GmQ/G99>>00umWg r-   c                 z    t        | j                         ||      }|r|nd gt        |      z  }t        | |||      S r6   )r  r!  rw  _upsample_linear)r   r  r  r  r  r	  s         r.   _upsample_linear_vecr  m  s=     ){MRE+]$#e*1DFE5-@@r-   r  c                      t        | |||g      S r6   r  )r   r  r  r$  s       r.   upsample_linear1dr  {  s     E;zJJr-   c                 "    t        | ||||g      S r6   r  )r   r  r  r#  r$  s        r.   upsample_bilinear2dr    s     E;(?STTr-   c                 $    t        | |||||g      S r6   r  )r   r  r  r*  r#  r$  s         r.   upsample_trilinear3dr    s!     {MHh+I r-   c                 L    |r|dkD  r| dz
  |dz
  z  S dS |
|dkD  rd|z  S | |z  S )Nr$   rd   r   r,   )rz  r{  r  rl   s       r.   _compute_scaler    sB    5=\#(S.1HqH#/EAIsU{U7XCUUr-   c                 &    |r| |z  S | |dz   z  dz
  S Nr   r,   )rl   	dst_indexr  s      r.   _compute_source_indexr    s$    y  	C(3..r-   weightsweights_precisionc                     t        d t        | |      D              d|dz
  z  z   }||z	  }t        j                  |dd      j	                  t        j
                        S )Nc              3      K   | ]F  \  }}|j                  t        j                        |j                  t        j                        z   H y wr6   )r9   rf   r*  )r  r  r  s      r.   r  z%_sum_tensors_uint8.<locals>.<genexpr>  s8      26!QU[[ADD--s   AAr$   r      )_sum_tensorsr  rf   r   r9   r  )r  r  r  r  s       r.   _sum_tensors_uint8r    sd      :=c7:K 	
 1$	%'F ((F;;vq#&))%++66r-   c                     t        j                  |       j                         }d}t        j                  ||j                        }d|d|dz   z  z  z   }|dk\  }||j                         z
  S )N   r  r   r$   i   )rf   r  r   r  r  r   )r  
max_weightmax_weight_precision
precisionsvaluesr  s         r.   _compute_weight_precisionr    si    W%))+J2:;L;LMJ:zA~!677FgD$((*,,r-   c                      j                   d d \  }} j                   dd  }t        |      }t        j                   t        j                  j
                        \  } fd}	t        t        |||            D 
cg c]  \  }
\  }}} |	||||dz
  |
z
         }}}}
}t        t        |       \  }}}g }t        ddgg|z   D ]c  }d d gt        |      D cg c]  }||   dk(  r||   n||    c}z   }t        j                   |      }t        |      }|j                  |       e t        t        |            D ]p  }
||
   ||
   z
  j!                  dd      j#                        }t        |d d d   |dd d         D cg c]!  \  }}|t%        j&                  ||z
  |      z   # }}}r t        |      dk(  sJ |d   }t        j(                         } j*                  j,                  dk(  r|d	k  rt$        j.                  }t1        |t$        j2                        sJ |j5                  |
      } j7                         s|j9                         }|S c c}}}}
w c c}w c c}}w )Nr%   rC  c                 |   t        | |	|      }t        j                  |j                        j	                  
      }t        ||	      j                  d      } |j                  |j                  d   gdg|z   }|j	                  t        j                        }|dz   j                  | dz
        }|||fS )Nr  r   r   r   r   r$   r   )
r  rf   r  r  r9   r  r   r  r  r  )inp_sizer{  r	  nsqueezescale_factorr^  x_f32r:   xp1r  r   r   s            r.   
get_valuesz$_upsample_linear.<locals>.get_values  s    %h-P LL%,,7:::G%lA}EKKPSKTekk!n@sh/?@HHU[[!1umm1m-a}r-   r$   r   r   rd   r1     rL  )r  rw  rB   rC   rE  INT_TO_FLOATr  r  rj  r   rQ   r   r2  r   rd  reversedr   r9   rf   r   r   r  r  rQ  r8   r   r  r<  round)r   r  r  r	  n_batchr4  	inp_sizesn_dimsrS   r  r^  r  r{  r  xs_f32xsxp1svsr  kr  vxscalev1v2r  rM  r   s   ` `                        @r.   r  r    s     ++bq/GZABI^F''!AANNHAu
 09	;/0
 +A+(F 	8XvvzA~>F  CL)FB	B1vh'( TluV}U!qtqybed1g=UUuc*#Au-
		!	 eFm$ 
)be#**3477> b1gr!$Q$x0
 B 27F++
 

 r7a<<UF //6M ||F"zB//fell+++];F""$MQ V
s   I"
 I*&I/r  r  c                 4    | j                   |j                   k(  S r6   r  )r  r  s     r.   is_same_sizer
    s    77aggr-   c                 .    t         j                  | |      S r6   )r   r  )r:   r  rD   s      r.   _reshape_aliasr    s     99Qr-   c                 .    t         j                  | |      S r6   )r   r  )r:   r  s     r.   r2  r2    s    ::a!!r-   c                 2    t         j                  | |||      S r6   )r   r  )r:   r  rz   r  s       r.   r  r  !  s    >>!WeZ88r-   c                    |D ]F  }|t        j                  |j                  t         j                  t         j                  fv d        H t        j                  |j                  t         j
                  k(  d        ddlm}  || j                         dk(        r<t         j                  j                  | |      }| j                  |j                  |      S t        t        |            D ]2  }||   }||j                  d| j!                  |      dz
        ||<   4 t"        j%                  | |      j'                  | |      S )Nc                       yNz3tensors used as indices must be long or int tensorsr,   r,   r-   r.   r   z&_unsafe_masked_index.<locals>.<lambda>,  r  r-   c                       yNz*tensors used as masks must be bool tensorsr,   r,   r-   r.   r   z&_unsafe_masked_index.<locals>.<lambda>1  r  r-   r   r  r$   r{  )rf   r   r   rW  r  r  rh  rf  r   _meta_registrationsmeta_index_Tensorr;  r  rQ   rw  r   r!  r   r2  rX  )r:   r  r  fillr  rf  meta_resultr^  s           r.   r  r  &  s    LL

EII66M 
LL

ejj <
 KAGGIN+//AA!WMzz+++T223w<  ?
q	A>GAJ?
 a)55teTBBr-   c                 L   |D ]F  }|t        j                  |j                  t         j                  t         j                  fv d        H t        j                  |j                  t         j
                  k(  d        | j                         dk(  r| j                         S t        t        |            D ]B  }||   }||j                  | j                  |       | j                  |      dz
        ||<   D |j                  | d      }t        j                  | ||d      S )Nc                       yr  r,   r,   r-   r.   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>H  r  r-   c                       yr  r,   r,   r-   r.   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>M  r  r-   r   r$   r{  Tr  )rf   r   r   rW  r  r  r   r  rQ   rw  r   r!  rX  r   r  )r:   r  r  r  r  r^  masked_values          r.   #_unsafe_masked_index_put_accumulater  B  s     LL

EII66M 
LL

ejj <
 	wwyA~wwy3w<  H
!&&)QGGAJH
 %%teQ/L!!!Wlt!LLr-   c                    | j                         }d}|dk  rd}|6|dkD  r*dg|z  }|j                  d   ||<   |j                  |      }n|}| |z  } t        j                  ||k7  |d      }	|	j                  |      }
t        j                  | ||
      j                  |       }t        j                  ||k7  |d      }|t        j                  j                  k(  r|dkD  r| j                  dd      }||fS |lj                  | j                        }t        j                  |||
      j                  |      }t        j                  ||k7  |d      }|j                         }n"||k7  j                         j                  |       }|t        j                  j                  k(  r|j                         }||fS |t        j                   j                  k(  r|j                         |z  }||fS )Nr$   r%   r   r,   r   )rL   r  r  rf   rg   rR   gatherr  r#   r)   rz   r;  r  r   r9   r+   r*   )ry   r   r   r   r  r  r  r  wr  safe_target_r  r  wsums                 r.   _nll_loss_forwardr"  \  s    XXZFKzA:E "(aE+E"AAax++f4fa@K((5L ll4l;CCKPPF[[</;FINN(((VaZ}}R-|##HHTZZ ||A{L9AA+N{{6\14;xxz,.33588>IMM''' < 
inn**	*,<r-   c                     | j                         dkD  r| j                         dk  sJ d       |j                         dk  sJ d       | j                         dk(  xr |j                         dk(  }|sA| j                  d   |j                  d   k(  s"J d| j                   d|j                   d       | j                  d	   }|=|j                         dk(  r|j                         |k(  sJ d
| d|j                          t        | ||||      S )Nr   r%   r-  r$   r.  r/  r0  r1  rP   z/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rL   r  r   r"  )ry   r   r   r   r  r3  	n_classess          r.   nll_loss_forwardr%    s1    88:>dhhjAoP/PP-

EDE 88:?8vzz|q'8L

1a(J	#DJJ<z&,,qIJ  

2I>

flln	9J	8Cz  |B  |H  |H  {I  JJ  T669lKKr-   c                      t        | ||||      S r6   )r"  )ry   r   r   r   r  s        r.   nll_loss2d_forwardr'    s     T669lKKr-   Ac                 0    |dz   | z  |dz   z
  | z  | z  dz   S )Nr%   r   r$   r,   r:   r(  s     r.   _upsample_cubic_convolution1r+    s(    UaK1q5!Q&*Q..r-   c                 <    || z  d|z  z
  | z  d|z  z   | z  d|z  z
  S )Nr     r6  r,   r*  s     r.   _upsample_cubic_convolution2r.    s0    UQU]a!a%'1,q1u44r-   r  c                    d}| j                   t        j                   d      k(  rt        j                  | d| z
  gd      }t        j                  | dz   d| z
  gd      }t        ||      }t	        ||      }t        j
                  |d      \  }}t        j
                  |d      \  }}	|||	|fS t        | dz   |      t	        | |      t	        d| z
  |      t        d| z
  |      fS )Ng      r  rd   r   r~   r   )r  rf   r  r.  r+  r  )
r  r(  tt1tt2w03w12rk  rn  rl  rm  s
             r.    _upsample_get_cubic_coefficientsr4    s    Axx5<<&&kk1cAg,A.kk1s7C!G,!4*32*32cq)Bcq)B2r2~ )S!4(A.(q!4(q!4	
 	
r-   coeffstsc                 P    t        |      }t        d t        | |      D              S )Nc              3   ,   K   | ]  \  }}||z    y wr6   r,   r  r  r  s      r.   r  z+_upsample_cubic_interp1d.<locals>.<genexpr>  s     EHRRErk  )r4  r  r  )r5  r6  coeffs2s      r.   _upsample_cubic_interp1dr;    s$    .r2GEFG0DEEEr-   c                 6    t        t        j                  |       S r6   )r   rf   add)r6  s    r.   r  r    s    %))R  r-   	num_stepsc                     | dk  rt        j                  d||      S |s| dz
  | z  nd}t        j                  | || ||      S )Nr$   r   r  )stepsr  r   )rf   rf  linspace)r>  r  r   r  r  s        r.   _linspace_from_neg_onerB    sI     A~||AfE::-:)a-9	$A>>1"ayuMMr-   thetahr  c                    | j                   }| j                  }t        ||||      j                  d|d      }t        ||||      j                  |dd      }t	        j
                  d||      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |d	dd      }||z   |z   S )
Nr$   )r$   r$   r$   r  )r   r%   constantr   r  rq  rz   r$   r$   )r%   r   	r   r  rB  r  rf   r  r  r  r  )	rC  rD  r  r  r   r  grid_xgrid_ygrid_ones	            r.   _make_base_grid_4drM    s    KKE\\F $A}eVDII!QPQRF#A}eVDII!QPQRFzz)5@H XX  $$VjPQ$RFXX  $$VjPQ$RFxx""&&xV*TU&VHF?X%%r-   r  c                    | j                   }| j                  }t        ||||      j                  dd|d      }t        ||||      j                  d|dd      }t        ||||      j                  |ddd      }	t	        j
                  d||      }
t        j                  j                  j                  |ddd      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |	d	dd      }	t        j                  j                  j                  |
d
dd      }
||z   |	z   |
z   S )Nr$   )r$   r$   r$   r$   r  )r   r   rF  r   rG  )r$   r%   )r%   r$   )r   r   rI  )rC  r  rD  r  r  r   r  rJ  rK  grid_zrL  s              r.   _make_base_grid_5drP    s5   KKE\\F#A}eVDII!QPQSTUF#A}eVDII!QPQSTUF#A}eVDII!QPQSTUFzz,eFCH XX  $$VjPQ$RFXX  $$VjPQ$RFXX  $$VjPQ$RFxx""&&xV*TU&VHF?V#h..r-   c                     |\  }}}}t        | |||      }|j                  ddd      | j                  j                  d      z  j	                  d      }|j                  |||d      S )Nr  rP   r   r$   rN  r%   )rM  r  rS  rR   r   )	rC  r!  r  r  rS   rD  r  	base_gridgrids	            r.   _affine_grid_generator_4drU    sg    JAq!Q"5!QmLI NN2q!$uxx'9'9!'<<AA"ED99Q1a  r-   c                     |\  }}}}}t        | ||||      }|j                  ddd      | j                  j                  d      z  j	                  d      }	|	j                  ||||d      S )NrR  rP   r6  r$   rN  r   )rP  r  rS  rR   r   )
rC  r!  r  r  rS   r  rD  r  rS  rT  s
             r.   _affine_grid_generator_5drW    sm    MAq!Q"5!QOI NN2q!$uxx'9'9!'<<AA"ED99Q1a##r-   c                     t        j                  t        |      dv d        t        |      dk(  rt        | ||      S t	        | ||      S )Nr  c                       y)NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r,   r,   r-   r.   r   z'affine_grid_generator.<locals>.<lambda>'  r  r-   r6  rR  )rf   r   rw  rU  rW  )rC  r!  r  s      r.   affine_grid_generatorrZ  !  sJ     
LLD	VU 4yA~(MRR(MRRr-   rT  interpolation_modepadding_mode_expand_gridc           	          !"#$%&'()*+,- t        j                  dv fd       t        j                  dv fd       dt        dt        dt        ffd-dt        dt        d	t        dt        fd
+dt        dt        dt        f+fd dt        dt        dt        f -fd} j                  \  $%|j                  \  })*}|dk(  sJ r(|j                  d)*|      j                  )*d      }dt        dt        dt        f$%fd&t        j                   j                        j                  ddd      t        j                   j                        j                  ddd      dt        dt        dt        dt        f&)*fddt        dt        dt        f fd"|d   }	|d   }
dk(  r ||	%      } ||
$      }|j                         |j                         c'('dz   (}}'(dz   }}||}}||z
  ||z
  z  }||z
  ||z
  z  }||z
  ||z
  z  }|'z
  |(z
  z  }t        "fd'(|f|||f|||f|||ffD              S dk(  r< ||	%      } ||
$      }|j                         }|j                         } "||d      S  -|	%      } -|
$      }|j                         '|j                         (|'z
  ,|(z
  }s",j                  d      ,|j                  d      }dt        dt        dt        f "$%fd#dt        dt        f#'(,fd!t        !fdt        d       D              }t!        ||      S )!N)r   r$   r%   c                      d  S )NzInvalid interpolation mode r,   )r[  s   r.   r   z"_grid_sampler_2d.<locals>.<lambda>@  s    -.@-AB r-   c                      d  S )NzInvalid padding mode r,   )r\  s   r.   r   z"_grid_sampler_2d.<locals>.<lambda>C  s    -B<.+Q r-   coordsr!  rM   c                 B    r|dz  dz
  n|dz  }|dz  dz
  }| |z  |z   S r  r,   )ra  r!  r   ofsr  s       r.   unnormalizez%_grid_sampler_2d.<locals>.unnormalizeF  s8     %2tczCs
Sj3|c!!r-   	twice_low
twice_highc                 P   ||k(  rt        j                  |       S |dz  }||z
  dz  }| |z
  j                         }t        j                  ||      }||z  j	                         j                  t         j                        }t        j                  |dz  dk(  ||z   ||z   |z
        S )Nr%   r   r$   r   )rf   r   r   fmodfloorr9   int8rg   )ra  re  rf  
coords_mincoords_spancoords2extraflipss           r.   reflect_coordinatesz-_grid_sampler_2d.<locals>.reflect_coordinatesQ  s    
"##F++]
!I-2J&++-

7K0;&--/222D{{AINEJ.j0H50P
 	
r-   c                     dk(  r| S dk(  rt        j                  | d|dz
        S r | dd|dz
  z        }n | dd|z  dz
        }t        j                  |d|dz
        S )Nr   r$   r%   rP   r   )ra  r!  coords_reflectedr  r\  rp  s      r.   compute_coordinatesz-_grid_sampler_2d.<locals>.compute_coordinates]  sx    1MQ;;vq$(33#6vq!tax.#Q #6vr1t8a<#P ;;/D1H==r-   c                 (     | |      } ||      S r6   r,   )ra  r!  	coords_unrs  rd  s      r.   compute_source_indexz._grid_sampler_2d.<locals>.compute_source_indexi  s    -	"9d33r-   r%   r$   r  ysc                     t        j                  d| k  t        j                  | k  t        j                  d|k  |k                    S rq   rf   r  )r  rw  iHiWs     r.   in_bounds_condz(_grid_sampler_2d.<locals>.in_bounds_condy  sF      GU&&rBw0A0A!r'2PR70ST
 	
r-   r  wsc                      | |      rndt        	
fd| j                  t        j                        |j                  t        j                        |fD              S )Nr$   c              3   p   K   | ]-  }t        j                  |d       j                         / ywr  )rf   rg   r  )r  r  r  r  r  oHoWs     r.   r  z1_grid_sampler_2d.<locals>.clip.<locals>.<genexpr>  s7      
 KKa#((Ar26
s   36r   )r  r9   rf   r  )r  rw  r}  r  r  r  r  r]  r|  r  r  s      @@r.   clipz_grid_sampler_2d.<locals>.clip  sY    b"%
 A1 
ee%++e.EKK0H"M
 
 	
r-   ixiyc                 8     	| ||      \  }}}||f   |z  S r6   r,   )
r  r  r  idx_xidx_yw_C_idxN_idxr  r  s
         r.   get_summandz%_grid_sampler_2d.<locals>.get_summand  s0    B?ubue+,r11r-   ).r   ).r$   r   c              3   :   K   | ]  \  }}} |||        y wr6   r,   )r  r  r  r  r  s       r.   r  z#_grid_sampler_2d.<locals>.<genexpr>  s(      
R B"
s   c                 <     |       } |      } ||d      S rY   r,   )r  r  r:   rW   rs  r  rz  r{  s       r.   get_value_boundedz+_grid_sampler_2d.<locals>.get_value_bounded  s*    #B+A#B+Aq!Q''r-   rc  c                     | dz
  z   } dz
  |       |       dz   |       dz   |      f}t        |      S )Nr$   r%   )r;  )rc  iy_ofscsr  ix_nwiy_nwtxs      r.   	get_coeffz#_grid_sampler_2d.<locals>.get_coeff  s[    cAg&F!%!)V4!%0!%!)V4!%!)V4	B ,B33r-   c              3   .   K   | ]  } |        y wr6   r,   )r  rc  r  s     r.   r  z#_grid_sampler_2d.<locals>.<genexpr>  s     :#y~:   r6  )rf   r   r   r  r  r  r  r  r  r   ri  r  r  rR   r  rQ   r;  ).r  rT  r[  r\  r  r]  rv  rS   twor:   rW   r  r  ix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se
ix_nearest
iy_nearesttyr5  r  r  r  r  r  rs  r  r  r  rz  r{  r|  r  r  r  r  rp  r  rd  s.   ` ````                     @@@@@@@@@@@@@@@@@@@r.   _grid_sampler_2dr  /  s    
LLi'B 
LL	!#Q"F "# "& "

F 

s 

 

PV 


>F 
># 
>& 
>4V 43 46 4 77LAq"bZZNAr2s!8O8 yyAr2s+221aRC
6 
v 
& 

 LL188,11!Q1=ELL188,11!Q1=E

 

V 

 

4F 

 

2 2F 2& 2 2
 	VAVAQ!!R(!!R(xxz288:uqy%ueaiueu
urz*U
urz*
rEz*U
rEz* 
 t$t$t$t$	 
 
 	
 
q	 !!R(!!R(XXZ
XXZ
:z155BB

%Z%ZaBaB	(& 	(f 	( 	( 	(
	43 	46 	4 	4 :q::'33r-   c                 "    t        | ||||      S )N)rT  r[  r\  r  )r  )r  rT  r[  r\  r  s        r.   grid_sampler_2dr    s      	-!# r-   c                 &    t        j                   j                         dk(  xr j                         dk(   fd       t        j                   j                  d      j                  d      k(   fd        z  j	                  d      S )Nr%   r$   c                  L    d j                          dj                          S )Nzmatrix @ vector expected, got r  r~   ry   r  s   r.   r   zmv.<locals>.<lambda>  s!    0BswwykJ r-   r   c                  v    d j                  d       d j                  d       dj                  d       dS )Nzsize mismatch, got input (r   r:   r$   z), vec (r1  r  r  s   r.   r   zmv.<locals>.<lambda>  s<    ,TYYq\N!DIIaL>RURZRZ[\R]Q^^_` r-   r~   )rf   r   rL   r!  r   r  s   ``r.   r  r    ss     
LL
a*CGGINJ 
LL		!#` 3J""r-   c                     |-|dz
  |z  dz   }d|z
  | z  |t        j                  |       z  z
  }nd|z
  | z  t        j                  |       z
  }|||z  }t        ||      S rY   )r   
logsigmoidr   )ry   r   r   
pos_weightr   
log_weightr   s          r.    binary_cross_entropy_with_logitsr    sv    
  1n.2
F
d"j1<<3E&EFF
d"Q\\$%77f}i00r-   tensor1tensor2is_outc                   	 | j                   |j                   k\  r| |fn|| f\  }}ddlm	 |j                   dk\  r|j                   dk  sy|j                  r|sy| j                   dk(  ry 	|j	                         dk(        ry|j
                  }|j                         }dg}t        |dd        D ]  }|j                  ||d   z          t        	fd	t        |t        t        |            |      D              S )
Nr   r  r   r%   FTr$   rP   c              3   J   K   | ]  \  }}} |d k(        xs ||k(    ywrm  r,   )r  r   r   r!  rf  s       r.   r  zshould_fold.<locals>.<genexpr>!  s4      D% 	TQY'845=8s    #)rr  rh  rf  r  r   r  rk  r  rd  r   r  rj  )
r  r  r  t1t2t1_shape	t1_strideexpected_strider!  rf  s
            @r.   should_foldr    s     $+<<7<<#?gwgwEWFBJGGqLRWW\	||qBHHJ!O,xxH		I
 cO!"& ;tob&99:; !$tH_56"
  r-   )pass_is_out)r  c                :	   | j                         }|j                         }|dk7  r|dk7  sJ |dk(  r|dk(  rt        j                  | |      S |dk(  r|dk(  rt        j                  | |      S |dk(  rC|dk(  r>t        j                  t        j
                  t        j                  | d      |      d      S |dk(  r|dk(  rt        j
                  | |      S t        | ||      r>||kD  }|r|j                  n| }|s|n|dk(  r| j                         n| }|j                  }t        |d d       }	t        t        j                  |	      }
|j                         dk(  }|r|	j                  |j                  d          |j!                  |
|d         }|rWt        j"                  j$                  j'                  |j                  |      |	      }|r|j                  j)                         S |S t        j"                  j$                  j'                  |j                  |      |	      S |dk\  r^|dk\  rX|dkD  r| j+                  d      nd}| j+                  d      }| j                  d d }|dkD  r|j+                  d      n|j+                  d      }|dkD  r|j+                  d      nd}g }t-        |dz
        D ]"  }|j                  |j+                  |             $ |dk(  rn|dk(  ri|d   |d   k7  r^|d   dk(  r'| j.                  rt1        | j	                  d      |      S |d   dk(  r'|j.                  rt1        | |j	                  d            S t        t        j2                  ||            }|||gz   }t5        |      }| j7                  |      j!                  |||      }|dk(  }|r7||gz   }|j7                  |      j!                  ||      j                  d      }n)|||gz   }|j7                  |      j!                  |||      }|}	|dkD  r|	j                  |       |dkD  r|	j                  |       |r/|j9                  |      j	                  d      j;                  |	      S |j9                  |      j;                  |	      S t        j<                  dd        y )	Nr   r$   r%   rP   rN  r   Fc                       y)Nz/both arguments to matmul need to be at least 1Dr,   r,   r-   r.   r   zmatmul.<locals>.<lambda>  r  r-   )rL   rf   dotr  r  r  rR   r  rS  r  r  rj  r   r  r   rd  r  r  r   _unsafe_viewr  r!  rQ   r  rR  broadcast_shapesr  r  bmmr  r   )r  r  r  dim_tensor1dim_tensor2ry  r  r  sizes_1output_shapefolded_dim1t2_is_matrix	t1_foldedr  r  m1batch_tensor1m2rF  batch_tensor2r^  expand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded
vector_rhstensor2_expand_sizetensor2_expandeds                               r.   rR  rR  )  s[    ++-K++-K!q 000aK1,yy'**		kQ.xx))		kQ.}}UXXeoogq&A7KQOO		kQ.xx))	Wgv	.  +-	$WZZ'$G+:J799;PW 	 ((GCRL)X\\<8 vvx1}, JJ{GBK8	 YY^^00b1A<PF-6699'')BFB99>>..y||B/?NN		kQ. !,aGLLQ\\"cr*!,qW\\"gll26F +aGLLQ#%{Q' 	2A  a1	2 1q a M!$44Q1$)>)>gooa0'::Q1$)>)>gwq'9::  $""=-@ 
 3aW<#$89 #>>*=>FF !R
 !A%
"6""=23-r21  #7"a"@&~~.ABJJ$b!  ,?"?"#''(89AA"EJJ<XX#''(89>>|LLUUVr-   r  r  c                 \     j                   \  }}t        |d   ||      }t        |d   ||      }t        j                   t        j                  j
                        \  }}t        j                  |d    j                        j                  |      }	t        j                  |d    j                        j                  |      }
t        ||
|      }t        ||	|      }|j                  d      }|j                         }|j                         }||z
  j                  dd      }||z
  j                  dd      }|j                  t        j                        }|j                  t        j                        }|dz
  ||dz   |d	z   f}|dz
  ||dz   |d	z   ft        |      t        |      }d
\  } j                   t        j"                  k(  rt%              t%        |      }D cg c]@  }|dz  z  t        j&                  |      dz  z   j                  t        j(                        B c}|D cg c]@  }|d|z  z  t        j&                  |      dz  z   j                  t        j(                        B }} fd fdt+        fd|D              } j                   t        j"                  k(  r|J t-        |||      }nt/        d t1        ||      D              }t        j2                         }|j5                  |      }|S c c}w c c}w )Nr   r$   rC  r  r   rP   r   rd   r%   r<  r   c                     t        j                  | ddz
        }t        j                  |ddz
        }t        j                  d d ||g      }|S r  )rf   r   r   r2  )rw  r  y_idxx_idxr  in_hin_wr   s        r.   load_boundedz0upsample_bicubic2d_default.<locals>.load_bounded  sO    B4!8,B4!8,utT5%&@Ar-   c                      t         fdD              }j                  t        j                  k(  rJ t	        |      S t        d t        |      D              S )Nc              3   0   K   | ]  } |        y wr6   r,   )r  x_ofsr  rW   s     r.   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s     Bl1e,Bs   c              3   ,   K   | ]  \  }}||z    y wr6   r,   r9  s      r.   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s     JRBGJrk  )r  r   rf   r  r  r  r  )rW   src_xr   ixs_ofsr  weights_precision_x	weights_xs   ` r.   get_x_interpz0upsample_bicubic2d_default.<locals>.get_x_interp  sW    B'BB;;%++%&222%eY8KLLJCy4IJJJr-   c              3   .   K   | ]  } |        y wr6   r,   )r  y_ofsr  s     r.   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s     ;%,u%;r  c              3   ,   K   | ]  \  }}||z    y wr6   r,   r9  s      r.   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s     L(2rb2gLrk  rL  )r  r  rB   rC   rE  r  rf   r  r  r9   r  rR   ri  r   r  r4  r   r  r  r   int16r  r  r  r  r   r  )r   r  r  r  r  rS   h_scale_factorw_scale_factorr   r^  r  x_floaty_floatr:   rW   yscaler  iys_ofs	weights_yweights_precision_yr  src_yr  rM  r  r  r  r  r  r  r  s   `                       @@@@@@@r.   upsample_bicubic2d_defaultr    s    {{Aq$ $D+a.-QN#D+a.-QN''5#H#H#U#UHAu 	[^ELL9<<5<IA[^ELL9<<5<IA#NA}EG#NA}EG#GAA k  c*Fk  c*F	U[[A	U[[A1uaQA&G1uaQA&G08I08I/9,,{{ekk!7	B7	B 
 !**+ejjmc.AAEEekkR
	 
 !**+ejjmc.AAEEekkR
	 

K K ;7;;E{{ekk!"...#E96IJLc%6KLL //6M];FMA

s   9AL$AL)c                 $   t        j                  t        |      t        |      z   dk(  d        |H|J t        t        t
        t
        f   t        d t        | j                  dd  |      D                    }|r|nd\  }}t        | ||||      S )Nr$   c                       y)Nz:Must specify exactly one of output_size and scale_factors.r,   r,   r-   r.   r   z(upsample_bicubic2d_vec.<locals>.<lambda>  r  r-   c              3   P   K   | ]  \  }}t        t        |      |z           y wr6   )r   r   )r  r  rl   s      r.   r  z)upsample_bicubic2d_vec.<locals>.<genexpr>  s*      Au 	!u,-s   $&r%   r<  )
rf   r   r  r
   r   r  r  r  r  r  )r  r  r  r  r  r  s         r.   upsample_bicubic2d_vecr    s     
LL[D//14L (((#s(O  #AGGABK ? 
 )6}<GW%amWgVVr-   c                 (      fd}t         ||      S )Nc                     t        j                  |  ||z   j                        }|dz
  |dz
  |j                         z
  j                         z
  S )Nr  r$   )rf   r  r  r   r   middler   dim_idxr  s       r.   r  z_reflection_pad.<locals>.idx  sF    ,,ufunQXXFzVaZ'++-7<<>>>r-   _reflection_or_replication_padr  r  r  s   `  r.   _reflection_padr     s     ? *	 r-   c                 (      fd}t         ||      S )Nc                     t        j                  |  ||z   j                        }t        j                  |d|dz
        S )Nr  r   r$   )rf   r  r  r   r  s       r.   r  z_replication_pad.<locals>.idx-  s6    ,,ufunQXXF{{7Avz22r-   r  r  s   `  r.   _replication_padr  '  s     3 *	 r-   idx_fnc                 n   t        |      dz  t        j                  | j                         dz   dz   fv fd       | j                   d  }| j                         z
  }t              D cg c]  }|ddz
  |z
  z      }}t              D cg c]  }|ddz
  |z
  z  dz       }}| }t              D ]E  }d g|j                         z  }	 |||   ||   ||         |	||z   <   t        j                  ||	      }G t        j                  |      }
|j                  |
      }|S c c}w c c}w )Nr%   r$   c                  (    d  d dz    d dz    dS )Nreflection_padzd requires r$   zD or r%   zD inputr,   r~   s   r.   r   z0_reflection_or_replication_pad.<locals>.<lambda>@  s$    .[q	sQwiwO r-   rL  )rw  rf   r   rL   r  rQ   r   r2  rB   r   r  )r  r  r  	inp_shapenc_dimr^  padding_leftpadding_rightr  r  rM  rL   s              @r.   r  r  8  sH   
 g,!
C	LL	C!GS1W%%O IUUWs]F8=c
C1GAq1-.CLC=B3ZHWQ#'A+.23HMHF3Z 1&**,. a)A,a@PQAJ##FC01 //7M];FM DHs   1D-D2c           
          t        |      dz  |j                   d  D cg c]  }|dz
  	 }}t              D cg c]  }|ddz
  |z
  z      }}t              D cg c]  }|ddz
  |z
  z  dz       }}g }t        |j                        D ]c  }dg|j                  z  }	d|	|<   |j	                  t        j                  |j                  |   |j                        j                  |	             e |d   | d  }
d t              D cg c]  }|
|   ||   z    }}t              D cg c]  }||   |
|   z
   }}t              D cg c]  }d||   z  ||   z   |
|   z
   }}t              D cg c]  }||   d||   ||   z   ||   z   f }}t        j                  t        j                  t              D cg c]  } ||          c}      }t        j                   ||z   d      } fd}t        j                  t              D cg c]  }g d	 c} D ]  }|t!        dgz        k(  rg }g }t              D ]t  }||   dk(  r||   }||   }n=||   dk(  r||   }|
|   d||   f}n$||   dk(  r||   }|
|   ||   ||   z
  ||   dz
  f}|j	                         |j	                         v  ||||      } |S c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )
Nr%   r$   rP   r  c                 F    | \  }}}t        j                  ||k\  ||k        S r6   ry  )index_ranger^  lbubs       r.   index_range_conditionz7_reflection_pad_backward.<locals>.index_range_conditioni  s(    	2r  b!r'22r-   r   r   c           	      .   t        	      D ]*  }||   d   ||   d   k  }t        |t              s%|s(| c S  t        j                  t
        j                  |D cg c]
  } |       c}      }t
        j                  
||z   d      }| |z   S c c}w )Nr%   r$   r   )rQ   r8   r  rI   r   r   r  r  )r   r   index_rangesr^  upper_less_than_lowerr  r  gr  rL   rj   r  s           r.   r  z,_reflection_pad_backward.<locals>.accumulate  s     s 	A$0OA$6a9K$K!/6;P	
 COPK";/P
 %%k4S#Fax Qs   B
)rP   r   r$   )rw  r  rQ   rr  rd  rf   r  r  r  rI   r   r   r  r  	itertoolsr   r  )rj   r:   r  rD  dhwr^  r
  r  r  
view_shapexyzcenterleft_reflectright_reflectrange_cr  r   r  rS   areaoutsr  r   r  r  rL   r  s   `                       @@@r.   _reflection_pad_backwardr   T  s`   
 g,!
C''3$%.
)Q1q5
)C
)8=c
C1GAq1-.CLC=B3ZHWQ#'A+.23HMHG166] SS166\

1u||AGGAJqxx@EEjQRS
 	#A
3$%.C3 16c
;1c!f|A&;F;6;CjALOc!f,ALADI#JOqQQZ,q/1CF:OMO
 NSSVZHIAs1vQ/-2BBCG  eCjQ0<QD $$[$F
CHD !!c
#C1J#CD 45!s##s 	-AAw!|Qi%ajaB"1o"1vq,q/:aA#A&"1vs1va0@'@#a&1*MKK,	- $l3-40 Kc *CH2 <AO R& $Ds5   K K$$K)K.8K3K8K=L
&Lr   r   r   c                j    t        j                  | ||      }t        j                  | ||      }||fS )Nr   )rf   aminrG  )ry   rL   r   r"  rG  s        r.   aminmaxr#    s2     ::dW5D::dW5D:r-   r   c                    t         j                  t        j                  t        j                  |       d|       |||      S )Nr   r   )r   r   rf   rg   isnan)ry   rL   r   r   s       r.   nansumr&    s2     88EKKD 11d;S'QV8WWr-   r   r  r  r1  r  c          	      N    t         j                  j                  d| d||||      S )Nr   r$   r'  r   r  
start_step)r_  r   r  r  r1  s        r.   arange_defaultr+    s/     ;;!!	3vf "  r-   c          	      N    t         j                  j                  | |d||||      S )Nr$   r'  r)  )r^  r_  r   r  r  r1  s         r.   arange_startr-    s/     ;;!!sAU6&Z "  r-   c                      ddl m}  || i |S )Nr   )out_dtype_dense)!torch._higher_order_ops.out_dtyper/  )rD   rE   r/  s      r.   out_dtype_decompr1    s    AD+F++r-   marginc                 t   	
 t        j                          t        j                         j                  d   
 j                  d   	t        j                  |dk(  xs |dk(  d        t        j                   j
                  dk(  xr 	dk7   fd       t        j                  j
                  dk(  xr j                         
k(  
fd       Qt        j                        t        j                  j
                  dk(  xr j                         	k(  	fd       j                  d      t        j                   d      }||z
   z   }|j                  d      }|dk(  r|n||z  }|   z  }t        j                  	 j                  	      }t        j                  |k7  |d      }|t        j                  j                  k(  r|j!                         S |t        j"                  j                  k(  r |j%                         |j                  d   z  S |j!                  d
      S )Nr   r$   r%   c                       y)Nz only p == 1 and p == 2 supportedr,   r,   r-   r.   r   z#multi_margin_loss.<locals>.<lambda>  r  r-   c                  "    d j                    S NzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: r  r  s   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    _`e`k`k_lm r-   c                  (    d  dj                    S )Nz#inconsistent target size, expected r  r  )nframer   s   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    5fXYv||nU r-   c                  (    d  dj                    S )Nz#inconsistent weight size, expected r  r  )rL   r   s   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    9#i~V r-   r  r  r~   )rf   
atleast_2d
atleast_1dr  r   rr  r   rR   r  rT  r  r  rg   r#   r*   rz   r   r+   r   )r   r   rF  r2  r   r   urh   r  rL   r8  s   ``  `    @@r.   multi_margin_lossr=    s    U#Ef%F[[^F
++a.C	LLa!16#MN	LL

a$C1Hm 
LLq5V\\^v5U !!&)KK163!6V	
 a FU0A
UA	AA!VQAv
,,s5<<
0CC6M1a(AINN(((vvx	imm))	)uuw##vv!v}r-   	is_targetc                    | j                   |j                   t        j                  |       } t        j                  |      }| j                   d   }t        j                  t	              dk  xr |dk7  fd       t        j                  t	              dk  xr k(  fd       t        j
                  ||j                        }|dk(  }t        j                  t        j                  |||      dd	      }||k  }t        j                  ||d      }t        j                  | d|
      }	t        j                  ||d      }
t        j                  ||
j                  d      k(  d      }d|	j                  j                  d      z
  | z   }|j                  d      }||z  }t        j                  |d|      }|t        j                  j                   k(  r!|j#                  d      j%                         }n@|t        j&                  j                   k(  r|j#                         }n|j#                  d      }|j)                  | j*                        j-                        }||fS )Nr$   r%   r   c                      d  S r6  r,   )orig_input_shapes   r.   r   z0multilabel_margin_loss_forward.<locals>.<lambda>   s    _`p_qr r-   c                      d d  S )Nzinconsistent target size: z for input of size: r,   )rA  orig_target_shapes   r.   r   z0multilabel_margin_loss_forward.<locals>.<lambda>$  s    ,->,??STdSef r-   r  rP   Tr   r  r~   rd   )r   rP   )r  rf   r:  r   rw  r  r  r"  rg   r  anyrR   TrT  r#   r*   rz   r   r   r+   r9   r   r  )r   r   r   rL   r  is_endend_idxtarget_masktidx0r<  tidx1r>  rh   rA  rC  s                @@r.   multilabel_margin_loss_forwardrK    s    {{U#Ef%F
++a.C	LL"/saxr 
LL!#M(9=M(Mf
 ,,s6==
1Cr\FjjVS#6BMG-KKKVQ/EU%0AKKVR0E		#R!88a@Iaccmmm##e+A	AA	CAIq!$AINN(((EEgE##%	imm))	)EEGEEgEU[[)112CDIi<r-   )	attn_maskrl   querykey	dropout_p	is_causalrL  c          
           j                   }t        j                  t        j                          fd       t        j                   j	                         dk(  xr( j	                         dk(  xr j	                         dk(   fd       t        j                  dk(  fd       t        j                   j
                  d   j
                  d   k(  xr j
                  d   j
                  d   k(  d        t        j                  j                   ||d |      \  }}	|j                  d	d
dd      j                  t        j                        j                  dd	d
d      }||	fS )Nc                  "    d j                    S )Nz-query must be FP32, FP64, BF16, FP16 but got r   )rM  s   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>^  s    ?}M r-   r6  c                  n    dj                          d j                          dj                          S )Nz,q, k, v must be a 4 dimensional tensor, got r  r~   )rN  rM  rz   s   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>b  s3    >uyy{m2cggi[XZ[`[d[d[fZgh r-   r   c                      d  S )Nz&dropout probability must be zero, got r,   )rO  s   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>e  s    $J9+"V r-   r   c                       y)Nz&q, k, v should have the same head sizer,   r,   r-   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>i  r  r-   )rL  rO  rP  dropout_maskrl   r%   r   r$   rL  )r   rf   r   r<  rL   r  r   "_scaled_dot_product_attention_mathr}  r  r  rQ  )
rM  rN  rz   rO  rP  rL  rl   r   r  attns
   ````      r.   *scaled_dot_product_flash_attention_for_cpurY  P  sG    KKE	LL&M 
LL		q@SWWY!^@		q0@h 
LLSV 
LLA%++a.(KSYYq\U[[^-K8
 ::BB C 	LFDD 	q!Q"	%"9"9	:	Aq!	 
 4<r-   c                 .    t        |       fd       }|S )Nc                  <     | i |}| d   j                  |      S rq   )r  )rD   rE   r   outplace_ops      r.   
inplace_opz$register_inplace.<locals>.inplace_op  s%    4*6*Aw}}S!!r-   r   )aten_opr\  r]  s    ` r.   register_inplacer_    s"    G$" %" r-   c                 B   | j                         s&| j                         st        |      }t        |      }t        j                  ||      }t        |t        j                        r|dk7  r||z  }|dk(  r|S t        |t        j                        r|dk7  r| |z  } | |z   S r  )r<  r  r  rf   r  r8   numbersNumber)ry   batch1batch2r`   rk   r  s         r.   baddbmmre    s     !!#DOO,=4yE
YYvv&FeW^^,
%qydGNN+tqyd{&=r-   c                 2    t        j                  | |d      S )Nri  rp  rr  )ry   rE  s     r.   floor_dividerg    s     99T588r-   c                 `    t        j                  t        j                  | j                  d      S rY   )rI   r   r  r   r  )r  s    r.   	sym_numelri    s    HLL!''155r-   r   r   c                    |"t         j                  j                  | g |      S t         j                  j                  | g ||      S )Nr   rj  )r   r   dim_IntListIntList_out)ry   r   r   s      r.   sum_defaultrn    sC     {xx##D"E#::xx##D"Es#CCr-   c           	          t        | t        j                        s| S |@t        j                  j                  | t        t        | j                                           S t        j                  j                  | |g      S r6   )	r8   rf   r   r   r  dimsrj  rQ   rL   )ry   rL   s     r.   squeeze_defaultrq    s\     dELL)
{||  tE$((*,='>??||  u--r-   c                 2   t        fdt        t        | j                              D              }|j                  t
        j                  k(  rt
        j                  nd }| j                  d|d|      }| ||j                  |j                        z  z  |fS )Nc              3   .   K   | ]  }|k7  s	|  y wr6   r,   )r  r^  rL   s     r.   r  z)_weight_norm_interface.<locals>.<genexpr>  s     @1qCxQ@s   
r%   T)r   r   )
r  rQ   rw  r  r   rf   r  r   r   r9   )r  r  rL   keep_dim
norm_dtyper   s     `   r.   _weight_norm_interfacerv    st     @c!''l 3@@H !5>> 9tJ66!Xt:6>DDGGAGG$$%t++r-   assume_uniqueinvertc                   t        | t        j                        s!t        j                  | |j                        } t        |t        j                        s!t        j                  || j                        }|j                         dt        | j                         d      z  k  rt        | ||      S t        | |||      S )Nr  g      $@g(\?ry  rw  )	r8   rf   r   rf  r  r   rO  isin_defaultisin_sorting)elementstest_elementsrx  ry  s       r.   isinr    s     h-<<1E1EFmU\\2]8??Ktc(..*:E&BBBHmFCCm=
 	
r-   )r/  c                B   |?t        j                  | j                         t         j                  | j                        }n?t        j                  | j                         |t         j                  | j                        }|| k  j                  | j                        }|S Nr  )r/  r   r  rf   randr!  r   r  r9   r   )ry   r/  raw_prF  s       r.   	bernoullir    sq     

499;emmDKKP

IIK--;;	
 
$**%AHr-   c                :   |?t        j                  | j                         t         j                  | j                        }n;t        j                  | j                         || j                  | j                        }||k  j                  | j                        }|S r  r  )ry   rF  r/  r  s       r.   bernoulli_pr    sm    

499;emmDKKP

IIK,,;;	
 
tzz"AHr-   r{  c                N   | j                         dk(  r%t        j                  | t        j                        S  | j                  g | j
                  d|j                  z   }|s||k(  }n||k7  }t        t        d|j                   dz
  d            }|j                  |      S )Nr   r   )r$   rP   r$   r~   )
r   rf   
empty_liker  r  r  rr  r  rQ   rD  )r~  r  ry  r:   cmprL   s         r.   r|  r|    s    ~~1

;;Dx~~D0B0B)BDA= = 
b=---126
7C77s7r-   c                   | j                         }|j                         }|rt        j                  ||g      }t        j                  |d      \  }}|dd  |d d k(  }	t        j                  |	ddgd      }	|r|	j                         }	t        j                  |	      }
|
j                  d||	      }
|
d| j                          S t        j                  |      \  }}t        j                  ||      }t        j                  ||j                         k  |d      }||   |k(  }|r|j                         n|}|j                  | j                        S )NT)stabler$   rP   r   F)r,  rf   r#  sortrb  logical_notr  r  r   searchsortedrg   r  r  )r~  r  rx  ry  elements_flattest_elements_flatall_elementssorted_elementssorted_orderduplicate_maskr  sorted_test_elementsrS   r  test_idxr  s                   r.   r}  r}    s?   $$&M&..0 yy-1C!DE(-

<(M%(,0DD..~1vuM+779N/q,?A())"'**-?"@a  !5}E;;s%9%?%?%AA3J"8,=#)coos{{8>>**r-   c                 .    | j                  d      }||   S rO   )r  )ry   r  	flatteneds      r.   taker  :  s     R IUr-   c                     |t         j                  }|t         j                  k(  rt        |      }t        j                  | |j                  |      S r#  )rf   rQ  preserve_formatr   r   resizer  )ry   rE  rM  s      r.   	resize_asr  A  sD    //----e4;;tU[[;FFr-   )F)noner6   )r%   )r   NNr$   )rP   FFr  r  rH  )r$   r$   F)Fr   )r   rd   N)r   r$   Nr<  )NNN)r   r   FT)r   r   Fr  )r   F(  rI   r  ra  r  rl  enumr   r   r   r   r   typingr   r	   r
   r   r   r   r   r   rf   torch._meta_registrationstorch._primsr8  r  torch._prims_common_prims_commonrB   torch.nn.functionalr  r  r   r   r   r   torch._decompr   r0  r   r   r   r   r   r   torch._prims_common.wrappersr   r   r   r   torch.utilsr   r@   torch.utils._pytreer    r  DispatchKeyr!   str__annotations___opsr  r   r#   rE  r  rK   rF  compute_only_pw_cast_for_opmathpw_cast_for_opmathr  pw_cast_for_int_to_realr  rT   r]   r_   ri   r   ru   r  Scalarr{   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r*   rz   r   _safe_softmaxr  r  r  r}  rU   r  r  r   r  r  r+  r4  r7  r=  r@  rB  rD  rH  r\  rd  slicerx  r  rc  r  r  r  r  r  r  r  r!  r%  r.  r4  r8  py_implCompositeImplicitAutogradAutogradr7  rJ  rN  rT  r\  r  rj  rm  rq  r|  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  unsafe_chunkr  r  r  no_statsr  r  r!  r'  r+  r-  _fused_dropoutr0  r=  r  rM  r  lift
lift_freshr@  rE  rH  rK  rJ  r^  rc  rf  _adaptive_avg_pool2dr  r  r  r  r  r  r  r  r  r  r  r  r  	Generatorr  r  r
  r   r  r'  r-  r  _upsample_nearest_exact1dr)  r/  r  r  r"  r  r:  rC  rG  rK  rY  ra  rd  rh  rw  r  rnn_tanhr   r  rnn_relur  r  r  r  r  r  r  r  lstmr  r  r  r  grur  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r2  r  r  r  r"  r%  r'  r+  r.  r4  r;  r  rB  rM  rP  rU  rW  rZ  r  r  r  r  r  rR  upsample_bicubic2dr  r  reflection_pad1dreflection_pad2dreflection_pad3dr   replication_pad1dreplication_pad2dreplication_pad3dr  r  reflection_pad1d_backwardreflection_pad2d_backwardreflection_pad3d_backwardr   r#  r&  r  r5  r  r+  r^  r-  r1  r=  rK  +_scaled_dot_product_flash_attention_for_cpurY  r_  re  rg  ri  r   rn  r  rL   rq  rv  r  r  rF  r  r|  r}  r  r  addbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r  
hardswish_	hardtanh_hardtanhhardsigmoid___iand____and____ilshift__
__lshift__r  r  index_reduce_index_reduce__ior____or____irshift__
__rshift____ixor____xor__leaky_relu_
leaky_relulogit_logitrelu_r  renorm_renormround_r  scatter_r  scatter_add_scatter_addscatter_reduce_scatter_reducesilu_r,   r-   r.   <module>r     s;M       
  % $ N N N     #   , , 0 7   * ( hh"" c zz~~   %!!99! !H #*88@@# 
 uDDLL  "uDDQQ  c f  **+\2F 2v 2   ,2 --.\4v 4& 4   /4 ../\S S6 S S5 S   0S
 ))*\


 
 	

 
 
   +
2 ))*+( ,( ))*+"V " ," (()@f @ @   *@ 112\f F    3 ../\PP%P05P@EP  0P 'GF Gv G   (G //0F & V    1 //0\:F :& :U :  1: 001\LL%L7<LNRL   2L **+\) )f )3 )   ,)< **+5v 5f 5  ,5 		"&v && &   #& **+\>v >V > >   ,>
 **+6 6 66 6 ,6 334%%
% % 66>	% 5% 667



 
 	

 
 
 
 
   8
$ 112\<f <F <F <v <   3<v #   &3<>>3G3G1
1 1-011   '1 ../\11 &1061CF1   01 **+0 ,0 ++, ^^))	1
11 1 	1   -1 44<<=%/5BEMR  > 44??@
R
R

R 
R 	
R
 
R 
R  A
R 00889		%	/5	BE	NS	  :	 00445
R
R

R 
R 	
R
 
R 
R  6
R$$
$ $ V	$
 $ $ $ $@ ))*\Nf NF N N N   +N$ ../\''
' ' V	'
 ' ' ' '  0'T 001\
  V	
      2B 112  $^^))	1
11 V1 	1
 1   31& ::;\
  $^^))
  V	
     <  --. ^^))111 1 	1   /1 667\
 ^^))			
	 	 		
 	   8	 		") )v )% )  #) ,,-& &F &v &  .& ++,	O	Oc	O 
	O 		O
 
	O 	O  -	O 

))* 6?
6? 
6? C=	6?
 
#6? 6? +6?r (4<SM
38_. **+ $U$U	$U 
$U C=	$U
 
#$U $U  ,$UN ,,-E Ed3i Ec ERU E  .E
 ../OO&*3iO9<ODGOORO  0O%+:?++ 334\ XX!'X.1X@EX !  5X 778 KK!'K.1K@EK !  9K*& $MMcM 3iM #Y	M
 IM M  %M` $]]c] c] 3i	]
 #Y] I] ]   %]@ 445
 
v 
e 
  6
 ,,-Y
Y"3iY47Y?BYJMYY  .Y" ++334>B

%
,4UO

  5
& %kCCDk2236 e HTN  4 E & ++,VVA& AU A8D> A  -A & S    '* ))*F  T   +, ' $  	
    (& 556  	
   7:DI &\	  
&\	*T&\ 
tF| 
# 
"&\"	" "J 00$//2E2EFG
 !	&\	  
&		
  H  	'')C)C)G)GH "&	
c 
 
$v,		
 d6l  ))0015 5C 5c 5%PSBT 5 25 44<<=67BB $S	B03B
63;B >B 

))*/ /C /c /%:L / +/" --55)) /
/ &/ 
/ 63;	//@ 

# f F # #    $" ../
 

  	
     0" 

# f 6      $ 77??@S&S&S& S& 	S&
 FS& S& S& 
S& S& dS& 8FXf-x/??@S&  AS&n 77;;<  	
 F   
  d ,, ,, ,, 8FXf-x/??@ =88F# x/?  77??@LLL 3iL 	L
 L VL 6
L dL 8FXf-x/??@L AL` 77;;< 3i 	
  V 6
 d ,, ,, ,, 8FXf-x/??@ =4OOVO 6
O 6"	O
 &!O O O 
O O 6668F#3Xf5EEFOd ../UK/((V( 6
( 6"	(
 &!( ( ( 
( 666!"( 0 0(4 ''(<(<=''(M(MN 
 
V 
 6
 
 6"	 

 &! 
  
  
 
 
 666!" 
 O > 
F "";#H#HIG4< G JG AAIIJV 6
 	
   
 666!" K* 55==>((V( 6
( 	(
 ( ( ( 
( 666!"( ?(  55>>?((V( 6
( 	(
 ( 
( 666!"( @( @@HHIKKVK 6
K 	K
 K K K 
K 6666612K JK0V 6
 	
  
  6 44<<=11V1 6
1 	1
 1 1 
1 6666)*1 >14 ??GGHCCVC 6
C 	C
 C C 
C 6666669:C IC4 22::;11V1 6
1 	1
 1 1 
1 6666)*1 <14 ++,VV   - & $(%)37&VZ & EKK &
 U\\"& & & E//0&  '&X diiAB  C &&{';';<--.VVVV, 6
 6"	
 &!  !&  - / => 00889 V 6"	
 &!  &!  
 d  68F#Xf%556 :6 77??@aaa Va 6"	a
 &!a a &!a a 
a da 68F#Xf%556a AaJ 77;;<""" V" 6"	"
 &!" " &!" " 
" d" ,," ,," ,," 68F#Xf%556" ="J 778VVV$  6"	
 &!  v  % 90 667VVV$  6"	
 &!  v   % 82 112c'v c'E#s(O c'   3c'L
)8<S	HK0 ))**8
*8*8 c*8  +*8Z ))*494949 c49 I	49
 #Y49  +49n ( HH	H H 	H H )H ' II	I I 	I I  (I$ %@%@	%@ %@ 	%@ %@ %@P ))112"";#H#HI J 32 (()<: <C <
 <J < *< (=* =3 =z =: =  )=@@@$.@8B@PT@, 001Xx (f (vv~)> (  ! 2( % $'$'+/		tS%	  c5 
! (	  &  &; ';
8 //334//334//334$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9:	3	3$s)$	3 DK(	3 		3 ; L ; L ; L 5 5 5	3 66::;66::;66::;##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A	?	?$s)$	? DK(	? 		? B S B S B S < < <	?@ 0088$:Q:Q:U:UVW  (()N)NO  (()=)=>Dd; #;;c; UO; 	; < ? P X; 	##++T-K-K-O-OP ''//0U0UV''//0D0DEDd; #GGcG UOG 	G < F WG 0088$:Q:Q:U:UVW  (()N)NO  (()=)=>Dd; !% $	GGcG uoG uo	G
 G < ? P XG 	##++T-K-K-O-OP ''//0U0UV''//0D0DEDd; !% $	SScS uoS uo	S
 S < F WS 0088$:Q:Q:U:UVW  (()N)NO  (()=)=>Dd; !% $ $QQcQ uoQ uo	Q
 uoQ Q < ? P XQ 	##++T-K-K-O-OP ''//0U0UV''//0D0DEDd; !% $ $		c	 uo	 uo		
 uo	 	 < F W	 
 	c %! 	
  4 >/ FK+\&,/-d& R ++,[BBC[112. 3 D -.8 ++,[BBC[112. 3 D -.8 **+KAABK001. 2 C ,.@ **+KAABK001. 2 C ,.@/6<~/d 		(>>?--.S / @ )S> 		'==>,,-S . ? (S<;; &{<<={++,. - > '.6 '==>,,-. . ? (.6 44889!!))+*O*OP!!))+*>*>? @ Q : 33778  (()N)NO  (()=)=> ? P 9 0044511556##K$I$IJ##K$8$89%%k&K&KL%%k&:&:;&&{'L'LM&&{';';<A = N < M : K 7 6A //779O9O9S9STU
 !%	KKcK K uo	K
 K  VK 	%%t'?'?'C'CD !!))+*>*>?
 !% $UUcU U uo	U
 uoU U  @
U 	&&(A(A(E(EF 
 !% $ $

c
 
 uo	

 uo
 uo
 
 
V/7	&	7$,V$47IO77-'9 -f - EEcE E %!	E
 E ER ))112F v $  3 ,,d.?.?@A  B ++,-" ." //019 29 2234C 5C6 AABCM DM22 
2 2  V2  	2 
 2  66>2 j --.X~&L
LL VL 	L
 L 66>L ' /L4 //0X~&L
LL VL 	L
 L 66>L ' 1L/F /u / /5F 5u 5 5
 
3E 
(F%7 FV F F!Xf% !& !NN#'N05NEJ\\N&f & & &T &"/f / / / /T /"!V !49 !T !$V $49 $T $ 223S StCy S S   4S  c4c4
c4 c4 	c4
 c4 c4 c4L ,,-  
  	
     .   	#   !	# ==>$)..:N:N1  ?1  d t B [BBC>>?', tW  @ DtWn 0088$:Q:Q:U:UVW  (()=)=>
  $#PPsCxP P e_	P
 e_P P   ? XPf //334$$[%J%JK$$[%9%9:
 48	WW%S/*W W E%,/0	W
 W   ; L 5
W. --.--.--.	v 	c3h 	F 	   / / /
	 ../../../	 	sCx 	V 	   0 0 0
	38_ c3_f,- 	8 667667667\T  8 8 8Tn %UEu   & $X4 X  %X ,,dkkoo>? $( ==%)
	
 EKK 
 LL	

 U\\"
 
  @
 **+,
 $( ==%)	 EKK 	
 LL U\\"  - 	", #, ../''(<(<= #^^))))) ) 	)
 V) ) )  > 0)X ;;<$$,,44[5I5IJX{#--- - 66>	- $ K =-x HHPPQ
 A #'!AA	A A 	A
 A A E?A 66>A RAH '   ( ))*9  +9 '6 (6 ))488<<89 $( 		D
	D EKK 	D 
&			D
 	D :	D --t||/?/?@A.& .x} . B. 		==>, ?, 		"38 
  #
 ../ ,0
,, ( \\	 0$ (()CG x'@  * 5: 
 <A +: 		"  #
 'G (G t{{ + djj ) djj )  - TYY ' TYY ' $.. 1  / ""D$4$4 5  - !!4?? 3 $.. 1 ##T%6%6 7 t{{ + !!4?? 3  - !!4?? 3 djj ) TYY ' t{{ + djj )  - ""D$4$4 5 %%t':': ; TYY 'r-   