zIg{ ddlmZddlZddlZddlZddlmZddlmZddl Z ddl Z ddl m Z m Z m Z ddl mZddlmZmZmZmZddlmZdd lmZmZmZ dd lmZ dd lmZmZd Z d Z!dZ"dZ#dZ$dZ%dZ&dZ'dZ(iZ)e*e Dcic]}e+e,e |e-se,e ||c}Z.GddeZ/GddeZ0GddeZ1GddeZ2ejjfe jhdejjje jhdejjle jhdejjne jhd ejjpeejjreejjteiZ;ejjje jxde jz!e jxd"e jz!fejjfe jxd#e j|!e jxd$e j|!fejjne jxde j~!e jxd%e j~!fejjle jxd&e j!e jxd'e j!fejjte jxde!e jxd(e!fejjre jxd)e!e jxd*e!fiZAejjfe jxd+e j|!e jxd$e j|!fejjle jxd,e j!e jxd'e j!fiZBejjje jxde jz!e jxd$e jz!fejjfe jxd-e j|!e jxd.e j|!fejjne jxde j~!e jxd'e j~!fejjle jxd/e j!e jxd0e j!fejjte jxde!e jxd*e!fejjre jxd1e!e jxd2e!fiZCd3d4d5ZDd[d6ZEd\d7ZFd8ZG d]d9ZHd^d:ZId^d;ZJd_d<ZKd`d=ZLGd>d?ZMGd@dAZNGdBdCZOdDZPdEZQdFZRdGZSdadHZTdIZUdbdJZVdcdKZWdddLZXdedMZYdfdNZZdgdOZ[dfdPZ\dgdQZ]dhdRZ^didSZ_djdTZ`dkdUZadkdVZbdldWZcdldXZddldYZedldZZfy#e$rdZYwxYw#e$rdZdZYwxYwcc}w)m) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4z onnx.quantizez0.1.0zai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedc*eZdZdZdZdZedZy)QuantizationModerc|jSNnameselfs a/var/www/html/answerous/venv/lib/python3.12/site-packages/onnxruntime/quantization/quant_utils.py__str__zQuantizationMode.__str__< yycD t|S#t$r twxYwr)rKeyError ValueError)modes r$ from_stringzQuantizationMode.from_string?s) #D) ) ,   N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr% staticmethodr,r'r$rr8s%JJr'rc*eZdZdZdZdZedZy)QuantizedValueTyperrc|jSrr r"s r$r%zQuantizedValueType.__str__Kr&r'cD t|S#t$r twxYwr)r6r)r*)vs r$r,zQuantizedValueType.from_stringNs) %a( ( ,  r-N)r.r/r0Input Initializerr%r3r,r4r'r$r6r6Gs% EKr'r6cNeZdZdZdZdZdZdZdZdZ dZ e d Z e d Zy ) QuantTyperrc|jSrr r"s r$r%zQuantType.__str___r&r'cD t|S#t$r twxYwr)r=r)r*)ts r$r,zQuantType.from_stringbs( Q<  ,  r-c |tjk(rtjS|tjk(rtj S|tj k(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjSt!d|d)NzUnexpected value qtype=.)r=QInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r*r"s r$ tensor_typezQuantType.tensor_typeis 9?? "## # 9## #$$ $ 9$$ $%% % 9## #$$ $ 9** *++ + 9## #$$ $ 9?? "## #24(!<==r'N)r.r/r0rHrJrPrNrLrTrRr%r3r,propertyrVr4r'r$r=r=VsR E FM FG E F >>r'r=c*eZdZdZdZdZedZy) QuantFormatrrc|jSrr r"s r$r%zQuantFormat.__str__r&r'cD t|S#t$r twxYwr)rYr)r*)formats r$r,zQuantFormat.from_strings) v& & ,  r-N)r.r/r0 QOperatorQDQr%r3r,r4r'r$rYrY|s%I Cr'rYint8uint8int16uint16dtypeiiiiiiii@ii@r?zero_point_indexc@g}t|D]\}}tjt|tjr%|j tj |n=t|tjr|j |ntd|d|||k(s|d}|jtjk(s|jtjk(std|jt|dkDr t|S|dS)Nzarg z is not an array: rkzzero_point cannot be rr) enumeratenumpy issubdtypetypenumberappendarray isinstancendarray TypeErrorrdfloat32float16lentuple)rmargsnew_argsiar9s r$ _check_typersH$ C1   DGU\\ 2 OOEKKN + 5== ) OOA d1#%7s;< <   Aww%--'177emm+C"7y ABB C"(ma/5?@Xa[@r'c|tvs Jd|d|tjjtjjtjj tjj fvr/|dk7rtd|d|jtjk(rtj}nG|jtjk(rtj}ntd|jdtt!t#dgdgt$j&j)d|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|j1d||ddSt|} t3|dd\} } | t5| |n| } | t7| |n| } tj8|j;tj|z j=|z}tj>|| | |t/|j;| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rGzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrFT) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protorrQFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrdrpryFLOATrzFLOAT16r*r r r onnxhelper make_tensorr rrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefrdqminqmaxcliplowcliphigharr_fp32s r$quantize_nparrayrs- %%e ug%cde% ++--))--  ?%(Z[eZhhi&jk k 99 %#))I YY%-- '#++I0 1=> > "Bdkk>U>UVbdikmpqor>s.0LseT  *3 4@*7ItD(UD9:  !,3774sU)CDQGHH%U+.u5TXY t$'O#dC.&*&63tT?D==#**U]]";e"C!J!J!Lz!YZ 8WhH=8??5122r'c ^|dkDs|dkrtd|d|tj|tjd|j}tj |tjd|j}|t |||z}|rBtj tj|tj|}| }|}||ks Jd|d|tj||z tj}tj|tjtj|tjz }tj||z } | dk\sJd| tj|jjkrFtjd|j} tjd|j} | | gS|r^tjtj||ztjd tjz |j} n:tjtj||| z z |j} | j|j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rczqmin=z > qmax=z scale isse?g@) r*rpminimumrurdmaximumrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r$compute_scale_zprs( ax4!8]^b]ccklpkqrss ==u{{1DJJ? @D ==u{{1DJJ? @D!4./uyy $@ww 4<55htf55< TD[ 6B T /%++d%--2X XB KKR E A:#|#: u{{4::&+++ Ctzz2[[$**5      TD[EKK5==,QQRZ^ZdZdJU[[u 1D%ETZZXJ TZZ(  r'cd}|tvr|tjk(rddlm}ddlm}|}tdDcgc] }|| }}tj|Dcgc]0}tj|rtj|r/|2c}tj}ntd|d|t|<n|tjk(rddlm}|}|td |d tjt|} tjd|} tj|| z |j } | | gScc}wcc}w) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)float8e4m3_to_float32rrczQuantization to element_type=z not implemented.zUnexpected element_type rG)FLOAT8_DISTRIBUTIONSrrQonnx.numpy_helperr#onnx.reference.custom_element_typesrrangerpruisnanisinfryr*rxstdrd) element_typerzp_dtyperrr all_valuesfvaluesstd_f8zerors r$compute_scale_zp_float8r0s%H// ;33 3 ? H#H :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point z%Weight must be given as an array not rGgrcrrz1Unsupported option reduce_range=True for float 8.rfz+One of the quantized value is NaN data in [z, z], quantized_data in [z].r>rlrzUnexpected value for qType=)rvrprwrxrrr{rrrurdrrQ RuntimeErrorrrranyrr`ravelrrrIrKrOrMrUrSrrr*)datarrrr rmin_override rmax_overriderrrrrquantized_datanp_datarrs r$ quantize_datarUsK8 dEMM *?T |1MNN  YtxxzC  YtxxzC ;;t4:: .D ;;t4:: .DJ KK4:: .E ((( RS Siio3E3? E)%ujI %%ekk288:S@SH ImmD)G=gkkm_Bw{{}o^&&4&8&8&:%;2n>P>P>R=SSUW 4z5.[\]]   t90 PYZJD$ 0tT4Tb c J)%ujI4z5.[\]] 25'; <rgr@)r{ bytearray)src_8bit num_elemsdst_sizedstsrc_idst_is r$pack_bytes_to_4bitrsH IA~{A !#H H CEE )a-  *S0Q68E?S;PQE      )a-   ye_s*E Jr'ceZdZdZggdfdZy)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_yr) r! initializerrminsrmaxs zero_pointsscalesrrr) r#r!rrrrrrrrs r$__init__zQuantizedInitializer.__init__sF &  &  , r'r.r/r0__doc__rr4r'r$rrs r'rc eZdZdZ ddZy)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_yr) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r#r!new_quantized_namerzero_point_namequantized_value_typerrrr s r$rzQuantizedValue.__init__sD"( $& . "$$r')NNNNrr4r'r$rrs%r'rceZdZdZdZy)BiasToQuantizez+ Represents a bias to be quantized c.||_||_||_yr) bias_name input_name weight_name)r#rrrs r$rzBiasToQuantize.__init__5s"$&r'Nrr4r'r$rr0s 'r'rc|jdk(rtd|jd|jdk(r|j}n#|jdk(r|j}n|jdk(r |j }n|jdk(r |j }n|jdk(r |j}n|jd k(r |j}n|jd k(r |j}nz|jd k(r |j}n^|jd k(r |j}nB|jd k(r |j}n&td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.rr>r?r@rArBrh z has unsupported type rG) rrr*r!rrsrEgfloatsintsstringstensorsgraphs) attributers r$attribute_to_kwargr;s; ~~:inn%55STUU~~  1   1   1   1   1    1  1 !! 1 !! 2   :inn%55KINNK[[\]^^ NNE ""r'ct|Dcgc]}|j|k(s|}}t|dkDr|dSdScc}w)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. rN)r!r{) item_name item_listitemitemss r$ find_by_namer%`sA( Bd499 +AT BE B5zA~58/4/ Cs55cRd}tt|D] }|||k(s |}|S)zC Helper function to return index of an item in a node list rk)rr{) elem_name elem_listelem_idxrs r$get_elem_indexr*ks9H 3y> " Q<9 $H Or'cHtjjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr )inputsoutputr!s r$ get_mul_noder/vs! ;; $ ??r'cl|jj|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)filename identifiers r$generate_identified_filenamer7s+ ?? # #HMMJ$>$P QQr'c`ddl}ddlm}ddl}|j|j t dt |t dt ||j||d|jd|jd|jd |jy) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotrpset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesr<pltrps r$ apply_plotrJs#ES[[1 , $K  *JJtZdJ+JJ~JJxII()HHJr'c ddl}ddl}ddlmcmcm}ddlmcmcm}tjd|ttjj|dd5}|j|j|ddd|j!d}g}t#|j%D]} || } t't)t+| dt+| d} |j-| } |j-| } |j/||j1|| |j3|| |j5|}|j7||j9|t;||D]}|j=||j?}|jA||jC|||jE|}|jG||jI}ttjj|dd 5}|j|dddtjJjMd dd vr|jjO|d}|jQ}tS|D]Y}|jU|}tj|jWtj|jY[ttjj|d d5}t#|j%D]\} || } | d zt't)t+| dt+| dz}|j||jd^ dddy#1swY$xYw#1swYyxYw#1swYyxYw)z> Helper function to write calibration table to files. rNzcalibration cache: zcalibration.jsonwirzcalibration.flatbufferswbQUANTIZATION_DEBUG)r1zcalibration.cache  )-json flatbuffers5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTablelogginginfoopenospathjoinwritedumpsBuildersortedkeysstrrr CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndrtTrtTableStartDictVectorr{PrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironrGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrRrSrWrYfilebuilderkey_value_listkeyrrflat_key flat_value key_value main_dict cal_tablebufdict_lenrrs r$write_calibration_tablers, LLLL LL&'8&9:; bggll3 23S 92T 4::/012!!$'GN',,./ )"3'CF1IF1I78'',))%0 w'2!!':6((1 i( ) $$Wc..AB#3 '' 23!!#I 7# Wi0$$W-I NN9 .. C bggll3 9:D AT 3 zz~~*A.(:%%77Q? '')x ,A!q)I LL ) LL* + , bggll3 34c :d+0023 C%c*Ec CCaM3uQx= ABBA JJqM JJt   [22@s%#!N&*N3#A9O&N03N=O c|dk(jtj}|dk7jtj}|j}|j|z }|sy|t |zt |z }|dks Jd|||fz|jtj}|||z| |zzz }|dkjdk(sJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrz"n_zeros=%d, n_nonzeros=%d, eps1=%f)rrprysumsizefloat)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rGs r$smooth_distributionrsQu}}-H6//%--0KllnG'!J  w % "3 3D #:; ?: 88EMM "DC(Nte{2 22D AI??  !! ! Kr'ctj|jd}|jjD]}t j |syy)NF)load_external_dataT)rloadas_posixgraphrruses_external_data) model_pathmodel intializers r$model_has_external_datarsJ IIj))+ FEkk--  2 2: > r'ct}|j|_tj|_i}dg|d<t |j|fddgi|}y)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)ropt_model_path sess_optionkwargs_s r$optimize_modelrsb!"K+9+B+B+DK(+A+R+RK( F%6$7F !,,. jH^G_jcijAr'cddi}|jr8|jD])}|j|j|ji+tj j ||y)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdater~rrrset_model_props)rrprops r$add_pre_process_metadatarsZ.0CDN (( :D  ! !488TZZ"8 9 :KK~6r'c|jr2|jD]#}|jdk(s|jdk(s#yy)zCCheck the model whether it went through quantization pre-processingrrTFrr~r)rrs r$model_has_pre_process_metadatarsA (( Dxx33 FY8Y  r'cddi}|jr8|jD])}|j|j|ji+tj j ||y)N onnx.inferrr)rrrs r$add_infer_metadatar$sZ"$78N %% 4A  ! !155!''"2 3 4KK~6r'c|jr2|jD]#}|jdk(s|jdk(s#yy)NrrTFr)rrs r$model_has_infer_metadatar,s@ %% Auu $4G)G  r'ct|d}tjjt |t |tj |j }t||j|S)Nz -inferred) r7rshape_inferenceinfer_shapes_pathrerrrunlink)rinferred_model_pathrs r$load_model_with_shape_inferr4s`6z;O**3z?C@Srs #  >>&QQ-PP@ ?      ,$2'47 4Dqq SZ[fhiSjloHp Q'*q  t   #>#>L $  V!4  +%++g"6  +%++g"6!!;5;;x#8''  %  ;5;;q #DkekkRU]b]h]hFi"j+%++d%**"E{u{{SV^c^h^hGi!j!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#>  BV[@\"]+%++b"={u{{1TX?Y!Z +%++d%**"E{u{{SV^c^h^hGi!j  ;5;;vU[[#I;5;;W\didodoKp"q!   ;5;;q #DkekkRU]b]h]hFi"j+%++c"DkekkRT\a\f\fFg!h!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#={u{{1TX?Y"Z+%++b"={u{{1TX?Y!Z )+ A 13h<~"LhlI=X@ <>%%8''"#J0@R$@F: k 777+,2-3/QL D E"rs0Y Y.Y> Y> Y+*Y+. Y;:Y;