From 5a06ea99fc8cea23f6cdc78803cfbd9fc4be2cd7 Mon Sep 17 00:00:00 2001 From: Alluri <53390631+disirulla@users.noreply.github.com> Date: Fri, 11 Aug 2023 21:26:45 -0500 Subject: [PATCH 1/7] feat: onnx input files --- python/converter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/converter.py b/python/converter.py index b8b4856..7bccccc 100644 --- a/python/converter.py +++ b/python/converter.py @@ -5,6 +5,8 @@ import numpy as np import tflite import msgpack +import onnxmltools +import os def get_shape(interpreter: tf.lite.Interpreter, tensor_idx): if tensor_idx == -1: @@ -28,6 +30,14 @@ def __init__( self, model_path, scale_factor, k, num_cols, num_randoms, use_selectors, commit, expose_output ): + + if model_path[-6:] != "tflite": # In the case of a non-tflite input model. + if model_path[-4:] == "onnx": + output_path = os.getcwd() + "/" + cmd = "onnx2tf -i" + model_path + " -o " + output_path + os.system(cmd) + model_path = model_path[:-5] + "_float32.tflite" + self.model_path = model_path self.scale_factor = scale_factor self.k = k @@ -43,6 +53,8 @@ def __init__( ) self.interpreter.allocate_tensors() + + with open(self.model_path, 'rb') as f: buf = f.read() self.model = tflite.Model.GetRootAsModel(buf, 0) @@ -522,6 +534,8 @@ def main(): parser.add_argument('--num_randoms', type=int, default=20001) args = parser.parse_args() + print(args.model) + converter = Converter( args.model, args.scale_factor, From 1d0b0c74fb8bbfd71928f4802a3574f7b082b717 Mon Sep 17 00:00:00 2001 From: Alluri <53390631+disirulla@users.noreply.github.com> Date: Sun, 13 Aug 2023 21:28:14 -0500 Subject: [PATCH 2/7] add: onnx converter --- .../first_transformed_onnx.msgpack | Bin 0 -> 10113 bytes python/onnx_converter/mnist-8.onnx | Bin 0 -> 25669 bytes python/onnx_converter/onnxconverter.py | 146 ++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 python/onnx_converter/first_transformed_onnx.msgpack create mode 100644 python/onnx_converter/mnist-8.onnx create mode 100644 python/onnx_converter/onnxconverter.py diff --git a/python/onnx_converter/first_transformed_onnx.msgpack b/python/onnx_converter/first_transformed_onnx.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..0afffdcd7bf8de557adba0e8578698f9f6b32a26 GIT binary patch literal 10113 zcma)i2UJzrvMq<4g9wrlFd`r#iXvh{%-ZJEW^J>WTidIxZO*o-&58*##>R+QQP;t_ zQB*_;k~4y2IQ!4-yZwIO9^?K0h6y=)ueGXb)~s2zSbb^1qzSX8&zU;zrwNl5hRs)N zYn`4oVd<3l3-+x&_x+1;3zyEFa=KgCoW*`25p%*OPZ>9Tvet#^bLK8uIBvnT33I0` z*sW4IIb94}wD9lWGIVk}J$J(V39}dM+^MIxQ>FU*awi8(={xkV#Z>=TjBozWi#_=J zGH3fuSn@+y*sTA1Fzx@vTIS~eeK2k9o!Z)eU-2Jy>ily#bMt>5&GK){{lD$h{l{W_ z^M79K|6`}#|6(m&-T$^zM`x#w&fo2f7&PU3?EZ>@{y*2#)&0-2?Nq5%TK`_}KhAa| zbkVF?OB)g&bEZsMIAwCfz6yH&TG!w|*8RT7Ki4(>Yt@K_Q|2rPo4??X-%fo$+ipXZ zp5DpH6BbVRq!m+x^^&Fr$-N|vB*|u3YSl)uUXpd4)Y`^sm6Ig9;F4P9tX8#@kq$;uV zTU+s)xTRK26z9b|@l>4smn2&v0e>O_zeyT%NwZVZ9~w7k|rfp#8uHQ zNt2AHy~H-#YP}U|RcEzo>{6WP4mjf2bF8Gvk~AA7%^MzBt+EkUB>7iK9)R8R)T)8v zPFKXZii1l<@q9nE$^|?!ltx%3-U^=9R+1+pPHgLdGm_jFSH-n)jk;%- z;cm;H#igO*;bal6R=MNJ@4$5=h&64vaWhGtP-xd71Ggo4 z33e4b;jARj%ENE1CnHvp7t_|aDknQ!GId+wY5?HnqUngks@{&@TBji)&B3c7?Ye1f z4q<_4fNgOjGXQdNYv3MPL=cv>sG!dXPBbn0fh4Hetcy!(dMbg96H zB3uo%Kq4aX-c$#-0)L@*?tcqghRwAo%Vus#vb@z_S6|Gmgx@Lw#>BeJBw)wF;v z<_IUQuku;JH&0+|3U=cS5BV6^T6bLZwZXua3(F=pvCq%O<^mghkmO$=F{ReEDb{3$ zhGpiZ;Ir=X9@U6y-xzP~vl+zmnTR*%$-{MV8*3$b0bXk1L!e)NS8uUt+Kl>)0`LNd z2pwE5@vE)G$uCU4?E&NpdIeDiMX5y7IXr>nhV>N|GjT?Q&&N7CV8>6x8sv05$c2lvP5R>G#4B}_;o8K1&5G%xn1`U54!7@hx7am7N#vEd6$wDjGJm?wW=Z@GhZ=29d(icR7QDXIm2!>b%~ZvuX^ zPKCJO0PzKvYOu$fhq;}JOP(%i)p#(Vuw3Du;n-?~&5b&D&Q$q{b+(Yi27NcN2Ycw~ zt43y$Jew5u6z7@ck~|RaaHAoT+gj!2;f5U@6I)V?<5o`2K1j4NGWBZGWJ(C1k;>eR z6gUMq_N`=~^YI#Mn6?jan3#zp#t!0HYHLj^hMd0xhH?va8qT(-Z#VNQ zmY$vfg2MJ%r;~+zvtZ1v7s$q^M;ZT-AhTj;mxi4bs3@mbSL!pO#*f=qSI}$20 zkeJbE*x($VF%oZU6-<#gNIswN(XNrcN^B%wpL3+05d`@*eBi~xfKJJXoN=|k~~_H9kDBy zxZJOh-KcC@b_#JU$!*2nmW;rY_Jbx?K+?sVdEx|ax#~GDu+eCV?X>C-#SU+R<9Ay9 zoz9XFZSG!%w?ji6>6g>f({Y6?*I=kPrtgZY=GjpD;9(lnC5WxXNhFsjRB=+0yBcD9 z1$pWlruLI@h8aTPlYSXo$-O^-8~0+vv7clHdo1AgNCIsta~>YOI86 z8J>#^t$by>cKF=5eq}zcbe>sj7~+m7`+>vA4>$3-6VydW%w%Fl25I%FYFdleWu3Y; z^Kr7zv~g|z)7X^C*3*dQOW1_ZNN8%CO1rMu8{k&wmJGm0Q?X0zW}MWMDm`$@fQ-g` z2sPu{2eC$>9Y$^{F`E1x1RpAKw>6Fj^}sz7eIpi-hyd)B@xF2{A3w*3?uXDrFOk$r zq%vQR(DZv>@P|SoHw^|mbL*%qw^A;5cc$1c$!%1F`m%Uz#Y0OH@LCedGljBsX}E7H zKX|>=f)x0~ZITf`h|}bm6Y?(_YlY&y13p^28`sKxm~tAM5FD=GG=Ew;A3ocnUWhZU zXPUM0#&Ja^oe?YB;~}A&E-qH%&NoDc;JmfCT}M2f6Y(wTxX&q)C=a{SQj#Z;?l15m zY92$eTD&K0F7|U^VH{cRrJrlovi2Kd?7UcVUSYkVIAn!$0lfx7ENoh@X1Q$$p4bu7 z_r%&h1}shw!@6M=KC~5A`r~BNJaL>!q%P~Ko0ezPv_PTVg)TaD{Xrx+6-e^(ULJ*v z;{K5}h%hW{(s!Y+nVFqFo^`YE#A$xqnm*Xff2W9j%!{)Kck#r-pyh(3`Uhev>A)Sb zA_iCuTDB5Wn4wl#wW*yio;I|@+P-e% zyesgskJ#O=qzp+v_rtLYF0&Z#+vX?*=R`YPY~bIj8qZUW6<~G^9ypGmIWR2%e=3h1 zr+|K??6R6!w^PrqhQW-;w5Y*p23uW@WDSn!I*u0)yDb)5CnBM&OeJc`>&nUd9Y#=p*JYk~Qr2-)eVouq z=|~!kX@kgKV!tV_)^!tEjmcqmv}fq~S8#(QybYdIT0w8bf2D$AQiL)|1<;Qe*}^ri zwKiH3P?&+w-G|_C5Z>6)jgO3Rfq5~T&@jf+Ozg>2M8PjMB)ZoGZ?Z#k8gQM8MG45F zuOoQ4C%925?$e=(Ri-_BG)lDh<9-K=44RWg|L_A+z%_k>>?Uq@DCSbaZ5->?Hi8tV zmLs+*BSq{f7^K%oob(ra`V10x$2amryiu#PmR7deVU_sPIM`OB!Lg>S3=c-(WD� z(tBRpwOK(GUIg@GnR$&*S-5Xz{uR5;yLZU4790Hsi1Wsz!8qK?^1!t^!<;sx#aMiF z`LW!wDH1x_wI8Bk9L_L#50vnnk@%2qlvA2&;*RKML#?r?*CLjSW0KrXE0Y#K>ZF~Y z+o;;*o4188%Y6nueIfLz_E+GQPc60Oc1Rl99u27U?8RD3d`_ES!0g@63U?`!R!C?_SZ-iwA9d6sIYiM$@XtrQ|5jo=ak>gmmd^854)z{vdn_WWCF4vMxhjFE zc$C_N6mZGU>}#OO(jH{Y)%|d#4lgYcZ3wlE7YW5!Q&!+ZO~MkQ)45oZ=Xv`&Oc%Rs zaMq--nFSAeuU^B>gxES=*C~dXy6grBs0f<$&Plqo?e5s;G?qoPpGCZPkyWh^|3w`iv_AH8m6_hr%0qiJN zs_3gv1pY2(11&oEck_H6?xC4=5LL!m#7>)#XO~XTyl0f|l`H4Yz#Za0yFOn&igmu0 zBl@`;VU>qxAEkUj+m33+isnA(SfhizjWme2H(>M8ps7ZGaXm!UwI4)J9LSRM^NqA| zOsv(V)ZS|!sLTjvBfOMMc#AZ4iyIy#IBuljhP_Lf;W#HFyinnM3De~U4(JslKFyM; zWec@C?fc+r(R{8hPS;d@!H%3FoYGEdm6}^zVPKcTp0+5@Vj1>p2H=#Ya#B4cIf_+3 zP+)yl8VADiqotB=gmBD{lbS3*vYDq?)4f*Te~D5-mLjnsv*#E;zDhU58I{skdwlbG z#zREdlfg{bZO%y4nXXp3{e-(>4@G}?P@86A?O-bOG(1qkebXvcZz&GATZ+^DDt^N0 zAGq8ZZ9KDN+V9b=D4Xh}JGJ zqNk%(ijrD_&6)ODA)ZKV>FBSICkEX+Fke6Rwx^RX7V5GZ#(yLDKa*Tu2DLE6-MsGR zdcnc?Y~P`YJ%RqTir9EF6!!-x^O`w0*qiuOWPOyUjwPdgsDS@Fs1Wr!65sgm9NxR} zG-l{3{7zd19@%op!ZiOipoG<%LS}{E;>Mji4z$3V;R6Y~y_s!rvk&z&it%=~YxAtH z`K5STzZmNZ5$j&zPiDD=D6*jl&%>@f>RNlBfS;*sH!*J#G*0+fI=sbbN6X%=#d<4U zu`LV<-J8#vhuD-FT6&0eF2=am&_cFUIXcB$YL6l~PvEtU7E?a1MHymsS-=>jqxcYP zf&}I(UN0Y8j+IqGlm->q+$yPYHe$5BY{#<}T`$FM7MbUxz!}y~sK>DszAK?j7LIg9 zvY&-YtaZjlZA47*P&NEMH@J@DeTa|?N_^H-n-5EFL!E>u=hLqxnR{8_&_Kv%lo^NFn*1<8$u}EOFA%foDIF z)sd@H*!+t$z(tSq+z)X?k$BP;a7!-=i<7nE_P*DIq(n(h-y$D(dBq!h35wJ*=81gMfJq$vxuxLMWmjg zZIxM1MAYJWAUFm5l_z}8WQrmJH?vf{!5tmWJF2X{`QZ=SUM+E~&(LmqxF~K?u9*s!aH;HUfCljWxvB!a`8qfZkm0A9Kj%mr>H--=XK>+NeeTOK1 z6)z+huoQdYWd2_-Xn%iU$G`u(lfzfz|60OWt7Whk8w)(ALc#TVS)qOKcqX=p1Q#VI zyW)tiZ_Q#!wo_FzLna%si5P*8M)cM@S1nu#4)DdN(sqcaaK*W{B*u55&;pl!Qu0kl zoE86z>p%jL%a9MWxQh{OP_ zHk9NcMTk_ZTHvEvWjGl7S={IxBb!pOAHREw4HRl(1F7hN&Vo?(adS=GaKj(Bo$*c= zdjoM&+{7jV0HUSXj6c2cDKw}q-^;GRvqkq5oo3(g@n2JMUy?i1;lq)rRVY@8eWQ6u zr5`jNBhhV&{nUPLgIB_KB;wEq2ZE z!`eaEVx&LHp;?YurX=el$zN>D_sd6u7Gp5pM;Quy_*Jd?fnO*i-^M9641AO|?&;^1 z;*`B^As*Go!T#>%JlOtUdsTO-s18&{#`o~&T(MiM9OkjViPtln=NUI@L|?)waOX?>c}$>Jn+bgt%KGGwQ3X3 z>4K^#+NzmhHL%D-W6Plg~J2CnFm#*QBY9)*eVh2?FIzR5xXS0C%2_LT2YVY;4QBnR$1VP9xnJdX-u*> zqHR#As_Ec?R~-z^zj_ZRkv_$(>NZHU^)wP|$X-$2i}Uc2G*`x9urkSwM3T6nYt3Xa zHu0F+3QvM79NOiU%a~`*cnht7H<}1=j4Rcrnzw1E0U@ z99mrM>6VJ?U66}Y`z2nn+N^t}CY#*BpT%Opyx?zbi5^Rd_DHAswDg#>Mb{?dqL{pB1qxKK} zltd7#aaEFo`?bLiEj($dR#}c4KxizJWP9%6k9(yadG0B8)jaP;x}oa3#_?nCCN5;= zaV4=|q`1<;j@ThqjU-!M86%={MKt+!d=+JaCIQhS#ghTJWL;5+)9sq{&-1{QA&cz3 zw!zWvyk=QX6{A-5#cekWu?^b>v5IZ25;;|hh*>zgF$pBG9ho?8#FxOKypAcP>+Z0i zu5U*+iAZnY>D3%-*slp(c3|t=T~!>+ihXpb<$w$`s`xwNFl$b@P-FXuuY5(**I|UQ z(vob!pP6C}Q@DjGuX!{z$BwS0TowowOy* zLmu?P*LH!}_b(>*XqLm|pRB4)ty**P)3mDs*V$s@OdJtssSRa3_+`aZJX5MC{~AH^ zO%PA9UcBWcfs06_J}XydRB?EvjR*O>h%zNjv+gf%-@2=ISG;1O8sEop3=#~PJ9J4C zv8BGD2rsj*^3cHvAB%!c4ckE)t{mj1SkpN}$D($u{ukZB4n??;XPU<%!sPzY+1b0R z2112Tt$t|J8waLhl~I6T*cL0DhNJTDizze5`pUBXG*ejh=$jT;s&lZ7Lk?4O5OJq;o^jB#mwL6Nz!?OTWo8VQ&i9j zQHTTMw5qemU{8(W`E;>oDpQO7lO!98T|LG2g633J4`_{et{hq|q#;YFCpIi|}N!^YDiB59&N-8J-pAvev`3m z7J0yrlIEkh5j?#(g!LP|v*VRyHg5}_;<#FS97prFDZMZ(kZ|ff5#jDrx&0A(?`<9S zcxhX6Jl@6>F)ryE#5(iapQz(z6;ABd7VEjvgyHz0R_O(e#TnMesp4@}W1ePec4s7c zAj!EX8_(tP_>|o*qkeAR@slB+)%5wkZmM6NUds4@T*mHz^{1M zCG}?uy7{TqM10P|=L{k+9B&3%dp0P=rvP1?8HD$aixt|zXkh3mYUhBa`TeW zGR`)jWwChSmc=@F-_)cT&Mz2`N3Q(tMkJ~#**+W%omE!Ry|u_uM7|O8Gj^(2H>bBw zCp<93eJ^bF%E_x`A$f+#`3&=!@=ATPzC*FwP@(SOO~_AbYjKoCQ;19){)N+aQtAOj z7vp8VWeVPhczRmuSx9pIu_kzLl+(1wl5%%^a$5 znCu=jegNK$UW${>V^ZrOzJ?$bO$HBE@Zdgqq%ua^Ws>Kam7B3dY_by-j! zd*Pvo(ayq-{@pnldpYwfw)kOtC){G&?`GkJgMKCK*M3pv9Fif~;3Zz$W@1;6Bd(YD zLVRPy@=E_7N;7^cri?ioJUM81}I$^Byn9CB96i4bn#7ull223 z`dNDUlpvr9W6lfbSA%OpKS3CX<&?2_AyOc?GeHkRPtC|v7?v(1=^ogjr`pZ0i* z8dI8hk~w$LzZ{=(bu)<7`>s?IuWVTIM&YzrGpwAo<9c7L4`78}Azs$9So03U9%mfg zO|*(r-8p|^Dt*DH6vTwK<@E1}Vqg-BUAPdYE@5JwyN;x(F6bt<@w{F1l?G`&j<|^w zwj+2VHaEjd=h7m?=DHhq#XW!VQn`1fmzI^{n2lOxRAj}~?s+MfiX*1k6o}Q#J~nlC z#Wj-6xBk>)BDRi+RLqmzbio@RJsq7|R*6ie+-~>5NLG=rpF!=1{W{_{n=Wfto!s{d-@59H{2Sm1uo^sE-?{|7zQ6cGRb literal 0 HcmV?d00001 diff --git a/python/onnx_converter/mnist-8.onnx b/python/onnx_converter/mnist-8.onnx new file mode 100644 index 0000000000000000000000000000000000000000..3a2798ceead6d0ac7a3847033436fd741a6012c9 GIT binary patch literal 25669 zcmce-c{r9)*Eej)TxLQVOoft=!ga1qlp!SvMS~QT2102r^GpaO85#-ArMS+uNHnX6 zk_M%b1}c^2&;3mI{XXyWzQ_0d^Bvb8*Rj{Rhqc$*aEtJ(ALV_bgRFLd2O|0o)($qrr>7#1A3+An;C|Jp!12|I~mQ7L0ZS*h@KVL<_b;dY{S zqW_?1%1ZmMUl-!H);~a3cFvTU)BL>L-KTjJi;5c$mX%l@7P5Y=#y~|eV`W*HfS}d> z{}}y08{3G;TFd?)*iu&T2%O<%Vg64(CI6kEJIEt&_47ug#y^gZ7dq~LYpy1_` zCjGOW%zx2zrDpnX{7313o1U1R*ngYee^8WU2Mx?D+)m`*fB&eY_AfJ*berWlXPW20 zYzG+PMP>ho@nz-yYaOg64=lt)<)Qx~a5J+B|KwEufBo?P^HOlK|0l+j75^c6GqZ{R zWL4@vDF3w@nWD1)TP|q<@*E>_3VB=LHeB6aQ}u@*kA{FAK6>_J3kPR`;I_ z{MUs1!UF$oqfhwv{jRM2f2*UVgdd`m2{#s0qrw_`M({RELEzGe7eMQnlt8; zYi3=Qsd0HZyk>3wOB!A0N=3)Y$f)`>El*aXWZP&ou-Z)l=4JHeLkZ#J1NgmuDLH8M z(eH-2lzng&8kdh@`Hu|PjEt+au(+0&3jYO%%1z*OTo4N@ya-3O5R)%i%+^Fr<{{pi z9XMIS;v3E|Gp!GtAM<8mVaM6vlkVx7KsOgvg5}HEjMp4R%}UGZ z*h__)#0wFqrQgi@7p&pu8GWYSK*gHw@^^SYN{LmTOk=iN$HQrB;D zY?pM8x81~tvbR#kgk5E{@u~wo9yK3+^((RrdjV6NHMnNx1wqZ}J?b^$RrMs}E^C`U z`4lS{7e$h5PSNNebIB;gpPe{vX#1zWnx$_Z#AxwTj5`?tI~DTT#)>d@a$+y*ITFB} z4qt@;8wFeS?D=%mvzuPLnMdz-RkLY}zcX*)9SW*_NTOS}QEF%lo0p%*L_@pj&fa|T zS$rNRE=*^SeY;_3_yjUgI7^|&hEad)X_h#JF;$aH*cB+lIAg%k@6Ynic(2d>!Zqsmtr(7BOM zXv1+K937I(=ZuR(*H@Bs&c1{@B;Ak3djq5u2Tt0*pR>pS+2@Y9Ag~%wOujEP+bT|K zQXOzRz5<+s$5G7NWSaRfpHF;{2K$r}Hbc)!Ye52`)BC=gVMP zfHfJP`GnKb7Yd@vBAKRxBl|YT4=pE)(_`%i9juQQ<+JYHpj^X;B zQ)uNof4XsLA!#I>!b9(4;aFWPr95mwnJr~h^h}1v7K!4<)33SrtImUT$rBXmwI$K< zfz&qpE-e3JkD96vcuw^s?>77`=PxxCcFev}&Ca!eznL(@xk;$G-5>r)Td+an;&IJv zd1@3Zq;C@UP<9i7Zgw8Kw`?=)bO@ve-xd+K`xM4(90JAB-{HX{Wja%>NQ=+4(Nb|G z#Opq=V$5YOwUbiW_NE}d+K9kb#fEBZbKtt+D2u_`E4ZVN7P-C(WY zi{5MG==Z`z@*VF>JvG1ihkF;n?ZWY7IzO9=N{rBQ=T;ixcLO{=I5CYO*Dz~aG7Vj* zN!3$MplOb%ATe<>bkmyaGZRw0)$8~Qv>!3)(LplvyV@}Dcv zy28VpR!}FWuXhI{7ro_LRK+PfYz2iBB$3-%Y4*3X2!6KwLe1x@q%3bnH-qxP{^@*H z-@b>}n#cG#X*$&MX%s4cUCEc~D4_J6!A$IBBy$*J0r7Iz=xJIJ2JX^iPC9egYIj9S zy)unr_L^b%)eGR{yof2tRA7hdEGn#$q?qKJINs@5^~W(h#a}6;rKWf3%ZWi)U86#e zL&~t|Lo8m033JN7tM*e<1wL!;=TA<%QzW%xSp65N{Cf))==C|^&9;K2tQ zTgR({PrK@cP4eDs&EsfLU0}qV52UeW+g4%Vq|>m=aVUdoIi}~V&EM!0V-Id;@SO=o zIQNGHR!kj6StsO}rsG#`&yp0%AGPQ49pG*&Y`#d21Wq)S1LFZtFF!jVWOCKzF7sQ0)yZOKRmS8tShpjf@LFDxcR(|+2|IKA8vw52fIm?{kQQj!b zthPgupq1n$t%~KxjbO(2LVSHHj<#N@#wryv5-6^t*5j?Pc>Q1A=v}lh-fIT_9gy$S z@I}mf+e~cg9g8#HM&VAKG;HdT<0RH?Vx~4j*jt>$%Pml*8;f7S?JGH;fBg(@GjV}o zX}0)L#*Th1HKyl^?%0`q4Abf!a>jb6Ftv}-ipd`ZAtiD&vO9uMC5=+GAK+1=xz&(X zh$RKDp#Ak#NO06eT^(`ugq?xi>yp{#)LASb(v$TK(t}5jG-yz14Te8^ge#Uf(V8R4 zl} ztW`6dq@RqZR2x5fI>nXA{T#>KOtymBd{e9qN}{Zr39x^C4r{)t3Hd%_+0xsGX>M{X z`JT_lLKSC<7?uf{%L_Q?^*NZl#ve0W+jz%FJ2JK-D7-g?@2cPk-t_aSgN|T}>nEsE zk%KRSVH9Zn7Zs;mU^!X&?Dal9mbIBHp2V zZxf7Hk4DJhSf5lJJ3gA--FXGPazn7aXfSofL{WEp0(4lGu+mLVl>gkk&Zbh)1ny{q3qHM z);8l3$Tf~eorB?IyGntAZ1kc3<9znUG#$qO%4Z#`k}+3r15RDJl+P{+U`vmFg+}p6 zylz&6DNQ>C_>Xsi^j^|Zlqc;;OUTdMfWmxIs3{zO5l`Or zjOgJqrXPEZoBqzzX2i6~TW0)A z%8Eb7pt$&aSVBqc<=2z!d%$eA23tsH$X1g3+ls0Qld-P&5m)Jc8aCVvB+2b7slcIC zAU!mj|6*0abOdwY%)us1y|)_wXzJ6hHzW8ATW{juorVgB>p17O9KCdLN3B_Up|1|LOyvV>kIh>WeQ^a?XC49_LBWeggO8I*N$w)qjD_R;vyl6V?N&O2q zeE&d2;Y{=zPBtrQufUCGacucl6VQGyM9WEu%+7NfXC@U5-xubvRc+p6`0y}())+=D z!elN<>OMEzeHTeA8cDq#?~%PTq9!#bG<6qZ*X$D%`Jogqb*WKa%~mMQioxa$a03H=!c-cqdLvEXOsD7Px^Su1I*c>94jw0+g!gRgplDed#jZ`Gf{zcOyvvx$NspwxR~s=%Z8@G= zZ$ct*op9%pIBAavqODv5Zp1s-)YpV#L#pwJ`*mz66{T+_JMngc3}&{w)3L+y6k@!L z>PMAh*t$#HmTG<0U#deFRKjS5iY;Avaun*H{SruRu&2|H)X^$SiOSvzu-IWJT1~h`&S^(%vDsLJ_)ZFd6MRc9eAVu0e|po6`pyQ!4K;4r6{TKG^yJScqKX_VQikZ2G+e^h^{fq&_-awGNjGeGm~8?n>vERrd@%qV<%DT zkQPjHdwIu#31If<`qbam&58iU$OPGDSL9z zmx)}KyOt8>Gjq>>f5(}S2b%AS0SAHs!tdeGWXNFPyK7Z=;$fei^yYO|eH#)dyK(N+W zX5BNIMO0ej32!6fEOkhbe-iKKD)5^w#^KESJs2ULO@d2GR3g}kXWeq~R-PRM)@MLw zE#nTST(zk+O=7#|w=m(mqilGD5A<|ZW8$`I41IeDy_*WLuyq(2TczWVG(8$REQ4FL zDv7%g(~37%YLUn>S!z@D=jF$C@a^CF;p(61)H}bGkC$lXem90<@!w^FX19yz+bTh^ zheE03=_9-}B#C64cH;N>PdLdDxh!SWMkcxZ4HR9jgRQf&g*PJ8z=He4X?&5y0H1Qa zr#=c@N2$Ux+X!_3eFCOzPs03Vmh~U59^463FzqI|D^@db3wsKC%CB6R(-7HI{)5S$3~g7$l5oFaP#&pldj@iy`k?wNE&0vn8Cw zxaU~+do`KV3E+8RDg10-Mh``rF;A}p#g<*gvgn`u;lopy?h{M6n6j6jl_*OVcBZs- zkTvu2ehg!7Me`zyp?e?Npi?WCUAfD{RIx`m%d!X8YQF@}ZTif-+<}STP3E(dtFcm2o{Muli(^+V zVJ-I$u-?x@$n(So*u77m%UZb%zKaL4V?|@2+9g&v?754LAK(MF+wR1nv^6@fsb-a#qH@)I_-@J@-GT#cNCs|R0b_}{DJmEdhC6n!D zE6N&5aq9|jUUn5SR(voSAf+6|^=(0~sU&5-sLohMgg--rG zN5?P9vDu6K*h#N%+^JnQyywRi6d{P>o$XE7YyaUa?PVlrO&m4QzdILLpaADL-Q^vs z8+bn7ny=_HfJNnh1>;*Dz{^f!Qh9uW)-)VI>CuU3y6U1(QO}qbJteA%^&>Z_A0Sbl z1;u|qV8n_z`ZVnY?&~aPRuY%si4}oD$m!~YYaHAPdjg|xr(%;*7Oc2560$eP@KY*B zq3E6wWOV2+oZsusZaK*Cr>*0-rF~LNcVrUeYbn!p#c81SSeJFo9m{-1eMG7Cij=S} z4R1#Of}NWiQQBP!H)v;}x65oa@moNy+jMB@)gBO>H?ltZP@TOmT+I)=QUF86Bze8o zI*fZ)&UgO&1!EUTut@JhcqFKVg=GsstYA853+7;PxjxA6i2!cIAy`~9gN6R$*x-Xv zI45)?|MdKI*rYv+o$jtfdr^HlHEuN>-JF7F?&YKUgM4_>Zw}AHA~4ZtAth=Jr*cbI zn0(8hn|NmkHLez6&WnYB8;_w?cph%tNu277EcR_%HfwLahI*rJ;7r$P?8^47{NdNB z@X>Y>{hS@h75>uTYuj&8pmsG`h+X46j`Uzfh6@E3Zlv9!yJ}OQR4+YH#O9ERiP9FO3j?kfRuz0l8SoGgBi;3fNS}eMolTDlR2*&0Bxr zyaY2cXlaL=FozyY86$L>c?fp|M)OPU{MfhB>Ey6@EM3-%!9kW{WVt|!HBZu{S+Co$ zZdyFvXsCxD_Rm;)*gI_Pb)f58jL2i01Vvp6Lqk0e8q=Id)=w2^dxr=2b*};2AGjP| zxi2Jc0uv3E!Ju&w3SDP;A28;h;_?AHwqwo!Qk zMWhE(!^t0*{`DuP`|~*Ne6yZHO*}~*UZeWVA~cNsNn1C^(VK``IN`hj4OJ$PBv*%x z9-+*xw~y=mB2Al3dT>;DG)zfzWL=IqT>8pa_&ekR#@DVBt{iNPYxz#@sCyFjc|As1 z^&B|=xZe654Q5{U2QmBHO!#?r4<0C1g*^>P(90Kd?WbP}mkiete$Y^71-TdS&&O2W z_ldpmRnBM-Mo6l6!2&~eH&c{o=gS&Mo6)z zJriSc@xb}YH@Qhsbui-48HnoPD7&=6WYDwJ_qsb*~Djic^K!+Rqxuh`#lqoTkzNn3%w}-3vt5v#m;=oCAzMjCj z-Hc`{n(nd1VwNQLcr=B{YqC(@4Q%U70MtvUggd~WZ#)SrTCBKx^=5dZxRvXB$%FA7 zS>7sJ$e)yaiZ|08;NXpua9HOw9__n}@0-IY++Y(Ic`u(mdoKmL^j4IVBXCa=5s0L8T?1SAm4DfCC zeil4El;#iUXn{!?Z%`}F{s#XQI4cC;uw$cfrc*Ze_)I=j_726gm5x+omr6!fYaz7w zfS`8B6}%JRLeWPHP~-Iyu#i{7Q&VCnL0_5qG^c<;?O=56UqEV(Gim7-b0!|G!}GRg z^yzC9w>wQ9?`z~j*3F?zL~1$fHm-$+svF$&?0ek3U^_T|aU!>3^&1Sdkzg}KKXI=X zmO$&aiIo2^nzku7g4^iVxX?0)RD5D+$e`&=bi^LEbBioB%pXs=^I^aT8U{Dl9fj!N zA6O;V3n%3_GrWI{ZMmS&zbf}2@gdJpT`Y!oq?2f^v6yUL?W;bx?lMY8WYK-cC|a+) zn=Yrj(!pWd==A$oDieQz??Z>thc+o%{reOxYdcG^Cm-=1lGez6%CK|e_?nxFU_UDqU2c2XI(wM?QT4^s#H z|7JAh%qsNkjHa6lPLjBE8Aa@tA_pHWD!ebi=ew<_V01D6aR@om_0mb$=0Z7vDw~}>F9)9 zI2NwNR&3GZ4Jv(UM4mHBADzN1H;Y$4uh>X6I%;IPKpLMMZ-Ys3`ziCF7AcC(5nMey zlwourjYx~a{3;*X=sF& zrD1_IHmMeuuDXa$zveL81=6-38=K*rt~_7zbSc~B)4&Grh=QYEY$15oda^mF!4l(R zVRI}nl>xnSLoAIqqz8D(O^vPnvWu<$-U0dtp76YJ1^+!X5hJ^Une{I%CicRPJX6#8 zvv1qrn?o`!pLiHoO>6_3eUtEXR2&VzWkdHbb^xDhKp6p+a5(%njMAxxSUp$X>{lml znYkG+q~~*evGoY+h)ClbEb!A|<$D;Dxj%>e@@G)$mRU?<_(RAT@Oj$^k}0D=K+>o4 z5fkqs-;hW14s0U-gyHo4)B>8DJA%_V;{X*++PuNnp;TO8PERh+M*7mt-BB3EfB0!Z zdmS#|n$tHB_Z;HIjAO9$@t@Xw!8KDw(Jo^5z2+`Huoe{=9ImaL$T1>(!t*g6~b=7JujXvV`k`+KM!%V<_g z4j%P2#Y4sAAaVRKRYfgE;x_dQWnv*bDz+_x*VPlQYHD7c_g(q6h*Z?nYFwqztUp_OI4EL77WiA z*rN||^gbu#{Ni{8v$x>+#SY}oo<^bCHhL|wpTfI*$meb%Kd8Qy{Kd*hWXLDV5_v`| z_GQy2!;^FYc)AvNn5HdkMgzHz@O7dq?Os`hiMm02cg7KXyi^H$3@)K-=rR;BJ_QES zisYixf=zzI7}FTd7Eiv5#TF;AOTv}z&b-b`e2&MLmNOxI@Lg!0Jd#}yIMB%EyZCFr zI@|Z-I((Vk%IVszqa9&q`EMcVlss39l0wF?)FcHsoCgu}_f`Ipw$O*#v_!3SD{uXP;7|5fl8VJS~?yFc9~M$raN3(pGZz{D2wu z<@_SI&+z`N1v_Ft3NC6}lN^6p;BPhH`%e7?m0@?;$h95F9yY_^9v$L?l$griJ%4GoRvL=@mSqX$ zhAcY5k(>{Hgzb;d;I-|x@F8>rJQ8$E&fe6GaTpFiM}P<2{4bQ@pYPVAuT1JtG*Jl(jEmvFko zE!rv19Q-Rmuw}0Bd0rv8oOh#6Z&3DdML5I6+x`dVuS^Mjtc>?PYry|h%+$7^aCe;wFYE<9>HwK%iOV3 z^Kr3sCjK=INBh(bw8C`A0PZA;I(M~T?2;;w9`{t(`rDQkm@lCze}wq!X|OPJ$V_@} zGKhXZyoh*ijG$X{ICo=^3y6=1V|Lp`;C68`HX5(OC7bNo&cyvu+o;`Q3_WIHaK&mWHrN#LlHn#aWpkC_)cd`-)Lqr z(J&lkKaP$0JCE%hxd!G1%A-R`GwR-Nhrpi$adWAuxchh%HY+QalF_<1KRjX1{W5(aNdz(Ad~kR_AK0kX$Fc|ak-PfHE1w< zyFXfZ=EN85ZgLY0ALb+8d!SewT>?m(Rfe*7xM?KoP`#f%zG@0wT8 zvj^t%!Syueww7R2VQx|oByJhE${y#>{sjH zF8W;N4(mMR&+HJR{7a7oP6sbR=GTo-*K!(9sl4R;H9Iih?=Xn|p3lu3@YmlF-O6^9 z)SRExRe>MA`JmmJ?ReIz8WOuTxXT$PkQ$^<7emJJlO}uPr{iYq*ru7VUF8&{ido1tq_zhJ;maSq0~^Oh~t2QJ3KA$ze{<{Mbp<`09L8+2Tr5Fvvdv}FU^2IzPUobtrbXmQ(@#Jd%y*Ddv<}ov?d)F z-9@#IK)vbrd0CHT|%LJ2{?!v4)%1eLx%3jsnPzR<@n@)I(=I_6>e`2 zhKlkE-l#Z++rF8xX6pqo#x9v#y5tc`HVhNwIT@3ID*gHL^PGN}n=yxMePX zd5@L~bg~aZFTL-a;xlzxxL`By^W6;g|GbVON0VS|_+UEcRE&D-H^HFZPn>VgVQ{&- zn3cKz6^#5ckz^!?@%fYH^Rf!TSh(4b%JMBSeV7~`3$>&qyF@|ghbVk#dI@v~uS3;% zQ{2@pj-Dwy`K=@TxbY$O@O=Jc9Fi+T1q!C<&@6=)LJiS>ZZG#TGlIngx>0awtgyFr zI9pmciON*$gi2%A^E;P0GV5G<*r5}TlMKA!ZdWAh$sWlhN4-Fs*#kI}0snScc`_z_ zUP2#y_F#?D4{q|OSv0+8DV^W<4nJ0PVCbaDEK}W)iFW0JMc6V{NVTPj)#2IEe7AFlxai9QVVH<`urji_3;l zUH!nFwB$VI?8@f!A6@1DoVkTlk9kw1QVj&#k3`?)o@8h2JYa`TBn#tSlzeZ%o|aFh zeuJenciBq#Kw0?wVT(Xbd^XF>RJ0B8n86YfrD#@mHZ%H=#O59?X0y(IfZCreAdUNB zlT94kxX6H=JL8P|JA|k=W;t>j{-DvCc)HS{K!e9g!`)4rQ9MJ3s~H={5|_BrskupT zZG$*G`gNF<4AEkJD>8Vg^)s0Ia|==p8pVb_dyk}GN)<+a0-1m}c-L4Ix>rlnm&e|; zw@`vRBb&>66%Js5>eWf-Odws7{SIxbYhgsqFo-y?3;R^^>7q_8III}N4r=$|_>bWSgDEJr5H5$UfJd9B!P>9JRO2&_s!C%h!u~8b@5}|r-F+FN zq^dx6{vc92l`MFyu8PJxk~zU}Q&Q6qC%L&gf*^5m(tnjee`JH`SC|{Fn&$*Z_B&&E zvozCOSpo?gcj4VT2qur`!Ms~d(DmMoOa0t~>7BDV%N`k~zwRACy$yYtl84>DRM`2I zQ=#km8N8=uK)nh|m{qRe?2c6G=D?x?|7E-w2!ggJi)@#D$(YmOtlSlSH_Z{>peF&FT*DMRZW z-RRxt&R2i?j^uR~9iAJpZ-%1eG-^1znRy4NJk6vRy)I-^sLB>)^`M=X?m%28lmw2- z)Yw`I_9iz3%3Z&?tO?%qyxNKOE>*+zJ9TK>l<9P@><~GKhm*>XA6$P&5luHPBQ^g@ zihG&^CvLg3gJZ_gx`FqgZ~a1A+MCTP2QA~L zd_|}Wb!xkjtaCI?HNFDvCswm@yLZ8$>ff+n|7Wmp&|w?ozH#RcoyGR{@$BYyUA8M; zl=w%UWEe0L|G2b(yqO~1pQXm0^ZW7GXu`#JfAf>x4B%Jy`ml!z5wM~A0I1Gz#MYvB ze4o+?%!#|s7l^&XC-!?p59zI#n$Y(m4Fd5Bqn2>uUg1n|Yk)t>DIHr*5a2slrY{0P= z#;|Zhy7f$%Dy+I(iFHSg!3(iUeh<5hzioO2P7S6oF-wu|CQqjOT~lc0-(uVq-VbVV zr?^Y6_u&(#oiu4)1EyTr#5yMk*tA=D{1@Fwa%v3bJ!^X5*~46l6x+pfDN}H>rXfw; zl88?6Ij|$1VgJMq%+ih)^pqTj=iA4Dtd1Qst(igoN*ZJ}`x>8pbuQf-knw_*exP&i zInH@Ik@Y9;MVDvO;Ovw6)IIh(-e>|QDh_nb@+dZyxAQk{Kf)J3Z{cp{1Ea6#kpBB< z{MC02Tt4@44|K2COkW?zR7Nj@8^eOBIR9|K@pP(AJ{g*=}#|myy?LiQv)i9Z{-e4Xuj`X8S z;AqbqXgs9NLLaVWyLQ>K)ft1C1H4D`_oC$4vyg^PA48iXSCYzY2a15Jn5AZnyIy(V z7L5=5a>?tsIk5`mYo^nc+yp!$`wsV9EP-cjS}?kG6J>rGNoQM>P@yM*oHbkdv8K`x zb6bs+PL71&#|LOyl_J~JxQL~1tKmB~O(!iW30fI3fi3ScV;!aK{LpjjurySQt?{*F zS6!E*ilGy&PI-mHU*CaIb7O!)&vP2yPS{tJgdMi}WF?lzSa>bmPw*VTJ9y9#xnwMj zt!38nnXuLU0`^`whw4_(;Gts?+c-=QHTxz|=Fzh_+Nl)krdzV#_F4Si-*J?(Xf#@z z$ur+qk*JgsLdxa?v6$iUIOEt+%o{$38?oXiS2D(moJDqU4U=ZVruxU&aNdT>x#>9M zP#*|8H36(~K0`vWnNhl2#<{PG7HD*wQTFC3`6cpsGp3o*iLHR^>+ za0=u4F!OdP^+!yBFH54JX8Y^`{Uk%L?aHv3`v&%R??B!0b!6Ci72n&5B1_I^&dCR1 zOw4*JahpV^M`!YV8>3jJ)ixVFkDc@>x*U(>=CbniFfO(H0k#?s=pXAizFhq@?Y-Pe z-@*su$dO<9YXkWTljSn#g!oYUn!OpS6W`&Igh7u z&*q%O$_G1W?Bu)riy9qrRUX6j?TzGeY|UXv*(!ck?OiTCM}cV%F{cCb6nN#p7r2Gq zK-++A7$bgB^c}d!Z#K9B-=55-S}+Am6J-`B&%+W2TPkx?XJ-?PnLkQ0?J6B^eVGAOEvaT* zG49Mp^%8970)+Br>LgnIRWN!WP8+ps9W)1~!m5u2w4rD_eQFQpTE9!NS3NH5mfz!n ze4li97aWJDLlx;te=UwzZo(3mI-ywF%j&v!USwNWhg-6|XvHTFDAZOQcrE#H8dJK^ z{$G;=ji*a1{#zo+#OZd*VWqg|>k~Y&T9K@;%_i@V$@DCL0hteyA&J$_Gk z%5V8tbJ&OTKa?r#n%6>7YLn@p*9bH=krq>wvJn@J6tn+-k?&@_-(5OVOj>qe`2RUM zZF;1;M41qlp6LgVo5#UT^C!gHUdQ+$@Z!|JPvWh9FxIp;Z8MH z;PM5trM=jJoE5aSrj13yr$CCA9q=oHMsoVG>Y>V=-b z8(sM}BQcnDGoLKVcX9n)m${yiyVxiVQ)aec4{i}=aMW-gFQk^>PlxqzhV@%dG8<0o z27iO=TOYy8@QZM9$j$*@M>B62*#SNC%FuG;QMmNZg3Kn4qpUgiVcHIJ%8e<-g3s^S z;FsUHY9l=g@|(dn_T0j-iNEPpO)1lDuw=K2o7w8Mzu>oN8fno7`Z$11>G)O7A5#`5 z-Pex52VG)~DQigk-W~S7po-ItC<61$u{BM*HshakTTtni#Fa;t*oTASWO%iO8(Oc- zr=6b6Du>9R>p+ZU`DzI^$=e18b<79DKc~5sTQ3RUg}9R`%V^LTmWa3pAN)fr$hOgP&#IF z4Zr5yrrKXbRY{xJ%mj&=;*mEOGoSlH1Gd+@NWA=aAeW;l98}L+v67xg5Ol$rqUMTo2M_+aqEItkh~IJ@GvW&odM0%iwQhR%HOz zvMs!~Pad20WD(g}AA!8hiOfAKn{sv}vf6jE=*rGNn7QeaU|~%lTiGPXO7dRvxh*@% z^K~_MZ*ClPr>N1i(M>QqWiZ{n5eTnF9YN2*U4p-d=TKSe>j9j<71`KaU`st$(a-un zf~OaKsJW*JYcr~0|A%BYH(r$%mz;*&8#@O;v2yI<^K!tiE`j9Ut(a95}TRV@oSRw&pkLZTU)(SH9tzoV|GB zmZCLQHfnssytmpgX|)RlJQ>FR zIw!&ghjuP2--c>;&ZJMfwz2Th%B;HEjhB@ZWk=3t^A84a<^`YLz^GGYbOtA9~K8VBNwxnEWV>W>!yQv%hSlZMFf-=!+E#b~{c*k=M9!3SaT|l`^*YTsW1l z{>hm)m2oK=tMSWRSDtl8k!xNBy(s>}_V?am4v7~y92!W&eZ>TRa}Lt5(|w#vwgvl? zIiDHUC9>bHPr<%7gNz$W*iz**_EGsFRqPp8GuZM0`8>0r#DScXPlb}UM(pnZRz*;= ze0eq=(Gf$hNyBYV^rf*`bE}hvqJHhAFsqAK%5^cF#3*yB})KchxHzJ3y zc(E=pJftLC`Pq@PA1Y1_ujew~&TG)|u~&FUPl&J68Pz|i0b2uq_N39Dd(g3)J7n7k z8kp4-rOq<8fiq{2oOhMbugi&aME+p%vSc`M;5+ID`$3e>UQ&6ZP+eST%07>u zW?OhBnbh1uLFThE?Q^@xk94f$?aM>x{e(0!nR5&7jFl$Ej{~@tky^~#NRAcEDo3^J z%FJ?N2V4t!ipRe17JQsg3ip=;p-O_pfc`dSLvMdT{f!dr(31CHIJ5%7EfmPvG+h{U zW-nf~*bB-7_C`!=4vu(XNO`lT;iCR77?x{9w^rQ1udZgK+Ikh-gS}zaTV+^2Il+3b zWG70UJpf)YLHwbC_u%02dX!%qhjvf)aCD>%{yu#P=l#u?rBgh5y%UF7+Y+Fzrx`6f zY*5%;h3nK$fYrxTyw_67K6noym2Z*QIA#KKt6zmPE9FT(DV?6|4HkAMkE1fXMrM(B zAM~uh@@H-oQ_cH(!o`hydEsQ_WqLiSZpjC@JdHzlz8?f9Kf>&nm298P6q0x;MfwvL zGVyDf*!U^}F9)yWcfYy{83Wk1h*Q5gzt>lAcl}N7cw+?ZAM%vTSCADmUo6U`(D5#xR3l+sUtV5a=GLgIn7Na67ZFv1==DlDCyTxy{~0jW>w3b)Ti$z~h34 z8dj*190F%;CT5aOnrVb%PU>LBt_0suZl9i^Z)7VOXIO>yS_6|B}6hr zxy*!8lye_TgD8chB#Kf<8femRC9^18p^}oTB2gqtVXvi9N=cfNN>oykqB-&G>wTa1 z!~Hz>{eC!p=a;pQbD#Tn?!DJq|6#e*)7bJQoWrZfoKmJTtX}FynFT2zKDxO=QD-`N zSL;FLY~+phwsTMOKd|1cG~RYa5YBmX4BfT%W24c1th{EyBs)vMN4%W0#>TR9%7hY1 zADDi+5A5(whZ`Oj$guNoI%;#4Z7ph`ftH!f+$#>w)!SjDOEG>?xe99UBH_EsS8nn! z2Qpu3#{IG}rt)34nUr}Zdy|pIJf_G|rI!O0IP4R>@A2TIof7z)IpaiT5o)|R!9{HYtfvF0SagIY)t2-5^}{GTAp`wP z^=Oy(Xp(xDfJ0}WgYPP>BKJm3wk{!+5~L1rXJ;31+ph0sN|CN$WpM- z)yaJB9yhi-w}tsFv_SQ)cK-UZW#H@~1#&80cxCbci@#jLX7)QXohnB#56h?dY%Wdi zTTk+W7lHbTEiCA=6O0*kjK!=`ry)76WGVC|q=pvpcRh6N9poiQX?i7s?qfW8D+9Ll z2B7bl)okeb0{r9AVNUW*5#$Ur6|_Fq~^LD-Q2+*CCj*URmLK{MlN_z54(mMg|5E#FyB@*rL&X+M28 z&1VwVu0ZsyGG??eo;5ZOhi_hkpibauPkR&z7TR@{MTaNSkW6E$x%d$!MO}DvZZmE( zHwBF+=a?A(jv3fHlIBn?`Z+Ivm0K%-qSJ3ayTzH0EI+}tLgQF@)&?v&JA%H8OG2B% zR>(MPMu88@+47|-tm%EZpi|x=)vBO*5mf9X9aV`VyAkIlvct42Go5lpY0gMh6l9r zhOS3AJ?BF>X0{P~+#rW~n^G~^aW-~;8BST(UD@^5;k;VPGnQ|DlPzyiAkQ8HEEHAZ z$c15$^*NS08z-Ql<$G?p#5{KTWrWE1;s`eqof8Ar<^4nX^~7htn>0}F^1<2TN!A+fnjXyKbkhTM zBY`^9Z8iy>o1TaGAZt8oNg!VKhJM%D!wzl%U$IJsmd_bhZ8T4k7H%91TJLM%PnnT$ zd-fl+x>ud4s-^IomJPuhGS+0tTG*(}158KAlkSO|pmswN+eJEbdb1H^&kcrMuk6`< z@6WvU@B0k6sdV?NJWRfR0d?x*1}g z-i0?Y^4@OdEn}XiYCk(;Fl+T zXB{8JC~5Rm+H=Yjj^+H|%&zFbb#X78Zu^Kk@NNZZ9gw5nv4_AsgiviPquuHC&==)T z;j^ru_G%ArDfG3re!75twM(mlB}TwLcMm41Z3zn%^0^M9-Sm|UqQ1?l6#d8rLoUU@ z8e3a%v3fw6p_e(u%T{EXBVHX--^YB89HJH9MlhN>9i+s2!2RrVeERj8sLdJBfAJ5B zG%;WatBROyoP>R$z<*!!P6NV|_0eJcV)DAS6?&)2)7kkCg}Q5^{d;S1+GCbL?cTrG zM&m%y=jpoD^*5rK>z$9Bvf?S|N&3Va&S+s5ZU}XmwHx{0+YFJxEi7wm9$lTMOe+Rr z@KM!lSod9)E$W}bhulVP>If-jr+brGv<~IELz<{dQUdH3oB{uoG#b1i5q_NVgnMtN zkq;Dr-|~Z$)Mrwat3QZp=C6nJ@@=qwqzg!?wqu!76xmGZz&n#q(9|j36mFe`9qw0Q zOYl0H`|cjNmiA!KTLZS#CzS16&;NNq)#{QKYo6rpzkC9Vj1B@1_vtH7W5yF(kV zZM#D=f{3fJwk4&kTo^bLO|xgcWY4GFV()-n?8+gvw`wqG z>^#;KG6_PB4q=LF3|C>ig*yH`2=-&1kW%a}*rTV-jypzir6EVaKdP8lx;KMK-|*xo zEly_l&Rr3uzO$nnx6djV#}GQv^PeZk%M>dh~npMkMQK05$b`0k&vvFq6tFV}1f=Hg~2&)x_(K=)T)hH2G*M zE7DnPn(;6Ycb`R4n_S@A@d!GTvWJ_vqM2zg?F0SjL|WF@aq z*ohB`)I0qtDlT+}MVseC=eNo5`#>ieyB(ta)4FlHYAG#nm9meR`H3__*U&>x9mrFQ zfo^URoVij;Yhw!WrTPh+5#|O>_JVG)cszaR982-RN$~PP9)A70hn?$gr?k(z*ukFd z__snL^tme{dp40h*6iSBG%Z5Sg}cCF_*XV)izg-?TP|pL((O_(p7saQ)uv% z4EoG|VH2s)d8e-eOFNgh((L6vs!OAOOEG&_Lse7mCPRfY;j(9jETAbCMIRHH@9suw zm6-rPxyaUKWzPT&luODp%inau(LLX;W)PI$M3T7*_a3@_D@nxgGwkIHz_f zDNH}XbT=GG(Rx0bbBIy6u<_jqiuTYJ*hRe@qSaQ}bAvq9r}M z*^gdpk|_Rc~70OiQaPKs$0Lr70_um}3nS z6-Ke+T}`y+1EEnx5*wa(pBXGV2kPD?oNIm_E_HT8`FAeFR~B=3S8?#y@-fsOQO*^& zyuiC_GttXtSat5Gb5N%AjfJh43*$>9@Ia;vgc+|Bv~YyS|B|G_5zpECs2&=XJ{c9p zZKE~4@A1Gl5$WICZ_m$JKzsijOrwVv;YW)`_;Jx3mTY^=wBPqJoww)cd!PgC-o)sv zo8SSU~!vi||Q zG5R8R$EOvSEDaTU2Fq~EDA%ev;&qsA7Q_nO3o-W79!!-iAd8<~F!{|8)crXPb}2|f zd|DHG9J`K`&7QOPkQ{ndGY0go9mahn;`W&apIA_+2&Nser0=sA!1Rtx`XlEuIB%WD zpI({4JS`Hb_KYjs^_hZ4M)fo2-w6nm#Af=$wzVQ{h&abXYubaws zcAn(>rbdEX!V7f$EWs~pUPpCbc2-qZECAiFcX3#7F>agWhnlO#!6d=2?DE}F^zN+> ze=}T;I)Clv#I#$PSJn^Kl0TAO=zM1HB=wor4{um;+=g-tO4-ET9V9(11iMwHGYjhu zzG?0-CYrSvKl}c{BHbNf;8qE?&ZuEoYx6L|M-1u%TX3z@K0&MW7droH#oH^7gDm%g zogHV6nko!#_LX7P7kAV?S;Zz^+lx~|p5pt53&Akm7eo(>S@ouHihLEp`K(uifzT)z z$ttlvPZLf~u|ZZK2VxE4WLPkR#z*GE)Q?CvPh6ny4~#&%%9(n4eCX#Q4=CFtNlAmJ zW3X{AYn4{vcVACrZTV{{1YNY!%)y zPW3nNrwRuqLM|vuj-V4YLNE4{1nO;&g_<+r=s^eB$hxtlIee1P<2gjLuRLY@0tQ5r z+)CM^*i@S0p+b$$64fUk2g3NdVsK1j7sNA5Z1uITzT)o4Ov5apKOmNx8>ZX;VN_3+ zH8qqIplUztN;KW+nP7kFcqCmscpM*FiVIwvEqL_J8pxD>$7wiT!iBz_n4BlaMwCp0 zCmqRHykaKCyY0qbPV;a=?08zH0W@Fh9Ok~(gW5IftVz0n73{HLZCf7kKKK{%kw5IpxaBbU0b!ED(( zn4r=QXBIiq#mEh?!a|a&6`peK{xM)1H;eu%lAw@bVrX65gr0-6(RP*un-sDG#}3a! z?WMA?bW;KQYu*W#`lz3qvBQ!p-6r5g5$|cIG6$mq1#ZHxWB5yM7nd?)0vgXg$c|MH z0j27-FnjSaKD{lGg*`~Yw5_h}ce56oVEq&Mlu}$D6^-v2$G~LEHhlFcjA;()goKb` zbmxi|)S0S6?SezN+#rhvMR*~vyb2mK0?5VTIVu!P!ArA-fa^E`7u`P+4r%1$hdLX$ zaUcwq#ZIIV4+2?WwjBhW+l<9ml-R|yCsAE|6t7Y77+Vb+P}O=rQwT7GmM}w*YRF(A zA9vDQ+eDT)e+Df6V;o$P5xfkyEu@>`?B}p++gOM04?R$6dSWLesEYyD zmkZL{B*}Y>F*Dn(P4@%4A$e9L9bcV?{E!h8>AIiYuRTs`3zE<~zE2d@5Xnnxt%FBe zU0`0DLZOdG&^DWFyy6}P8%#aWE`5;wu31vmGqgr<))iK~X2um5T0ENEU7ER^;7Ic8 zk_BC_Aa2^eU;%Tx3}32Fun*vhV+>MhO@;<|X-U%Ax<(dLQjRKZ$8d>WtLV#ORVv|n z@a3R0>|Smk%DuY=6B1^ME?3OLK*c3gRi_OHG~{6Qw)bpXiUeNPe1u=md%{xV_k7`z zAZoX&W#ca`rQ#>Qm};mZEt0WfP6anGy(Eskl={dyH=Sl{hdR@d)a&e7?=yaT=m@Ic zbDLlCWCJYRwwRx8T90L=l^9(X!>#E}2B+DN&~)_`+#WxM#aOywVa-Ca(f!22{1!5) zt?KlLOEa%Bsha(2zKw>7wY=m#7p6B!haV@_#FTS$neHeJiZ4iKgZ)piFKv>YcvTQ1%fbwu@}#-v3;zP28p^73rl=-ptD@T{s!tPcIee(}D7cF%fyGIL&1 z%#UgCplV}%=(HDbwA%x0TW=@s{T4_%??ZUc4< zq2oa~w8zbZx}nzSyDSzOiY9>8D>-OSJVwgX>Ukr#?PSyuOha7ufr8KnO&ga{FV%}K&N+Kdx_CX)AzYLv1*N2cmo6gU4Y=sZji zyvdHzGwWbFXXcN|FM7e<`3Ur4x9EP_RhnkkfU}2afMH!fr}L_qt^9VLX>7mEHx*sS zhQc@)d~YU%!mld(Uwz<^AmpP1FB!bNfKv<1VARt!BqP0qHWxTk`=}r6v*a!|bF?b7 zcrPVgf51Db1^lfe`F3$h%CLB2B$-HV=Wg>Qkmh-Y`m9cJwV$$KzI7-Y)Cg?iS%0Ah z>&Nqd|V*XgZoKUq_B7 zPeF{53KYs`LP>uj4f^fI{v5NBFK(5mil9ht^Ws3x$W7ip{Av@~VJripJ$3m2-O{%9S3jJFrOfM-B;sVpsU_CwxnFu(JZJB7b?C&%q$=}3qy6FV1* z5o$SAwX>$uMw?!yHM|mkC|+X5Pn6)eoN}0k*+(-&;z2R#EEa9>royXjFju|- z)w=7zb-3VIBE5md^=XrDzAWTh_CUUxh)LK?B~hD#eQ|v$EvG^DGP*nI#QLK+V0x8Y z9~}bfHiejo8?2>ZIG@_MoLv?2%H~x{>|m@PI#?X7l1rAM4=)?B^<_Wz$>uw6qgI5Q z{IxLd?tQFws6|ocZM3h>gNccW6nlFX^E}+d>_--gdY&85>2VD#J76)E1umfeOP`sE zOuXR7sXz-qigRg4D`93$2H7-ef{Dd%PR*&3`TuxNSzmqWyiIulR{+J6_ z4l9v#?`Y7wrN~!zUPrOZN7-qKp)~f1;19Mck!^QN;_HKMF!z_sal@W%-19f`pr|p3 ztglMq?9^RMY@`Qk4?B-GYL28Gq|XK$i9yiZlYHe1ZKk|3kzy*$NO!p`1b%b^%?Ho; zd8?N2RwqY5eqAf-OC(ZRt_56qb(jJ}r6}{mOcvdxN9Mgdc!gW;EaH!$pi%P(&w8oT z(|U6f4a>&4LXD<)-=QjOnhot;G=s~C3qd>WHvE!um$yz&XUQEB5as(6yUuH3;eMh2 z_h}_RWN16;UnEHXJ)bt@E5g&4&#@%f10InA$(_sPEf+3gt*n!U$gRiLi)jL%+nIDK zW5IjkM2s0G2}38#;OfB-SYX#f+&H9ya_={i>q=Ssg(KU@e^L*eu3QDZZ~4HtClg7eEsu-MFu`U@}Nn`PgyeohwmEy)nhb|;~3*IMT1 zVukxUmBB&GkUU2KZNLr8WK<$dEPKgooROyF`PT5KpcCW@r68x+f@HD;Pe9o;%5wP0 zI?51FMGc15W;ars6ev2fb25}Kd&{|P(B#K`uHgI~h5PHxL@M3V4cFZbX~3$1h3Njx zN3%V+G};_bS@nqcR4){p*oBhQ#7JYyMmFn&3Kexf5cDc0Y(!@_+tKj_*H0b~#|0k% zy{|EF-NYRBPRizg{cs=+S9LOyRfn3Ob#!mvGnVD(#io3TqCcP4!M63iRf8P9p@*a* zeeJo1gCw1por@K{c;dr`7$-8%F@iv}P=pj72zU5is9l`79 zz^81^`7cS(IxJ1SnS!(QSwXC%ebh8b4Bsom ze%wyLEj9?z-6c#c+n0RK=hKF%b08_zlwG+$0=1oDAo@fcNqA;LD9H%8b}4LrP)`9N zHM~KxHhhdOBdI;Nz|gLu>hP~KVkC=9Dn!^3ZWFjEi=;su7Fp7G~OTO8{?APU>Oi>*EPg-Zw%qbPyBxoVw1o_2Ib z`7@{3)c`-7`Y(?5-lP8CXbP3#@~X08DoXO=vSR3?)uLW|7X{gCjWa*#D5Ak|92A{{>SP5v$5hoYvtJgJTc*4zu=!=`hO2G z{Wnt*{!{Fje>=Y5f1J%fTTA}4R$Bi5rw*tLl^-HpNI4-^OkP4@drp; Date: Tue, 15 Aug 2023 19:00:17 -0500 Subject: [PATCH 3/7] feat: changed output format of onnx converter --- python/onnx_converter/dataprinter.py | 6 + .../first_transformed_onnx.msgpack | Bin 10113 -> 10409 bytes python/onnx_converter/onnxconverter.py | 207 ++++++++++++------ src/utils/loader.rs | 2 +- 4 files changed, 151 insertions(+), 64 deletions(-) create mode 100644 python/onnx_converter/dataprinter.py diff --git a/python/onnx_converter/dataprinter.py b/python/onnx_converter/dataprinter.py new file mode 100644 index 0000000..64037c3 --- /dev/null +++ b/python/onnx_converter/dataprinter.py @@ -0,0 +1,6 @@ +import onnx + +onnx_model = onnx.load("mnist-8.onnx") + +print(onnx_model.graph.value_info) + diff --git a/python/onnx_converter/first_transformed_onnx.msgpack b/python/onnx_converter/first_transformed_onnx.msgpack index 0afffdcd7bf8de557adba0e8578698f9f6b32a26..97fef7a40c4ce72ac78074df035cf5467d117a38 100644 GIT binary patch delta 1036 zcma)*zi-n(6vuPtvukB=BXy{RvXm}J3`Gq5128b~tCiRg?j^Z^RoiFz9Mn!$#lXT~ zJ4RFt2t=CJ5t%Y0P%CwdWc~t{N{GLJ5iS=;y`&*xdp^JS`M&pkcl!^s&G2>h%%(ML zo2)s6rH?x+$8O6p>(pjiX1SheHFgP$OBrniMXYyiyhnU?@Tw;QGuZP;_cnE3+`QBE zu#X*iv|22#Ip5jiBj=s&xUcOOw|UyK0ZA^|7~C^G_vak=$>`4bif>4uS3bZ$^ZnnIC8 z(6mTpp2#A9c!5Ml6LQFxtNsW2vOkJ4?a#D;IuYP}ZHAA~g+EXDO%e+@+9G24^VXZB zpn^~gr2#aFza;Y$Fa)K7-@XOcp3VcdMOMNy%Lyn;*d+<8Ph-nt=utpiMt%17J-8-fa3}_d^-co^_#p3p|8sTYcm5;)QqE6A1{`c6V?`6O(%xRZaYclvD1@Rg z5@+h*!|K_xC1P#-pLZ$}-u*)B%lX&4=RdEX_gjpZjM$_e@SaZlb?P`)VAe>J`Xu^X geBx$6{HR~E0%m&LZ=Uab>y6&IkKG1!h7Z>M0`GvS!vFvP delta 716 zcmZ1(*yvy1yskJoF()%GJw7clxg@{nEEB`BoW#o1qT(5?tH6x-lFEYAWzP9|WkxQ` z^YT+t<1FAr%`~7^+@>8sHfN=8VnslHeh$KAOhlN)#|L*=JrmO;CMIMf(R|8` zZXO>Wy4wVh%){+dkVje23=`n1$731+pR!^YhG`ZH3*4v7%#)azkv+ORD784l3K)N| zNM%Dekc9=+tCJYdGBQ97JkUD%I+LOuOfNf%Ua&%Ry<9Mz%S%%8it~$#r#Ed5W1h#z z*fjYUiwtAaW*JsKVIapq!a$&DDL7?KWLlDvSduuQX>*mtF@9;N=p;riu<)#xEXw>B L_Io^euKff6xYinm diff --git a/python/onnx_converter/onnxconverter.py b/python/onnx_converter/onnxconverter.py index c90425b..87ef517 100644 --- a/python/onnx_converter/onnxconverter.py +++ b/python/onnx_converter/onnxconverter.py @@ -3,30 +3,16 @@ import numpy as np import msgpack from onnx import numpy_helper +import argparse -scale_factor = 512 - -# Loading ONNX model - -onnx_model = onnx.load("mnist-8.onnx") -model_graph = onnx_model.graph -model_input = model_graph.input -model_nodes = model_graph.node -model_init = onnx_model.graph.initializer # Helper Functions - def get_shape(container, node_id): dim = container.__getitem__(node_id).type.tensor_type.shape.dim dim_list = list() for i in range(len(dim)): - dim_list.append(dim.pop(0).dim_value) - return dim_list - - - for i in range(len(dim)): - dim_list.append(dim.pop(i).dim_value) + dim_list.append(int(dim.pop(0).dim_value)) return dim_list @@ -41,106 +27,201 @@ def get_output_dim(node_id, graph): def get_input_dim(layers): - return layers[-1]['output_shapes'] + return layers[-1]['out_shapes'][0] + + +def create_wbdim_map(graph): + wbdim_map = {} + for init in graph.initializer: + n = init.name + dim = init.dims + wbdim_map[n] = [int(_) for _ in init.dims] + return wbdim_map + +parser = argparse.ArgumentParser() +# parser.add_argument('--model', type=str, required=True, default="mnist-8.onnx") +# parser.add_argument('--model_output', type=str, required=True) +# parser.add_argument('--config_output', type=str, required=True) +parser.add_argument('--scale_factor', type=int, default=2**9) +parser.add_argument('--k', type=int, default=19) +parser.add_argument('--eta', type=float, default=0.001) +parser.add_argument('--num_cols', type=int, default=6) +parser.add_argument('--use_selectors', action=argparse.BooleanOptionalAction, required=False, default=True) +parser.add_argument('--commit', action=argparse.BooleanOptionalAction, required=False, default=False) +parser.add_argument('--expose_output', action=argparse.BooleanOptionalAction, required=False, default=True) +parser.add_argument('--start_layer', type=int, default=0) +parser.add_argument('--end_layer', type=int, default=10000) +parser.add_argument('--num_randoms', type=int, default=20001) +args = parser.parse_args() +# Loading ONNX model + +onnx_model = onnx.load("mnist-8.onnx") +scale_factor = args.scale_factor +model_graph = onnx_model.graph +model_input = model_graph.input +model_nodes = model_graph.node +model_init = onnx_model.graph.initializer + # Converting Layers layers = list() +commit_before = list() +commit_after = list() +wbdim_map = create_wbdim_map(model_graph) node_id = 0 for node in model_nodes: - if node.op_type == "Conv": layer_type = "Conv2D" - output_id = node_id - if node_id == 0: - input_dim = get_shape(model_input, 0) - else: - input_dim = get_input_dim(layers) + + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) node_attr = node.attribute - kernel = [_ for _ in node_attr.pop(0).ints] - stride = [_ for _ in node_attr.pop(0).ints] + kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] + stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] # padding = str(node_attr.pop(0).s) # TODO - params = [kernel, stride] + params = [kernel[0], stride[0]] elif node.op_type == "MaxPool": layer_type = "MaxPool2D" - output_id = node_id - if node_id == 0: - input_dim = get_shape(model_input, 0) - else: - input_dim = get_input_dim(layers) + + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) node_attr = node.attribute - kernel = [_ for _ in node_attr.pop(0).ints] - stride = [_ for _ in node_attr.pop(0).ints] - params = [kernel, stride] + kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] + stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] + params = [kernel[0], stride[0]] elif node.op_type == "Relu": layer_type = "ReLU" - output_id = node_id - if node_id == 0: - input_dim = get_shape(model_input, 0) - else: - input_dim = get_input_dim(layers) + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) - params = None + params = [[]] elif node.op_type == "Reshape": layer_type = "Reshape" - output_id = node_id - if node_id == 0: - input_dim = get_shape(model_input, 0) - else: - input_dim = get_input_dim(layers) + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) output_dim = get_output_dim(node_id, model_graph) - params = None + params = [[]] elif node.op_type == "Gemm": layer_type = "Fully Connected Layer" - output_id = node_id - if node_id == 0: - input_dim = get_shape(model_input, 0) - else: - input_dim = get_input_dim(layers) + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) output_dim = get_output_dim(node_id, model_graph) - params = None + params = [[]] else: node_id += 1 continue layer = { "layer_type": layer_type, - "node_id": node_id, - "input_shapes": input_dim, - "output_shapes": output_dim, - "params": params + "params": params, ## Change params HELP HELP HELP HELP + "inp_shapes": inputs_dim, + "inp_idxes": [node_id], ### RANDOM COME BACK HERE HELP HELP HELP HELP + "out_idxes": [node_id+1], ### RANDOM COME BACK HERE HELP HELP HELP HELP + "out_shapes": [output_dim], + "mask": [] } layers.append(layer) node_id += 1 +for layer in layers: + for l in layer.keys(): + print(layer[l]) + break # print(layers) -# Converting W&B +# Converting W&B +init_id = 0 tensors = list() for init in model_init: - shape = [dim for dim in init.dims] + shape = [np.int64(dim).item() for dim in init.dims] raw_data = numpy_helper.to_array(init).ravel().tolist() ## Orientation of ravel ????? data = [] for i in raw_data: if isinstance(i, float): - buf = int(np.round(i * scale_factor)) + buf = np.int64(np.round(i * scale_factor)).item() data.append(buf) - tensor = {"shape": shape, "data": data} + elif isinstance(i, int): + buf = np.int64(i).item() + else: + None + tensor = { + "idx": init_id , + "shape": shape, + "data": data + } tensors.append(tensor) + init_id+=1 # print(tensors) + # Converting to msgpack final_dict = { - "scaling_factor": scale_factor, - "layers": layers, - "tensors": tensors + 'global_sf': scale_factor, + 'k': args.k, + 'num_cols': args.num_cols, + 'inp_idxes': [0], ### RANDOM COME BACK HERE + 'out_idxes': [node_id], ### RANDOM COME BACK HERE + 'layers': layers, + 'tensors': tensors, + 'use_selectors': args.use_selectors, + 'commit_before': commit_before, + 'commit_after': commit_after, + 'bits_per_elem': None, + 'num_random': args.num_randoms, + } +# print(final_dict) + with open("first_transformed_onnx.msgpack", "wb") as mfile: mfile.write(msgpack.packb(final_dict)) diff --git a/src/utils/loader.rs b/src/utils/loader.rs index f3a4721..2a2890c 100644 --- a/src/utils/loader.rs +++ b/src/utils/loader.rs @@ -74,4 +74,4 @@ pub fn load_model_msgpack(config_path: &str, inp_path: &str) -> ModelMsgpack { }; model -} +} \ No newline at end of file From 04c50eb2ff8337b646bffcc5464f8d8ef390f8b0 Mon Sep 17 00:00:00 2001 From: Alluri <53390631+disirulla@users.noreply.github.com> Date: Sun, 20 Aug 2023 11:51:45 -0500 Subject: [PATCH 4/7] feat: functional onnx converter --- public_vals | Bin 0 -> 320 bytes python/onnx_converter/batch_size_modifier.py | 42 +++++ python/onnx_converter/dataprinter.py | 6 - .../first_transformed_onnx.msgpack | Bin 10409 -> 10403 bytes python/onnx_converter/onnxconverter.py | 159 +++++++++++++----- src/layers.rs | 1 + src/layers/conv2d.rs | 2 +- src/layers/dag.rs | 16 +- src/layers/fully_connected.rs | 6 + src/layers/layer.rs | 2 + src/layers/max_pool_2d.rs | 4 + src/layers/relu.rs | 45 +++++ src/model.rs | 9 + src/utils/loader.rs | 3 +- 14 files changed, 243 insertions(+), 52 deletions(-) create mode 100644 public_vals create mode 100644 python/onnx_converter/batch_size_modifier.py delete mode 100644 python/onnx_converter/dataprinter.py create mode 100644 src/layers/relu.rs diff --git a/public_vals b/public_vals new file mode 100644 index 0000000000000000000000000000000000000000..ab6c8307475494e349d76f6ac8e7df17ff6ee0bd GIT binary patch literal 320 zcmZqv{r~;suMeFk7VNC_cwwv&8`0Rf%{5?$<^scqMSdv;=lJ&u47_ literal 0 HcmV?d00001 diff --git a/python/onnx_converter/batch_size_modifier.py b/python/onnx_converter/batch_size_modifier.py new file mode 100644 index 0000000..53d77da --- /dev/null +++ b/python/onnx_converter/batch_size_modifier.py @@ -0,0 +1,42 @@ +import onnx +import os +import struct + +from argparse import ArgumentParser + + +def rebatch(infile, batch_size): + model = onnx.load(infile) + graph = model.graph + + # Change batch size in input, output and value_info + for tensor in list(graph.input) + list(graph.value_info) + list(graph.output): + tensor.type.tensor_type.shape.dim[0].dim_param = batch_size + + # Set dynamic batch size in reshapes (-1) + for node in graph.node: + if node.op_type != 'Reshape': + continue + for init in graph.initializer: + # node.input[1] is expected to be a reshape + if init.name != node.input[1]: + continue + # Shape is stored as a list of ints + if len(init.int64_data) > 0: + # This overwrites bias nodes' reshape shape but should be fine + init.int64_data[0] = -1 + # Shape is stored as bytes + elif len(init.raw_data) > 0: + shape = bytearray(init.raw_data) + struct.pack_into('q', shape, 0, -1) + init.raw_data = bytes(shape) + + onnx.save(model, "/Users/siddharthaalluri/Desktop/sid-alluri/zkml/python/onnx_converter/new_mnist.onnx") + +if __name__ == '__main__': + parser = ArgumentParser('Replace batch size with \'N\'') + parser.add_argument('infile') + # parser.add_argument('outfile') + args = parser.parse_args() + + rebatch(args.infile, '1') \ No newline at end of file diff --git a/python/onnx_converter/dataprinter.py b/python/onnx_converter/dataprinter.py deleted file mode 100644 index 64037c3..0000000 --- a/python/onnx_converter/dataprinter.py +++ /dev/null @@ -1,6 +0,0 @@ -import onnx - -onnx_model = onnx.load("mnist-8.onnx") - -print(onnx_model.graph.value_info) - diff --git a/python/onnx_converter/first_transformed_onnx.msgpack b/python/onnx_converter/first_transformed_onnx.msgpack index 97fef7a40c4ce72ac78074df035cf5467d117a38..e8c4117a0ab90be9c3b6cd2bf60b96333dc6b842 100644 GIT binary patch delta 675 zcmZ1(xHxcv5hMRZW2O433=E8njH@#93gU}15(`p`Cr@Eyl#!Xj!OF@wkz*xDJTs*N zD9pgfv@*Z61j3uhyb8pI=$JSKsDWciZenrvg!WZAiIu5E@gSmyVDFUpl z91{hQ+|0_xj^t(zR5$bS36Sh&pq`bP>aZ{Yh7Te{I5A8I`N@6?GZ3Kq32X@53{E67 zxKPbtVG)>I$fT%?9B`ASFam`qG9i39iHjS_2%gFNnUw3-xs~SRR66J9<)tQs8LSX7u&tXLBo6R%vOz@{Cwpi_0{}u-;;8@t diff --git a/python/onnx_converter/onnxconverter.py b/python/onnx_converter/onnxconverter.py index 87ef517..c710c05 100644 --- a/python/onnx_converter/onnxconverter.py +++ b/python/onnx_converter/onnxconverter.py @@ -23,6 +23,9 @@ def get_output_dim(node_id, graph): output_dim = get_shape(model_output, 0) else: output_dim = get_shape(model_valueinfo, node_id) + buf = output_dim[1] + output_dim.remove(output_dim[1]) + output_dim.append(buf) return output_dim @@ -35,9 +38,16 @@ def create_wbdim_map(graph): for init in graph.initializer: n = init.name dim = init.dims - wbdim_map[n] = [int(_) for _ in init.dims] + l = [int(_) for _ in init.dims] + if len(l)>1: ##Check here while adding more OPS + buf = l[1] + l.remove(l[1]) + l.append(buf) + wbdim_map[n] = l return wbdim_map + + parser = argparse.ArgumentParser() # parser.add_argument('--model', type=str, required=True, default="mnist-8.onnx") # parser.add_argument('--model_output', type=str, required=True) @@ -64,17 +74,32 @@ def create_wbdim_map(graph): model_nodes = model_graph.node model_init = onnx_model.graph.initializer -# Converting Layers - + layers = list() +tensors = list() commit_before = list() commit_after = list() wbdim_map = create_wbdim_map(model_graph) + +# Converting Layers +init_ids = [] node_id = 0 +init_id = 0 +wb_ids = [] +fullc_wb_ids = [] for node in model_nodes: if node.op_type == "Conv": layer_type = "Conv2D" + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + + # print(inp_idxes, out_idxes) inputs_dim = [] for input in node.input: if input in wbdim_map.keys(): @@ -90,11 +115,23 @@ def create_wbdim_map(graph): node_attr = node.attribute kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] - # padding = str(node_attr.pop(0).s) # TODO - params = [kernel[0], stride[0]] + padding = str(node_attr.pop(0).s) + if padding == b'SAME_UPPER': + padding_code = 1 + else: + padding_code = 0 + params = [0, padding_code, 1, stride[0], stride[1]] # 1 == ReLU elif node.op_type == "MaxPool": layer_type = "MaxPool2D" + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + # print(inp_idxes, out_idxes) inputs_dim = [] for input in node.input: if input in wbdim_map.keys(): @@ -105,15 +142,24 @@ def create_wbdim_map(graph): elif input not in wbdim_map.keys() and node_id != 0: inp_dim = get_input_dim(layers) inputs_dim.append(inp_dim) - output_dim = get_output_dim(node_id, model_graph) node_attr = node.attribute kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] - params = [kernel[0], stride[0]] + params = [kernel[0], kernel[1], stride[0], stride[1]] elif node.op_type == "Relu": - layer_type = "ReLU" + layer_type = "ReLUONNX" + inp_idxes = [] + inp_idxes.append(init_id) + + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + + # print(inp_idxes, out_idxes) inputs_dim = [] for input in node.input: if input in wbdim_map.keys(): @@ -126,11 +172,20 @@ def create_wbdim_map(graph): inputs_dim.append(inp_dim) output_dim = get_output_dim(node_id, model_graph) - params = [[]] - + params = [] + elif node.op_type == "Reshape": layer_type = "Reshape" inputs_dim = [] + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + # wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + + # print(inp_idxes, out_idxes) for input in node.input: if input in wbdim_map.keys(): inputs_dim.append(wbdim_map[input]) @@ -141,22 +196,36 @@ def create_wbdim_map(graph): inp_dim = get_input_dim(layers) inputs_dim.append(inp_dim) output_dim = get_output_dim(node_id, model_graph) - params = [[]] + params = [] elif node.op_type == "Gemm": - layer_type = "Fully Connected Layer" + layer_type = "FullyConnected" inputs_dim = [] + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + fullc_wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + + # print(inp_idxes, out_idxes) + # inp_idxes = list(range(init_id, init_id+len(node.input))) + # init_id = init_id + len(node.input) + # out_idxes = [np.random.randint(0,1000)] for input in node.input: if input in wbdim_map.keys(): - inputs_dim.append(wbdim_map[input]) + inputs_dim.append(wbdim_map[input][::-1]) #Specific to FullyConnected, to match with rest of zkml elif input not in wbdim_map.keys() and node_id == 0: inp_dim = get_shape(model_input, 0) inputs_dim.append(inp_dim) elif input not in wbdim_map.keys() and node_id != 0: inp_dim = get_input_dim(layers) inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) - params = [[]] + params = [0] else: node_id += 1 continue @@ -164,43 +233,51 @@ def create_wbdim_map(graph): "layer_type": layer_type, "params": params, ## Change params HELP HELP HELP HELP "inp_shapes": inputs_dim, - "inp_idxes": [node_id], ### RANDOM COME BACK HERE HELP HELP HELP HELP - "out_idxes": [node_id+1], ### RANDOM COME BACK HERE HELP HELP HELP HELP + "inp_idxes": inp_idxes, ### RANDOM COME BACK HERE HELP HELP HELP HELP + "out_idxes": out_idxes, ### RANDOM COME BACK HERE HELP HELP HELP HELP "out_shapes": [output_dim], "mask": [] } layers.append(layer) node_id += 1 -for layer in layers: - for l in layer.keys(): - print(layer[l]) - break -# print(layers) # Converting W&B -init_id = 0 -tensors = list() +init_ct = 0 for init in model_init: - shape = [np.int64(dim).item() for dim in init.dims] - raw_data = numpy_helper.to_array(init).ravel().tolist() ## Orientation of ravel ????? - data = [] - for i in raw_data: - if isinstance(i, float): - buf = np.int64(np.round(i * scale_factor)).item() - data.append(buf) - elif isinstance(i, int): - buf = np.int64(i).item() - else: - None - tensor = { - "idx": init_id , + if init.data_type == 1: # Tried avoiding unwanted init with init_id, but not working, used data_type instead + + shape = [np.int64(dim).item() for dim in init.dims] + if len(shape)>1 : + buf = shape[1] + shape.remove(shape[1]) + shape.append(buf) + + if wb_ids[init_ct] in fullc_wb_ids: ## Not good + shape = shape[::-1] + for i in range(1,len(fullc_wb_ids),2): + if wb_ids[init_ct] == fullc_wb_ids[i]: + shape = [shape[0]] + + + raw_data = numpy_helper.to_array(init).ravel().tolist() ## Orientation of ravel ????? + data = [] + for i in raw_data: + if isinstance(i, float): + buf = np.int64(np.round(i * scale_factor)).item() + data.append(buf) + elif isinstance(i, int): + buf = np.int64(i).item() + tensor = {"idx": wb_ids[init_ct] , "shape": shape, "data": data } - tensors.append(tensor) - init_id+=1 -# print(tensors) + tensors.append(tensor) + init_ct+=1 + +# for l in layers: +# print("ids: {}".format(l["inp_idxes"])) +# print("shapes: {}".format(l["inp_shapes"])) # Converting to msgpack @@ -210,7 +287,7 @@ def create_wbdim_map(graph): 'k': args.k, 'num_cols': args.num_cols, 'inp_idxes': [0], ### RANDOM COME BACK HERE - 'out_idxes': [node_id], ### RANDOM COME BACK HERE + 'out_idxes': [init_id], ### RANDOM COME BACK HERE 'layers': layers, 'tensors': tensors, 'use_selectors': args.use_selectors, @@ -221,7 +298,5 @@ def create_wbdim_map(graph): } -# print(final_dict) - with open("first_transformed_onnx.msgpack", "wb") as mfile: mfile.write(msgpack.packb(final_dict)) diff --git a/src/layers.rs b/src/layers.rs index 49c3eff..a84c558 100644 --- a/src/layers.rs +++ b/src/layers.rs @@ -22,6 +22,7 @@ pub mod square; pub mod squared_diff; pub mod tanh; pub mod update; +pub mod relu; // Special: dag pub mod dag; diff --git a/src/layers/conv2d.rs b/src/layers/conv2d.rs index c8f7286..4ed54d8 100644 --- a/src/layers/conv2d.rs +++ b/src/layers/conv2d.rs @@ -299,7 +299,7 @@ impl Layer for Conv2DChip { let inp = &tensors[0]; let weights = &tensors[1]; - + println!("wt shape: {:?}", weights.shape()); let (oh, ow) = Self::out_hw( inp.shape()[1], inp.shape()[2], diff --git a/src/layers/dag.rs b/src/layers/dag.rs index b73a51a..8a4f0f9 100644 --- a/src/layers/dag.rs +++ b/src/layers/dag.rs @@ -26,7 +26,7 @@ use crate::{ square::SquareChip, squared_diff::SquaredDiffChip, tanh::TanhChip, - update::UpdateChip, + update::UpdateChip, relu::ReluLayerChip, }, utils::helpers::print_assigned_arr, }; @@ -82,11 +82,13 @@ impl DAGLayerChip { "Processing layer {}, type: {:?}, inp_idxes: {:?}, out_idxes: {:?}, layer_params: {:?}", layer_idx, layer_type, inp_idxes, out_idxes, layer_config.layer_params ); + + let vec_inps = inp_idxes .iter() .map(|idx| tensor_map.get(idx).unwrap().clone()) .collect::>(); - + // println!("vec_inps: {:?}", vec_inps); let out = match layer_type { LayerType::Add => { let add_chip = AddChip {}; @@ -286,6 +288,16 @@ impl DAGLayerChip { &layer_config, )? } + LayerType::Relu => { + let relu_chip = ReluLayerChip {}; + relu_chip.forward( + layouter.namespace(|| "dag relu"), + &vec_inps, + constants, + gadget_config.clone(), + &layer_config, + )? + } LayerType::Mul => { let mul_chip = MulChip {}; mul_chip.forward( diff --git a/src/layers/fully_connected.rs b/src/layers/fully_connected.rs index 6dbf529..3e87fed 100644 --- a/src/layers/fully_connected.rs +++ b/src/layers/fully_connected.rs @@ -44,6 +44,9 @@ impl FullyConnectedChip { ) -> Array, IxDyn> { assert_eq!(input.ndim(), 2); assert_eq!(weight.ndim(), 2); + println!("input shape: {:?}", input.shape()); + println!("weight shape: {:?}", weight.shape()); + assert_eq!(input.shape()[1], weight.shape()[0]); let mut outp = vec![]; @@ -264,6 +267,9 @@ impl Layer for FullyConnectedChip { .unwrap(); let mm_div = if tensors.len() == 3 { + println!("shape vec: {:?}", shape); + println!("tensors shape: {:?}", tensors[2].shape()); + let bias = tensors[2].broadcast(shape.clone()).unwrap(); let bias = bias.iter().map(|x| x.as_ref()).collect::>(); let mm_div = mm_div.iter().collect::>(); diff --git a/src/layers/layer.rs b/src/layers/layer.rs index 676d6eb..65971c0 100644 --- a/src/layers/layer.rs +++ b/src/layers/layer.rs @@ -45,6 +45,8 @@ pub enum LayerType { Tanh, Transpose, Update, + + Relu } // NOTE: This is the same order as the TFLite schema diff --git a/src/layers/max_pool_2d.rs b/src/layers/max_pool_2d.rs index c929092..382d3d0 100644 --- a/src/layers/max_pool_2d.rs +++ b/src/layers/max_pool_2d.rs @@ -52,6 +52,10 @@ impl MaxPool2DChip { let (sx, sy) = (sx as usize, sy as usize); // Only support batch size 1 for now + + println!("params: {:?} ", params); + println!("inp: {:?} ", inp.shape()); + println!("config: {:?} ", layer_config); assert_eq!(inp.shape()[0], 1); let out_shape = Self::shape(inp, layer_config); diff --git a/src/layers/relu.rs b/src/layers/relu.rs new file mode 100644 index 0000000..541b6d6 --- /dev/null +++ b/src/layers/relu.rs @@ -0,0 +1,45 @@ +use std::{collections::HashMap, rc::Rc, vec}; + +use halo2_proofs::{circuit::Layouter, halo2curves::ff::PrimeField, plonk::Error}; +use ndarray::{Array, IxDyn}; + +use crate::gadgets::{ + gadget::{Gadget, GadgetConfig, GadgetType}, + nonlinear::relu::ReluChip, +}; + +use super::layer::{AssignedTensor, CellRc, GadgetConsumer, Layer, LayerConfig}; + +#[derive(Clone, Debug)] +pub struct ReluLayerChip {} + +impl Layer for ReluLayerChip { + fn forward( + &self, + mut layouter: impl Layouter, + tensors: &Vec>, + constants: &HashMap>, + gadget_config: Rc, + _layer_config: &LayerConfig, + ) -> Result>, Error> { + let inp = &tensors[0]; + let inp_vec = inp.iter().map(|x| x.as_ref()).collect::>(); + let zero = constants.get(&0).unwrap().as_ref(); + + let ReluLayerChip = ReluChip::::construct(gadget_config.clone()); + let vec_inps = vec![inp_vec]; + let constants = vec![zero]; + let out = ReluLayerChip.forward(layouter.namespace(|| "relu chip"), &vec_inps, &constants)?; + + let out = out.into_iter().map(|x| Rc::new(x)).collect::>(); + let out = Array::from_shape_vec(IxDyn(inp.shape()), out).unwrap(); + + Ok(vec![out]) + } +} + +impl GadgetConsumer for ReluLayerChip { + fn used_gadgets(&self, _layer_params: Vec) -> Vec { + vec![GadgetType::Relu, GadgetType::InputLookup] + } +} diff --git a/src/model.rs b/src/model.rs index d88b59e..e8230c3 100644 --- a/src/model.rs +++ b/src/model.rs @@ -66,6 +66,7 @@ use crate::{ squared_diff::SquaredDiffChip, tanh::TanhChip, update::UpdateChip, + relu::ReluLayerChip }, utils::{ helpers::{convert_to_bigint, RAND_START_IDX}, @@ -333,6 +334,7 @@ impl> ModelCircuit { "SquaredDifference" => LayerType::SquaredDifference, "Sub" => LayerType::Sub, "Tanh" => LayerType::Tanh, + "ReLUONNX" => LayerType::Relu, "Transpose" => LayerType::Transpose, "Update" => LayerType::Update, _ => panic!("unknown op: {}", x), @@ -344,6 +346,7 @@ impl> ModelCircuit { let shape = flat.shape.iter().map(|x| *x as usize).collect::>(); let num_el: usize = shape.iter().product(); if panic_empty_tensor && num_el != value_flat.len() { + // println!("Shape: {:?}", shape); panic!("tensor shape and data length mismatch"); } if num_el == value_flat.len() { @@ -404,6 +407,7 @@ impl> ModelCircuit { LayerType::SquaredDifference => Box::new(SquaredDiffChip {}) as Box, LayerType::Sub => Box::new(SubChip {}) as Box, LayerType::Tanh => Box::new(TanhChip {}) as Box, + LayerType::Relu => Box::new(ReluLayerChip {}) as Box, LayerType::Transpose => Box::new(TransposeChip {}) as Box, LayerType::Update => Box::new(UpdateChip {}) as Box, } @@ -605,6 +609,7 @@ impl> Circuit for ModelCircuit GadgetType::SquaredDiff => SquaredDiffGadgetChip::::configure(meta, gadget_config), GadgetType::SubPairs => SubPairsChip::::configure(meta, gadget_config), GadgetType::Tanh => TanhGadgetChip::::configure(meta, gadget_config), + GadgetType::Relu => ReluChip::::configure(meta, gadget_config), GadgetType::VarDivRound => VarDivRoundChip::::configure(meta, gadget_config), GadgetType::VarDivRoundBig => VarDivRoundBigChip::::configure(meta, gadget_config), GadgetType::VarDivRoundBig3 => VarDivRoundBig3Chip::::configure(meta, gadget_config), @@ -686,6 +691,10 @@ impl> Circuit for ModelCircuit let chip = TanhGadgetChip::::construct(gadget_rc.clone()); chip.load_lookups(layouter.namespace(|| "tanh lookup"))?; } + GadgetType::Relu => { + let chip = ReluChip::::construct(gadget_rc.clone()); + chip.load_lookups(layouter.namespace(|| "relu lookup"))?; + } GadgetType::Exp => { let chip = ExpGadgetChip::::construct(gadget_rc.clone()); chip.load_lookups(layouter.namespace(|| "exp lookup"))?; diff --git a/src/utils/loader.rs b/src/utils/loader.rs index 2a2890c..3f3bda8 100644 --- a/src/utils/loader.rs +++ b/src/utils/loader.rs @@ -53,7 +53,8 @@ pub fn load_model_msgpack(config_path: &str, inp_path: &str) -> ModelMsgpack { rmp_serde::from_read(&mut reader).unwrap() }; for tensor in inp { - model.tensors.push(tensor); + model.tensors.push(tensor.clone()); + // println!("tensor: {:?}", tensor); } // Default to using selectors, commit if use_selectors is not specified From c22255b5d708f21523b710d0397cd0e8ffa5f971 Mon Sep 17 00:00:00 2001 From: Alluri <53390631+disirulla@users.noreply.github.com> Date: Sun, 20 Aug 2023 16:18:30 -0500 Subject: [PATCH 5/7] code cleaning --- python/onnx_converter/config.msgpack | Bin 0 -> 921 bytes .../first_transformed_onnx.msgpack | Bin 10403 -> 8685 bytes python/onnx_converter/onnxconverter.py | 481 +++++++++--------- src/layers/conv2d.rs | 1 - src/layers/dag.rs | 5 +- src/layers/fully_connected.rs | 3 - src/layers/max_pool_2d.rs | 3 - src/model.rs | 1 - src/utils/loader.rs | 3 +- 9 files changed, 243 insertions(+), 254 deletions(-) create mode 100644 python/onnx_converter/config.msgpack diff --git a/python/onnx_converter/config.msgpack b/python/onnx_converter/config.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..6609bc1413fa10b8856398b55c0e470763fb6a02 GIT binary patch literal 921 zcmb7@%}&BV6oos`VyRe2+_Lc%dglVVgv<9BQg`57Fur)D28}TJv z^G*)X*r^c3S>0Kj^UXc?&U124Xa^I6^_NI`9Ujju&osJ}uu3wpY-7+H0SlyrdTv(N zrv#6|VUwqHT^R1zhUpo#9#7AwHg>SdUL*-2l&n}@q-h)0%ms^4hxrSSfx&=q48MLZy3{lWuF-)$b-;hmuDI`lbPA;9!gc1PCOsjsXKs3?`T)u)8UwC!0-TN|yAH z1k*d%fWf`WBkLIzt4lMZ>Am-v@18OIF3#Qj?%gZ>p^@f$-}k)dJm)#jIivr0>HfX@ z?%uU`=l=T!*sSNiR`KFP-*|B6J^S|V7reCRp@(`z|4@?z(sk3YQo;HG^KeRJ{VgAea|WY>fHpJB1sZ1&6L_4^;#_3-ZfPd&|M zn@mrO_epAD^2wUdwm-nVYIENIVDp!5*!9?r`}Xbq-xqnBLtp>56L?ph!2ib+6pDXh{{P!g zwCrgWpD#L~`s^Z4@dd)G4if#J2dVzQ{iIM_wLOL8v+#=l2b=$QKXK{nAN}-?Cy1{) zLGq6$@Ob~k{1qB+5@d&88&>IG?%aNs08T?PXVQ|0J7G{olRw`nBKKyZ3Rz z>_fZnd35)^pPq7nb=4J~y0Y|139Wxp!v5^Ar(|-)Ri`Qc@ifUNXT9*~?uYj8du0D_ ze(*O+AolZD?s5hGf4_Ivqr0YA=Sw(cVZIY?wlELi$8qzSP58cIfXz4m62HQj`B&y= z@t%dbN3sg%@ngJ~T4Da)UYQFYFy^1!j|seru{(EKn0@#oMpoU5KjTcea9z%L&0XfV zZu{Cjx2(fYgP6hyPFNVBg}KhceAB|*jsL^|Ucn2Pir`&(@oE6)X!HJPM?3zItCy}a z>|9Y?c*~cB%f5&UHU?)grtiZz&RUp&<^Lzp)0Rw8@~NxyQmh zYGF28m_7KD`R8}w7#-2S!u--*4G!gwwU4c=GXDw}@E_>E>)Uu1PvXQ^Z=m1a_HL~q z3lC!AjzyZrL=Ln1)nDxR%Ho^M2keU*WsO2i+c02ZzMIA%p2A@aUhyU86JH(>U2){I z7{dX5A&gnR?aJGNPhDNkpif*a`(DCZ!kkG*<{rswB*h3O<~1xWhV1MOi({Ch&5z)y zR?oqih#=m%CXbn9c+o;ki7;Jblq<;tvz4`&vEm}8BR+saa>1v@Q72u0HY3zK=uhXW z6WN?l#xi&!-C<1QEvMlI6?y!;Ph3aAna0y;g`i-=Ww)5y z!pF<>)>!AGTdcJGWN>4(8c*ZRw5%qE8GM8b83j4(P!cDq7Q4ybNgrDw!Ay=Km2ToR?rYhB-+tRmGf=Q19m&jq`*{V_jL(u@vn<(+gy$)cnwE`xTN6+V_CpZgeBvz6=TAB=Z^N; z?iIOs-e|9m*fATd==WoSC&VF)+i^MJ#fyGSsUk*EDv2o> z2Hn{bEFy(MAC&L~x{8X$jebvF&7*MtLD5>Eu%z5L>9CT4{oZ*D%txyj13th9#!ig8 zVi+}*;puj{yl!hKO(2}cS-+s29C8ZKaxj?2bV7{x$v6wnBylm4_2vmLBk^`8EPlaF z`E)_^Krbfzz1OfKIiVxK$@EpLoRzFzCHft0saUMDi!fQ`lh`nvvW1I4_o^DLt!uer zVDC5jH)7wbsJdgK6aZP7Of(#k=mBMy% z$aE+u({Nc^zIv;|>-7gP5yBXs=cK1+3fVdgWQ9?-hpuT|_3LU3j7F64r^HcR zNK(VZ!>RI3KEmmARZshR>5__-(SqBaX^Gco#RNyruW?G$LY@O> zWLlhcIoz2=Qer8LEHoA9$9cOKb5Rd2hZI_C!zD)x9^Emxq^I~2CozE_mltV zMrZL_f{YyY;x*&-28X?-y|>`ycvs$p86{V-Qo^u4Cn{s;0{)A_cmcD7n`4yCLUrEP z*(5J7?Tb-_j!8RiCOt|&7m^x~CgeQK)a!jg!FAi* z?%sk1L%b4xi@yo%WY0;w>!DQb4^xZ+1fxa|r$V=AG}p&zqLx8wJ$_EsW}b2gEW(0g${fbZjDLlh&PH6o%k z)nfjD;xFCe#1_R})yR|Zs!GH=A&0-v`6!ERW&Tg{5qPd6*@kK0(?o`@{H!#6&#W^Y} zQawgX6gHxjh#GGa;LdqGOU*y6F0H&rCBvy!b&sy#Y}sbY=EG~+fP#@sF(!#rEhdaf ztE-BR2;Yt8>J}`b@DKDEi5`x?5_Hj?C+S8PII4L-5iZM6Q#+*QyLBFuEHX+$kz2H`U$V>H4O_${*jX2{DDi`MEmBBh$ zO$Xy65R+udI!VpM_=#+Uuf|+uhZtv?SKUlf;}5L(yse&{N@|MAoK9%)3u2)t;eg7# zQ(vM7&UqO^`<#c(k#7=(qrK%j4LC3sryyQdCh}xk5TlgIj#TF|W!hzwB`{d#mpRJf z59gV55o2kts3w@sHZeA~JRNZdTscud<-utT+iF<^;3==i8}#S0{N6s6ea%W8jsYIi5>G&+?gkN^c7XIrSNQpf||-u9IR^ zkggQjF(+2lJ8_DmT+EAe2n`hKVV_?>7kZQO;i5H1DLd(^mUP5YCgpCiuyN00C7~dT zIgUZ)6FRh9$$FMA7SVI07%RrR@~0%JU>iCFhI%Nuf`U&&*UCoc$7=(h!M%zNn>|x6!UJ*vDbtN_~Y8ao4D-B318SZK^ zrb?!E%9}VVmEw~I+sG+1xV(lSeh8K z%!P3-C@O>9?pV0EVpUkE-Ply(RW${LB$e^g^+PRIjM13tpNn>da z%)Xo_MATEOE(ydBmy7xBTR7CVLHG^Vs15oPA3G(l*9M9C=; zVd@n;U(rX$p1_4|_t~Vk+=XMtEZxwIV^ym)^|cE`-mZ3Pyy8Yow*)*H-!f%oBAd$D zB^jJbyVZqiXPGeY!Hr2=NK%28>5Xd$GiNA!8AUMHsa>cLwNz~ubGEJsHdZcM;f@r^ zb(dgq@S8i+u4s~-q2>KlcPwrx%j?zU+=Sg}dIo_yK>+W&*Y)Iayikkx@}vO66$uAh zCtXB0n55#bS2((ZGF))?kRr`GTndawszj_!3Y=zRc(IzLDz{#fu(S{+huOsXf*9AD zT3)4%I9b~a34Y~Q zjvl7&bk0(0Sa{!mJ*nDL1*+N4q%#!8 z_c5vBpg$&(djpt~N+|QZqIq6fp;Lu=!_D>Kc$PFCPg)Q*Qx?qEgh@coyD(*=Ogc&Z z9S3g}MY$@Nr7Z@awX>0nvAg=HuupQ-N)45SW~BZmFo5@BqK3s2nA zy{DcE?9cFA9&@t0apnO*rG>dABFi+BWG*dCXB!dl9u#Jarx_& z__dtm7R^JRTW4YZ)qKGGOXtG*)I7tu_@RhSxsW2nQSm}XoZ#VQa=rOCLGusWF`%P_ z{GlX!1I}wVqQR{EOl-DJZMeqcV9Oet_!O_}iS&8|B!6VY4oBU{k?{dRHDL{Id0 zEj4z;w^J{LDTZ*~bFJ4dyp7;}2*=#!r}5q*obD;6yyk+SKJC z!NB^KVtu|?b43(Rt=P7qyBbI5qvDjeAa-d!r^cunm-nb5INbBy?f9^SCvK-svlk~< zF2qX$+pWYk{AiIX%FExqUx0ILye{fqzS)HLZ`q1JTbM7<1Za4r`6uS@(G)9~G5_&e z3~{&0dNFOF7PP5`;bLxCRmn>OfE7G`wNi(pa!~~#&awq()-Ja&-wG`2GXD$%42dMQ zO@Vh~A*$o4@tx4Gv}2}d3`w=TjSfud zSEqt_BZsLV@xyrd%Nz;b=kqmvRrb1q+?33&Y>3tC4JwA^4>PmQ5d$&&AS|8iMA_!i#+nt^I8p`tt zMN_7zRYx(|upXm&F=?(@oRMI3p(o#j*H_$%AMYnFndtF$NPF0LyDdx^KBG&eF(j=| zMlhpqS)7rn6M9J;Fk)f8U5~dVy^lLM3H(5!GygE$n!<(km@%5)FIWCLwuRZH z65Wf^U2Nu0BRi`p}%QLs> zjrB2(u++nia@yq?OeO`X5XsyB}*60<83=$ zF#m{pq|*!Nk}vrEedPd*Z1HRGpX$h#v`iSgwl1ztgbB?T3w$a!+M8}-L_wU<-b#5;YhOoc;&!Dm z(8DSw7UCk2qHnDCPX zPS{}?U@syOGvj-pv!));_rjuH;>V!R$Iz^gEtc4f0*5ljxE1~oURjpzZ7|}L32$}= z%s;5@!N@YayQ^4%BYlbbusWB zY?2diSqsVgF}%53lEfJrv2*npR^YPk0h$#YZ)w0Y3QVlh;<8s(pYvkU-BW?%TQZH& z3L1LhZ4*aFQiQVU+pFrYO69*&h0r@K0M6rhpOSK5+oH)$R3$xwSTFI<#r>fjZ%n1aQ7+tx@rnKU8kX4`? zplMmI&{X|sS*7tv^!068-omc4@-Rp9Rb4V!a?E3FC8_Ovfh%b3L_e?0R9zKzxA;c4 zYdO>6S%kX&N(MLHeIZ(^#gC4hHgt8vbe zXu`2J|H3sGQuIW`{-V=W(a}+XkJO91sUr3ZQrdVjwq~njEzhNk3-P<#gH>%f)1%0G z9R7}8V&%h(V{^uXY0aiC9E(=asPHU}@Hf}rT%JTsJVwsKWCdOea|&y}MM*FjSQ9rt zVPXD_Q@k7FLL6`8;p{GL9PhVcuG$;mV9Ktu-ph{Td|P8}#<`5Tq$-->VVYVXkq!e@ zSyIPqi!Lv3tJ5uwqra90TUmoECodVbnAB)J2E1SFkTD^E`Y~k(4TLp*Re9oq@s-<~ zg@CiBw~kGMo@0$g#n+bHnQ9I7cS@|k&{Gf5@4*`jFk@nMZ)*p9G>7u?!sMxS zOOqSB8}WLMwC>sKF~(?a)-cXa))ps@ce<-8-o(KlU9BRWKIh)Fxbz*G z_7L}+rg`r`ow^0*6M0rMiP78f0{`<$*-9%_%3w-WgPBf|DaCxA=;QF07xc7HSvnbR z$>7g->4PB-p7h*{H|N)C4fHX}>&pm7r!k~TuE515+Lab&Zvv-oYuSz?a+=%nc;=sz z@;Iv8L=0Z1|MtSDg}I4*F!Q;ltt6}d#!bwx4sEF2hBs?0%)j4B^?W$7;9uN{#ndd9 zNb{O8B979msn%;~UMY;VW2%Gd)Tq>B$WlGM4nM?uo4mO2b=PhSv!8~wjgQmN-uw$9 z$S3eo0Ds)kSyfW3633`?x_x(oz>T5DNUP*SNlwgeCE4Ya*4nx;ysFbkG>R}}`>ch%r$D*VO#qXrssFU5yjaBfYy?n>{X z__WuNTxnc>m*l8L*5+7T7klj+`*-i$zk4rzbWI;IpL*?{eGfjk=h2;a@4j!}BfEe7 f>ZiZ%y6@55kNo=O@_%zYvg@IH_dPhUqv3x6==)*c literal 10403 zcmb7q2Ut~Cw=IX>iwM#Yup%HLiXx&>#MonRQDY%?QBk8YCi>fAFQ~Ca#okNQV8yPe z>)_ZZDxwtW9YH#r{l+HwUi{zvzjyDoqG#{5)|_+9F~^*Xji;v03ZFD_)`W=f)!JH7 zGp$d~SvY&bSn z*zum>a~Ao99G^RJ{>0f4JGHb_D%EM_{fKE3=Y~Z@?og?moOT%M>8Z9E{8aQy&-&<|{Vwb<(`DgiUI$tcV`#)Ix z)PRYLzX=bY^?wc7q0Jxvza4bH*g@~#cQ7~q56A!S{lwRHSnBB+ZnONykVrj!gD*BR z{C_vH{9pUY-298>8Ipg(YxF-@{J;81hd=(^PyfDy(HA=y|N9QQy8q$$Cx(PI;NU+* zdHW6(U)iSpk9)Qon|!gn>6goISE*H6|J6I^Ll@4PwS<_R6E=B4*p$C_*{1b{36Tx2 zZa3!p+l^KKY!+#1X8y%)7XQAR@n3tLSP(WRB7A7l17qbvuw3$omeZ$x=w0sZs{b+E;z4NIjdD|C3(7}dE9sbwo0<8B+o3E zD`b&q4SYk>o+xF;A@;EN=EdtgG0!M3l_; zQ>$FCU6O6FL);WxmriMh72=iPX>BEW3gX4)E;uR4{qcvmJfT^yoO0Z1`-3v z`hQ1JNzT8bXu3QbQaxfiwE?}ou)?EpO)4(qaWkzpID!Y-V#VYP#7J^WekjRHTYKv< zASouXph%MNLS8jX}&lv{XEHU>% zEDm8)hn&)K3wyZ(US_m3mt;nB4t5%O6e*6lQyVlA$xdUCjO&s-3D4S!y{_1#EY&C& z(YVIr6giq!(GBMi*T|r+u_O=0W}F)#vc)=XzZF-IC{}gg#;qiIVv${!Ox%#<#fU6% z!YN6fm5+b5pMp3^UPN18shsR^-qh`9R|5dg7EVVzR`hlJ+&&$N=?-2^Y1cJla|jE> z5Ek;HkXE>-;I74-iW*!fs4fZ>D;U54yd5wD*YJCuPdIJgjoVPmk{rSVNV0b^-$+p4 zxcDQ4b!Z`=YrLoyCvnsim#T~LN`@Z881` zwLlUs;Ekyct_S`|@7(zuz7!kku%9PBrl5Wjp5TFvKL2@+KjFVfoWX_CfmYK3zA{G? z5&j3C6@2puHl<<*Uh(BsZ+Bw3vyLB5FR?&;eV&AJH^Vl7Da)JFS_% zV-yG_VSh7YBxRK1kZ%nRm*90*D_V4yq_KZU(CF~8%!-b9d___ayXZ^w9>BC%DD5*}hF!JrY&Vx=Va#L?azpvFZ> z_G=wvKck~Y8wcCsWe(02WM{gY;0}H#8hjVD%jb!&+01Ds_TxiZ_59#eywygsB!ACL z-J)b6GbfzRf2X)&f#TSx2oo)R_hRPBZ`f7gpq;e@DUM>jxJ0sQgMZ<9uDLe>zd@%` zoOOWsi1W4B>CMC3$ijI~m-HIk9aL1Iu+T_sGQx&t-MeS0{KOhtNa9z0H?b2t>F7U< z%p`d>$?Or%Fv}%*Fka(oQ>1jX%FoADJ31z=v<`=@oSc1-WMgFN)uKgM2%nM4-Ha4C z1vvJvVxSA~0;`y|ckw$h6Hkoo!{fB}nsy91e+3NZ7V31I>P+8m;Io21B_8$;-WM|u zuG5kP(;-CeS{(A!#lb>E=UOW+IYUgY&&96PQp8sx5~s3;;293%wW&BB5#m<-s7EJGkuj_$TMEyZhp=v#g$BKLT<*% zaRF7#%dJfCc*Laf{j>Ps?1?)q#fc&L%$hagXpq<~ULvI(1D0fj`yP@!R+1eNnMYji zQOIsg4lO%|coyf5Vpm&6;9=(>lPV$U;^jPX1lL{loFi;BT4D>W`c1LJo8Z_+s~hMn z88PPW<#;tb)RBHUK0O1M$Z}1Fi$nUZ_`^I0Y9HK7hq@GTbvTNYQiUpxN^%cFY^fyg zo>5BA!u-LVTeINFeM9G3)L>)P$U#kU(V$V;=NcQ`j!F`I!w@9bD_?zut=*a{p<0S3 z;%qx#*{%~lG;ds0fJ@zH))|JlBierO2=c=~v`wR3m+TF2y?a{*;GL<66gwCv_2eoK95W!JF&{$B zxb{A*QfP;fn@WtPG=Si36>hc1;h^5QZK7|)0&*e%yJWnnn#;#e^PxK-^w2XTw-ag1 z*8?>D_DB4tkjOQIq0ZboI@_&`%iWqO_DFIE)sX%y9-DB_k_5b(MDkdnY+V}eo5l}b zEU_R3K6IOG#1G;%`R0WD)8<;Cc;kR~*6zl2azCb=#wG;6H*Q%lErSoAYSSph8CSB* z+Ii!!BA8B!<(+Yl(9IC%YH;&2E`;EWwYX7FJe?K^ZR)wtF_I_`yWLWfCzI~a@*%1p zL$Ol4A#BbKaA08^UE!slXV$jvGh*$$SaP0Yt)bX&h0_6j216`p*{F7zZ3rIP5!1KD z>V5_+PWQrlVg=rI6qg3#Xv=(Ym`S8A@1dKXZ`86-q1^>8I&}R$BsvvJ^0Gc2MU3K} z(X}{XSk$8b0$(#TJAFLvY2k_E{J1@Ruz`PviQUYLQ;2f$#J!+pf~5KzVyo!Dt+FBp zSRSa|reiNwC;)m!=--m$fs)*>G`E*~(0D}`Ue9r}#}TD$JyFy{g5rS)oJi;L#G#^b zZsBq+j#l^@V~1%marPd^d$g0uf^q#@A*BR+C@8*eMEYxGWlS8r8$Nabj<)Dii^%MW zTIAWQ#T_MiJT7bZS88;mZCjkKQ(&+b2U96%CK4^12r0}^tE@WI%@>cF+F^Bnw+Y^r zc-Bws=u}#c8(U4ax04(K|5C+_uJBsNV#VtI%9QW04gF%IY5>8L0q z<+UMOw8L#%T+2XmUJYU^v{%W!fi8OuDA^k+9(@O{Wzm;kZv

{0s#sZgz`8d&mbHCQcvmW|OmEgp)C)jEPb%QSR#KZEgV;bX#PHI=u zU~C6m=p*)+;*a{CBD*;`?56e%J^xB>kc?NslgldUjf78BP)v$YCaC}h5F;D8=H>QA zivx->@uAl+{2qjtc68%?W1MAP%qBF9@hA&BvlLPAqYa7f1;Lx*(3%EZp<+=2vhdR= z9_}HoR*5@wXi~LlZy${k?LD~D#UhjDWYa(VfD~|9pCG%2Yh6mXlyIBCy0w`g#j$0G ztIkXnI}3;CH4{hu#m;_1#I5g|`60omU3yz9+nn$!{AnC)tI^<4OIC)vV{o*XVPok% zqwU(Nuo_PT2C&S$!24|6F*E;!9p=5dWLt}M{)5CBW71$eu4jATa=l@02hw63-no2R z;n)(1-RwFK(=ZMvnY?>TdCm)Xn_-k&mS*CPm{!BBvA)kjmWxA@+(|2o7T@otU69wT z#^tlOg)hr}CfKGA^LyN4C-d=|;!tV?zU zPU#T0r|Fn8h%Fp3%cTg<1MISif`|NlTajK1@K`Rc#kpdA%RsyUTvqX#`)HYru6k|@PCg&$gXgnnQ55`4X=59}j$*lr zzIspKM>-p5(ZSoy3wXGDX4XMe8K)38ZDPJ%20inJQMz5BoI4XYiT|9&eEA^O_*#w{ z;BJH!9-jS_@&#=>su?Sqd$((?4t6!uAi>^%%}0}#8vR9$5MAGS2tBbkTP`Rt(#9dN zT9;CLmwlizEtrj{GBV+H(%5xwc#z<@nvSdXF6D+3oQ&{Pg)^m0m#f&TSAv9eOQx1B z)b6zJt*b@rxw<%BTm2DRbBl3IJGEU}UP+~aT`qgt;(Uvx*rOSQW16bTjgagpR(wN& z^+9PI2+MbtO1crkvENQ^5rGslPqC_3oxcBKrGhL$Qd4HnA%6S^-4Jh7MqlmpE#MjV z6JZaBGGRA6BS~kvTIKdVZi$@~{ZT<3T8Y&|snFALR|)qut2DhO*z0a7jt{8(9>>4o zawoO%*pg|#Q@65^Hf;*_wMLR8hxzuP=XXO*_MM?*@{Lx`_RNOUY^hCr{Cp6jU1&s4 z$EXw~wHO<+?6X5Wk<`}FUmp(*dUavGzUymGC!Z_QWi?FrOz?joxjYMMV~AV%z0CE3 zgYm(>OAC7f{ZTcs@p3rs3{oaHbFi;3@vF%CXiptWMtgrD|Fusc>NO<3?;1F~Z}Vx) z&=uH5TLtdha@@i+|0ST5)tf@*XTL?wyLBCGftMo(6L!0@I^bGA>S>hV)oj<+*`Er^ z@ThSK))XPmz0#k|avjlRLlKpa$b9Nrd!K+GsBG6WZxS_5cvm*E%~(gvzU{?YD_yZU z9ErVJ&zgt0)LL4)pLH(QxX53FT z3zb;yjCI;L5$2(4+Aue`g2VlYkQ3Uto?)7ykGuJ^{o16mbrw5)b4*%`yN)>R88~q9 zcS?!5!%}&cVuPJpe%2>uXguAPj7OfJi!C(oi!3K7r66<1wEW?DJ$5Q&Ix(ikxncxiTupsZS~m?zuER_i$nc}_te8VabsFL4X*0C zxstagVTD0Rm?V2NBWYhFvPHd2n7+VH2dZj3ZkUM~{5>_Z{Cliv$p$!~Y^Ra? zselz%BpR?Dd*Nt7!&_#5z11*V{p&?zt+v5l>?!bs;sjS3Wry~|gPGVU5?z#Z?12Nm zzO{=a*-ll%~$>{7M$Os}m8*7H+Po8?O4}hBIF4VpkxJ zifdTU*h91x8}O$$-iHR&7kJqfdba76s?+K--u*NccORWT18@o*gckqR9Zm; zwd&`*d}r!q1~h^>D2_tS1#hIWb(CaIHj$`S=?=B3?!-jiIdW!)VVTtV4_K(9Wn4@f zFE$cz56n3Uqu`vyVuU!gietf<6R>VvZ2{uds(vHBLfjm&)=APV#cuHjZivV^epo#O z8;$hGIJC+&%aUY$B>RhX1%3rc)M5<2^HJsjZ+}s%zTp?j+_!nE4Fex-jobS9WjJQ9 zTZH?Kv2UQeIS;nymp(OLRn`P5bK~3ibFSDSmXG#VPV?krqkO!qY=q~{#8!8lac8Kc zrpl-=e7vX^yU9@0|Jvdr+spZrv1rQMRZQ@Kw20x%dd_V zKFW8Vfl+&Bs-C3|mMxhDQDQyTEuAfCit%>VNVUoud+SLtcRg_5iCu%%D79)m&*_5b z&*dN-BK%pZZVm`VLZ43LO3(D-;dt1yOB-C%u#@h`vV1l>kY!4Z6Ml}3wD6>Lbrbq7 z#kgbV(MYy*=Ku`x9FT}pxSl#HT@i-!G;BK69Y2vM>MK5DS5QuHkccKv-wMb3v}!** zwM3j5R-3E3`cY|~8HN>B?c4-MFaCUJC9f6aemc1>q`Z%uB+L-&=SXs&T7GI^-iI;Y zqJ`fFe>M-QOsAHh?y*%O+Sd&Tnj<15`D<=VceJAv&BZI;Kdi980X>}cZ_%6-aX{Ok zN>$s%1JAn{nt$>hNe+F4>opyaWb0`pR*|@(y%*)<9T~2ib6{nNdjZMfs;)JY#n{AS zYCAj(u5{>>S319EW_K_9$;w^GZ&<^CyH07%R%{R6GF7`#f=GBKcBZ@FwXry#kK-ZL zc<7PW$BmP@$29?BAMO^4U9>Ti4GU2-%~L$`cJJ3cL#&WVXJ=WiAK~e6?=F1)59iR5 z8c(-0Tb*Ktxj~n@rmJm(37LMBA_)`r* zti&IZ96X={wrb&FTeZq^!XQFpsU+KT4}aV)^T>BkwX5NI*U=5t4f4j1ZJRiomCu#L z9+B!w3%g>gSTUMxd1;)8&J!`@*YB#S5HyL187&?T!g=e;A{_73Vqm@pE)82~_o)L8 z_TpX3S_&Aosy}YHS%}TpJcI>oQ?t^DBI7L|~gT%^l$_-uuUQZjv zFq|L(y=g{pc%+JTMNKR+#D?HZXBFanmJ*9+zpX7rOyfF{KxoDElF1edIp0HcaD_#& zS3Ls^8>G0?%&}RL>>Q%u9izC4KbngRRmMuS_Y_g$h-}5oU*}2Ed5G)mXqHh=&ik2t>XC%v2!X@i>;F+8;Z!U#g@X>6jXOV@hUFyi|C^hr< zf}NtMuFb&V!QD$8Ok9vs)t`m`4$hh5)Zkz;KD)6W5Z$gRJq>tt5Lfl^CJTE^#s!iYe^sqZ92xliTx2k5<5_1Nj9 zZOzGe2UEnlWM~lQ%>RB*2{)@~QqPW9%ataM#9OsWFK9eYvOZ1~52~BA8s|@>z6_5tx_M!*&i<1;IlH7Y3mFD&th{^6+>yBTw z8?@sj+^NhLaX2Q{jT#9JJSjF3QFDIn0LOs3( zuU0pdgrv2^2?A+KfG*j4uEGn|5nqW7d6H&@{V2)}wQ3{TMv{Zns@4R*T15x^g4bWA z{a`^iKeC#H583#TNd!jWiCS`aZpo^13@WyeWvaXYer+)RfU`xnSGh0<*UOHCB zsV1~64o}^(SqJZ!nzX{1i0^RUmH)dANvbM#4hKVLl~?v^FLD)qZ^ZnJn=01K>8sNX zcMWmJ3!A)h^Xphh9^=A%hWTVgmA+a3;n-oQQ1{3d7Mkk%M+p>R!Z@ z;8}rXD&B;6dRppPNOI%xCU|3%+p_oK3U|DBux3`gM9N4!aS{3Wb6{Vk$N4^U5j*MF zoZd8k5MGa6f}_sk(i$P5mLL@^h7MKm;67!vGC_+p$@k38%UmorS^sDV(I(4GlG}@m zc;`0=>nAvs(rr(WFnqG2`R8aD{a=guU)P(7)%5qW^yXyB&#N8UL}rbP(#9_5p{&$u4N^Go0T#3I&Id%jy#% zu=MgNL!4L>RwwFlxHIBM5Mf)fFW93mw&awn`e3=CQX2-DhyyNm^EkJ9OOktvcZF_& zxG~%+AYW#wVx12c(cg^gN0G1+$^A-Ln#E0TChU9e(~P%@JP2uesb^b8itX6gSp@d4 zIO>`gj$Kn*M@X8t*{m+-`Y3vJkwX=38dA91kj+@DUBp@z%U~Z-`)!(}Iiie7u8S?9 zVvEwr_tGQ1EW&%SqN~_Jl@%yys;KLJ$MNhz;uhj^rr=ga9o5ipIAQfsgM(cm`F=gm zJbf_!9L6cjX&XrG%5qsDwzGaJ{lOXnJW`}HF`h}9EZ#GPh|(rFs8-pLoE1QE4)TWI zt|5LO-oDm3qESXYZ4s6n5*3o#;`rWi+F;*N2j~1Vu%jy%yrC;@vSS!v*;bORv8}$} zoawkWxDIg~;Wl-zY@cq}3wMh>l-Bllo3F|nD`b8;WJ+l@uNV0>%j87dw#S7khp&mf zE9|!u#c~af>8ImKS@Rxv$}7=oJf@!5*&DGlaHGPh%SX})_EB&4DDHq~BuG{-j{Nyu zHKLT&c^JQE<^tl<$>vGljN^*nzxa9RA6Gpj;b)~pH2A6x{R4Zm;H%Ot_t^J)v zttm}B%A7mrUxD{|x|zi49apM}=Qb>PV{lxo8d1U8aiu@j2Czc^ES}Y|So8YA9w!~$ zO|(kV+&Ow;Dt*NJRK$jMn< zY)9}=Y-ojN&Sk}j%X2sGf!qG#nR4YyFD)&@Ase;IsMw0D-S$%M6Gu(6DHJQ2eQfG( ziOVFL&jYE)L|i=+se~uH=7N_#dOA9FtP)vFxgG99k)k4BKZe>5dvwGNHeJ@PJcnAv z-5%ij47MuFt<8%IgByBz^S>tm#tn`9-dtS|3*7lSBfCw;xrGs76C%Q9@jI;eolpyQh`!L7YEv&Xn-k I>VaH9{>OV diff --git a/python/onnx_converter/onnxconverter.py b/python/onnx_converter/onnxconverter.py index c710c05..7657a19 100644 --- a/python/onnx_converter/onnxconverter.py +++ b/python/onnx_converter/onnxconverter.py @@ -7,7 +7,6 @@ # Helper Functions - def get_shape(container, node_id): dim = container.__getitem__(node_id).type.tensor_type.shape.dim dim_list = list() @@ -15,7 +14,6 @@ def get_shape(container, node_id): dim_list.append(int(dim.pop(0).dim_value)) return dim_list - def get_output_dim(node_id, graph): model_output = graph.output model_valueinfo = graph.value_info @@ -28,11 +26,9 @@ def get_output_dim(node_id, graph): output_dim.append(buf) return output_dim - def get_input_dim(layers): return layers[-1]['out_shapes'][0] - def create_wbdim_map(graph): wbdim_map = {} for init in graph.initializer: @@ -46,257 +42,260 @@ def create_wbdim_map(graph): wbdim_map[n] = l return wbdim_map - - -parser = argparse.ArgumentParser() -# parser.add_argument('--model', type=str, required=True, default="mnist-8.onnx") -# parser.add_argument('--model_output', type=str, required=True) -# parser.add_argument('--config_output', type=str, required=True) -parser.add_argument('--scale_factor', type=int, default=2**9) -parser.add_argument('--k', type=int, default=19) -parser.add_argument('--eta', type=float, default=0.001) -parser.add_argument('--num_cols', type=int, default=6) -parser.add_argument('--use_selectors', action=argparse.BooleanOptionalAction, required=False, default=True) -parser.add_argument('--commit', action=argparse.BooleanOptionalAction, required=False, default=False) -parser.add_argument('--expose_output', action=argparse.BooleanOptionalAction, required=False, default=True) -parser.add_argument('--start_layer', type=int, default=0) -parser.add_argument('--end_layer', type=int, default=10000) -parser.add_argument('--num_randoms', type=int, default=20001) -args = parser.parse_args() +class Converter(): + def __init__( self, model_path, scale_factor, k, num_cols, num_randoms, use_selectors, commit, expose_output): + self.model = onnx.load(model_path) + self.scale_factor = scale_factor + self.k = k + self.num_cols = num_cols + self.num_randoms = num_randoms + self.use_selectors = use_selectors + self.commit = commit + self.expose_output = expose_output + def to_dict(self): + model_graph = self.model.graph + model_input = model_graph.input + model_nodes = model_graph.node + model_init = self.model.graph.initializer + + layers = list() + tensors = list() + commit_before = list() + commit_after = list() + wbdim_map = create_wbdim_map(model_graph) -# Loading ONNX model + node_id = 0 + init_id = 0 + wb_ids = [] + fullc_wb_ids = [] -onnx_model = onnx.load("mnist-8.onnx") -scale_factor = args.scale_factor -model_graph = onnx_model.graph -model_input = model_graph.input -model_nodes = model_graph.node -model_init = onnx_model.graph.initializer - - -layers = list() -tensors = list() -commit_before = list() -commit_after = list() -wbdim_map = create_wbdim_map(model_graph) - -# Converting Layers -init_ids = [] -node_id = 0 -init_id = 0 -wb_ids = [] -fullc_wb_ids = [] -for node in model_nodes: - if node.op_type == "Conv": - layer_type = "Conv2D" - inp_idxes = [] - - inp_idxes.append(init_id) - for i in range(init_id+1,init_id+len(node.input)): - inp_idxes.append(i) - wb_ids.append(i) - out_idxes = [init_id+len(node.input)] - init_id = out_idxes[0] - - # print(inp_idxes, out_idxes) - inputs_dim = [] - for input in node.input: - if input in wbdim_map.keys(): - inputs_dim.append(wbdim_map[input]) - elif input not in wbdim_map.keys() and node_id == 0: - inp_dim = get_shape(model_input, 0) - inputs_dim.append(inp_dim) - elif input not in wbdim_map.keys() and node_id != 0: - inp_dim = get_input_dim(layers) - inputs_dim.append(inp_dim) + for node in model_nodes: + if node.op_type == "Conv": + layer_type = "Conv2D" + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) + node_attr = node.attribute + kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] + stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] + padding = str(node_attr.pop(0).s) + if padding == b'SAME_UPPER': + padding_code = 1 + else: + padding_code = 0 + params = [0, padding_code, 1, stride[0], stride[1]] # 1 == ReLU + elif node.op_type == "MaxPool": + layer_type = "MaxPool2D" + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) + node_attr = node.attribute + kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] + stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] + params = [kernel[0], kernel[1], stride[0], stride[1]] - output_dim = get_output_dim(node_id, model_graph) - node_attr = node.attribute - kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] - stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] - padding = str(node_attr.pop(0).s) - if padding == b'SAME_UPPER': - padding_code = 1 - else: - padding_code = 0 - params = [0, padding_code, 1, stride[0], stride[1]] # 1 == ReLU - elif node.op_type == "MaxPool": - layer_type = "MaxPool2D" - inp_idxes = [] - inp_idxes.append(init_id) - for i in range(init_id+1,init_id+len(node.input)): - inp_idxes.append(i) - wb_ids.append(i) - out_idxes = [init_id+len(node.input)] - init_id = out_idxes[0] - - # print(inp_idxes, out_idxes) - inputs_dim = [] - for input in node.input: - if input in wbdim_map.keys(): - inputs_dim.append(wbdim_map[input]) - elif input not in wbdim_map.keys() and node_id == 0: - inp_dim = get_shape(model_input, 0) - inputs_dim.append(inp_dim) - elif input not in wbdim_map.keys() and node_id != 0: - inp_dim = get_input_dim(layers) - inputs_dim.append(inp_dim) - output_dim = get_output_dim(node_id, model_graph) - node_attr = node.attribute - kernel = [np.int64(_).item() for _ in node_attr.pop(0).ints] - stride = [np.int64(_).item() for _ in node_attr.pop(0).ints] - params = [kernel[0], kernel[1], stride[0], stride[1]] + elif node.op_type == "Relu": + layer_type = "ReLUONNX" + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] + inputs_dim = [] + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) - elif node.op_type == "Relu": - layer_type = "ReLUONNX" - inp_idxes = [] - inp_idxes.append(init_id) - - for i in range(init_id+1,init_id+len(node.input)): - inp_idxes.append(i) - wb_ids.append(i) - out_idxes = [init_id+len(node.input)] - init_id = out_idxes[0] + output_dim = get_output_dim(node_id, model_graph) + params = [] + + elif node.op_type == "Reshape": + layer_type = "Reshape" + inputs_dim = [] + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + # wb_ids.append(i) // No wb for reshape + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] - # print(inp_idxes, out_idxes) - inputs_dim = [] - for input in node.input: - if input in wbdim_map.keys(): - inputs_dim.append(wbdim_map[input]) - elif input not in wbdim_map.keys() and node_id == 0: - inp_dim = get_shape(model_input, 0) - inputs_dim.append(inp_dim) - elif input not in wbdim_map.keys() and node_id != 0: - inp_dim = get_input_dim(layers) - inputs_dim.append(inp_dim) + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input]) + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) + output_dim = get_output_dim(node_id, model_graph) + params = [] - output_dim = get_output_dim(node_id, model_graph) - params = [] - - elif node.op_type == "Reshape": - layer_type = "Reshape" - inputs_dim = [] - inp_idxes = [] - inp_idxes.append(init_id) - for i in range(init_id+1,init_id+len(node.input)): - inp_idxes.append(i) - # wb_ids.append(i) - out_idxes = [init_id+len(node.input)] - init_id = out_idxes[0] - - # print(inp_idxes, out_idxes) - for input in node.input: - if input in wbdim_map.keys(): - inputs_dim.append(wbdim_map[input]) - elif input not in wbdim_map.keys() and node_id == 0: - inp_dim = get_shape(model_input, 0) - inputs_dim.append(inp_dim) - elif input not in wbdim_map.keys() and node_id != 0: - inp_dim = get_input_dim(layers) - inputs_dim.append(inp_dim) - output_dim = get_output_dim(node_id, model_graph) - params = [] + elif node.op_type == "Gemm": + layer_type = "FullyConnected" + inputs_dim = [] + inp_idxes = [] + inp_idxes.append(init_id) + for i in range(init_id+1,init_id+len(node.input)): + inp_idxes.append(i) + wb_ids.append(i) + fullc_wb_ids.append(i) + out_idxes = [init_id+len(node.input)] + init_id = out_idxes[0] - elif node.op_type == "Gemm": - layer_type = "FullyConnected" - inputs_dim = [] - inp_idxes = [] - inp_idxes.append(init_id) - for i in range(init_id+1,init_id+len(node.input)): - inp_idxes.append(i) - wb_ids.append(i) - fullc_wb_ids.append(i) - out_idxes = [init_id+len(node.input)] - init_id = out_idxes[0] - - # print(inp_idxes, out_idxes) - # inp_idxes = list(range(init_id, init_id+len(node.input))) - # init_id = init_id + len(node.input) - # out_idxes = [np.random.randint(0,1000)] - for input in node.input: - if input in wbdim_map.keys(): - inputs_dim.append(wbdim_map[input][::-1]) #Specific to FullyConnected, to match with rest of zkml - elif input not in wbdim_map.keys() and node_id == 0: - inp_dim = get_shape(model_input, 0) - inputs_dim.append(inp_dim) - elif input not in wbdim_map.keys() and node_id != 0: - inp_dim = get_input_dim(layers) - inputs_dim.append(inp_dim) + for input in node.input: + if input in wbdim_map.keys(): + inputs_dim.append(wbdim_map[input][::-1]) #Specific to FullyConnected, to match with rest of zkml + elif input not in wbdim_map.keys() and node_id == 0: + inp_dim = get_shape(model_input, 0) + inputs_dim.append(inp_dim) + elif input not in wbdim_map.keys() and node_id != 0: + inp_dim = get_input_dim(layers) + inputs_dim.append(inp_dim) - output_dim = get_output_dim(node_id, model_graph) - params = [0] - else: - node_id += 1 - continue - layer = { - "layer_type": layer_type, - "params": params, ## Change params HELP HELP HELP HELP - "inp_shapes": inputs_dim, - "inp_idxes": inp_idxes, ### RANDOM COME BACK HERE HELP HELP HELP HELP - "out_idxes": out_idxes, ### RANDOM COME BACK HERE HELP HELP HELP HELP - "out_shapes": [output_dim], - "mask": [] - } - layers.append(layer) - node_id += 1 - + output_dim = get_output_dim(node_id, model_graph) + params = [0] + else: + node_id += 1 + continue + layer = { + "layer_type": layer_type, + "params": params, ## Change params HELP HELP HELP HELP + "inp_shapes": inputs_dim, + "inp_idxes": inp_idxes, ### RANDOM COME BACK HERE HELP HELP HELP HELP + "out_idxes": out_idxes, ### RANDOM COME BACK HERE HELP HELP HELP HELP + "out_shapes": [output_dim], + "mask": [] + } + layers.append(layer) + node_id += 1 -# Converting W&B -init_ct = 0 -for init in model_init: - if init.data_type == 1: # Tried avoiding unwanted init with init_id, but not working, used data_type instead - - shape = [np.int64(dim).item() for dim in init.dims] - if len(shape)>1 : - buf = shape[1] - shape.remove(shape[1]) - shape.append(buf) - - if wb_ids[init_ct] in fullc_wb_ids: ## Not good - shape = shape[::-1] - for i in range(1,len(fullc_wb_ids),2): - if wb_ids[init_ct] == fullc_wb_ids[i]: - shape = [shape[0]] + # Converting W&B + init_ct = 0 + for init in model_init: + if init.data_type == 1: # Tried avoiding unwanted init with init_id, but not working, used data_type instead + shape = [np.int64(dim).item() for dim in init.dims] + if len(shape)>1 : + buf = shape[1] + shape.remove(shape[1]) + shape.append(buf) + if wb_ids[init_ct] in fullc_wb_ids: ## Not good + shape = shape[::-1] + for i in range(1,len(fullc_wb_ids),2): + if wb_ids[init_ct] == fullc_wb_ids[i]: + shape = [shape[0]] + raw_data = numpy_helper.to_array(init).ravel().tolist() ## Orientation of ravel ????? + data = [] + for i in raw_data: + if isinstance(i, float): + buf = np.int64(np.round(i * self.scale_factor)).item() + data.append(buf) + elif isinstance(i, int): + buf = np.int64(i).item() + tensor = {"idx": wb_ids[init_ct] , + "shape": shape, + "data": data + } + tensors.append(tensor) + init_ct+=1 + final_dict = { + 'global_sf': self.scale_factor, + 'k': self.k, + 'num_cols': self.num_cols, + 'inp_idxes': [0], + 'out_idxes': [init_id], + 'layers': layers, + 'tensors': tensors, + 'use_selectors': self.use_selectors, + 'commit_before': commit_before, + 'commit_after': commit_after, + 'num_random': self.num_randoms, + } + return final_dict + def to_msgpack(self): + final_dict = self.to_dict() + model_packed = msgpack.packb(final_dict, use_bin_type=True) + final_dict['tensors']=[] + config_packed = msgpack.packb(final_dict, use_bin_type=True) + return model_packed, config_packed + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--model', type=str, required=True) + parser.add_argument('--model_output', type=str, required=True) + parser.add_argument('--config_output', type=str, required=True) + parser.add_argument('--scale_factor', type=int, default=2**8) + parser.add_argument('--k', type=int, default=19) + parser.add_argument('--eta', type=float, default=0.001) + parser.add_argument('--num_cols', type=int, default=6) + parser.add_argument('--use_selectors', action=argparse.BooleanOptionalAction, required=False, default=True) + parser.add_argument('--commit', action=argparse.BooleanOptionalAction, required=False, default=False) + parser.add_argument('--expose_output', action=argparse.BooleanOptionalAction, required=False, default=True) + parser.add_argument('--num_randoms', type=int, default=20001) + args = parser.parse_args() - raw_data = numpy_helper.to_array(init).ravel().tolist() ## Orientation of ravel ????? - data = [] - for i in raw_data: - if isinstance(i, float): - buf = np.int64(np.round(i * scale_factor)).item() - data.append(buf) - elif isinstance(i, int): - buf = np.int64(i).item() - tensor = {"idx": wb_ids[init_ct] , - "shape": shape, - "data": data - } - tensors.append(tensor) - init_ct+=1 - -# for l in layers: -# print("ids: {}".format(l["inp_idxes"])) -# print("shapes: {}".format(l["inp_shapes"])) + converter = Converter( + args.model, + args.scale_factor, + args.k, + args.num_cols, + args.num_randoms, + args.use_selectors, + args.commit, + args.expose_output, + ) + model_packed, config_packed = converter.to_msgpack() -# Converting to msgpack + if model_packed is None: + raise Exception('Failed to convert model') -final_dict = { - 'global_sf': scale_factor, - 'k': args.k, - 'num_cols': args.num_cols, - 'inp_idxes': [0], ### RANDOM COME BACK HERE - 'out_idxes': [init_id], ### RANDOM COME BACK HERE - 'layers': layers, - 'tensors': tensors, - 'use_selectors': args.use_selectors, - 'commit_before': commit_before, - 'commit_after': commit_after, - 'bits_per_elem': None, - 'num_random': args.num_randoms, + with open(args.model_output, 'wb') as f: + f.write(model_packed) + with open(args.config_output, 'wb') as f: + f.write(config_packed) -} +if __name__ == '__main__': + main() -with open("first_transformed_onnx.msgpack", "wb") as mfile: - mfile.write(msgpack.packb(final_dict)) diff --git a/src/layers/conv2d.rs b/src/layers/conv2d.rs index 4ed54d8..10aba60 100644 --- a/src/layers/conv2d.rs +++ b/src/layers/conv2d.rs @@ -299,7 +299,6 @@ impl Layer for Conv2DChip { let inp = &tensors[0]; let weights = &tensors[1]; - println!("wt shape: {:?}", weights.shape()); let (oh, ow) = Self::out_hw( inp.shape()[1], inp.shape()[2], diff --git a/src/layers/dag.rs b/src/layers/dag.rs index 8a4f0f9..2af17d0 100644 --- a/src/layers/dag.rs +++ b/src/layers/dag.rs @@ -26,7 +26,8 @@ use crate::{ square::SquareChip, squared_diff::SquaredDiffChip, tanh::TanhChip, - update::UpdateChip, relu::ReluLayerChip, + update::UpdateChip, + relu::ReluLayerChip, }, utils::helpers::print_assigned_arr, }; @@ -83,12 +84,10 @@ impl DAGLayerChip { layer_idx, layer_type, inp_idxes, out_idxes, layer_config.layer_params ); - let vec_inps = inp_idxes .iter() .map(|idx| tensor_map.get(idx).unwrap().clone()) .collect::>(); - // println!("vec_inps: {:?}", vec_inps); let out = match layer_type { LayerType::Add => { let add_chip = AddChip {}; diff --git a/src/layers/fully_connected.rs b/src/layers/fully_connected.rs index 3e87fed..05642f2 100644 --- a/src/layers/fully_connected.rs +++ b/src/layers/fully_connected.rs @@ -267,9 +267,6 @@ impl Layer for FullyConnectedChip { .unwrap(); let mm_div = if tensors.len() == 3 { - println!("shape vec: {:?}", shape); - println!("tensors shape: {:?}", tensors[2].shape()); - let bias = tensors[2].broadcast(shape.clone()).unwrap(); let bias = bias.iter().map(|x| x.as_ref()).collect::>(); let mm_div = mm_div.iter().collect::>(); diff --git a/src/layers/max_pool_2d.rs b/src/layers/max_pool_2d.rs index 382d3d0..73af4c3 100644 --- a/src/layers/max_pool_2d.rs +++ b/src/layers/max_pool_2d.rs @@ -53,9 +53,6 @@ impl MaxPool2DChip { // Only support batch size 1 for now - println!("params: {:?} ", params); - println!("inp: {:?} ", inp.shape()); - println!("config: {:?} ", layer_config); assert_eq!(inp.shape()[0], 1); let out_shape = Self::shape(inp, layer_config); diff --git a/src/model.rs b/src/model.rs index e8230c3..36272e6 100644 --- a/src/model.rs +++ b/src/model.rs @@ -346,7 +346,6 @@ impl> ModelCircuit { let shape = flat.shape.iter().map(|x| *x as usize).collect::>(); let num_el: usize = shape.iter().product(); if panic_empty_tensor && num_el != value_flat.len() { - // println!("Shape: {:?}", shape); panic!("tensor shape and data length mismatch"); } if num_el == value_flat.len() { diff --git a/src/utils/loader.rs b/src/utils/loader.rs index 3f3bda8..2a2890c 100644 --- a/src/utils/loader.rs +++ b/src/utils/loader.rs @@ -53,8 +53,7 @@ pub fn load_model_msgpack(config_path: &str, inp_path: &str) -> ModelMsgpack { rmp_serde::from_read(&mut reader).unwrap() }; for tensor in inp { - model.tensors.push(tensor.clone()); - // println!("tensor: {:?}", tensor); + model.tensors.push(tensor); } // Default to using selectors, commit if use_selectors is not specified From 0f0625339856232eafbe37890d3f88b80aad70b5 Mon Sep 17 00:00:00 2001 From: Alluri <53390631+disirulla@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:32:40 -0500 Subject: [PATCH 6/7] code cleaning 2 --- public_vals | Bin 320 -> 320 bytes python/converter.py | 16 +------ python/onnx_converter/batch_size_modifier.py | 42 ------------------ python/onnx_converter/config.msgpack | Bin 921 -> 913 bytes .../first_transformed_onnx.msgpack | Bin 8685 -> 8677 bytes python/onnx_converter/onnxconverter.py | 10 ++--- src/layers.rs | 2 +- src/layers/conv2d.rs | 1 + src/layers/dag.rs | 6 +-- src/layers/fully_connected.rs | 3 -- src/layers/layer.rs | 3 +- src/layers/max_pool_2d.rs | 1 - src/model.rs | 5 +-- src/utils/loader.rs | 2 +- 14 files changed, 15 insertions(+), 76 deletions(-) delete mode 100644 python/onnx_converter/batch_size_modifier.py diff --git a/public_vals b/public_vals index ab6c8307475494e349d76f6ac8e7df17ff6ee0bd..853dcc278bc0e158a88a0e1c9eed86b2fb9bc350 100644 GIT binary patch literal 320 zcmX^7`~UmNUmrS8EZAA;@xoXmHlnd{n`^)h%>{-Ji~LdyKK;QjZ^p@h7qk-4e~=Nc bc7~o``0cM_#;;$8fcrKPaG%p({PqI?{QqX& literal 320 zcmZqv{r~;suMeFk7VNC_cwwv&8`0Rf%{5?$<^scqMSdv;=lJ&u47_ diff --git a/python/converter.py b/python/converter.py index 7bccccc..02b37f8 100644 --- a/python/converter.py +++ b/python/converter.py @@ -5,8 +5,6 @@ import numpy as np import tflite import msgpack -import onnxmltools -import os def get_shape(interpreter: tf.lite.Interpreter, tensor_idx): if tensor_idx == -1: @@ -30,14 +28,6 @@ def __init__( self, model_path, scale_factor, k, num_cols, num_randoms, use_selectors, commit, expose_output ): - - if model_path[-6:] != "tflite": # In the case of a non-tflite input model. - if model_path[-4:] == "onnx": - output_path = os.getcwd() + "/" - cmd = "onnx2tf -i" + model_path + " -o " + output_path - os.system(cmd) - model_path = model_path[:-5] + "_float32.tflite" - self.model_path = model_path self.scale_factor = scale_factor self.k = k @@ -53,8 +43,6 @@ def __init__( ) self.interpreter.allocate_tensors() - - with open(self.model_path, 'rb') as f: buf = f.read() self.model = tflite.Model.GetRootAsModel(buf, 0) @@ -534,8 +522,6 @@ def main(): parser.add_argument('--num_randoms', type=int, default=20001) args = parser.parse_args() - print(args.model) - converter = Converter( args.model, args.scale_factor, @@ -560,4 +546,4 @@ def main(): f.write(config_packed) if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/python/onnx_converter/batch_size_modifier.py b/python/onnx_converter/batch_size_modifier.py deleted file mode 100644 index 53d77da..0000000 --- a/python/onnx_converter/batch_size_modifier.py +++ /dev/null @@ -1,42 +0,0 @@ -import onnx -import os -import struct - -from argparse import ArgumentParser - - -def rebatch(infile, batch_size): - model = onnx.load(infile) - graph = model.graph - - # Change batch size in input, output and value_info - for tensor in list(graph.input) + list(graph.value_info) + list(graph.output): - tensor.type.tensor_type.shape.dim[0].dim_param = batch_size - - # Set dynamic batch size in reshapes (-1) - for node in graph.node: - if node.op_type != 'Reshape': - continue - for init in graph.initializer: - # node.input[1] is expected to be a reshape - if init.name != node.input[1]: - continue - # Shape is stored as a list of ints - if len(init.int64_data) > 0: - # This overwrites bias nodes' reshape shape but should be fine - init.int64_data[0] = -1 - # Shape is stored as bytes - elif len(init.raw_data) > 0: - shape = bytearray(init.raw_data) - struct.pack_into('q', shape, 0, -1) - init.raw_data = bytes(shape) - - onnx.save(model, "/Users/siddharthaalluri/Desktop/sid-alluri/zkml/python/onnx_converter/new_mnist.onnx") - -if __name__ == '__main__': - parser = ArgumentParser('Replace batch size with \'N\'') - parser.add_argument('infile') - # parser.add_argument('outfile') - args = parser.parse_args() - - rebatch(args.infile, '1') \ No newline at end of file diff --git a/python/onnx_converter/config.msgpack b/python/onnx_converter/config.msgpack index 6609bc1413fa10b8856398b55c0e470763fb6a02..302e08b4f759001d3ac4934d8b683772b9ee83fd 100644 GIT binary patch delta 31 jcmbQqK9POG2G%7(sXn2TA2Nzcic=Icic=I Layer for Conv2DChip { let inp = &tensors[0]; let weights = &tensors[1]; + let (oh, ow) = Self::out_hw( inp.shape()[1], inp.shape()[2], diff --git a/src/layers/dag.rs b/src/layers/dag.rs index 2af17d0..12d0f2a 100644 --- a/src/layers/dag.rs +++ b/src/layers/dag.rs @@ -14,6 +14,7 @@ use crate::{ mean::MeanChip, noop::NoopChip, pow::PowChip, + relu::ReluLayerChip, rsqrt::RsqrtChip, shape::{ broadcast::BroadcastChip, concatenation::ConcatenationChip, mask_neg_inf::MaskNegInfChip, @@ -26,8 +27,7 @@ use crate::{ square::SquareChip, squared_diff::SquaredDiffChip, tanh::TanhChip, - update::UpdateChip, - relu::ReluLayerChip, + update::UpdateChip, }, utils::helpers::print_assigned_arr, }; @@ -83,7 +83,7 @@ impl DAGLayerChip { "Processing layer {}, type: {:?}, inp_idxes: {:?}, out_idxes: {:?}, layer_params: {:?}", layer_idx, layer_type, inp_idxes, out_idxes, layer_config.layer_params ); - + let vec_inps = inp_idxes .iter() .map(|idx| tensor_map.get(idx).unwrap().clone()) diff --git a/src/layers/fully_connected.rs b/src/layers/fully_connected.rs index 05642f2..6dbf529 100644 --- a/src/layers/fully_connected.rs +++ b/src/layers/fully_connected.rs @@ -44,9 +44,6 @@ impl FullyConnectedChip { ) -> Array, IxDyn> { assert_eq!(input.ndim(), 2); assert_eq!(weight.ndim(), 2); - println!("input shape: {:?}", input.shape()); - println!("weight shape: {:?}", weight.shape()); - assert_eq!(input.shape()[1], weight.shape()[0]); let mut outp = vec![]; diff --git a/src/layers/layer.rs b/src/layers/layer.rs index 65971c0..c84d943 100644 --- a/src/layers/layer.rs +++ b/src/layers/layer.rs @@ -31,6 +31,7 @@ pub enum LayerType { Pad, Pow, Permute, + Relu, Reshape, ResizeNN, Rotate, @@ -45,8 +46,6 @@ pub enum LayerType { Tanh, Transpose, Update, - - Relu } // NOTE: This is the same order as the TFLite schema diff --git a/src/layers/max_pool_2d.rs b/src/layers/max_pool_2d.rs index 73af4c3..c929092 100644 --- a/src/layers/max_pool_2d.rs +++ b/src/layers/max_pool_2d.rs @@ -52,7 +52,6 @@ impl MaxPool2DChip { let (sx, sy) = (sx as usize, sy as usize); // Only support batch size 1 for now - assert_eq!(inp.shape()[0], 1); let out_shape = Self::shape(inp, layer_config); diff --git a/src/model.rs b/src/model.rs index 36272e6..07b2da3 100644 --- a/src/model.rs +++ b/src/model.rs @@ -53,6 +53,7 @@ use crate::{ mean::MeanChip, noop::NoopChip, pow::PowChip, + relu::ReluLayerChip, rsqrt::RsqrtChip, shape::{ broadcast::BroadcastChip, concatenation::ConcatenationChip, mask_neg_inf::MaskNegInfChip, @@ -66,7 +67,6 @@ use crate::{ squared_diff::SquaredDiffChip, tanh::TanhChip, update::UpdateChip, - relu::ReluLayerChip }, utils::{ helpers::{convert_to_bigint, RAND_START_IDX}, @@ -334,7 +334,7 @@ impl> ModelCircuit { "SquaredDifference" => LayerType::SquaredDifference, "Sub" => LayerType::Sub, "Tanh" => LayerType::Tanh, - "ReLUONNX" => LayerType::Relu, + "ReLU" => LayerType::Relu, "Transpose" => LayerType::Transpose, "Update" => LayerType::Update, _ => panic!("unknown op: {}", x), @@ -608,7 +608,6 @@ impl> Circuit for ModelCircuit GadgetType::SquaredDiff => SquaredDiffGadgetChip::::configure(meta, gadget_config), GadgetType::SubPairs => SubPairsChip::::configure(meta, gadget_config), GadgetType::Tanh => TanhGadgetChip::::configure(meta, gadget_config), - GadgetType::Relu => ReluChip::::configure(meta, gadget_config), GadgetType::VarDivRound => VarDivRoundChip::::configure(meta, gadget_config), GadgetType::VarDivRoundBig => VarDivRoundBigChip::::configure(meta, gadget_config), GadgetType::VarDivRoundBig3 => VarDivRoundBig3Chip::::configure(meta, gadget_config), diff --git a/src/utils/loader.rs b/src/utils/loader.rs index 2a2890c..f3a4721 100644 --- a/src/utils/loader.rs +++ b/src/utils/loader.rs @@ -74,4 +74,4 @@ pub fn load_model_msgpack(config_path: &str, inp_path: &str) -> ModelMsgpack { }; model -} \ No newline at end of file +} From 81245447e40e9ec8b8365deb813e460da40f3c71 Mon Sep 17 00:00:00 2001 From: Alluri Date: Mon, 21 Aug 2023 11:55:43 -0500 Subject: [PATCH 7/7] code cleaning --- src/layers/dag.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/dag.rs b/src/layers/dag.rs index 12d0f2a..aacadd1 100644 --- a/src/layers/dag.rs +++ b/src/layers/dag.rs @@ -83,11 +83,11 @@ impl DAGLayerChip { "Processing layer {}, type: {:?}, inp_idxes: {:?}, out_idxes: {:?}, layer_params: {:?}", layer_idx, layer_type, inp_idxes, out_idxes, layer_config.layer_params ); - let vec_inps = inp_idxes .iter() .map(|idx| tensor_map.get(idx).unwrap().clone()) .collect::>(); + let out = match layer_type { LayerType::Add => { let add_chip = AddChip {};