From 3a17a307ec89737ceaabc07060516ac3f46caeb3 Mon Sep 17 00:00:00 2001 From: AllenTiTaiWang Date: Thu, 14 Sep 2023 22:48:00 +0000 Subject: [PATCH] [ONNX 3] Add ONNX registry tutorial --- .../img/onnx/custom_addandround_function.png | Bin 0 -> 34769 bytes _static/img/onnx/custom_addandround_model.png | Bin 0 -> 7544 bytes _static/img/onnx/custom_aten_add_function.png | Bin 0 -> 12731 bytes _static/img/onnx/custom_aten_add_model.png | Bin 0 -> 8613 bytes .../img/onnx/custom_aten_gelu_function.png | Bin 0 -> 22646 bytes _static/img/onnx/custom_aten_gelu_model.png | Bin 0 -> 6960 bytes beginner_source/onnx/README.txt | 4 + .../export_simple_model_to_onnx_tutorial.py | 3 +- beginner_source/onnx/intro_onnx.py | 11 +- .../onnx/onnx_registry_tutorial.py | 451 ++++++++++++++++++ beginner_source/onnx/onnx_toc.txt | 3 +- index.rst | 7 + 12 files changed, 476 insertions(+), 3 deletions(-) create mode 100644 _static/img/onnx/custom_addandround_function.png create mode 100644 _static/img/onnx/custom_addandround_model.png create mode 100644 _static/img/onnx/custom_aten_add_function.png create mode 100644 _static/img/onnx/custom_aten_add_model.png create mode 100644 _static/img/onnx/custom_aten_gelu_function.png create mode 100644 _static/img/onnx/custom_aten_gelu_model.png create mode 100644 beginner_source/onnx/onnx_registry_tutorial.py diff --git a/_static/img/onnx/custom_addandround_function.png b/_static/img/onnx/custom_addandround_function.png new file mode 100644 index 0000000000000000000000000000000000000000..a0c7000161e9585859c331b73497557814345855 GIT binary patch literal 34769 zcmd?RcT^PLwl3O;ib_VwNkBk}A~_>bL_kGyk_?h_Y`{QHB9b!*NY344RG`UCk{p`Y zWSX2B=vVD;pFQ?Dd*5@<*!R9a-gq^Jqek0QuT`t&n)93AH)n;tR8u0nLvsfN0ud@d zlYb2Y-5>>laLI1n1penz@nZyV!f}1ABnv8o)2#y^u3J4-eF_3q#^7I=+yFk`c6z4k z3Ib8EV*lYNzkaX_0)<#A%RhbNWwbSmpQO10=DlJJwZo zf0hsWa9xj^KPMPtWLqv-ZAB%-|5*Ff@4Tes1Uzn0-aPN!aJg^fv)z3g6v&WOR>tG8 zJrzF+0tLeN-`f!b(*+GOGcjY&FTaogHwpqhR1Up`J%4>5e;s>%PR4uqtPD9j**`_NZ0*~>`$Pbe&tVj7^ z%rTNt@-;mIGk3ChMs$ZP=+7?=8aAO(d*z$Z{EkvyEtK@L7r5;~dvWRY`{huWy0JIQ z+yJAE_TNV1k#dtw^IZRQYJl*oHUF$bcOQ9pIrFLRaOUpf$Fp~h8@250oVXx2TX%Q& zO55?WlCc{Qp6EwrR`T{*^Z|(QI_GCv-FBFtU#T?WrE~L^3N>9#Cu;Glp8q`ca_WfU z6DbzdqjwE9r0*Yac)aja$GL4zJHExi>xI6AILuP|h5p#sri%2c6hh&hFIUDBHp_cP zc9vc%Uy?36=5$M+RZEBDAN$921hDx&@(*A|O>c=ON+jE*Xs zRa@B=psEK>Zh#J7967f0Bg&&G57NmFa2CAT<#k)8Tmp6cn$p+9Zh@QvDI}UiAXY15 z%D<{ixUYwX-EcmtFyULlvu`weV@%)XGTyHnKPs+Z-$ykKu`+bpmV&Ea8#VF83k{np z;;T1vHFei=Ouz88!<@uo;OXd$)LzWxq}Ba9AUeGsy|viWnX!wPD{4d}cY!4i#MKmM z$%l$PvmKk}S;v8zD){S5a0grUD?333vN+a9wd&ZQk|IpD1G-MzUv<(hIggA|456kS zS3XQa8h)4HrE1u+Zm~Ymsh+QY97D3Qdh1ro%-HEHMf<46^G~%3C}(Ah1#V>7l+usX z!%(iYOb71c#lA@d7#$rosc>TBHii2s`N$CvtoV8nu9rRJzAZu{ac$L(+7vRPxtPIP{9&R&|d{4EHG zlrNrn*Qcbh&xQ8kQE2pz96TansdT&YbYT%xX;UG38q6A%sUgl%q(6jPaY9*ccTqB> zx~Hfmq0=lK;2_sE9X~fQF@fuRLEitXn@O0ahX52()a>&>YGfV9R2JAVPM~0IwccyR zQ%QT{XyQvPUf%1Lq)SRQgXmI`v|>Y=fm*htK59Q3GhWzt|NJPqfX2bGRZ|rHwCMV`wrJWyfY4W(UFW1#(PKR5{3CPZIk}i=@f9T~7+k5AQftueh9YOPh zv0PRAn5$N=rP-2p1YTJeFvO&=F9&Aa^L`AsMeLnQJ*GXgbys}RhG&XK{+}+Y@d=R9mCmHj0rzM^2zL2p_ z-q|lMIfob15tP2ln#Qxo@y$4aJLTSiUdQdHJ>IRecG2zd+kdEi5qM4O3ICr zX7zV;pA7X640`2UB7!B@&6@Qu`jFYvKaXX&x^|{3oK%JJ*L?z~o zSXbdpj(i}~cPVr6wk2h}`)Ll=9J2%smFrmhackLJC+rRBG+-3Jevr8m3ROx{sN1Bu zR%nC}K=t;VP{;#CqG(!L9?L+5?6iF*it;L;(-QqT6?MMOL z7x}q9IoCkxo3qC47-%b~GAr-?UVZ7Ff6G)E){wSMx}u`wjP-wu!{82EqhobWN)EQS zNa94%&-(;*ntmt`ePCRdWzGwx60YZIb)OoXX)#;s2|gLNo9mhpx220ty&e`dpiLLe zV^hx5BigTRk(GXr7;HZ|UspLP;XzpCeSLariKq2t*HfOa z!#`7+5-r$eOns6~26; zuumLs&kCTaKnISqY(%kkdC5(XlhYRjdC>jJQmio#s@^8S2|V`5YUTaU=<45ET)K0u z{xFj^WcxuhT^!?hmB%6I&J(jyM1^_{)?E?h~)-zeX6h)6agNy}Zsi2%B74b@q_zRLY#^bPj4+ z%GuP<-5|3hub#k z4!Yo*o!)nu-`QqxZfMp?Qb;%52arb>J@@yhE9@I<*O z4t8Cro1IZ7Q880R8*2Jp7MlGRp~{ps8jfv#s3whK+K^)@u}3!QJ_5*%(YvlIVUwTN z(Mw1RoQAbrpDu|i-}mlcPV&P%s+4Fv9#Zbj**}I=92{vUZl9o! zsHMadrOVIT{_G7&NUP5@@fp7<|_4yHOZwjd*Elz3Q0J-_*YM}(z#m7qTF&}mIJ zrX^pJDWttrrPy@22_>+Zr|oCdQeZ4W%Xc2Fk~7HY)U{H-N%Nr*_tL+Y`Rc9r%+#{) zvRp6fo5|eMCa7ysPEq!WX=a`Ie*284wclLO348WVu0m$IFXiZ6v&ou}l*3Qg12g2Y z#<%c@5SKPYz!8&}_e%U?Mx-%&+$1O9jX)vP?q{unD;wMN1of*_`b}qe-Th7a3BQSh z7CV&?G-M*T91$oYn9!k7tKj+0!zeROz5fzb$ajBxXre_;vi|Jx3X}8CZ=n|DEg7T4 zeO=O2lO$gGY7(r9$d=XMWr)5D;$$*3EH|Y&>M%{)3>+sz4)P(wy7!R6Ip^c=3zMIb z+j|D8_5=XdD{~K+3stQpy;SE&6P3v4=IIO3X(A{7xZ{OK8!1Rb24!oD%q)#0UH9`> zT_v4KOz=HSrz)U3>=^``{;5?;tkQl);wl3-P1~v zBLn#`{JF~(aQ`cvTR4nHG_`l(oO&aa-YtUeLq)*AiAxcxzXROox(ev3N5LB|m)!;#`#MEzQ`-nwOZT`K%$ zNEKbtSe}~K0s?hG^_Rnsd8n^u%vq5a90Pp0Mf5%ha&RlMSkh6KDsE)JrXlpX>10h$ z4#U^_6DMB_F7X16GA^UtWzdeko_}_{`HfXJ6komZ!2m!Vd>hY5D38N$hVaxo4oPtWu_&g+9QewrFW~Mp96| z!2x9@VINhSdeyG-6VHX(RMV-&vQ3*?pg|2PI05L8DwyB3jxX&DL=AjGTd2u{=0CH_ zFFUaSzDq1z^e=+M0~_jr<3AB6S*cIJqM*S!qQjFQUC0?GJ3ydK)ebjR4frjfz-i zQ9fI}l@B(BQ~FGEsv|_7Oqj1d-hl5R0SZ$_@B?zFoMcZmF$X zh!qI*7n=IvkDbGs3*iAlS<2nE;FX?rA@<>9ywn=iunzqZm6T`Yjw0;PS^=wtu@1hx zwX{dlUM%7-D*&J}wJLx*Oz`(V=0aDEH$$y}l_w&^qMU}1jbgKj{R7jEaPs=?pHz(a ziyMzLOncfc&6KAg%nWHSY5A{woCOr!&gM}YUXL4j8ine$oj)gCe-DsR81$Xhl2A@z zBYOUK)k}D0<-XEXm&GtQqtB;*aGLxZDcVZAsZfLZ$z8;uR~h#-fYEBLAlxZ4%U(X) z2|;|K+AVfQJDngjd^qiG&E-8={!+Q6eY2Fh!K?X5NUY*Y>a%&t_EJ zB_46d2exrm06avvu4+nuJ<^+Dh+s4#Y2!?6kCGd{`}(8fvkyHV<6AvncvY;;)ep2f z`u*0@Ox$UnFoM?!j+N>&Xs*gfrd3L48&PgLbW6XW`KszASd@UA`f{A7xwr8t=ksP> zGN0*G;hwb%*DOO+n6fVx*cn_7DcA3RQ#$BmsC#h!pfg1o$z|sgGgFm4jm z)5w+6;H2tMoI!OE(W=jOTZ=Q7BS38j9peXi!cm?~$6|E(&cl9x2Y_R~H_A&J11>6_ z+#EBgz#M?GLhb_gH7NN)gf~3*#~l07z@;PR`&w!_%HCH(TcdH@y|9J6p3~NBz11#e z;@p6%KYa}IP4=Fylk6d5208H?57Zt@isaARmu6me;ZfWK%{vmsi`h24rpu8ED~ zexts8PvLulL}a3(phcGc?ceZo`1I+?pq4o1#Z3bIJgLG@iW27Z++qN1TQ`Qt-5vly z_gtpIf%S}!;@DS_r7<0&!&#a%4RXz;s#E@x%T}>RrjykLuw25-Sl_wsyDUh_C429H65diruu^~^RDHqfU{NucBV{>P}Ab4k~byZG9nV^P&YksT_^h`ztD9d55?iI zD%<8m^S+o^J|C}hI|<0HCs#DyOy^N3yL;BrosYZo7ty(v;jLN_zh{7#0h1dPa(Uh;m2K)k=gh z6Lpo=+3|g2QI(H^JL@x1vu@U|cl$rW{7@aL<${uqXxr4P*ZAid^s7XU>vOXm?K$-~ zBcM48oywt!HLM3W*zl9S>d54^0XLh_5uDI)*4)D6`Aw2;=JG&NK;_WmdLYBIKMOmG zj?jjkPEqsad$A(TJCK$9%p8urKG%?xHie}r8Qe8rCIGKxK#yOgXC(F2uIhdd;8mK7 z-Xy~2_>RNPN#ptW3hHnTcN!k4W)$M5IX`_7dA4V6OL_W?;z^;y^y8U}m&?BlKKODn zh|R7YP3+of*_8<49VO<y@W&f=fhzKEXVCxLE>G863qVZ8W0AstQDg!gdJb0rXdpg_pu}#iwS1qW4c2GZK~t*xjm&4@O5c;6 zlDs2q;2TtBmEMhGJO{|KY?Avsz?#|GG#mL|jIaTF=J)6*{U9*|2ngG2fXo0K|3ks& zp8|K`KS4vf^JuB_HIe-;mTDT%kDt@cz9%@B`D4Jn0l4RZ%%S5Ia0ANz+7DWJzXW`@ z><_Y%zXFL2er2`qk?n2)C8n$7?CB_vAW z&`ku~{e3LXX_F_rZ**K;q|@SHSki0XYbF0CL!4jBq7w&ntng>^L(X>qYGY@Vv>4f5 zVN&k`>9f{tClBY}IqQdRHuz&3+q3#Itx zYeFrg4Jb_IL0R{(PbMUAZ*QLPW3@~Byb3Ui4#|Lkx%I~-*rZvmr&ff4%@+9*OHroH zRI$@(P2G0Hq%OD<7LO$;(3G={XYK$Q)P~Q@CP5`kq)hALlIe~({a#u$*Vt(qlERos z==X%DE107kh~a#ws-&~mS8JQ)JP{)HGIG4_AArC@y|)$YC0UFMs=;Q~G6TX3P1cbl zuIF0vm>Gu=TUB3NA?gIb?GGijL%Xl6C~riOvcDl15Il~#TXk*(k)l+wWr4+5@2}V6 zqdRve?$JUmkJO%2o{w2~JsGHbQUni6GS={cwMnsOro@9l$Ev^}KeBmKtDRo#eMX{T zR(Qm7A8ct?0e$_zY+%|!y)r$ zuD;YjjH5)#am`tFPP9xunw@}Tw$eDmA32+9;eo8f{$v@$8!NuVRu0vFNDVB~aBCY) zN>yH!n$a$Ar$9cx4tn^96#1M8x7TS3?I{vZSjJw&i{qPoM6o6Gms)$dwsLF4K)G z|HH=%>D%QM=i!%fMf6Nl3LPrWK)yc#c$y;EddMx%*D+glHLaVVwXuzj4b$;5LtR25 z@XE@Hu8p)&3Nip`XK2$+~dQnf+=OSk_$Ml$4NiFH5!qzQwN?aHd(=yKFcNs4Km!pS;S_s31geJoznJz`;|8|oW;4E z2OKm}bOtap(BBpAe`{j@pS}E_!mRuIza;YE#Z$Bv0Qp^lN+ZmAu>2K-D8?qj3b1b* z)@m^%L8I7kF|f@!9jIouQY`PRKF~XFx*2ssZ&d%lGT^EnjHrIPcR((wUAZt~+Pzgd zBwl(Ky4ZOqqcD_L+v~nL+DXlCr`5N&2AT-u7TdUXdo1PQvG2*Z_ z-_kD8iJeYpuUXJF+hG_ju5z(GqNlAMK?^i$4_8-YORfiDX;Fhv;<${kAhYh{w~+(U z!1Le(oSBSxacK%e1k^HK3s42its{4Zn#Z!ir2Ln;Io*SsKb>>40Yacg2C$YvHLRfP zvp))pM3t+l%mOIMXCiIwwmQJ6k7JY5n$+qg7`kfi6X2}Vgb0;+7%b_AFyEp#h$qBh z00H|j(gThc)@C|U@d$6wmkbCl*vvmmlCYkSA?-6SPn`#*`QlwMti3{@o%SyO9TiS> zuZqd%ecN2=(v{Y8?QZ}@Uz{ty#zaa9FiJ*XGY5JIod*B9#tNjGjhV_SrGi&279T{Amsntj!uAXBU7`+F3O^jt{6+oaaY@RLlY^ef?tJ!qxxj+0D@_?PFJdI>> zepTopSANyWrRlGDK8PdWPXZ&ctIDPo`T?>5nd*MJW1%$|Ahn}2V zw=$XSfDbT}SR-GTswW^BsFt=iC6*F-2Ru-#AuOW!FZc-f{GW&y(7(q#06+8Z5-b0G zz0ChDZ}PfaQdE~3l%-!>TpVH8=wTr!a!3z89CBWcRkFllC@25~anDstvD%=UhH~H< z;sTJEzWiaE9$;QhYGI?pTK*Tt^JbqT?0Vv{pBcWoMfNvd=PQU5WbKG|j>Z-@0LQ1x~>H~^*le_-{f7e+PQwz!~%yTIlI(M3LxY4Xr`q2I); zRnlc2QK0Z9DonJvYH16RV6peR`fRa^ktf1|l%++0|0Yoy9X{0D9VF6+rGw@J?PEHK zc|#ZCp&b(~x*7o2s=ua zFW#-R9vaq392Ngl0Y&s1TbHrDD{^wsTpip5Py;dFf$|Oeewqnvs*by9I@{pxjHxS& z1|+)#tijLAG*>=)mmBcro(~g6+MsmsYP6jXOR;B-+uC-YPCU9b8TY^yKYu!1wewAZ znhp%*e|Aoc9^ zv0adBcxGd_ca$sVj=Ry&0lBo&lDKk<#??9LNP>k@B9K1^ndxfahi8`V&2Ii@& zGqK6Tls+PSEt6b);G)iMKF;}PgI`RG`zSyi7$)3w@&*VC#^~skpry?OCCxXmP|K%g zF8x1J@T*(_7N#2vFeq^->AZz0C3bEA8KmCuUe$sXjsjqkra+B$$RXleU*_Y-JgJGA zWCL1?7WgL_(kAk6EL}PCT~?#4DZYAqVuCKuSqRbj0_sTB;8Aw4Rk98^z+jly_-5e8 zYHlMh(^p)2Ul9qI9I5f+Vx72AN^T!!qqVG960B{C$iFCQ5ezn85n1XHm`E|!xOE#) zZ-@qEfK)PZGtG-gCC5iI7S9Ys&CJ`YNpn%wO&(n%|Kino-KTwo0LG}Hp@xhLxo)hcV7KKe~O+#;7VFiP8PqdtrvW( zi@4Y*!*BI>*LS^?8OH#ox5qph2TS4+S}O?UoNiy7z>lNs@FpjNNZLe%qF)u)xQAU@ z#%O0$<72p;lx$8q)-JM|?PsY{O}!4KnTkEb!ALNkrlI0*?a(hTmoMRR=e z={0TJ)U9Mmcea&ts<)f6(B*Akl%W9m2>sbc-`#fFR4XexOys!x;wR#(L~)ev5!Qt6iqCD!ug)4iSlu>4U_PrMl(GR1CdQk&)pw%~ z9+icM&}f-Dd}aKVZr%2BEzwel>*|S%qUIP&<>WP>V&mf<|8V~hh;cfvsrk&dDyW3G zpizc2t2`O>W{o2+k!e7lG3iDcpEI~Qzf4K4D`YP}^n6gXdw2a;&%or%jbpu1@60dp zc_RzIqlyJ$(wr@XzMZy<JE}lwp(sf=$#_4l=sS(4 zvj5W*`JHM84u4)C=l*LVtWOt#ysoMgeIqnXYN;q^Bkjyt*986xtsb_rEy%t|4b4B` z6iTMtn<+3oTuB#anQ!07cO{EXt8B7XSTkQ)X^s&tYd83YV0|_E+BX8y@ynK`t?Sn! z4_3ljs79(!7~i@Lnl=6x0U|yz(TeqWJujKC_ywS}P(p=tvuQvRiRt2=mU&};!s%OS zty$gQy;YNA=2XHUx|~A*3-}O70^Gz}*u)(`bNGStx3Fuija48dgEKUrZg|}!$PsmENp4=l27y^z(3U#c{d3^}z`y>L;dM*Dfy4uM zw_lDqFJHf^8BvQM_WQwT@2Qc5U%9(_9aJIRP5e*Xl8x9WyU8lnSz@Zk_h(0%?_C+( z2~Y#l4Hg-IOPhXw)k4C|APcddPS^OX(&j z?-V>n$W%Q9z5u>q1~vk4{0|_gckwEWkw`))cy8Cru0I8P)qO0(73fiQiKk&Y@wKB; zvUo&q+H^q>7i+wH-b0WQLeQGpr_g1>7LF@agK%D`f?+8IUI$|x6i5KT|h8qn6fXNO;0Ay9AW z`();Hm`doejkr*-k(1fX(Z@`f_?ZT6vzNBd1!qicQ;ABPFWa2~^t$ch-nXlMgnBuz zdrWil-2s2_+ntp6d$oDKlCLf<7F0&epjW%nRTNlNlimkl_g1bPkv}eGKCFAJ7HxfM zstGaN+q0z*q84k~fN!N7`Uf3+K_dlhY)z6-0M%T-;8k5uSZ#*RKvk^O;|~}ddh;9i zWRyCo?YRnzmODFiG{ZT4S{7HW{n)!bdyJ(`r>eHrs=iA=ow#h@J>9DooNGMQC5X70 zwR)F%nB$0dhBm+pp4rh#CoWWWZj1=sndhUJ=yh#^h)U0v3eV3I6pW&ximMq*w8DmDt?9o5P(`PTuLtaN)vug< zdal4C@cO--x+2tAohNMORCz3A>gr9|cHx@udb;xg;{~(?NX5ic|Dxk`5>Z3YG9T0S zrru!?kE5o>F4N|{%x$YLDVpPsodVv}_NC&<;SGYpN{`9`_gM-FPjxqBig@}w$ zO1)`lQPiiA7 zF?dVDt>}Tu#h;Gf#tlvq`IpeW{`a6@p2Z+XE!fQHWFg*SYdUqnml@aL8_q7Tr4M2b zGGnK&mR;>{UHZgCr(%#_*3gTyd#$uSbS-xW;Pp&M7_0N@xX*+vKj?>vz3EXY-viE0 zYVG`=V;3p1&qN+LfakyDoZUpXv&O|`?RERDjlMON@KIUdkz4GcW?{*Qq5>-|)ynad zhsO>89PIF%JFwj^4_MeFX`mUg4DfES)=@i5rWC>IjA{4?@U@MeG-^EkV282ai`TeT z-;m!(*VS)xg(&*#9PKZQ7bq%@4lnzo@o}m>lT#QGx-}daH8RyXd=OoUlru7D$#kaf zLTEA0q8?(lbXX$gFcai#iFboqR$kA<)(;7rUi_kNIr=+fsqeOSr$YU3^#JC9r=qKV zF2dat_S>ZYT60VqwFc^3@xr5tBmkqXdVb}e+=97;e7|Te}&zhnAai4W;jSCk31AT`#ponpus&R z&#~a-RHxJQ(}af=w0C(4jzQZV&jtHZa{n0q5*8Qn0sboJrK7H$mn5ghH+~2ETRe%X zja-{SI&@vvWCqyezmcEAR`CihFH^)8mM>l|o+sR7JBf<$7nW>*{>+RSs-IQAS}EZE z6l1usd>U5{W`P?>N?LOJ2~ohW^qO{kT^FNO^iSb{il!;VJ|DWs)ZhJi)#D3+sKYJx zxb2yqhkaZVqCnXbyJd$wjVZGE{2-cPFim(l!Ti)S*77|U=+GQk%LUKH zg*8B{O_k0%e$`zT$8@f(^Di~%P{kwsnokNE{iiH}>CMKxau?tHoUAP_cVCHdyU2vG3Jn1`QcM0I95+KeJYF{g*WMP!(E2VsD#a{Gy6m@Z+l zAfPU&p?KlZqRdyN$a8s7Gt@Zj zTEgjZPRo%i3Y@a+oV+M-lQS!l`I0BBC}B#!1|Yyks1zwEJlFYmiU~6X1Abk}J_D^? z{UKNno#NQlqZ5sM9{s^oicvK2wUA+2{3XuJ7B&V4{uHd-U4nMR zWTeqpQM_*V%!pdgJZr1tOff!C+wnJZJ@IAIc32RFNHL5PTs-GW+ijlkI#DRynwoSgmywRaT;^pvFR7P>qojrT6`>;mH|*w`M9b_UHHja_I#p4~Ns#@P~f{rczl1C2#yE3>3RgL~m_e6lF+1qEv_oSKc z5~*Ug_Sx3dB<(JA)*WmXm~^@;y2n1k8`h#9_j!+a3FbJL+Gy!BbDn47d=^~1^j^Q} zAY~Ih?F>h)9FIcVwXdx9MU5jG{Re~JcTGMjWs&P=yJ9LuqmXG30hNg z?&d#&CJk(Rzt9)*`6CT{Ci*$?dnrGl;LK^4zzXSPCgRG8twqCYcd`t{gl6&t2xZI; z9LEVCJ_bnJy)O!ehqtZ=60yrKlN^qCU9@6ME*AVDjTzAMCji2f0kRL#>*>@QKnid= z>JLE~8c5&+%o~7vnE^~o2M~zs{y+}0p0Sg+ySg76FzFp&n7-eGS=Deu)+tA2-egX_M zQmoDU6u^NFf0dE}$lr>)6p)cV5zq>8&0-cQTSKu44)+N7o)7aC9Ce}AfCrk${9ojR z65kA*JIE_7`%{ucfP(6?USqi4`a9m%3zv(tOpLdf;|g)WXiOSDJQ2y*yuLj5`C0x*|J zQeYpNw?nlpXZ?=(0Yse)zIKs}(o(kA?$0&RPoW`MzNlv5n?!U}`VhOY;<5$~Bd>*^ zty}5qquR!*2#gAE^VhPp%OAMJ-+*-hhEO+CYnOZT#Q68uM;ePAl*TVMbCqJEX&vgA zqQy48*Qj+W`@HATsklG;<%WaQUYPo~pF6kN(>|x;{CNN554}ssw8*rR;ky%t@;_ZrT`y|eL=@#5{_`Uv~^(I#ak z=3=#43pADbx3o0YVRIUBzB2xupPJmT_)Ag46LL8ApQZR4&>Ox z16azxb*28_V%h(_p4fjH{jU;<*&b-?<#yeAe(K_|3U8DWBUr%p#=O(tB@IK{quI8_ zjg68o)PFRyN>447l4#MkvgjMi&c8lll^z@0r)`qK>Jh8~q%@DA3DT8(OTRG1NYz->vB$=!_nwgH~29{SVa8tWgC<4ZY>rU&x0hl7BC zlEsZJf^Ito8bE6`&Hk2mdIlX*?--?ziYb!$??^iVmH98RnXPs0C|^M~^zAUGlLA1N zAQIjXsq5!@C`O8YCMicvB&E;Vc7(I?aW@)HsZ?fzJtDVLYpuR81fQ7^!%j=Mix|T@ z^s8NL0R^m4z1o*m@*(Y%W8_buG(s*2RNGy>0zuQ1T?5UNU#aO)>d-d$0_PjCUg3Tp?YRgxLSkNR{( z0IBLp>)W|`(x42l+R8)Y2A7wv?1e22Z6Rtv9jPRi25x6zJNmLa&D6*q5YdjUxIP6c&ML8^NKSmP4Ox+E`o%FQ+Q#WP zZM)Nc{FPwVsM1Sso5oWjpFDros)p>8wfO277^Z8o86*dJH!p3ynk4_Dv(kKQ0@`jF_^TWu zmXrwKlz@x?=zzaEwN-h2=76Tzax3#@HhrT9mYG=F>8I3~1S%wF5q_~iq_76IZ{XPT z;N7|Lx@!lKl|g5>B7oLJK=UPx%6^ODof{ieDTlm!(ROBbCn9SZo;jF6Qm*RLInny? zdS=Ifcs|n)IwMi0DLNAh3XMm0X87;T+fO-R3{K_G0TzoFa8xq-1oc3FU0~F3c1Lw; zh;_FUf_(2SK>aELMX5DT{^BaDJbsVw3|Zw`9*0u`{k_B;@GH7lW)nF62ZRpx&;HFO`hRj5|C8E> z|M$J!Z*1j9FwX(ttvcp^eq8EeiRsg0Zn@lc>{Jz=y$8To8;SF^TwU8a*8xdNt4y#5 zz;Zgh`VWcLX-ypXHuXa48c0Ov@BMMZ+=nQHG=O-?Q-vLY%8X;UII7(u(GV$pA1AO) z_FqwZu@2dusJT7y)&udq4B}0T)Zya6WbiQ(k*SU_W%l&1aPY(zs|;^8xa7%9 z$;}@NuwNW>4n4iS)np(U^i;j8mDW3YNz!$#Xn=99TL`AD($i4pmB;ym16D`w#yYT5 zL(&cvOUla!^!5{Y3~5-DqWj)~=%NpSdP0CU1r5Fe@N=gMQn6_dMq90CG>rQy@D_F+ zf`{^3AYX_>)Cp!Av3QwzCE$PAYiwCsqcA*TK31Z;w`q(KD>Zf>)pTAAW6!skLgIqa zFg!fMaN;zdcYgD6nSFXVK5Tzc{kK4FUm?fx_n0AZC*vhy4bu)0C&oU?PS2pD#p;=- z^954mj8`MlQ7oM?1He^4k5LFMur&u?eF`;vdm=~N5ulouG@4~4vj0V8ktG0gqDzi%$*xFhyJYm2@L z1dvE4z2hkDdueb2$A<6jfwl{aBhr-^(?^eweH_rv4*Kb1S(|j3)9^0)*7z~K<4UW{ zC>F)Yu6Ll>#Eeik73Dy$?ffK&7dAE(J7Rm4;h|S2rJ=MT%j%*peV8G#WXymxOu?>I z9-$w{D8;QT+wRI~rduZB0^6?(Fv#>12hrsdSnWK(cq5bdt|E9XYZX0LAOT(Ww!<`T zjO@!0=^lII0ehtfgbF_3HMv(UzMfjRYb~Eb-K4RNk$jbIzyEq&gnro(K=i%hcj@p?nqOKA40rvA-*Sum{*?rju-AUXH5C-Qvlcbu z;pm{^^FS!3^`r^y@EhUp3-qGu{l(sTrJ>LAUzwod&Xs=iM6E(B34sWZMp79yXWS^X zVb$OB&ikm|-r50eqxW%#I~ix~!%V_q61Ub#lw*6NN`ko{C`)s0;$H2LG{+Z{z7DlD z#x_?aAy!QIIQ6hMjuotE2Gie%yi|dTYg@1{+v^L|@99(-RzJS9e_4U$O>?|&9-Q>s z3}Xu{m3eL62&}%X5={1dEUxvA=htC%M439&av;$I?oSFL$7oz4}e_# zrW7@0i9Y4#m(oAnWjF<`E;;d~lN%Q=oiS9R$fea>Nq~66Wt^a-{=WF?P*T#&NlG{0 z*_8%O8Q5*O>U&f3!@w8U8eKV;esBD0W{lD(h_Y_G#x`wwIZ$OgWAo! zKJ{PJ?slQ`qw^!(M<{f?Jt~uX@Ul66I)5dC&>v_@NQej26>ET79I8m@1-Qkh69Bh( zOZX4BXq|SuLfznr=xv%zyRlTq|DId)!g7oC#_e$o+$k}SPjpA`(&ERX1 zJU-zfzIUxl_GL0XhXf)aXfNH{M&(@h?0}}NMG+_zkR9|=(|om??D#HyYxWzm)03jK zC!kY44s{!pUaFqvy--Ib%+nfmYOY(hJs~1l97F4Ak~Zo0NnV^3c!ubD#`_)qN!!`% zMg7M&TKy0DPml6dfu^xcGjt{`R^o<+@S!n`sHtmv&&Yn4f8EZSVEh zIw<3KtO=N1&(r+g?F6Vt;h;CXp%%IlM|L|SN`m^~=j|G%=1nx4(*j#iXOUxzpdXJL zJ0VjART-aHayRyWS&@f;lM^%SPL;00;!Wl_tQ!b&rG~B6i<{>d6{k3-K=e2S4~E zK-Q$?hh49htV6o2mzOS-W`!T11VsQ!5P)5DfM8=k==bm-cxFu3)k03mW)8JG4uB-< zWgS z{lWYhKWI?zAH;TP@_!QBw+B(>*h(2VR&B41_`bD$q)of}xakS~muBvS%L*H5*hR%g zd)0TpZ;vCEijpS%o_UBxiaB0g26Y$%GW7KHF#nTPKwOq9Qi6Z)0sNBwOW>^$EZBTb zhTRW3qb zeT<6*3kDLrAWsy(wWTNtlzf`Gum6xOzGSMYd|?3K+sY{@UOjSe{SyAG8)(ygH^X{% z!;-AtSL)4$B+mTfKcd>luh+vkfubVF4&{Be^T^`xB3Epa_VzLKdrK+}i#-TNSTXGzET52gQt)yko+#*mIB&7Sg@yxdDz9uztqAne zp(Tj}eLlYQz4g4f%msTP5B3KY;}tpmY#kBp%_TDm0x!0KDhM7?U;MpLYwLMtBM)+{B&=V2gE={+s$z6|g3dOhj-3t6BOa44CH$y} zqPgkB*Vk65q%fIQW1zT8Z3|?0Hi?C>hVr2ktQ?CCA>-}QtEe#LRl;!p6^mCG4{)27 z)*SCm@IphiWOn5}8A zc6>RbwA5y3D!Z@)1_~@=lixBYuV#!WI_P297>70S(lfS|*K@ubv%e$lxd7e_Y89uU94IOKTKa6;qI1pZ+TWyAy z%PCC@43I@j!t7C0$am6{{fa_77kSL*Ji4Uo%K4&nG8Wzpwe%kwZ2#@$qx3`=k&={J zdyk+wTg>QcCHlLsaB3@RwqatgcPc6Vrh%)Os#pQy8`al0y-FP=cG0bpwyCo=KTexh zuf5o*^-%6RcftSYxmvC~F!_pKkfTL{Gh;3(;L5ys8p2l8nqufWRH9P_Sv#AXqT`kJ zn>A{C&)%O*BG=2jp*=%x#wHb83$a^DU+pXjD2fp=eHmGA^Li}Bb!4QHNr2vpag?B|j&OxC?I2<{h9q_iE>&pZ86t zJF@S3pq$Y@28{!n8u=rad*qArL@%t=U!Jp@o~P3u*we55I(bJCDt^rrco^(2CTeCj1v!rAj*U+9ceJv5u~~0#KXhOy!1*q-96oSEM+@NuhoxJ* zlG^>1)G)u|b)Eq3yLMWMZ`OTL$afLie`Y%u`B2>9Wn^aa1EjyZL&;fVq4R=ry>D6X ztgFV(g47gmu#psJg2ZEv#;2x!k+f61Pz0GE@f&1@4<>)0ygQwK$Z@T1=KtyLJHwjF z+IE?jcVrv}kwFARKtMpHiAs^KQbZ{-NN-B0QX@zw(NU3((xnR`NR{4+1&Gp{v;)Lj;H+c_kdz*A_xBi&TFz607ht16o&B$mNFWrp@y+Rb` zl1-vc{<_hjL5=i{z-}j*qQ@iunFN#sum0C$^r@ICA>}j4o*XJwIs8hLH%9{PZZqLq zp+c~c=AwoDD<$Za$EHc<)RTPq9dJjD1GoEwJI$(-LDX-O6o0~c25zoC57T38YI|Qk zes$U$SAi1`lF}q4s!5rA8N6IPbXM{Eiw4hMlvfkc&X$~yEGc;XZp9&c(Gp~#|3GGD zQJiNs%N zVbr_)5=gY;7R=Z^IuM^HROhQE)4LXcj6#OpIB7mYut^f)OXChC7yARk>I?dNQ^YbT z#XP(Euo<#$E8Y!TCz)nf?h!|eNqeDq*M?|huMcdy+wyH#r@XI_<#8`j0$+(!ic;er z7ji-;LKkcHUSS(}>v|4-@n5o0EQ|2zUwaT{pVlaOEls++pea9TMN~*6%@fn~-V`nDmV{ zWH0(MpYSoC(t{G^Es1zLeSY;Pa#x>Mb=)hzrcL(={kQmI^+aF2B&)?C*bJFA}DqUHLRrVA8K2RkDTK-nRJ%bO4+Z zbC&Mq&~qKBKG0g?_O{|&SkTVp_ga(aedfR);55anKA|NX;3tqJ?3X))5Etwds@ZP% zo&6oMMM5|cBk)jZrbQxwGuzfFBmZ$t199*wy0%tT%f}-Dwov*bbgq0z+pb1MrKiIN zj%p>;$t+}^zSh5vK9IG$Rs1D=C(tNSRFqHUG>f*BD4|QxjdR4jAj%wa${?y^aaP?c zkB5@7MI*FO1-y~YH&}(7d{eF8E5^(1PBpc5(7?srb%X2OLlu}VCyH1V3`dG zW?`C(jFqw>`J~!Q(XzG#H&MGScTRA-!+;Z$>w%LP!t@=5lp7FUSZ8OxAUQr#> zRwXD>a&k;YD;!rimAPX?$3B1${n?RD_ZaE*&h()eN)Bo-mC~QjQxk905o5Egk~`xV z&f`iJ;gxsVn_^2_-EI>J(Q@UYm~fu#TG*cB=IM*2rlTFz;Hh8xV7^YDu1tTLVL*t+ zADlrI+~~t>HlKUh2H0S11?-1;)W^)zYVPPocJc5j68$o|#*bTGoXZ&S!*j5S&ZC}w z4dK+B--9jA@}nu-RYrYsgHBU_hG$5ZE8qu36&pOd7o7Rbl)^kth*#OqtzP-)qa3-! z>&a%P?0ru|xg=1#_mxw&&?Hx%rWvERG;j3dA7>cPdR{MBJm)-w<;@q)#Gsm#9TBSY zsUDL3v(MXltvtH1uL2;Wg)&MMAlsU_n~+yK!twcZUFnQ9_BkW9D9j*S&lFS(=FIaPHL%lh^#}O^f{3 zJQs@4x_aApwuQC}rEGqA<3tSg@{N0Q@tkGXJ}Of7jnu2@Y$dZu*vwEZOalS8Ke+X~ zM|$_LkVqVt>Fw_Qr6ILD5w|sK#O`fex4beZsMO?iQl)w*QOS%4w{%;jLo3*47nkC^ z%+nHF*REEj&{^f6-L~tCXqL(~hcADR=K*(h%nV2EIdG3U9KVYPHSPv? zQ+PLGvnE$1dO5oHUz(z$g4YU(l|4Q~oB_B-?Gwk)C9o$K=SyQ(AGCP&9_oIrr|$e| zRjI~pH)^UT$nrO2os=a}DLLepeTex3?^2nZY>YB7S`ul`V}?=|p-nhUhsOL|bK!7% zqQHsCj(1|3!Q0yb+s~XG<6ETc3KrYTLRyv)LXAGrtJOymYO#zZV-4-tjVYeIOjRkV zNOxC4M78X>PJg@G2i~S*5 zEIIC{XMyRt;Wz4=KYM(>^;}urOfvIjlGF3>9w#?s;+O4C+I?&QrQox{N9bt_-;AE( z6G2@bJm}hNUd@hkUi4BQ6Y+Hw9cZj|k^GTXvz%?!{Um0Z@>Y7GEY3IaX9eGwTr&~A ze34_14~g*ZmLKmL0F+LOq+P70XhSz%$_^DRryw7_W{@LdrJ0wTUKVrM9IPN?!#}KF z*ld_NLP>z|P#m+vnRTGRU5j}s@&lzaZvW_X_rnAlZR2u*Ja@T=OKyKl!B zl&jej-AE`ORMU!@w{ZU%Sru1V13`OKJ(#5Ow4LGf-E1#T6m`3q*H4&wsD zKKQVixmhXgyq>*lhWKL-ue!O975y<2vOvVnzwoLxhe}s!w;-bKOGZfW%4i6!eY@7S zOM9`gh9a){!Ppd?I9*g2G26bZv|Lv4qPt~1c)R=d^dw5efbrGIRp}wI_&?_Z=l$3b zU}R}qTInDK_o6n1ZJ_YbuwU&`-v8!V;#%B3^h5cIGxt{IHZIe#1Jrf<&rWxKV`o3) z0(@-Q9%_g=3+<>aW$wR^k#=inzX*UB-TZ!^MT#>!K>ffb2LVf3=l$8N-~R#P+y^RkO2k28Wr~6dzR=`7bQZ{6E?O`&=WQ1sks;5 z3EteAU-Fk4myp>>^aXx^n}T&x#M1v%?l?J7{~$1FsG)nI0Rfmwig;i+z=LM<6qHpL z2z5Bv!Hy{a8pF%}*2I8~t`6{RhX<1)W%t?{L6&`(98B`}0qyHv!;JWT2&cO?YHN-n z;Y86KfVNwALWc}hULIBfil@#76Q|vS;djjZniZzh#5mp5@f>3*%{Bbg|7c?S zLhKnNZGC^UVbV=*IN-EQ0+L{$4LPQOeNR$X<9dhI)(cgt`g}~1to2dR($deiufk>8 zC`MCiu9G`^KrPW8NW40^|K3o#o5wwuPn1r_ zk3BU{-0ycUoJ4Eq3IDuo^BbtFLO1vtL)~ja39HNon%nM89(xEmivphT_x`|e2->%V zdQ$))lHl)uwEg{mv|{mp_bhIGDreW|Qas_gMuCR#q^g+s>_eTxIxbVyEY4y8 z6cioiSy3)4D|4o7Eh@Tve!fpiLgq9q`2fm{e!Kt=c{bPyi=eHOK5!$>t3zX~V*u8S z&*=^xh0gtjkoAVigD_`7Q{VMY-(ilIKub}%+)%gEV$a-x@shp=)HvdSz37jcF*cT?2#mNc=wn3MJ zh|Rdfit*c{zVZNF0PRbI31YaUPsf3ws7Mc9eOt2nlvQZZ15ozOgEb_hx*|2B;xqjb z_x{?i;48y?vK6Br01bfuVAts&?W;$eS5=D?poioaFJ@(B1wP`Nk+&&b`XtKq2kT*4 zwgTKdVAf9!<{_vnflt4?gfFI9E#wBzeM!`r6x0auZ)Gg`G2L<^_omGp9rJ{LTDJLn zm94w1O;jqn59aKD=SF5W=#T@qzDT33I=9jU)PuYy{xKZz7lHj39?XZ);+r^7oiXY2 z>)5p)1*2eMU4T$ig%G)P^`fD~gH7eeX=+m#Mknze!4ZNWT# z=P^GdmDnx+BP~d zn>ygCJNaM_`UY{;abKkKjryuP_H0|9<(;9l5@Qqw>z!T&AO#YpBut7pkSfH99w;*6`W~p{*+PlxtTmoYt-Zta3p0VeR=O zHu_G9wlVpsi2ZO%2V)Pjf?!a0SAeSoTft?qg`f5p6_789DNij5N@R9)Wo2uj=?=~c zXwv-`jK;gtZn*kQ-7t~{Z?D_3;i{awaSMb4}2rsyo62WDj7 zk??X1PqtHtY+2RGtlW^zHxmSp_5)^PLqJ_Q^t(sou69-3AE^il!)28+XPQ;Zk)f%2 zk>jXbpN%xV(1SW>dOgP3J}AsU=s?HEd(MpCYssB!_I9_GXdG90exi+4QtTCUn%+2n zN0UTv(LBDV+h;pqelZ49s9D%WZ3@0Lr;YhFtBD`T-U0BL`U_9nN4)^pz`SvS8Kch` zFB9iX?HbYtM_+7_R=SVDZtT#UszRPq(?vCzbT!2Lf@Lem^slXNvqj=&rg4Sh~nNz5}XA*)#jUc1%4!g>C(!Z&WKSI9rwgT3p9_N`sV2}X$(39C};E$jVj zV_vMgRl3?`qH-31AZHIrUYWY6f_CR{I+?~@*`KIn8%nlNq*uh@<|<5x9kXikRHgQ& z!|=(ALUpj8Wngm&cr?i=i!RK)RT(cWVt7ndI0qoSD!pQM#D)02)yJz)O}+X&e!gP6cZ^yf!--#y ztbV1^pHPxL?}c6pp?qJn6;RVq%A0$UQzx~wl;}v_%$hRGB$H-^YhJlo+Gpljy-?q~ z?`<+P;HN$zA7jH`%pr8UzeWCFPFZf|AUvF2*31-525{gwANkTsf;BzwT&JzD!a=Ra z;HOsMnx&YJ0S&ViX%t$;OH}XyniQvkN7->A5f(#}i-}+L)`e}Zog)v{hKdU1DZG@~ z&QLJewa9LDWkN&s zkH*O${b~4P=jCto%h{87j7sOrYLy(T@$^HH$R@FSc%xXeFCC38_zKMLq!EPT&Q^F~ z&^k8qiJz;PnK>^xg0esLBEpiGMLm-gyZuIgH^m%wZ=er8w0Nz^+nt$bmn`yxT_OJM zc7EbQ^Li@}<(u~+PNiwJ<13b_TM?mPwZF63EI&U!K8$UnU`UGf-Ugq74VP}8+8|)x%r9}1YUv8*zV?NT)g=T|~G)U8Muv-UM;(D6dc0$BWv>>0wWPI$_ z>38&=vv8yC6*%=9b}t6DkV*paa{0EqT(euq3}Rz7un;x1e>-UI+jsa^2~YFe*b6FU z%x3Z4Rh6DX?Sa`f-s#Ja6O)Pt%i^O6FDiHxitFdLC=!J;XuxM+tmLci zmggegLhoy+Yt0r>k!vWJ{w;eu;E0IIIjMdVzvmZ~qNFzDlzV0_R}$J<5N6&Ha!uNW z;!#AU*DV_Q4q6(@+84x3RswF|f(0`~4>>6NU%frxOY@Euav#?}A+aYqbDAw-gdbgi z;871=>a<2(fs>XiG~tv|Rn3)uwlWvJm&5doA`8y-aGi;xVuPCz!-{Zv0U;8(P`Xc8 z|I#KCcg1QjPSWZPKQvx}s$$OWCnAkr-^>`F>|UyWCBc<|KugjFYC=QDzWLw|S}yxC zEiI?3uL=m(Y<<|qEo@xkBsu1Y;7#q1+1>9(;EiK3Tb0J%5%i^ICaEI1jx@MpR(IA) zwNSR#av(0wPj=O=nOT+Dq|suh+g-w`Nli?GX=PzLnl36=4>b{-j0tBN&KVN}(DpS2 zEAN)k)v=1)QrqSr)7`cupsZ)8Ki4Qz92ZN_rG4ARF|i8Vv9Va8Fbd^|X@~f^&gXun zx8p^Q8H#o%l-IcnI`B~HZl2-W2!m6P-4{Xb-@Z#|V5{j5}?(VO@=cHL#2bFxXbdZFjC z*Y^5AIB(+(V~s{N(M_&akK3JRS8P-RZ*>hSk-D0;$RdH37R+0N4<4szXC#0TCJ0B0 z9X=_s>I8zTd4oIDF75W5s76k_XEeP#YDjCN^re~ANfvs) zZ_c!+6I%tKxkeCr|I+Jw4SgKlB*TZ~y+rF2KhF<;PUQ^!Q)2H|T2R-Lw{_sn)p?A; z`RT41IwR>!5{~1C-JaiVAhF&P+~o>TKTe1rpxuB^PA*od`g7*lq#XHLC2%{Fve_!Y zT)~=GgliGQR%-$sKh-{Rcm2uL;y2UIq4WmQl)2}%cv1g|*7+8nc|UcL>74NL%PqkU z(Kf8)GWlqYx$9l)y6;uOZVU%Fla!xO-q~_yqSHpyRhlaS?o^(rBm|(Ji75D@>+Gpg z+ham0Rl#Nehw*m(r`jF++c~KiREMHv`Y(!@*`|BW>zJ0vgE5J9zYz4MK%j19MMT*J zE1RQAUIECTenm1>a`+x=ou~=$F(zK0Q*?E{0v>+>jR%xT+6tG_Ugz7P1Q3hkjk~|d zz8zkxk_2Z=eH?B_%b=LEOe25i$r|$RZO!ps$_{Zb%oMScsjBJYVffY&xqcTWYyEPg zMs=sqrqcE@<)a{f&+u!jAJm!a3vF+D7fJ)|Lpgm*qw5vE4d%9}1fP0?KZw^>J2TjN zcA;@QO5bLrw=bA6HiNZW<$tVi)m6&FxAIZN>tLmd{vjt^WzoG)cB5{iX;!t#EmFF_ z@-XQ2DP2O~n7h`C!8j{~qbmTkl;GBG?uh3;6ltxt;C<`m2Th`1{yg`oM02EvvU9wv z*TuIp=|}=6xxtyFH@RqKI=6LScD29}snnzpjUN+1;GE}pi`l1Q6|B$|amg1OyLbb9 z2UV0`$mWdRk5*gZEGdcm>}^BSy_El@rr}(^Y0r%e^a_3;)+}E>E~Hr5F7z^fZ^oMD zNEGBSO#oV=XuY3cs~c8tklMXb_5yW#BX&C<%b&fONA~sVacK(t{q*ZJSxDBXP^AV? z8d`gowy>&z69wY>vlmqEsUWh<`YGX$Gh;8<*CM#d63jelP8W|Gt+qpi?p=&A5{nw^nx%DE* zOCeM*1gqKnMmR=KwXS=*3K^wiG~q~(in@gc<5 zma1;FX3{?9m4=sB!NY^EX=n!gk)!lZL)%-4blVk6Gg5oy9G}%9rCja6>t%`G^OK7) zM1R#EaFn5lh`Kw3ngYB(F)CoPV)mpLbE^WY8LU4*U(x$u+11gPa_8dVW#X576x>+0 z7{5O4gE=hv^qF%w+ENSDI_#VFI2_g(LEK?cZER09STOTlRi;pe9>wZ{4xedfsCaK-1}w!DP8ORQ5+)#|}^p0Vs~yx2?SoAvAK zwN7md+W)z`V_j{q*DpTC{E%t%%E7w}jRY*M!-rn-Gkm*r440}fYqu4uJaph~fAG0? z$|f%|152Kz%=B9O<2F?z^S{HqzIJsexa4Y$W7CUgn9=e!^~MUc7!xrZJ8L-Gv`>i>a7+Qq&JrT4Qys~~DZ-Z^Q-waV8K zW^=oE%FIAGjc5wg*8e@}`b$b7O8IOss=4DL%2+lSV?6wceUv3m4!GLI?hmrz5*p;@ zJA2L4aycfX_iV{uI_)Kp&lMmKm@8U`d5}02goNBR)161$5e2Ud*58)oWf7s3lEo`7 z@(`j%!aw}X|9UaSMC~kSCzbAC@^SOSV-ED^7ZQNL4={o^pfwKwoUD92_vO|wqyw;# z**WRz>dF=O7VH0sD+&*#K(x~UOG66>+Z*MoDanz63;<9Z#xCsq)9}u|9ctIkld=jt z6d@gnry#hzK$7*6izDyzLBib)n57x%^C7r|srJwS+bH5{7q4kN$TC4Okl!gnQRb&W zq34>^bl2IYH0uWpDS zz(nI`vB63^yc)s&fD{0r`Jix;I{0&o8cH`Zu;n(D{P=ar%*S zWMt&TRbY^8L2D7RLRdMT2hbqvMsFKS86N;OkY+a!D!@Phtke;JN^utHC&WU!8Da}QucFx1 zaCpBBMQbA&Aa@LanFII;8%JMn8U{E>Ge9fGPQe$tVpyo3!3oWs;cIXv1&?0La1w!L zE8qnGjm_DEH4Ue(l&Cy;y1TpUNz2m=U!bm4n||ts3=Ej`p#u#csEY*wfb?_v(y%do zcAnv2AGT8Ld3nWWAD~bA`C5_Am_{E@45aa?Y9rq0V>^lXw`2}TX=JbJwx&Ul6e$}; z$zEAFJ0RI{)CtLWaP8$!M_^Y0CbxVd>f%^s#5Z=)Xuui*C0}EQ@b!j&u+TdB0%`GC zlbanpDUQ+E%098UBp^uo>6`!Szw|+XaU28~&1q?AHgX$awH-eNAu)kG;Q#4!0LNU$ zH2D#xY9uOAfK&{A3u3)Llo!{1ZGj-gId%1>$>0Epl~8Hd^ya&x!NFIXIyd7_W&MDg zY(y4bfpD`3I!-hx#Z>FE*7nPxuJ@1F>m=9$|8gijaaH%mjgOzIk&Lw(U{+eOxXB_Q z$F=_^tai!kr-JgoEiR@@ApH^?m`b=bL{q8MeF)mz(Vj~_h*lpz9oGcO(?~2$+OItc zLj2}$jACkQYb$`dU}y4RD#o#rAOF+~j?jGz@V{2)6uqk0W#peMiac-?V+ET5Wx|*5kB@oK^(0SzT?dM2IG}^*)06tzX13IWOL%H zW6-{We0;7Tc+oaWDjy{98AK9Fd)BYzC#->TKS+u49Y!7{mPcQaPBWP1v0_7w^ zY2R{s$>H*uUT61lEOlBn(%!W{Tv2`Ejw?2NU{iou9A2mP?=GG{w=@@acc(8+S?hw> zASwXX*^(ADcU~pWv90I}J>~&sqAU3&znl9pp%asu^!or_oOG$`9_Vvl%oE4IDS;V` zDu>@rmEL`na$*nz=w6=CEGPh02PfM{Cc8@?wJzYh=!RLy6s3I=Lt zTAlPRRV><@iF}d%>SrTlOjECl4==eL6@3&M78p>odR$9Vmzv#}YEhnTa?p!!UCb** z6e>?t%pFyqD~oFlmhLY2af?EtL@t03Ry^11heRq zlYmqS41_^jK;N(lI56y}tt&;&5uL?p`g*&Bw{S}ob!w}>1OD}8N8zNILAPSQs-}r{ zos>5E?nP6heqo^p0sXWR`k&_>4}zdy10~^@H#T|$xDWu$pF>QXV`DJOnik|c?-yd! zS{$OZ$PkEh$Wc1gH2(*$xq*Y|cGf^#vw~25i%iuC_>Gg>uZDVd58f%R0*v963YPb4 zQ5`STl!w(--I)(NrBt+wVCt5D(t1rZzA$190f3p&>0`mZb6#etv6u6Gaya4nmOUM~ z!KBdgv(|T>mf#tON%QQt=vWu9X_S^$I>a?-N~OXS?t6zBH4n<1@~jeL$hg zsKX3uhZ?N-Ig_i2<9QjgM<=XgKG?NUTyCzVOPbCKDOHafoCKT; z{`F|C^2x5OhU_BWk#T`~N!0HFypI7c%Bie7+sIWTby=m!XR7gZ4>0g^B3om3D+%pg zvrok)_|gAI)b6Fl;O^IP_qIEZ#mQACq*brSpQ(<%@};F^ zeP5!ntJk0^>HFiVSn1latvBc=cKls40M4XJ;w^QjF>-QWp_BMMlf@Rk9MAY6tsN7? zE8Rn!ZG(Ywo{h{v?)~1g@P0*f7l<63>HCe9dR*^6{M+>rtb)4#y@9u(@Pl!)6=t+V z;&pO;#U=vsqhnGO$9wVJ6z<%hr>uW#{6m6Iy1-!Aq!~bN+GDTCzdMh3t?9wW0Ry!U z#cxo`VQc!aD|jwc8kFdSW{yZ2CiR^@ZPW-w5H?<2`y?C;WKK3*53Bx83SPg4s>zDM zMC?AO4f{_q3^25VCQJiC6CZNHOgJmAhJ7v|Wv}9(~zB`$iuW zI*vuUtAaA_!k?V-(MgD%A`tV={$0QPzsjsf*BwB>Y6F2^533;4UGVw;MfSLI+!5Rx z^iV7@D12VH{9meZR?Yy0b!J8S-Ae=6b?NCHz}5>s?V7=2;~$5{fK~thPYDqJm4yh6 zy)lk!nhTQ|1ZCuAFq!Hm+ClwW;DXC_I+*^6D`O?@tO{&k6pPjp*kb3~A_*J}($clb zEkoUL3Z0pICJ|6EoI##94HqYtmx!}HgH(v-fcK%pe^h+5vYoP&4Ied>E)EcF{PEH~ zF9syaAQSRu3G8$x&*!}KT?7UFI4Bz`S4!Ktdh8S{#YgdKjW@P-(9PE=VEaP zcRa`8inG5z&WL}$$Yd?*!wGm_kDiWE!k^>T{0Xs9u6uc7kHrS>%%hMwz*rF8#be-76nJQ( zgX{=tQhmrb5cz;BU5nYTZ$=4R68eSoOLY2A;<>W z_ojbSEC0d001}7p$J|;1qDVSiEojmqSkKtl*p_`0tbR7OV^K$Ictf@aRK|uk%d0zM zAF*@v2nS<%ARphsKh7m<$BTw7zX}0_jnU3iJl}M#f(-G@Z(`0bZYrC$EM<9(^4$YK zMcmBH4B@xp)u;T9(U2&%a&2u5N8cVY^>DHNN?s9sok2plGvvHv+;$QuRzMy+dICJ8f?H3b^}GQkgS0bdTCxka$d{d*FRH{s4_uQ(Gz%LV zl=6UB()$>6>3QfHY}67nk0hq>md^sMCVlabRuLtZ!HsX7TUE z=V|)If5nJ*V%yc097BVukU_Gx`Np5?mE_`VCVxw)OBv7!I}&Tw5Lfse8RX{9J5X~* zQ{~)hGSTmiO8|LR!s+FTM=bE%mD7ARyXrdH5U)Ef-b^07>seVV02rL{`MS!YiKM>ceHG z9&|(2^TzK@SQ1kBli~RZRpjlQ$i<2f1Ub5yNu($u-7I#j?f6w3>D$&jGjqRGil0jf z2sVu^d#&yt(SM!R0=uK9h8qg{98P#w9%?uqzn>k_fN#!BZ6l7C_ZB`_#A2<9bK1R9 zk)~-GCegiECCm19-~8vBLY37Oi9Oi(nJgZ&^n?aYMa@GDjo`|m!8j5)_=o+>Wod0@ z@R9b&Px@d`AC8{Zm4zoH(Jw3vX5exg_dNXkk_7pK;-x=W4`?^_?9&jUC}hNy{QeK2!$zB_jMae%0^jjYXB)&MQkFN~y}na6+ERL+#ouFT1ZDq<5*K)hV|5)mcO?t>41D3b^r|F;l$%N4)5*yxqNX zN!w8lsa`^b7Oc6S3d+?n*vCs3UZ#$)-rv=u2dOgp`!H!?$MWc0Wjf6wTr2kq$V9Im$#MMB~G;1=E z!;^+n+t)J2#M@mwpH?n$N2 z>u*nN38-#SGb6U6y&+>69ZxWi>$g;?%N1C3h7;3oCb9Xh3Hs34XR7zOL!OY){+r=+ zxP|Uli^)jZizR~D61%0IZ{(#n^sj|eA$(I6}VJn6x(F{FTqh2Mh>Uz4)cSoDl{LFf}5vs4+rkwhGgwlz+ zR*xJ&xAv+t$kO&o?0JSMyuUgnP~&7W)hN-jtu-RUh@i~~t~nAzAQ+HM=Pl@&-lBx_ zM?!CO8hBe|*@U;hsJcq`Diu%0>^94W{;KM{mZ-XE9J4s$cgv#6C%U*i3)Ns_*xsh* zHpldsI4k7}ANee3Hqt1bwU(0L2J-%58u1JERiV|^+@~lH59%Jde)}mhR!^pml@rH+ zUo-?Wv6{ap=Q~fipYXBAWhH2A_f2O*Vrg5X^KWV}4^qfk1?hURP8L9A_L5 z5U}P9 zzE<9;Rc=fDF99&?2SI)?P}-P2qcgb+jH0^m`N+IoFy-=N)1p(KC^R+b!pw2OC$nRH zGdw_(4+lXCXE@aoWHxSZvn_0J&AWFAs>p0I4)dHn%x9(f`}+F&JiI^p!*m4Hg-&e{ zCES>Mnz;`!;?eY~v!P_GLSX#+_i1ST8nXSZh=pn~mLq8d@a~m4KxM5b0_WlxXjqT^ zd9>y_lyi=NcRtn637sBksBdh9lG!mTDBKwRb-o`3=X-B&Z~bho>rHUJ6C568Rb16r zuOIs>m^cbW2s_Iop0tHHg&vUVU3(gUAE+{o~W=GY%gBTnz)k2pkv}qXBH4o51QCh?=5>Lw~%0>_F;$eSH(5JfQNd z&llWUUxtF|8pyi!q1^Z10S$4hPaz48>nYv^sUd%o=AJdRRHoQ-eWpPX#47b#_?&=3 z;C)xg3Kdq~Y5#Ch(j!1zU@ON^$x0e|LljU*6U_nheSM4~nN*?R*scntA@%dXlNzdF zfP0i!XCM9!0{-YB?E{T9(2_PEP)8-V^TWTES=dxdc&M#a)Imw^2#AYS7osIVfmyHJF>@LAZV7+VnBM_%v@M|aOV}WHERF(36@&{DI zK!f@fY`oNLCF&g8N$7=kthn~IvM*Ku1U}v@k}9w`0mtOf;IPd}B%QQE#DKIpS`pUs zTIT3apZ|`9f#dhT!cljr{}>KIlj#)X)dN1yGydXxf%doL!2qP XYk$35U2BP@A2QT4(XG0A=gG((XlQUe0giP95F1W{1wAiaqcDFPzB3MkS$ z(wi81sG){3m;cPHcjo&$Yt4tdvhK;b_pE!?e)ita@4S7at8ty2iJX9d;JTJ3)PR71 zP#L%$fJuOnpWMn1fFD9n0}T~|(tegT;0ENNtOLyW8clI#Lk!%LJ=Zk#Bp_hmy}Ags z40v`32w1OcL6wdCtgy4x&y6rmm%E6+Wlv$6uw%n?3ndPb1R`4h*6sw6)C-w>Mx}I= zpel{#-9g!#bg1`>qO4(^xHxNJR~H%dM#H6=iQJb$HR@B{QW zy4s@R2$>luDcvy1lctzvcZ6(t*1ws#lK##AaC6HcOVPoXF5n%(%TMlKlL!`gR3I-$ zz{&)=+#tdSkr2>=_`iKX5w7t=H%meAwq9m}?|h(%5WLwH#(a>FWZ>aZ5y2|^Tbo-w zqC8K@FOSp?-J9!{P5v}ft8g|niL}O;v5gt)s3>V zx1W@FsUFIadazzx);(3_f+>cd7u|z5&wy-7q&=4-&Ndws;}tJY2KXQ&!^4}j-=iVW zCpU$y62jO}n(9$3{KUk>)#8$Csf)vTFwAdr#0Sd;pGkx7(L*+&2e=Wvo?qo z`m!)i-llBy_wQ7v?e&1JCza-cgfdYO=zz~LaaRzp-eIZpSwCW`s44E~VRhs42?hdI zC2&r=&<$anxbfHx1K8c^>~6^fiP9BQwtBb4MNxjEB#QUvE(xriH%;XYL(qpDY6cb1 zJ?gc410R1z|Ozt=AR_F#N?%x9~Ql+ zH5xk><;~+H__D|^+GRytPsAA>CEa4qP_zI$kbDa_lxTfKXCPPg^QD%Ks)&m52j0FJ zzClSKZZtJD%g4=34trerO`B>yW|euK;4dBu6mJHU=Lf!yU2JII;Q|e<^@h8<7=4p6 zln95Z>Um1TdABEuQ`hhmSh$gA-iEQf=xY?m)s}~elNGkVj{8VdWZcf4Kwqr3w|x|Z z|D;WRyC^j;VaGVxH9J~=@D1LLBdcD86M;u$z)ufih#(SJ=b?e zp&N9R+-@Jv4>hc(a}82V8%$njnxy_+tvF!3eodG$Cv{$#yH5$fs{wP&DJkhatLf~B zWX8OQpXCT*1y1eOC}YQ+Wt;MkI+zii(Lp$Fwug=inQ?6^h_C7x#O!9kTXX;SXunzWtK^U*IHa%L}G$I*CgGw^I+%HWWjhNF~47p zs}ZN`Es!zy2CSqfnIo`?w`6@v>f{1XD}m{Mk+Pe|=#+MpbEm!>PcF=7=>7s5-w{LRY zD(K8MrB*yEw`lOudH3#}Hdf^E+9jh^y;oswUS5}8&>o#R`LRA7^qYsLr_RfgCtp>r zk`2tZCk@feFD)&-j9Q@G$>9rUINyy%CxnC$oh-bf=DB+pw$-?=I|6^;w`1L)K455R z>2`XF%S~&gyR9ouIB{@0h?@Zt;y4z`EY(}15p;>4?-?sLadR3-N>0Axxzhgu$ODtK z+59RHH;?6BMrq_;?&#RqB<%s0bGshm^=S|Mg6k^lVBX%&T&*mtSTkEszHe)H{Jg(b zeFLoQb@yC;4oFX(L!UJ4_x^TO-&mEQj3D@*S|>?H$XyGAD@xxI`}CK=TY|1sp{1)^&8vh1)LAoF)!N&uExS?^aNrY65RU=V10nqVXqYc>41?dDb2 zBI!28Q&?E&qjmNcKeTxDs=ZM|y%fEHmWzWj5CvV zphFU}YIIbP_vRo9kJu6|Dk>5UTvbz3s}{fb;SFBj8M($eHa6yg>83X)=P;<}J08+t z?>fXzy8+8=K;(erQ^!LPd^obre%_-1yB(VwA?AW(w+t7U{TV>+8V{ro=A5uU6F#dym-b;>eJJS{=v*{iunbWG6H!f=S#hjYz&+?_d29w<`71FT zAVN~t4~&IeNm0!Yk6$Z;-%Oafr&0RsNx^Hn$-K{hNM6cZ*KQdntE$wW!1kfm;~(hr zym%apPN`9J#3&T*yN=`v_uU(}&*Y0tJ;O6t{7wwYPe1pLoL=L%5aYL-Uq6hlp$H*X z+*+`ug`iwRj;g*kaQL;f;qF$~>qXX#J1q7 zt=8T4%P!#;htbqy%@`8aOH@sm1FOjLD%P=ry7{qk}y>ySwIgRoeN;hgEFLn@- znT=(i7al{kexe%Jna~iLgxsWbMM(% zXTSs=jcZz322`@N$>u^_ox7Zgo>8=oqPCc3Ky^3`-4mv4h%il!`!3U`2jO zEOKma2|GU?@;q9W2aFi%+4OEP??r6#&OAl`kWlVBI!*5t@KXs`me${w#$)!fy}G7X z>hh{L5#q@NR`*UPZ?$N?e$K4GlkqWgu(ry5R%U**(C}cJw_2u9#K1g#r!Yw~8Dl<1 zTjgc0c_L}(T+_8%D3z?&RQPJ#DY;GXXx_cn&FFeZfmQj%$p?G@YP-#Cx;!c)q}pZU z#?TU3-F!As)Gv2S_6s?v1&(x|Wz+{M&@(#Eq0y21KB6`3wHe?e{+#JO-PQrbP7rZE z%RF<{LG#qL;(#W1t(~rxrY-!=v!=4GmW$c(nj^tytn|;9kBg_WCm$2LhT@qgM~h8_ z=Pi9_G0dWNniH5l-99;uSl!bXYxvn}x3sy9T2-AvCPnQ)aTAss=<^bJa?IGLdY{J- z8y)RQN!36VcpNvcP8vUEtY|7(rhTM@(z46~Cy-RsgAKqZr#faAE`MqIa#pWx9G2@u z*Ntu=G%qN26EZFywtjtSyZ&9rFYO2Mx75dBsW|g<-zwDKx1yWx*HNMP(d=cfLAR5m zgYd@FNQ z<`(_9eZBTJ1&tkUP$UbZbmda8L9#-pxS`HB)h14(>)(Z(w5rnG_wrp8pfZuz0&-_2 zcj~~iq}{w!GsLqpoxRFGfyw20Wk}09KK!Y}gESsaMN4nz^#jyr>_RsewwZ+}% z&fxxid%$W#PZbupkx?vaU>{M|&|szeuQ<3y>F)8}T?yDTTC3Hg1C3~IG{eWo6o#KF!*D7HMzeDhh%rDK|zU;F{l5sk(E z{>l2Qyl6oG%Kn$++wwimwr8QUvoC(&=qP3>pReTm#uCx-G5=^t+hyWW z+MR2kV z!$4DGPM$4M&zK$;$7S0Q5xX6*B70TJ1HtRJMTLd+NGNY{(D7-DWH}Rl4s?p=e-G+MdyuYGY!HJ41OAC_w3me?`~fm1h>BT?bxP&8h6@<2;$yf8Az&`51MY=wV?CEBcoxS zU+64l-}A{6ZE_W;LT0xyN$|w~@-I5qL95O$6h(*#1UdiDnA5*N){hvb$&*JRJUpQM z0BO4<0<0QfGSX*-3s#KY53Gn89`tjIE`7`pUluf>wy@7>K z4zsF=m1Kd||AnFbU$8&DBPn1lHURJwfzLI49cCt&Xn#|TWHr(FM-_8o)ik9082iVaREypQfOzQw+tEI0Wn_EzD z)gS9j)_Jb{Ep6C}1B|ievWi3KZPEMlU>qd+c&Iu7L(!-^M$1aiEEtk#dGTzyqFc{1pz%5Lc`_dd>!HI73&6Yk}GyGx>{o?F7 z;Z*8jpEgq*GVc=tKIDpt;LgA$=J^=^?Isg)3JOebidglHjJJ@89NS+}DLr|xq*Pe8JcE#h<0-{Fa)s4A zq#p`ak5#zN$IGjBda&UDds$7<_y)H@mJtn!;5yG4TL~m&&4X)fH{(k)R2{fLpES{Q zPZz!i`%pP5p3Bq+o!9>}{NLbMq{H%#)g3LZ*9Om4+8a0;Zt`jkzeyZl7Z0NtQxHs; zUBL!W-<-S;ZRGlw?!##U`V?irt(kw$+)a}c>@Gap6<=={ehc&u_GZ8)3N-L&cDf1P zqx$yv?vY8g@`p7A=t#?Wt<|DULr9oujZKC~pv$tP5wblHwOy=D&Rl(+@w=Zy zd^3yw@X3VJw^Bt$)}<+a+h^67<|yk6sOcRZR#cR)#AkifEUq z>M7>2b$V;*z6MDK_?4AQ;}`l5jB)4hEZwVu(A}q$S~4QvY%-euA$6ro)Cm!E=DBU9 zhW>JDlCxQeB>yJvw7nv`*QSUtxd#^XDc`#xyPGap7v#D^TbNt1+3Y+x+J>T+qx;A6 zU!`kdkF~7g68fDj5LY0U1MJVDQXZyhZvU$FVP=hr5sPNw^;Z->(|8nNultI`(ZP~} zzZ!>_3m>5*;iB&acpOp%4Tc|Mjce#js_^?g@lBa{n7`e7JJh)JSqQmFRsSI=?<7Ol z4ynDWetyQTK&)1eXYTIrGrU0Y5ec?9NSodnf9Bv@EHPDtIgi}DRh_GC^(Z*Sw374h z1olx40PXJ}#2%>s z#DHI4qFIYy=XYGfb2Mgp^W3!90i^p%4Ay+cr(a@I`HR+nzXo z>x{C2waG_4Z)NY=f0_9bvPnkN*R2FMt@J)Pmneo7wT&J2o2+%%Q6K0|1U5gS?;5bM zj47hap!orC5oS}O)RnnxwUBhSVFD0wm1{Ze2w5FjzNQ%1U7YT+@yL&P#_w9-dsD7KJ zU)2Scu%26;Q+_MTPswN_k#SaL*CU`;P|sG_`vl<-A#VC8fu9#amq9+Apz5e~t> zhgys9w&T4U0fx*pmsN7+6xB5c0L7p@6wd*cb1k?jfqZQ9GoAg;v=PNz|2k_`dhDAZ0 zo>;Sk$UA_O-!9P2_;WjEkW)fJ z!sBGmejlZ>!VUU#M?%8beXfyTxCs|0NI11M-f~e@#{>HG3P`!Y$&Q|=sA%rz&zf`o z+w$h*(5=Cbsucj3x}sIu=?{hybHrX`S|_Z<{zL@rr+MJ9H9fY`@_ZxtyWWJ*?j3wr3&_!~EFbgqpQ*k7ET>|un_ra%dE3!&K=z75?_hOoW_ zj?F&s8|G`ey9?-}{bazQN(%ll$AT3IFNq2WQ>}#IqJ^1$i z*4iig+XpOQX6}3Dj_a5I`3hB##y}%RgFqk{GH>4~Lm*Ei!H)+D5_lw5*EJ3NdSa(6 zEeXmJUl}+qp|pt741Gd$x+F+kcPZKwZio zLdcrM*Zqm4fz^V=;;s0V*W#gv=jr{x-^04a^V|F8bYahX&&8q3x@phZ!fzyG=ld(U43V_Y{ollVe0=c8$XK}-iMh=RtNu9LU45CUFn}LO6HHXd z5Rw1*CoD_ExK?~h{sw^)6{rM=F(_Gfz71Gmc?{5MQ#b=>kpbY}p z)4ERck~C6bQM?N$>wRiPPOCFx=H<tldLkVQ$N78D159WWWI6LzVWQgj=JJ+$d-YJ`nlH!34Rq{4EJ$mjuPAExu zi#$-pag1`WI|9)0aB#9@=Ynq88ra81Ml79{Mw|~s*IM4_w|P;f@L0+UIvuFewmiu~ zIr&@;&GMCA*t@gVYjIC<+@HylZJN5+pWU9+wpNpF@oClfr@kV!qCVa#qEGfuh&DbL zI&y&u157@a@?x&q{o4|7v0JAS%z`9p}1nMCAb@Z@{lq3C^I_vozF5` z_i)MX#mo846vAc7qboDoZpsL_R30U8)5PFX$c{d^&%OS`MV$JYOUa|ZC#?sMKWmn| z{hNT0|B8v>$fh1nhU_a=`wVULNz>}=t-GFBW(F$Sz?GM;KR)xP{!5jLWNLjd{%N~W zOPV~7eGHB=8~*wa2Wl;9WQug;Ho*#J7;)CUm`v^&$wa=&!yB3nA;)uv@0WxF+C_P5 z{`R{s!}Y~Zv&&j$mf>rTa`)pbi&#&(eg~AmtB($0lmQTMM71E1iDoQ{vvPso$W<|u zg$o~UmdT28cvhD}@HH>^F!Q&@t8-#a=Qfu32YdBtrQ$WYVi`L;A~Ko z5^#u{muV&syO30O(KaV}Xcwxka6MvqBMIm`O)%z^b338Ip3qNrFt`-p&*2jf4>!;thy*H}Jz>esA!XI+)MSTQ5 z5G#Cz%h|v`-%Jzgp&41mD-o_m&~Z*xx8fGp>AqPFL8eAtkSU2aZqFG|A7Z_$3lWLD zIPBh>;N4tqgnc3{yQAheA*ruW;8{KZtj;`H-@oJjE0kfFDufYq8GQV?>1SGbjN`g2K69M4tm|p!Ezz!HTi@6yrdezgrEuDV zKadAo4W#fUx-IOS1*$Hp8BioK*$pd--oLnXT5u_tbK_h%u{&I7-sx^Ao9fy!b-UPO zytzNZN0DThfp`H#usu}J_c)KYu+u8l;#ON}<|Dm0T(k~OLrycezd5(R*wX|HIvt>> zN+h}P)^2&L<1UHkM6sIk>CR$nE7P%6baeD+-PMBIVG-`>$~;+YZ0sjlMSw=CMb@tw ze1k(nW1BMy6w-=ibwbT4IY0Mj4i7_%-~XT%zS+vjyEJuu4+vumU}S;iY^{>3Yah02TGP=t0LzR&%HGc6v1!sp{EXQxA=C7gp@SsWIBU*6nb zE}G8Oe;Kt0pwuw_mXD=bv(@vSK}aZd!Fd%US=gCVF{8*G;0$V5GLtVlyH=jB^S)K_6#^b@HP`6uiPHCveR73Cv!!YCq7mX;z++16MM40xvBE`b3Vdqp;Aun z#Xq!^9Vv1X3_z1M=hxtt@fzLx9I0>yKEC&Guc)cT(;CcnCri(k{IF!P%XNRMCzmX+ z2nwc%+@Gn;)LCZ;4fb&!m<8NvoemY4j^&?qK2wD5&(us1E&4(INmjIIc)#$Irf{3* z%jyO_rU9Ck9fvG;HgKP829BLNIRE^EDctf_2;M{zX!LLBvFmCh7Tud^7?jV}7;2{^<=D?v0QzzgaV zx()bDhAGuni01q^wgGXEXCae2XKrbkLNvq}I=FeP7v$b0utaN7^JWN#7i_to4)6l% zpl^3xYJ1{<)OL*>VuMje%b_@hdv4Wyu8t~|UOd)dXPjj6bO{Y$P=~>&oZc zeio~h$UaNfD0JD*O9Weg1Y7&l%bcQdFD%$knlH8p3*YWloec^e4v*wWt1-TwroziY zVQ1+>&nLzxCmcL|9vc&*iVAyJES29b(DQozxkMl}JNfa|YUJ42_SmZ;64F4haBALs z?t~f%;SRL7*l#PV!9f99W5ZO`?VoUxzUGmm6GNe+4vi)VA6ynn2glp?#8D!p05tqU zx66aLgoJ!qNG}q20yku#MRPp-=uY-QM5jTR`5T2e&KoPMGM1fQk}(qeL)`Al%E~qJ zOcv}A%KE@hYg*J*-|vH=2-1{zVn?<*3oTDPOqn#w_g;t*x~;$#Qf(^pa<~10N<$^znMA+H{Q0!N|R-&k(kbaq5Y|K^1_TD=s=(%*Dl}xbv^A zt!<&_{ml-F4hHyyzp}a$SheXcrJUBfas0DMz$dg%<~FC(u3^)uhsG050ge5MhRr|w zmJ+W$4hfT#!R@sj=S(*TCnqi8Vf`wm8eAE;A5>f)t~Wm!cND1<6ytuZ^uL*RT4cTa z=N!>4w3+>_FgOFNWppr2P_5|r?$GNow(nM}+9XT1Z2}y>)-yV}%*;&6o!L6;krQpH zDw7fForPw>GB-yWa5>e(iMfp`0-P@0mII!v7JWII8t`KQ2(L`Lgb9wj2H?^tiFK3M zVbdw5$JvN<@i?K#p%bn#keFQf;-O%a!Jm*2f38aa>_Y7K(@uGE>Xl$vd-!CVq$Q z(STaZ+5ABr@E7fQH)jNeXc;*UpiM=BIzld!D6mf;NrwtGtMa)r{QUg3eUQmB0T-8W z2B??mXjVIB{XEW;RaTA>+US`X%a_k7EG*2Hv}^ak4p~xRY47d}J?0(-7f#JM7!3{0 z7!G7DWQ;){-ZRsIO^=9xFkH9j$!yLpa`E>g z$4H7qw7E%9QPGa(!LP&@l`^q(BaIHbTi}}g94E~1yfIB`Dm}DDm|`lpSF?Z2kUSEA zq46yM^X}&ICCa_2atgo>P-o|Q7KJ|eS0(DD%0o6-3o$jo)17>)BVVecU8JU_Hfi@o z@czH<8+6+MKi4+9-`ymhl?>iy9J%zJo~3L_OG}Sg*3IWf=!AN0 z*6YVQ?S+2*`ej;54WCIh>hNm;?eSqwOCVYRcF1q{k}TrUC@mEe?hAjMm#7y*%&8LqRpmoibU&3ht@RHGkfC^H{e4iL zSsJkVSh}#A$yAwcB_3X+`$_9WzC6kK(;?1x3@1XZS69y46l+~~Gj{um`tWBCYQ?J2 zB!}4DeuBbo7jaWz1&B|?f4sG@_^z}3()s9vrdOmiMAs+LcjYrK@6^Un#vylKe+m`e zHAiz7g;y$&V(jt;{rk*h`me?^#o*0mYfJkh5uR3~l#zJ?YCMbdx|7H^u4ls5{1u3? zo*3yx)D=b~@tj>x)woT`of{2#1z16xU>*U-y>E9u8)!b7j-vK)LqXJvA@`f2kBst~ zEsgDD&R={1i4}2s98DT|xHvWZArB(RG)V$oa%UgdzlpMf@~j zH}z!QgU*4^3r?%CiXuw1OTj?1^<|>5Hy~d04GhG4I(Uj+31nbNrrZ7+!xNW~Q2Tm& zqP=~_LMlNID z=;|rv_(m0GA0YwC+Q(J+!TgND!5JmsDZQ+Mdsx4!^jb0!-p2{9v^Qxrq}7B8-vc5u z)S1Ty8*Dq1eX|+$5wq3azAM#0-=u2H=Uk+7n)IdUMawcNcMWGy7Un|FK>J13ptZ^$ zHxo~6Xg0WnJ<1(flW*!HFrdoWFjvThrrf?3!XM!2z<*PHWUj3oTmO2>esb}wM!Eu7 ze1uKo7DuDAKv~W=&uiq$`I;*u6LC6*rA#-TL&L^M(QLoV=UF~U6e_LND}9N8)h%ws zwJFA#Z98oA+6a0tR%dkQVW~w8zIDjDnS=N{JMvU`a3_VsYzIlu_{BHB%j6HFQdKX4 z={0#5UCv10Sw96rt4H#`J{h+Z&n=@kHhavT;3D&@M4+~QF!D5uIZM^q5z?G`-UasE z zq4;V%*-Xxk*y$B6*Zrz5k>k0);Csnf3{;oE ziLS__cIQ1q6Hz_=#U)VvcS0?`u+3OWTBD-nV9C0p5%W324<9f2o}CA z%!9;=-p6@Np0@~hXc@XjT%{zP;X+|qwwIji^~KuI*%j(l##XbuDhjY8IO0; z@%6}&F%L(-u*U1G_0?81Jv{@eV$jjPZ=@1R0kvOb_&Y?k&Po>?YoSJY`ggdAED2<_ zm7ne+c-BA*xSmgFYy)b~ude1&+tC6ToeM08?0&vthDh;vRTy~)5XNV7c0)?Qcjrx_ z*REzXx3-ElAO9ga0rn=P8MvHoY;5$hXVtxQWvB#N zQ6__s_esMfd|ru40VW{&utvvy*~^4VX(ut9O zeZXO*6Lsdt$Dnbku1E@Y?j%N){I~KW+N8p+aT^05lM*@!*6`51SbBJq-kFw|C=Fhf z*pR66XDqCKkr3^8SGnr5Jdlo56J@$7H@8Q>8fTVn@ENVyd3>1cnBVtj>q>Ap zm-Raf3mIHDl8jZqUh)B@JWinRb?@N19x8E~;Ty&7$*LGaV*Fh5fC;oHN*WviS0%V6oPYh(8*T_^l-K0j{%Y`=P! ziy8Csp0=+e7=$mqC(HE}k+H~CX_r_S85z&Eaw274*O^Vo#{2Siyhg`c=4DpIKj}=B zuc{hXZpcSk0190a$YiR^*;>n3H=A@$%ZLbWLV-X&z=QH_gl5|A6yjRd59IKIvI-yn7Rcr}&3;kdY?-{wxY&05!P5 zHkF?he|=v8fsdw-Gg_oV@1Gs}$;gNj4+Geysim0}5NexTpR5-nkhbqSkPAA70c$Fv z2Dbp#*whl#Vy^y`S*Sv##WY95;bdvboS%~fu-(pVWXLo@)JQs{UEBwPhfT3oKlge5 z-H5jGY5h^L2-I=TLa)XQAA^|dZM=6K%*;El$8~nT0S1Ji`FbOz7;lf;VMn#0dWb|r z$~T4zWCYHdjC%U7sHnuG6x&6b#D{_?Jl{$LVg8P1LGpz~7BkfB=Rc?9S7z)$E%DLT_5i+K>|}@@C#|0M=jDX%KWR)bhGth3)mq_7Ro2w=g;Y< z&Q7wcheU2!U`Uzvsx^tzSXoTWM}YcfEmm<0#13&&-kbKmKY+XMC-=NExY;Qw&-IGF zd=I1>FPVt@`!sI)ma}MyVcZaR557ZRLNYS>XY>kj94~qNsrg>;@qL3isF29WL^mr1 zee1=ovorhi?s@_?9ftOA85xRI#zXlqV4wsQhORJt7-klVLi?1oUF!@0fI@&iT!x(} zy?uQu#^^Axv0dn{-|m07KlFM>XgeilYg@VZh_L|zxB{1zuk^V*-9E6ePpK<{%ycAM za)(9fcL@HcwV6WacRm!JS7PL?r-Oo(-SF&JkPTn3(Mp30DV`9@DhP4zEkQ1lh;h+S zBn&Da)aNuS#98E#jT&V-NgW*>Th?B;Bpx7x+IdQ-*}(jJlfYZIe!8=UB0V2&@~Rlq zO8vA{ui4f9u=#QnXk$5m75MxY3Xe0Gg^aAVn9i2{3UhnCy}dEb zAO6lY*sX_P1GfS4L1pGsF3c%FC{Pe7uD@c>`Qs<0gS6oXwZ|6!Vbt_gls)CQPj}8R z?Zlw7Olg{+)2Kh{Gb8W7f(9GQu$1|QVypQ^)mBdrnDa#trtoW^#3S>ER%0M9e0@eA zVkNXKNKYRG5T0pFhwrIN4=c@5JO0A2B;K4A)3+bkP$ZG5@cJTdF|Gv+%4AMH(kzQT zZ|RI?-`|Brw}_l=Hbl<-zKH0#i$}v*62+Z-3TXng_jw6Db05foO#TKcJ-xct5tRCd zp{ds&+jZ@LOO{R*^-8z3v)cx>QE{g;i~cR((slBFP!5r|!!ysN?GMllS9IC%bs16D03=IP*D2Msyt3^iZTu zK~P|TuwGVB?F$WIDmt`YGYdiM$))Z2E&jzZP(7uup6;#jXS0DlIi0$vQ{oH)!SqLl zSyWCvxoq!hW(-TxRfPyX(rCx(Pr4H4N6hzdd?f*gxs{2A&orF(E^!>Eer{MZPAZMj zwN`+Dld(YtB0>Efsgyy^Xd&ebt^Sav;yVnc?3@=|oYqt?@ zZ8FEpbhR5P!g^0?c&!(74%$h<*;#?!ologwG^-U9%sEWyf*@EL7S1ufB6yfSom!|c zkWhpB>d9{o@Q)@|* z#`i@$?U)6#6Z9o!APjcc8ulUPwfa8jO+ivg=KM>#_1owlu zhJlB0BlT$T+VxN*=~sSBUH66=r@KhSd($6_lL5+oEx2ez|&EVUN9l~}5l1ODME4KT6YcCbrC94vR z@&vaQglBC4pPn!Au7DU-QLEAjwfUr%xfpNmEB)wHiH72&1p>^#Nv^D{7%hDQxf7T1 zAgLPKLEEj37dQw!BBCN$h(!?8J_}4DWWWNht>G*#5cGp~q}FO)09DF!|H)ne)cyyc z?JyW8qy8b{{9LGPLu)0V@%?=?OofB|TF^f`^q8DH!|Q<`)@D_j^`Wyjy-sA;S2u2- zb|xMK%ArU)k{mY-av{#p_@q3u%K6w35Oo0n?_Sw4L`1|L7QV6mB+l*bXc{gccELU= zz87qL2GL!S+{AKtB#gvE6MXG#SfhDyS4X_Q`(l0yNX||<-=9qc#3RmaHl7JUyXS?q zwWj0Z)&j_BZq^g^xAY%x5}BS&>bc4+w6SVcXM#laO+r&p~votc`9}{){DGVWC|1*s_we5n+eu1@dZdG75rfGiyrEO+Nn! z0U5b(VYI2KrF>Q7{ll;JXG^zd+cX|O-X*gAeq(7v?4O;uf@ltsEHxsv{l!*M`6LcW z9)nFELjClsdr*hTBg7@SR0!YV>o$(W%Ino=+zf)`H8uY5z} zfjET*4+l4nGmDCb-2kh5J(Z z@XoIy;gG3@B-@b`TMBto1axw9vh{u9ju z=mvT}E(nGGdd0*P!lYW5nXVRW{*CnI)X(oxn{X$TSckGEctU1FDS^jkn}lDHK*u8= z=!7k&ppYJ}K*2={wfYwe2U4Wf0r-ve?7ZfoA4sSDy_ZJ_(@VWU&B($WKljJikf z6L4mMJds^|5k$$Ga=q~O%8c?rf8A#79j%9{5qT=mY4FO*%4-}Q<*g4|5sxQ{t-~pQS-`bjzokWhz9w zEE)+0LdZuB9ZlVDsy+`GW_-s9HM0RaX-kxO3(K_MVG zQL3#`hBH~kl{8R&&V+@Rl*J60``?mK*eqE4K8QNEd*h!b7e%}ycvo^-J{;dqB8uKN z4T=o!q_{7V=w-_Am~DKZiFB-*RWcEy`$XWc{ByeaX&V?%fJ$kd);clZTck-=)e=BW z4kxrwy^n!yv5>mRo%BSkrse(`qg)5|(Kp}WS5B)kG&L;IelTPDg5Nbwq>+&`e=;Vk zw~GU-k)U*(9|eH0f{7u)Zy&!QG>7poY1RCt{eQjhJc!6eW%)Lb2t{Ek@Do9_1 zJ23o`xLK-Ia#s$u$CrB*9bTP?Kp=V4-HBe~h!Tw9u=23Vn!_27-eeVfW9XL0`zxy^ z=NZ3B@ZH75x6mz<+`tHXnor4O5eV-L4_?lg;99%lXvrYCdjlyq88A9PJTsqo2G91v z1kl`&#lbFf#m*yhhR{rLT5ucnHgW(!@x-7BA!45`bhQvFD!aM;sT-lp~ zC-uOs?r~I=64)YYn>3U?mUj?ef}kOPcyP-g|bu&}vb8_dBYv!<&Sdu1305A-0UCYPO7> zXSloU9vL$ih&v@QAO4tNr*rd}YDVC(wBjLu zk+}cMSfSA>le&AND9!32AOxlzlr~2c$tX)j>z3=!Dgh{7>`ja!-hizsouS#Firwk{ zTc7a0NSSuJ9lH(iJGRh`PL33Y$~eI;s8hKYySv~1!=2Q@D8r8*i@@0c>d69(8AwP+kY)kBD?2tr1Y{2VWqI>O@|>XW ztq812HJfjA7}0qLkp_)HDa*b4RSmp#z=*NU&*fi1CDM?E|H~Yv4RZ?{Q5z_E{vSp& zb(q-M6@{6mL7Oc`ub8eX4M`7zA|wjfZ9HwhS_zdTQ9C~HCZJbH%>?~Pujc?{0mMUW z6V;|F0`6BYCEvVxzfU!gascwI5KtJ>X~)Akqg#AIW2>`X6q(LQvL+1#vz`SGyOTZ8cL?!Gu6ERg~DAV`>50sZs(x0H!OS`d+0 zzo)DApm6ON_$RPYJV5#5)kx7%ug6<`0q3L4xtEva z7S10_mzF-D9zKCbg1&z3EdbvFmt|rDrcDa~Ey27$OV^xS^ZmsvBY@(Fc1Inzfx*H2 zX1B{xFeg!@Rb$5GdS;TOfB>d6O+IW6rsV)DIyxl!D5s>PbW-nnz5`U(N}CV-86eCt zPzKx=#4^?Jc0FJed41EXPbW2N?&K+l584kP8P;;9~Ku*Lmz@6DV#l4Ez$U&O!_+CzX|M%sAxc!Q)nJoeT(s-T^{EGM5c zsT%bSYFfv-&6uxhh^oHz@+F?kS1etU708qy^bED=4bHReAFeylDpNyV*|2{`Cd-D& z2)UzU+-OIuY4fILEn-gSiE%Ut)m>4UH@907Y>0CapMNq+)Y5($Ml6OJ_jbE6S$_ek zlx>zd38|9|sIJ$*ih;3=jgbC&ZKAzA$dS=1pdv#fxS#p2xGQN&qsC0lfnZMw3Z*Qk;Z1owD-e~T zv>hXI2NLr|{nxrLt5OX!>tHb79uzqaNEW|OMg4qz!EE8TWH7%C&Z>%WTjD?826#CP o_y6S^0{;WwH^@X8d4wlYVk{kt*!m8>Z~>8#RCrS+{=xTu0mL1IfdBvi literal 0 HcmV?d00001 diff --git a/_static/img/onnx/custom_aten_add_model.png b/_static/img/onnx/custom_aten_add_model.png new file mode 100644 index 0000000000000000000000000000000000000000..e5ef1c71742ccff347d84c65c6b7f5d15e960654 GIT binary patch literal 8613 zcmb_?WmH_vwk;$C2@o8DLkJ$Af#A>t57M~1Yk&lI0t8KPZyZA75Zoah+#NpL-QAnJ z^UiqhjPvi^``(WpyKC=SRcr5BRkh}v6|Sr(gZ+~1B@z-6w(LhqRp7dXgoMnEjtcyv zKk|7T}tUY==P}tCsX*q*qbdfzR5ZCV4id7YFn`dQ? z4*W#7v6`K)V;4=@oT<5m2u#pXkaWY<>yfhYPzXLqpfh|8!Bs%QrcFU269*x`|7QHm z8Xb-@iW`WLP5a-f5WLXW(DsDz7W8{)NLS1dl$t3wP>bH#Y}zl+1N+|f?!wX>RP*m{ zujUP0*Y9_x`8 zma! zZ{)QSfMz|KE#`7Q#HaS@Qz>BFz$@ysflkT)`Rm!)S)oMetAyBCF=l4wJ<@5@TVul} zmvAb-8HhHw$6X0aRe)yC& zIAyWPwfbRtkqRv+%}6+g=u?aE`r6v0zV*s+2RTJar3=UtMRD8r>yU{bHNpYj&Es0g zHnP;}J<%7>SOkMfCjt%!-7dN>t2DbE0%jn)C*z8b3r5>`M|Jk|#c`v*n<1}{?<(=- z(FkbN{~fM(DjG}bo=CO!x{CD06h5B?lVQ@3{DlcyumVkr-=7?X6wa8J#U(|$3Zct_ zAI$KEx8_?H>H?ZYBF{8b_{Bhvx!(}NbXi=YvW({+nsz2r6J&L2%<5T?IMh{+_H`1@ zrcH{$36N=FtVKQ(!a;mp!#YZXg}qJqtcsgCZ-a2BXWa|TqaU|{!a*y;(b zts$9QhY?Nx996C>?2{>Up!17oWt>IsC!O#^O<^!k1wy5IkT-Q;0r5l!WhgI$xvS<) z`P^Wn8EbAHW;aOGSu!ZS;4mN4u3Y2Cx2WhDGw~aq3pc$4r4>rfx>jV!ZknY-wce!M zuCMprrldBsnd%wo9{u_g5)tRR`Cg4}U~jJ@0b-M2LISqw)7G8Un)F%|u3ygy z_axaet*GI75r)hHmQ{; zDsrkjSc`h1?RL|RRU~HZz#r4c3vJUsVNJW+bp21&^7@}oSKpb%E-DS)cFo#vd|MQf zOzX5E#T@(cd`biqEDI`Oa))XMIE)k!R?F_G@AC}0N}WZV_mkqQ`}<9EO*&47>ZZ({?N#kgXyT3LV$QE81M@4?O55-_E}Hyf~Wp zsb?j+`UD-OPkRtZcLyasx|3uN88^!fPvYrzU|K%3Gsx0PtXG|8i555z$H1;576%`K zo-^D*&AOxuWh)76wlHrKl5Fbjx+Z2qQb8D7l3p?w%+=k60a^0{Hn33N!JSF>PtBT> zG%^Vuvs`>atr=O#C&`3@rEZPi(rI^dm1Gm#nspf}dQ<{{$~mNU2CNucXA9tk?+Cw} zUl)B1H~OL(g08uxr8IoO9xllpi|ODbH0fw6h>eYn(@SE6$ks2D4(q#%-HTqj=)!TD z)iHx5e~PczvsZ`nHm7S{XYamg3JOW2{OG?+J}@(N8FMtJ=6)+vk%j~RB;LnWV8k0% zCUjgq;3FuRR)pNuTy}udy54r9KGvg#MLZ zhscK@L7xz}?q#-=n}j#Qss5s%TxRyZ@^*HHc@8INHTeKHTdlCK#jf$0&3*gL8YU0P zYvDpOA*#vrXZ84;ZRXF`&j(P+jZ_0iSjj(1a33VoD+TNpqq^i|5Tdk{oGB2LglPRG80LNUcLx@MH$D&(smFur7f7^WBX@2(>vPU*Lf+Tt-B$N2C28%rtB%lU^~9^Vky7 zGcr#63C6UUfI-Z=7OR9moC5pQfSt-Pl;?YSJHqSo2ThoJkB0=L!h&sOj2VpY3U zKQ4bihkz3c>@ude<zMli#5Km+q|-oS|PIhM%HO3)rD z+jg2`TMmhlPpq7r+|H8saiSJg^`A>lPtP;b1JQ>Q37vcA$Ud!0U|2 z_;!|mx!gsMRoqjblHZ%Oil#`4vzKiSi-Ig1^~1+vP435v(cn9#$Ft;ib^2*SRrvc; zDbK6pWhDy>3zy414UN6D&4J_rgV%^AaZa40&xa1dtahgkks{t#qm^bo>fw$agpWgf z%L&M9vBv&8QzgmznSwKWOXz*ZsL!54&n-r7Uv{mKw-vuO+Dz?1xv2_T=|2k+Maub~ zQKz&$G{3)g!{f3e&3GiqU*j6?7i_FuzavG!*_!58Y3;kd#c)#`pWh zJrMoGhX^&kGoUA;lFfhFK@(IEzi;j6#IosAM&$tOzc$Qi-1F1EZHkE!(KgXbGA-_% zmG7V$#*?2Vd2_&eVF*QsEnjB$oilv*Pc)=sb{!8*xW=K@%&&0X+3gFm8GFV^`T z4Q9=-LH153)V-`FsEF=<<&^jw+d#RD@z_+)1;s%kzx8a5u%jxB8nrivO_R&onzKZcA<|A^dK!J!p{H7|RMRxE_}TjC}F#^(qeZA!^o?BA&_6WJzcD zE>J|JUEG)U_YAj0$8!P#a@725 z{OGoCydG&%sB9OTYMZu^D>W2(&{*rD-kP-C6164LiJ87euJ5F!DbzOy&3 zc25mH3J=!}S;VALd!Cx=s;Y0=d$xwsG{W5@h+|Hbqt3@`5+?$4o94ZUx|?LL21ab0 zJj$t*&AbXBX}m6*{X`>nLkzv({9@*KjUk)m?V1YuLD|yJZ#Cb1S)Qxi0VdQY8;6>I zkzHE^*;uK-{(X8~dC>aVySTin?-5dp3`>#^*F8@5SoK73OH=EL@^XejiM8zW4K{`6U%=oeS`49Kiy&i-ozML>hrou0Nb|26Z zZzn8mRT0Mb8pg_hr%mNrob>8DFP*On!<}mKpR=TO=#8PBEYaeaYxt6#j>ZGSqLleDqAZ!H^XUQ%LX7&%~1)4w3lC)s`E|)0kf94 zvo}>@`^qPPtl9=07CM@6N$o6<+x_Wv;BF3pGsfJPeG9gJlWaL{SiC%DH)j(fzN0_^vtBx*4A1N-yzSW zv)@f7|1gp2oAK zH8O%)3!p=el*c#KStv{EWCV``fNO@J`U~q}UTVilWiwuE_=R}8-{Vdt%n@ z46rX0e1E{L@ho%AIEq3bPS9m1TP~Xx9hThCy-^0PFkcY>V;vhk-6b#l68r0u>%l*< zAvn~E0PIkvU%_zpd_1rcjjtc&N2w+qA0OXYfd>`xh;$|=X}5S(E@XH0^pvPtW3k7M zVzu--v_HCeJ{)9Whks ziV4$C`!a5m1^z-SCGf3eqPQCFpe2_b1c5P;BN19CpUl38_lpmJhPqYe7`E~pInlEU z3qA(bme@e56p`t*$E@-4ivI^k`$N@1JYB+)SmZ7+KJTDH?(Aho?S{Pe@DM%x`b9T# z1NTK&mzCeho|u^uAm7gK$C=(ov1k;Z$c)zSk3fQ)~9~+V?pS*%rf!7!g z(WrfBa0DBU+wKbigd>9A)<6`SoGb%4CB>BX-A>_TQS|9nN#gy?55QNl?U5|4{8dIF zp)@_?ZnJ0+m|V61u7W5F6B9_c-oZW$R`z9goL#;IaoaF}1Fu9z+Mgv1bJOHGNM5FTf-Qaau%u<_3LQGsNU{_LB#sdt)Wcm=WT49f+ z^L0m(Z=ba0c#ig5aD)FbzTBI7tEuk!`zjcd#JsUk)@CeM>a1<*RE%hIbJI3zN6G(= z0FnX5)gtoG_vXUFk0Kq=hB^f~xj43G zyuc9LwgRBoT))LbB`M%o*$lP!=5$^5Tg+b|?7r|tW-&aMgk7J>u`I&>t)Z?ii=C7M z<%x<2N83&ALiBgvL`2YI3NnMOyqv6T4uGuFB_rPqcq}hviv?}?vaV_b^Kso@9mXf~ zuN=q6$J6nNuNW0_KK~Z&Gu!0KYGGxiR`0O5p_HY7N&HUvy3}GIiH$4ahyTqWni?S? z;V3{D$jd}iC2p=z16C_QQh3I%3)W2VC*C@GX_Do4S48oy5+8i{)04rK0LS(e{vZ*8 z6}x=~Hwj0K<-#{~Ee_{u&s<_;q9|hZOsoZb2;(33a7lOn$zEf%Jzrlvxu5vERzdg{ zB{48CoH_^&xH)RHK>|OjtwvyuZMV`_8UT56-voaBL9GE66%_?@-X4Bo-U{T5krab# zU(Z6q^q%_=hZx($8auUrXDsKuwNm>VG0{jKArc)ut4{T_qa*}d)LuY18iZUN`#&|a zV3%g&AWr(?1QL8+emtt6Nyrugrv#$a-^^v6Z;xo@2M|pfJE7n1kT}$7s;d{OB0Zi~ zgfzJJ$hg?g)yO8IDk=vx`rf6j`1ZE^5B!h=5~ z$aAd|Lj#{*44dJaqcP)^u?EE;K;odbOp#0)Q;GWNPp2ShcCFL>;4r562vkDyK>atv z^z3cM%i{9US^E*ylHQQ0vdHYP+4&uPdmj6={D==PmyOGr>DL!{EAO;>in)S@iFCIK z>TMSz$OeDL8&i=$4LuZ(?m$0Ob@TgO2~kC&-J$B2R-|N(BdpeuhwJzj{Z9J+}){ zV#Ibrxk~k?q@DJL7H~#g26v}sHLJ=@?8VDup_b%B-@T_Jfn)>@AM`6)WDQDTRLZ zy9rHGSreNjGaICoz(yhOf3IS<=Da5*T~{YUWgO0FN~=kgw`psrXG)7YMCra@(%IR0 z@}wRgzk?R0mCoHM2kX})&9^A|bUhQMv9iD`QA*)Dk{tbAE6jLq%gUah!eQmoH12r$ zw|7`#L)(w1q>lR-TFuWzM~+3?=>(3u`j$^k-LQ*4sx$2-*Pc?pvG71WBi-@m%U^Dr z>t;B6EY`PL$x<{E6Up9m+b%6`e>QN~;&Sp>+LMrG^RJu5lit^il{Vkd>xQ$=nCqB$ z+2v{cy!C@9`Jjp$X65GQE`X05xJ2VO&is#OCmWruZryB^`Ua2ZQN+W{7@yGhU9SeoGhC{oW*#UH*id>m7qZ@l~?+meZT|o!lr(8tH6QOn8 z4V5&QD<9^_72_TEU2j5uPlb(SAK5A_gJBhv7g{nZ^|Qh1@h3m+gU>+OL_G%H)C194 zCs1}%DyCw`CR5O1#!z0csqX%6{jnUK)<4h86qOhXLm$L1DGOrB`hH|v)i)cC4K6x4 zYl!t2)(+(3GvDWmfmkz5C#^;^+Ta&5xcuo(7P=4%3T2!>yX7I6Hy<-gZ~HqVmLsJ- zTgOlK)l&`&Bt;L=SEl75v?BAJrC)b6;GTizZ@Xzk-L9e~hvxzwmnr`s7{@{PvDTu__3c{h z(Sq3T_!=u!u2UaM4VW#zLbb-#jO;7x}5Eo$`P{;ufFLiYg20|u$qUqZKu0Q38&K7()&++ zI4VSZHux=RrOD^*C}@<5UIGAUYq^`)9;9c_aD{Bh$<`I^YNaf-u{ z?}_>dTymEwX4|W4rG-J@8<&0Rqdv_$>y|p{{2HPrf#7ZN{=HaOnh_$cXFyfmsw6TK zYcPWcQIb(0Po)4?+@v|kB<_o*J>Cm)6u?B&?p1a}7(%Z}Ds(oFY83~XT_W$>_38)d zPahB5?83ha3fEWm-K*$uJc*K}{PA(eU{HuuGDMrOXm>k@oh%vP2TDR{WfTkwlE5)@ zw_$Y6eQ9|?x=8-gR6Dlm0&601EYrMJpUW(0QpC7K#)f)}(sl1X?CmCOTsem`$2{(3 z>nP?m&9*7Uaol$Wy(@$tOcdY%=tXe|SsNz{IrDUr^ciy@TtQJWsUQ{Ygv4PORSh6I zHeA(i@V^BRP8y$CEHrcFkZcf-PP&)jLv}qcQoudtQ zk#tTo)UNJs)hEQGg*LKFck=2!K+aQ8V@zAz>0SJ(t+pJBv#RM1wu9?6e6~kQP4?no zWsRSq#=3a%5m~s4O@L}cRzU%)LcTTV>GV>@6SLiASGB*lS2=&RLqG9Q#?H=e@hSZs z7uQZJf5pz@E>g?y0GFlUKgce8yRZJe-~9iSJpPj?z%olb&Y#YBn_5tsv4?DQFnG4? z9n6Ws{YgW0HCDYCgym_2yuKkgmzFLkU{n&CmLKP&IjC_BCqcLTXHP7C8$$fV(kiXG z_Y;Ttm^!iew|C;P2t*C_U9OZZ6%+@Kf)=#CpUXmY{7&Hfz69pGEGcu$0<8QLZmb@pIp{B`gk4p#c3roo`uO~u~64C-8KHA2Yz^Jo1A!J!pG(0<#+4m9rA$Gx>xyXXDkTFtat0%AK4xOB};4M z8LJpb0%azqv9ZxM%KW^xrA0r4uoMk4i+c*lWqsVmX+WNjD~Bl&Nh%J^!w=NH7kIU{ zQxR>5J3Ew#Ke!6m>B63IQPiUB1>Z?HidBy_)YX*%A?osIF)h<$4MStEwYT?UL}X+> zpx!C__z2fJE*pwGg9jSdWz)$-jXIBr$S^9VE7RZEEH?2xZt=C}Fss1|@zGIHs)K!Q zRzqt%PEDH-cl*0zGSvA1FPVCGeXK+&=%PGgt|H9N@@)e01g1kedlxWfQw{npWSmVTV#FAI-J!e03I{{Plk z2bHV61*D|d3h;U6W8y4S2y07=ti>lUZ=9@-8AVXfzZ9;)ZHkDzM6;^NJcd^m&+_SsCR=A# z+gNQ%U8Aya2uFH)GqOHVi?{&hge|YuC8KFdO?TmCM&fngBBL%sDn4` zl4|;KO_;jkARdc-AqHi{b4%yC50fd1-90MlP4g)UZJENDE`i3jMG1am(~ zs~!fk6gauobKV+xp1m8z2rQzbH0&Hc| z7}nH^KNkR}E~8D+#fMCuO6trHacYmv3@YUL@Gy2M{!HcAP}8+^@ZE9aa3{5oRLqb~ zrOsE8HuQZjpCoXpB_@OpOUkub(;X|&=hDP}c&mInQT|s;31v$}|sJK%g-yo{{NSrK?X< z#+jo2hMBTU{Z_ZtFM(LP>DRel^5(FgtqNv0YAcm1qvPOE%3O84*cmHEdG-R1Ri#sH zsRDsuw?1CY7k84~RGqdoZ)>&RCDM<)apXa_jB(wcj?}2Hf!Vz7N*} z-4b&R6YI3gcu1acrx2Kc?_KxQbq4BfCZrR^vuRVvR!Y870ys}m@L+u-cKx4!v~}dW zdwSFWJU&H-`|j@<1JKR$|K@pu|6FkYbs~W>_X$m8llk04+h-U!T7e`hr6^f0ZXEbu D>9(|{ literal 0 HcmV?d00001 diff --git a/_static/img/onnx/custom_aten_gelu_function.png b/_static/img/onnx/custom_aten_gelu_function.png new file mode 100644 index 0000000000000000000000000000000000000000..5cb573e7dcb559480e00fb24e4878733d1aca9d0 GIT binary patch literal 22646 zcmeFZcT`l{n=QK0V?;zm1WBbLIa%Zkf|7%XWD1lh*&=5+DnU{qIR`;NqT~!mDXPdI zsmLfLp~yLhw>O?&cfZ@W@96jI_fL<$jN#Z;)lO^ewZ3o8`OUT8KUI|@xk`N%000sN z`Nz)y;KE%1IQQ%FMerL~N0KEuX=pT{7vj=McphK+i_|bDW!{rI$%@;g$WACP8` z?v&TnS!TYf!?gUdv`%O!`@DX{(g_@Qp6%2juqAfpl(@y{GizXS_CQ!xS|sxqmHRgs z^Y1XkoH|bRi1YSbJAbWn+|jGa6ET5vp4fGiyp*lo<25T1z07hM@H5P9Y?N47SSY;> z0KF8G5p+c0y#gZ1z)fh|8us6F>rslUYMJdSHVvzEpTRW+7ZzKDC11GqEbwu_DdeP0`-*mfKOlw^<2ts2335Pm*?esN&YB%Dk_-B zYQ50*m*Ze&$LE!qck|B!TWs6g+v?idaTT?H&6>fzmzHwd63OlI!YrH1WX%^R`c0HI z*Mo<;bA~!m8%np$!Y+~!TxJW2TC3D?M(+Af|etD&a`uzdS zKL}|ot{kE6$d0SJDZzBR<=4hWywC3BM9EL&46e9qG$k|JN;z81{_xPqsf}w= z|7JFYp>O^pi|t=5+-dY30v5{xtUmOcIMc4{604dp#?14XNB|Cm$a9!L_2IrPRj!x#@($m#W1zjyu9};!Lc9 zDy;{^dCaR!wASm@(j)!z*jYtb(_QMcinWt`eb2%+`TBr4Huo@o&B5;+Ot$by?mI7s z*A8N}1$^$qJ(rGVQS6!yzo0$z&ZThB9zJktcPh37d-2 z4NpqGkUX+K<>ltMA}5H(6pY%{xs#??;RCWipvbvx62GNT-|km#y69(Une!on)a4&n zeG}ZCpv$63Ue?q&_t)#?g=|Y4xokPlg54fEene>A{)yPYkR|pLD&)!fi|ZPw%@P56 zn`9U9d#bi(-mNd^VnVNZ|Hu$f+g%~gqgK=G0 zUj`?XNK2av1Cc8~Ma}miohFI=mHRd!^DvnJ)4|;LB)0U8+6UE$FnQ%{c$}Pjsg|kM zj&~TkLLI?FvKLLd>eD$fx|k&3{5dyShbP6&+om-SzCJLURKQ0i^x&e!U}fm5agW_^ zn7PqG@hE3nCCR|b0z!4iZbYEdL`Nh}r?%U?>`TqhDJOee>mPu}M^l-^Rq<+gFuN3! zVMrLMk9}O7$(#x}L3^)Q4cUjDSQrjcUFNJc6pt25T{h4#WQ*+l2d`Y z!~OGsM`?;nvqa`Th7Um|eHH2zl~bxiQj-36lr6thC*Rk8#cZpnPyT4uCvJ6_e3$Li z(9)n?ZC?%XcTL*`0Y3znBSm4*$po3J<;x$*sny_7_nZ?A_A; z&hy23wH15O)}0jb`6cB_-(3fk5*Rwg+t_}(?YvO{}Cq-A>WD?f-O&{w*+jD~X zqUv=qVO7f`c|B6IsuQOtc)sP}epNg8%`c8iJK|24&SNqDVRioJfrcm45+}YYbqHp# z3oZ%RK8l)e|J{7T0d|-;vA7fWtmAHey%XQW@2JXQlOL3D+UqcCyI46XF5b> z3$-ZrNn;?hMI?cN-MKmus2b zsFT?$eOzp}9`~1#%MG^|O_}-T6Ww{wU(GVD}b>R3R z#ypctKb3obMkIw0bZ#LJ#`H+;fqitOwbOoz=l zU0LitSN|!GuYL2Wl>gGMhhw}A33|ttEvnc&O{v4OvpI*t@N^^ldj?}9BFD8VL)l_< z*>j$wS#6@FN4rwiLaxyF3gA>IxP?L`OA1T)p16BngVAK&`?a5jQXrSSc`SO5j$=$c zVvq>%h&ebo$j#61&;ygB_QP}ktr5b18)Tz068@&(I?$UxfWyfl5Qw2SldBOj*WaJN zE_BzL2q>T0SsoiZP8q!{4*>Iwk!#=g&H?F}52n~7S1H?I!ecTJK52Nop(L%Vt2=ZK zc$N0c^NGym5&Cmy2fRZ8QgVwytCY}30JtmoU)jF0+$@bZln{@#?H*m4%ycxK?-ed# zvOWhy-vpDPlu>Lzpo}YH1ga|MB3fdFCD5q_C(JlOdJKC38A;a^&bnyjJ8QNuA9j z<)9}e5AQJQe%W5}_aAzv^Lpe5wM#-K;1l=ewsj;{b>4a*ypiPw=W{b--5F5!r^;s+213cG*1t_C*xlf zyuF~0&$wiDpD9&up3_<9T2avO+G01=`5MMlA2#EG@2lOTo)O`(v=o}S&=4YBoS$Rf zN2_1U$jtAdfD{b`2GUi69~tBMmiS4t8Sbdr3WR>Se-Rd|>6+u=&_?}1;SPD3b*Ur5 ztUoeduawxO`PPC{`LEXRaI2^5r5}lqoh4-KY3>r%=YgffvzvZ39CRhD{|m0jaG+ft zGh`FCy~w~8z{eOx5z|3~^X1^&?6wj1;32KMOnNCSB#9I?;US2E?E zjJw#6;a=CXjBwUbsS2>Wv|j{9rJx7KNkV(`Z6=ezgAM@!jpTrapQ6Mj>Mg8D2mCCr zidCL@OKF!bLnWrPxF*GU~P_#UsS3pV>J+LaLQToA`zM+hO>@V2H@BF znU)bcx(|P}W-@4r({y2VBw#&7?RkJy+IV1> zdoV$Ur!XsPgV>!C6HvlvEqYz|M#Uy3)XPR$AHoJIP)u2vi)(foQMKetlOb@Mx7DLnwc##|e@M_QMF zdCi5z#mu52F6&2^M(R1TQ3$dejwDJGB{`{;ye9JPMI3j2#YAFFVd0z*Tb|Z9EuLC1 zb>yZ}c4$z0A6aNy`BDJuFsl1~206p+;v}-LsM7w%>{jHn?FFHQSu*4lvlg~%j&CW4 z1lYPYR%zQ|=);u=r$1v4Tk@S z)_PnyR5}OH6*4p$oekzV7It=WY497SeZvo4O~Z}eoHe85NYPst!04^$?d?4 ze|%l|SA@_Hzw;npmpZhaYHXO8`Wyxg7Q_2jU}4-KbywVIxWDi7;pjt|_&@ITL3B)v z87JVkqtK|$_by`j-q``tdzS&H^6)($=p*>&!0=CO{RBomcDF!U2eU47Nh(Na+C>oo z^G~2FOZAjlDk#&5@Xd-PR(QVq9q9GkCS-CrSj&NQdx#(bPH#As*fV+ZjJLt?zY>dt zr}tKwuYd``Rb~Ag*qgsk#QVx%yxtDcDRj@;)l=ViW>*`Ns}`#GRt}g~g7(-F3tMC> z3frHV@%Jd{RJ#<(znbK;JG6E0Q?RwL0765;TesXnlOt*JC^2j#;p6mv%1V>FuJYex zRBPF5H6;xOq`N}5)GNao+%`js`CL(V4@rGPTbBr|+;&_uT< zFa*z?5{&wA@RQ1K`(v2ZBdWh2$(9beUE=NL(t|IX=CV%)9w)nKhSJ45Ur(drWw$bJ zQ7e08uj1TU^ch6X4+Rb=J+Fe;+MU=~x@#%7o99!ZD|hG<3SN_aGZI&BDhW;!;W73G zSLL7!jJxzMx|?%7M)13|r_U?tU+Ue)&M(=(3us{K6R(S&<}tS{lj~n}WY64)ZV$2! z#ucQBvuSk~gnrhLMVD!mVGDDE)vLm$n$6)=*u9Kh3}M8PvoW`q7islVLJ*_QUV3;d zRkFsl?|-JLkPSIX=r7IH;IL=!xi?2g*Y*8R7}yWnC0aX|0N#d*d0&xN&9eZRqy-QPm&&A=erWr(PsO*`RAkxs40Rlh=(aO+dv1UzVn zz)3AW@J`6tO-tn(810DzzXl3E3E?b6G_ZV}juxu9mu}+(ev~tmex3W{c?xr+;RuCG4jTLk0t=^Nq$s+OP(8>APcIY>D7HS_K z%}};+W{K}kt6&JvrLN&w$scK<`YJbv-W!vBG8mkFc*{`zr+CasL_ug~PHd|Em?vUW z%At$Eh@zqAE$Y-madALh(U84W#v!)K8Di3x*RE4N_JaaAxCph}TZ#RvG0)Y~-U;QF zh~`qg>v_zAg{SecaCjYqxog$;&7J}qvfP;Aa-J1^3OJ!WrRTcP#(a?GZc95xJt))I z=?Nje!4<3C#?q8Il~OTg^(@(t8n`aP6!~X}9BgD~-A_N&lgaqkJk{V0ol2|cw3-Kf zqkD#dX#%V%OcP$g9_rMpl?U}LJ;rKZ*@3MK5ClAk-Coq!wMJU^O<01emtY~M?z7pH zn#J!)bQVft#{;R--)=ubymf_X3iXSB%2+7DF1@_E`5oD*`sgLiNYsSS8QoF(2t4Fa=9r7ZbXMG$DC=?<-t(6ujpv_U;fNk4JtA{vZ*!7)=tBZ{xI9=>`WN`Gg) z)3NZem2KkeIDc?|)@oQo;#xiTD*wpS?AkSh{M4`qRmy3C`GUJ%Vs#;e37{7T<)e9- zNV;NcldlmY@NCoL7K$8B)My`F@ul!E_YQN}=V6uxImpffViR|>5S3SbDJ9m%JHr8< z;#4`e>8lP4tpZxgM=!uLMnodotmB7`O%ji?l3EJ#Ldm+aZx+92Bv1#QHy$6DcEX#B zYUhYPqLhdOvlj?HJ-ZQ~EiEIf@S5 zo@en^xo+2MV6e-af>o9+btw9Z5VB~X*Ykgq6A!i*;|53XUulzlJd;Ynq0_8Iky;>>)w)2^h5zN`yS(n(ON#?rngX({zhLnB>oK;1O5vr>ffR-KZG8y>#J*YqffIjkt!os06zhUngWH2->~aynk@^? zi^hnIf>b(n_x5C+sFX{1yNlGX zRW+Z7)r;F1)G{s-0mHm!*XjKjZ*cM=Sx=v)LZQY|`tkO!_$i@sd)w!duN67$g^cRN z%rQ$;{r9t!=`|V;29^>p!0*+pZV}+eqHk$nbuWxnI!`<-|BC-3&BS;o;=&02D()KV zcpa}yE0`sEGuq@aAo4(ZIA?U)Qjb-Cnr+k(c5n)%j&{zv2!=-4f}@l68;{Nbwv2v% z0O@LpulvEu!=MSM8|#vlymFTLONb-8i-zX7OzW1F3Aem-`=t;DS{jw03vvA8W;)6R z9OGF`)6u@KJeJN7Mpv*K3(bWZkhzxRLbqa2d6x-P^A}W8#=Ja3VIKC$sVc>W+TyI! zmFw`i@6#i5X}WiiOYpEv7)|}*ygC^ez2?dUfWK1DLe^U*^ylU>t32Opcb*9q7){yS zGA^?mj^k+eP*N7FU`2Dx`|kbt=1UrW?|jH|ifvdReeMVWNfy}hBRGhpX=VKVa8#bf zV?6ftH-B}0p6Q)SE@^L|1Xgen=oNr+&2It2klsWw=OuH0xu_zKbKSjR<&OAxL{!Qm zU)1y#d8t*NP=iRS$LN+7kS+t&l+yhxT)V>)-xGhV1UY87hI&ms@)+C4v)U<^T4-YH zzLF61L9V6lUz(9lGE`(|c6G;a5yqURMFvsNW0wZIdx%A(cIG{OU7DM~3vDPr^?rLP z!AczdgtNXT_*Wy0_kXF z_U$fv8^qzN4m&_**ef9m1p@%6A^RV?W&K~JUc`&8BbFvpb-k3=8n((bhPgI| zxBEe=omgIV-<%6n11@EUb&kdxNeK7JsTka-h$FFHkH1Y_q)@RjeABv^8`U=Guh`P5 zBeD9+?bShi{KSGv((~tBD0hn;?g7OW1X}}cSiT4V{D!EAK$dT2WxRunYZ)r2P|IVM z+uw4TG*OGZTT!t_1|}>>gXxX2GRGFwjnTuH7;aiyTN9hPcNVmY5bx$1Mh%V?_=cv$ zjpyiqHNA0;J24{zlB=CTW>c0y^hs&?g@_yErMv*}g&mygeq34_{Oq7mNo_kMi81WG zb!xlkTx4>W_pOP18DYBFIFWz@eyd06A)s?x@VB$YxK(6Qw z4X)bGHu!S3%`Fo!ST||+=T^lkhlNvlxf$Yb9AbxkTl$8R%R%EDxH`a;h-|BB$C?6O&N>Cj7fwXXc|heUxw%h509MmXpQZfGbw z)x2cXD4~4Pk6uXOUw(t&sH5Gl_VMO}B~|ki*r<835H#2=Nk>D4%ev$=0w6#JUim|~ zMU*8WxMB4G+m>VY^+nfGye5yh$Kpg8b&ic1)%SQhl)M-h*;kUL^}-6nvXyh}duZz7`il>lc`Nk?y^g^N{OaEYC2-2tz99nxCICp; zL4oss;u+1AY33pJU4>kk&z-_Vr&x5Vkr7@!{mk>Hrkrq_{u4Lt!XEc*uCB zY{{Rr{-YcaX>=2Wm1p7-SVEwdZr%AESnqj(QC08nI1BdgeR zrT<;-=e+CqyF`9R1tAnlaTsxwP1|US90ajb%x%SV=-@BWUkXx$ZC*b|e{_4TGIa58 zC8SqZpcqA>4wXkoNiJT0|JA=az{s_aZ|MvsIU1|{lynTZ&N)!VQ{{fH){HQa?WmAY z`alt@T#%3?&eUMk?A+!}oVcwL#+cL>7(EQio_xvN(S4rwdN(zQy*1)0C8Qa=Czp^a zxai5^in;SZZ7?K0){cXd^nSie;+V-%uP1&B!kf?Xn4gRbNPsd~N;rSX+F_=NbMwA) zekL2JWQRoXxOetmQtc_yrHzU_R864Pu$RbNFA3oW^Ry8w33t>F$H&j<>x4WFekPg_ z@7j^QUTI%rCoVQKEopR^{l>9oKoyhUR}}x;+cA{pdBJ(9w4Dx$5+_1mffzma?OMbA z!G-Kbip5~uj55?~-UC?+cfMEOnz=%moiVvg%J)ZTvk8V1Z!9$wENweJ@F1(cifWHP z@m5H&O?Agw_3NG7AWgE&VoQ&uwat9vfxJUwGXXYaM;{G-#~h_3-O>WL)HLHzKG*vQ zKRxTbI6do_nA%XYRFP$eqKPYd&nP%$Xk%C|WS;!A;G9(2ZllI@vSW8!`yqbWzvyi^ z)%O9{_TFh{VRC4I#EntWSb-&lNl9zh00pE+S-PDRt)tD)wSoRF*?R&>0pf7teMN+m zheptus!n|B?L=GFK^Al-8Ln(q2Z7oI14F8N5vR#rV%2!E5N7_ zTj&vqDAf*`sMrk0S0q`Q*|ru%7d8uhbfR|QQK`#(pixEhrmH;kRkHMi79qE-Aa#^= z;18hoJQ(l*r^IPJREQR6fJxvd#eAfqmAPoXZ1!(X~5MJ_A?FW<7!+Re+96Id{4%!LyhoNIi`zMzz@vZ@M@nF2anN(kci~M8t%>v#A*&0eb1q z01z)p+HFX^d=eKe=kETDNpAc(O@Hmk_)k}uc;0>hFXNv^o=|Cm5Ww5zIxq>2=T24$ zBY|Vjg+9cS_twU$%^uMV`5zaOku6>JH%VwlKsi0&th@AYN)HVQ#ID?yV7i#BL7~ZD zk6I_O=9~0-Ua<0o|85{Q)H%fDS()=)Ikb0L^%oLrA*M%W{R4{%8X+3!{-2yhebQ34^?isJ!x(RYmX?Y;M z`FF(>q^vz2J7=ypkN3ag-u{XMA3}KV4T&L8eiuK?wh5H{a1LmA4hr&hO&p-Azijdc z_~Zc~;*5F!58ft~(p}J{1JboW7Z;;yYikPzD~1e>&UCHpj}GpgqGi5AiUpuVA7!Hc zET#Ic6*o=nj=-yWxqkx=5gKAu>;y-zy^MGX7ngDmOTuPB0&FHOtB3x#L$2Q{n^3X5 zo*KqxJX`7xcX{!Q+uiahK+Ja+Mg2kQXY67_I`uVVQf9La&s_h z)0afy%ZMIW^^i)t3l5{!u2GO%D?YqKqC*22%IZt+FNcAO-0VR`%Yh*QW-?dd&~&?} z-|L+ja?5yqd?S8jE#jN?8PK%)F&rN!E?4Hs$UseV81)?m(tKCykJ#Db!b+D%%B|fX z2=atTm_sQ@e#0_Pn8pVlQq<&vzAodh=mPbIhamJ^NXhmMZAfQ58H4Jqfxyt(%!$nvsP6&Mh-+`IO*7rPk$AQ||ARrztkb^K3 z>Wg`qzA*bf;EU;{<@dyTOKuZi`?l{Z4ZZ&xeIBngKHMM&;U^S^e-)a`*mUM~*^Ecn zKJEYUg=)M_<~}nRPC)R_F5p1-VWiwrwLd^u;jT!BJcx4*Rl&p(5%Lmg)y%|O!a{Fj z7DuSaPjU&Am}%zBmMS)abzv{ssniAsTpO7S(@%8y$`y+U^BT9*NbFiPk)&hwmmrmPgaP6Sk%Tt= z)zm>DGmmdUHfUox8aIdJzO}6`Sth$MGo!SWFi?hOAd)naNjoNT$n99rM6mY z`VEFKrqV~~7e!2c?ISU{{aD6kGz(-``C;xBE6ZiOYUZI;c_Rl=AaT(9cM%d~v{(Nx zYyDpgO1+144z2f)e-T|Wh02~|VClB5NrMJtn4lHrrUbPSn5?5gAI!$_{e1?DzXeB7 z9sn!8kh^I)%B%R*<$Hg7XWoJ2bN&_G`Z2^%z`uV^!TrZ^DYlSD#*lu7Wagk^=P@mmj}JGh&Yw_U>VC`k!b5ki*G z>Cq-0G#|Bj5Y4y!5#c0vM(|6{V)^#1vrzq*BRAzXV}qv(FruYz+!jryUX%Bfj@9pv z^kN-K_E-seXSvWIHSbrer)g?{0;&p$$5hWIT2FtFqi^#Fj{N@q?M0GyJv#3FPg{hX zv~Im8=PzCh0x5~m{;#MqdccE?=)r^S$K5-{>Ye*bX}p>h)@Tct};_cA=%wB)#v^y4&=7X5+R<#`SN++*#fx*zmAByyoaf zm*}D^iRRtGHi=sHR^M0E72w8P?>u-$$4gSzMf3}@GNFT77FAK6G}3vGL(;7}`aZ*r zXWI)jd+!%0g~x9O#H0+4-X9u!ZHOot`W}qhDdm4wC)sUZL2MKSN6s}dedH0vxp_b8pX*(d zNDTB0Ax@LtFd&$O8w~F?GHVux^ut)(>1;2HDoJksTIW@j&mU>hU2kmLeE!62(>LjH z*&*>=qF0|6&&enC_I_&IOj$)&;0%*a-neajm&4jtP27%RR#VN9cGcR8cRM|tf~6fV zS(yC%=8qiGwRi%$#W?gUIkJ@#ZWcElSh-LDQihb@7RBkcHCBB|uoq;JQZ)$2p_?c&TeJ@&Yj6T0iU zRCJl82OvFZnB*trdsh=RO+0rkXfDl?OoHj^?o>|C4Zq37ICf(~-zQynF+&cghK>9| z6>s(-isFt0Ad+jdSBA9ZXn(WcgJ)qqJIOAXAzbLbt!s>FIlkPo-*{!bhH#O;t+iFR z%-qZZv_8fu4Dx$P_4d?lb~FBO`U_%3=^c9Y8f8HXJ$LQsq=S)9#S z6FqO8NH2PHlt)zoX|tb)Crh_5)eUC~c#RNKVP2@e0w8kcX_(-Wm0aPt!y11v!T-=! zS1%zfPg&O3{8*tnQQpwgV|9?=slu3fG0J-9ArBH4ss5p5ZiKMS#6TM5Sr((y3p2B? zqN*rsuZtKz)|g_I34y4eLLB68@mjVvQd(@AZw}#Z6e@J3gzf!xsqaJl`?{4{_K91U zfV+}3CNe7MAI)P`#5OTyrKhluy{RTRk|jz#b0W!7IgNIC*PY}79VxvtS_U?lqEtI3 zUs~o?%bhnKVAgy0K1mDXJ@R}Tu%tz;GCU)qhb-~Y08oYV=Aw9447Rr9I zz|%d{X}qfYie0zF;Syhvve%;ilp+d@={&FHuZ1TLe}P*IYMi;ECmxk*8Q-*lm=l%4 zFj?g1F%qJedWu7DJ5_ov&n0HREDk-iszk>o9+vX6(@5h3ew6S^@B*(Si5?)PSL#+( zQE>(i3Pp7WRk#>!!-RmWw#%)iId)*aVFg9`SKFfnDY~04ID)^zvqnH?xHZHBc@m`S zdo-e4xkuzbIbB7{ST2??id)6&O?>3DTj)!P35_?{@7z>b6c-4k_I7${vNpUY>|!c( zO^TbJPgQs>*?ZJ4y8(<+wWe_d<}@!Jy6wHsYTj&%%c+z5-L9DT%&GB1+Jzf)W=5 z<85l)jR(uV3$Zx#!apeARpV=Dg^VUKp+73MXV#0cMlnEZUAibA zFFCAAcs$=Ev7k>W>zGL09rAN*jIa~x_mW#S)mu-V-ZblP%2|1dogd2D?Otf(8VJ*< zSNj|qFwFI#vFUlGmT9f06Oyj|`NlyXHk{kw`@W)UGnbWKSy^?r`A5r?j_`vjB0sUq zcL`3Qxl9SwHvcU({im})J@|lJz|{<+qnN&lZ&>u1i_x@AmP-&ukuPeKxVjc!Vuw_= zs=WRysCt#0vT``y+~~nB*T_t*djG4hkJpdQ!eqhPHm5%>rs8JXtvL)`wx~u&n6;wd zvhYjUOKf40RMHi^LKC$W}INO;(4m+Ng!{-|vTMdnQ7wVMUH}6WM@ZHw- zrKlQrP#PkZQhWNZWz>I=-oeyAb!DacK=`_~__%q|>v$X2&j4zr*w{}RO0qrz+=o2N zv_P#Wi2VU6X6o17U0p7-T%2^3#6lpOhALXXf3m*&|5G(3WvOoJ*se2ymsvKHip*%g zUAuI$(v9onc$^edekg}Ml3NLF5@uUeJ!aMN-ae1q*_Ojpu>0PKyWUYN`d51lduCo9 zTL3AHzG~9L`tMGeTH#g#n|DY>q(JImdek+D!@NftJwG1vUau<oMWMC+65TQTm)r9YcV6lJdf@V3;#}A?q zQpXpxk~4YLPCuHF+x3hkHNBg4t5Y{LcVVzNqs>o3h6k6yJ$nnmvE)9<$5XV23o-ez z%VzR*SG%R~)XN%ehC@@#^vQuc`K`!;sK#lYxhSY+?umim&!zb7F(rKL9R`&KXrPW#<+eiJhVX;m0=0LD9*Fm)SvB3!?w+_6=W{$tb=+EzTR&5>r{Qw_ z?7T97LeC~P5TE4hixAr(@*Za1ULKbEBoHfxu!xkX%u??vy*% zBBZdSi{6mxbL3NQpm(!0;GfwuuQMb)TH=SF;>mW8m$i-Cic3|Y)72(}U5!^^#%Qt| zg;L@WGz?SG+x#LtC$n|_j@&+?7`hM6komE9=06tZooWT^Gxkh&7xIg*?Vh&lF36&yA7+Kuq#$A!4&IQ$rCt*&C#%$G;L~v~hIOh|< z@v4I}a8xPKEn|{c`;+vst*z}4&H7u|(cMbB3F=HaRCtZ!9A@hDcnS-V&lKni>AW$J z;ky@a;Pm|o@^HBd3-WLn#A7-1%2}2rkJg&;xTX{5t<@<>ad4U8LAGgII3%@#2RmH= zGstKF2~8hixXo#Cs542B`($PH1f1i){eJ0-32#(T0&Z(R;`uC}4OwyBIo|W<_PPhf z0C43uc*AZ`x;Ab)-s#X&DXLvy( zX}Dh4aH<;aVlx)I{ma7fXAK4ggONyf8y)n;O_!w|+vxQSKDyLGGr3*l*8D-voDOhe z9s-p$zalWI%N*~o)R`Q*HqrQ=Ozs+EJGdbM#BbVxvC~l(n zt^e_6*btH8-1`EI=iHNCr^o97&7?gZz`V`xP$ZC%fjD`+Gx80Ka9fUR3Z3$umsvmH z?Js^Ykpk~7KH~1${3&0r#>1i4TE3(dCH$o>a)N&qu=}Y z_4#1wDDC$NYW<%_04J_9=iB0yyCclLhP8Yrz3p86g16f>TddOfvY(z5KAnuLOr(A2 z<5f?PJYfJIl01;tW6Mjk0>xH$HubdeX-1h|H)@txO#7-tPbl9^lwoy#elXQvTwR7Q5n* zN3v!?}Lk-)Rq@IuXdHIL>(L zqg&7y9Q$)=kyAnv{MBz#6=*XYDY4bpV)qae?W>K&)4A5~8EZiabDrjph@}+J?ReF; zq(5hS)LQp!Zv0*@@PDj4_jMrXCA}==mWJ2e#uK-}a{;}Oq*F+WxTUFNJw3v?t$sT9 zfTgoNZ4v+J)LqPHFC{PjwXzE*w5xbHs0;8sU^*J}#VScQ@)sR+${EWyN#ody(d@ph zinOn>34Uwne&0@X4zaGSQ5141wOds{4@Sz78`^W- z(>HWKu7DUk!jE1KP++cOY23!aLf;zei><}Lk?RQET|?>(+*Ig{TKzz|-LEvG;3+W! zx;P(}x)-EezCS)~AyHGQ&U_|#YUmu{_x-XQ5#H*_0=cif4pQU!UA;@}xxKAaRgQK) zcHlK+4qMopeD0_2wUC$i=r>VqWn5V=Jl$-r9?w2YkzlDWQq`Fdgr83ThCJ@E3%L}G z@U++}T<>Z+l;83+^X537R$s5Yw~E>&_4K9X|B+ig{neIiRfv<#zDBH$3c80PCci9= zhDA<=BdU^R+~F-r%*^TDh>+cB(6*~LtAwWs;d1*naw2+RZzP}WSYq!pWJLT8SVacY z1(}I{v9Gtof|qCQ!mv@>wEEF;^sj=*xx>^*7PWrh! z9g$~G7oh?|M!njff&^si7o~m^N2A~-kB+a%*E0Jk=F*dk$J6P@#DUsXEeA%o1k*+| zb<=(&gZB$K34>R(I`%!{=!>BVOd~&a)*7v#ObwcP@OAoxy9FhwG47SU{fvxqO5F?H z=&nzj{6`9OTzL`@+3jp(HXgC0MJF?ztRIV=!QQGHo2}g&3HT-SAUtVk^wvt{VHYfT zvu=vNI%e65>*&QbcSmro7C6BX;7Bhuty;))`{6I`+?z91(513bqKN_FEt}d*kLOLvL`_M{sA2&`Rvid)MH2Qfz!s%OKuB+>s{YlsMbDF z1~1BHpUh0>*4k^JANgZBJp>F7(y;jT3I*1I)1#^N!cA0&6D(1@rjjiWN0Hj}IxQr& z@m7z(IR3r8Yzr|dx~JeCbkW{>V4K2nibLQslQ8mP1hwSk7fwU>?R25}E?r@k=@{Fj z>GzlHL)cYC*}_F!)pyd-2h}TE!SQ?CyJ&Lm`kT;<4L|IVzH?9%q@sov#~IO*=a|N= z42dJxb+yqSeYd z?~P-$lsTh)LHF39_H(p8y$6dmQAE{$Zees}L)ny%?x*>%kq<-5Tl-5*zFJVMJPUX+ zrLOf1`4l8w;pIg$bNw5eXQ6Nz423a**J$GhgRgT%XNG9%t^c_+{Db7}MX)9g-elt= zD2}ccpRj!_;IO73=?LeGQ*M$d(y=X{y25;hZh!B?D!SD>e3}XC(tBdWaZ6HObd2E0 z${uoF;+Rly1^@(G!-lOd89FFvu^&9mIPxx0oZXYI$|FMD)yawXyW%-@9 zd&HEteLNCR_r?uaLWs?zMi)GH*rT~A6;(3Iufa%br?38mCSNb}9ilb~PqUq++@`#l z$M~QriuMiZ&z#X|x7s!Q`}7+MpR!$@K}+wOX3*-ha4!4yDXhni{^|$c7IFuw@<6n{ z?r7aHEo50K)|6AvquFzk++MeWZStW|LOh9-#hIT3$9o~k#l7G6#DZ4L)eh~`_J^rr z!#c>`s)`WopyLT#e8NZcU^%wHes3u4R4m0x`KRymw-;NI8jqLVHaBL}UFI{+h?~@u z=~V%`g`VBi~NDo{|{ zic4Pq{00FzyP)~`^kJw#m)v#I-C}68!6Mb2$*%fmz`w!i-h0ika2or1_!$T` z^jxkeaJ7)!`%3z;zFzF)p!ie-^xQ*``^sHHN#lB$J-X>+u?h1zCzV>i%C5^J$-uFT zf4xo3hX-Q$RD7?V4nfKIj0BjZIQ1zYM=f(&76$W8_SSxuylf6AKONLI3M=atVQ)Mb zEXL4G?P0197V`Ujz@q>WqqAuv-Yr8aehU%;`wx5$m*Z5i62Bs3K7veq`EVnyDLFjy zk97OI;$r)wQ*iL5TZ7G9mqpRj?N0gn4p}*Xdb=5R@(}8AHOXTqo6&Mski{gX ze9CbHSrF~{OV>jgP+~j~(Sei>d=Nu;YxlWcYNEf-z`dIZJ7$^4V!T5pzgl{Fa)|Y( zG^C1RQVA9WyB?fDXUjZ$L6+Yz06X2o?qLKQ$#%yb)kb}=qLXB>leIv6!>YHvUby(q zaCurPzN+ciR&0AfV;xPdh6eSb&Al0J--I2Q^MLw#&*@PQ60C6L*aC@X&LB%bJ3&ZNyFKHb+;Ul`h zZy*o8PK&^3S2GjK=f%O!9+F?&sJA!V(Xec5uY}tZTl&%rzTbtNoxKb+C4+9Nw?HYS z`~FNEK3sG~ei5XnA-hed$CfFRoFjfrcL<5^y4$$s+6ExYyx)p6FcLEEy*oi*sot%h z6DO>+Zqc=2q|*+^=lTv@n+_+o&R81g{qp=Un6I6Kg_~fqGNN&a)7-m zndJ-$wr;*%1)KMSBFneO_pX8p*hx#`AY)sV8& zouc97-wa^)^4+$!S+}JFUzBFkp$6yx1&`rATlI6sCL%rDR>ttl_NQJPBUI}-IkD)$ zGeauX_2>PaChi=5rfHpRu- zm6r;~8pLlQw5w|=>W1UZ)QU^Cylf091Pxqdy}qnFPu|GiG2_}BrJ0WYCLUl|$EFAC z*f%&j6!xE^5@FZkH(aZ8TRHhQ=k&69lak(d!yS%~4J5XJI_LnTd0n>k>|CPOisN5I z6c(?}R2aM7%qy(j38gw9+(Ay|GvJSk5i~eMVUdGlH!ZJ|Uyk@%IqJTuKapMzE7p|Q z`7B`@1B28`VzXiJZC@^9Gu9j)Ge2pjmp&MUr!{(mzqw5+NqW8z)An}GC|(<_GN-H< z{6e-~^U~kqvmu?NCFh@BF3udfWXPjW7A>vkLl8-))_PuRz znP*maUhcu$uN>=o9a^`{HMo3jug1gFIO^Z);{#@T%y1Qh>&V$B48*7g2CY-4x%ssc z*f@3mRh-vKG`=kLjf@V|8UBcfu4X?G zZ@lHmEhKS$LUK`2oYvLASsRUI$LA4G(u_zoeH2WgqEtIf3aTJB9vm&;)~@piBN4z} z%0^J&!&?(SpU0!$N8t;kZ3j^>C5-Y)41=TymnQ;jQ@W3f!j~IO7NW(F3v}D zqu=HnCJCaxT*W6`sb0StbsRcK;ETaM*pW0Y)Q6~5MK$N4Ij9RD?(a*Pp)jF~e zyX{MkSG8=coLSZC3@$r$kdIFDF;uQG(-k{f%78E7=diyN7|Pheivb7Mi5@Us4|{vP zJ&*BZE60|A{B=Q_LNIFNNsUd8CD#4bG5@UX4_5rNeWFUU4~G`FJI8b>66tYTo;g3_ zi(;KKpj_{Nus zFwpY*H*Oy(!D3tdn-P*81#jPcJ&*5o8u($dK3?1xZtkBs7eV7dg~l5^kBwC6hAowP$No*}O| zlzr8UYmsBUj@hJ*bGKp`aqyR^WPFf?FO6vNHTPHbyKggzv315@Q%D7;bPu$ZrrW_0 zYwg5+$;Z^mK;pIIxcXtCj`P^8kk3)jG)92&Vk58a++E%_w}#$X5cdx#ilp`H=|@B}NWfC~s;!bhkBQ|Oh3P%kho3)F>^P4=p$F~PW@tupOt_6p{c$av zabe+mlO+DuacKKp?h$dOnknC-yR*vFbeB$qgnhlo*Pl04@0}jo8Jv9Z8F@0H@>AT~ z!Qt9?y|94qfg|!jUDx$jy(P8quu{!X~11V zz)>AV@VY$P8^?O3li%Ff_z~EEF?DkJNX# z9}1i81tuim0=`S?e={E^Z_N6-%65wS?)!OnK*rkkFif(0S#G~i@7%vHH>)+uk50Y` zuGE10d{aQ1-KNQbCT5nqm*qc5-!H4b@5dv#Q%?(%%4+>DfA|bC;lfp5o9f7H;H=OZ zv5l#xr>${||NQ6k`Q^t!Gc+%Oo2nQ<{e$Yx?R%c;zGO4$Tt2_f%J$NBP(A$-G=cUN zHh~6dBLLS|eP4RKO`vPv)B1nU>mLD|yuPo2C00xoI74JHorp_0(joZrBWMBH+uPfh zm;UP8{eGXd?LOdKS}!9r8&Z1%)DSriYz2$&|9Lk5k^R5N_M75e?;-Io<6-OC3q4oXR)+nccK!SM|G(ex?7a$XGrq0(bW;8DFVLvFd7c}rfUBOF!g2>VrzOuW zSCR1N{{P?g$(NRRe*AElKRx}WFK~iD1Tm; z_5fSwk^W0|$HrGaoeJuI04LNSCu@OPsf_l(0DXMU`hCw-?eH|)FD1_He7zf!j~@dj z!`WXM5Up|fPwn=96zhJTj!)CAS_Nw9Rp0wOZw>2#cd+I%V?&%bxTkqu?*=GbNA9jF gP&=Lu{`XV*SFa@B6&bliSRG`cr>mdKI;Vst0L;)ucK`qY literal 0 HcmV?d00001 diff --git a/_static/img/onnx/custom_aten_gelu_model.png b/_static/img/onnx/custom_aten_gelu_model.png new file mode 100644 index 0000000000000000000000000000000000000000..6bc46337b48b07ed2e617770c10046c2b1404f35 GIT binary patch literal 6960 zcmbuEcQ~7G+s9GApRLp?HDi@hl~SeDCZVXHYPS`8R;?h?DjHg1m5RM$OHq5Zw6+?h zsFB)4ZK1???>^6QyyK7OJ&xyn|2UJ}*GR5A$8~+r^ZU7@?i=bd(euz#P*5;IAzCmB z3QATWlxY73T4c{1gMl~7XE0q2ii)qiYrqAyle&RA1w|!>;pFjOz%`vW#NrtR1*hnr zpoGFiwkarhzCg9qO#^K=W*FSZmQK5NyeQwMacA30gBfV^ZwOv+7;<#sg};;XM`zOs z+S4XUyYCn=@R%EO`#V-Td|gW)>ZOgo;#ODhDYl*B+Wme0M%c%;FHg;#?@V0f;^wMc zISFGCeD-6wZbG%bXNHB3LLN{lDVeeYw8W z9N)XM(6uyCY41ZEl{h-t8yvuHF5%KV6Glf(xo^3@%VbfdW!06Jlk4_Ek*y>oB@^7n zD^g(!j68CX%)oiNp8kG0QEzUcc|o(i?!LZ6%-v%M?p`X0aI&o6Fj{2n)5RC?PP#Gc znt4Tcys+-#P>v?-!2>Y?)Jf&Nth+W3@V3&Y)>{ zxuik8kw*pZyla8_S(dJ{Zb0+>wUj|(`{6*42qVzvZK-G5n^ML!*pn`VbMEFvcxomu zwwcbCYlDQe=D&MNFuJ%FNt7jLr-34>ns8d2L=r9#cUwe3Qvp0);e6Wj{1}@7 zJ8(>f7hF7$x$@FP9NlaocIQHux`v?E{OH;Umv(I$E)rY20zG>ZmpydQ^BN((s2h}K zaAq0diCfgS?QT_73c|qK)(D!Vib4)xb=ZZPxaUH}2}|XQ>Sz&m4G}}8%oeE67MP*e zh5>%s50)rp^l%R}ey?mUmR9iS=F+2y)7I)K;W1f_5NRIIe|w`<6X9K46&axTn4@~qWW#&5)+@b@1g#``Cos#Vjq z(4X_Lmk>LXQYt4r-}Be5Hg4J5u^|O2!{~P2ut3>y#UoNPg{+fax#@9iIpkZFGi!Qy zcii~CY#)R%9HJb39G8`x(Cm)S7fp2d6e%TP)hG#P@0to2R!Q(mK6&E4G8Xzkn!_SX zRTmU&%0i%xua~p%sart^?0)Dp9AoRVCq-$Q-|mV_J(yVq*QTcq34R|L z*@Sbp3A21v-g6pjQ*~Qh+k{~=4-tdjO_i%@jsYPmny0TnZ-pKHWOq3Uanc z)NG>=3ZpI5=h8W0o$-|5O_6rMc&(?$^WVk*d$|!dz0{X709(0_I!{m)9ummK1;YuE zFMa&Ywwu4l`mc?MR)nqG(D*DwgryR}Ng2e7>CU!Px_cSJ7cY75uNosu`L9fCp>MT@?6U=i=gW%(*F~#p`o86VNAJHx06?^OP6$B>$=&+_yjKRA?2_ zjJ$E{mn87zu7QV@UHpk{XC(aYZ17epETFj0>HFAN&o?7g551otjnA^*a0ZF{`@TnO zMkvXGf`a*Zm%Bu4`=IK1bL+Y4?@t%u+m9k+Z901#nPI1(Kd1QQ?656`j!mq)uIsPlQC>3M!-vtMTVC3b5c@Lz84}m zThzCV>b(Y<;$K8I^rFgsT zkrXD}P|JWiWBx3@V<6pbrlPk92Q90*dXX_@TR5jlNNcFl&@(D2$n9LtYt`D-OKw3kTBcd`xm5qk*1}>^f9p{9V zWTIfteab3%eb*l?_oI83;WtG%UH?aV`0uIW|N0rdwmsM3 z$n4B7MD>wgDD3KsQgo498-=ux7E8poniUHOU2KlG{l|bXF(9SCQ_OY=r+q@d|2pLC zXyusGnf{*>9fy0OlY!At(km80XC@8m0T5&eNLbG~G^l0&(s6nYbUJemaDNc#A$QJQ zqw(4JA>Xjmd1U~0Ls^5sA3sGwFVE_1W@ZK?X3lDM!wm0(#YCfCK*bE`ShUb;tDWnsq$TeH3X4LBshvoOB4J1@2M=33uKTzNmS}`PD86LhqH;rCWXDfcNQGZ&rTbE z_1T?ORlG=7MkKJP-y78$a_&u%03(n0J1LC6rYj0#ly_+3Yezka;(dKGZSLP22YOg0n*#7s z7FAlPL_XC}YvTI)08jR*m;pB5t-8=K11y@cP-=Ht!pN`uFz9eo8bzKW098!-`a?6B z_w+1&o9&q@!L%&5R}zix#$TXe?tXjC9R4ZnFWjh&VFdbkD#onz!5U(WhN@nVv{H-< z^z*AZA)2>=3p{-!wfr}yi?ag#X2Xayl-B+`^V&y;ZRgTyw>%QQ6&j_%4sUaSUern& zT>kvpNcGf|P$CvvnoiF;7FB|kbETSBf20-9vIdfumw$|UOkZ#4S0Q;lYhlq8O|cqE z%S!R!pHaUM@3D(_b6VU;w;f3g@T~i!KWux=AJT%i^)I{8cRjF1FQipc0{r?bl$dkA znXd~AAnV6iPa1E1L;0?j66T~uIW?#^X&XYc1CC$gbBR=`VoOycO(^JvLj4>GBld9G z^pU(N8=680ctq>#2IAUWyRhI_&3s6SDM$5F@Bl`vgrq$-duuGGpz?5hsyt2XLUWeO z`pJ5Ut$b6UPv5%c!49rl(sN^vF@vvL37w=zum@ zxRg9~zvh#aU(An%C?265UQpxUlgj+&YS?)quZqXX$1-;&(kFTXYUxf*&_IIhtdC&> zQ0(_UJ6Z3%%RA8}{ui6-wz&uC!$|3>VPaTOZPoD&CrHF(#qVtMsv=wBx|4pnqT@)X zEBAJ}xaIiPyvpn(%ssRa8(pB}QAwAhk$QIsr5x`n@qL}9ci*mv&&(Myw$u2t^^2eE zCO$yQE=b$^3KC^C*_IpjiQzpl<_tMzimiHxW4f`<#Mk;7#*<}2;w_Y-XG%a@LNJ-f z0{jRoELB_NAU>_Z&~esY3EM*E=0Hca>jyi*ot(%z82<9smzM8*kh8BQS6Yf>zazet z&IYvLdskW(A1}=+JoL`uxD!H}#T-&HPvM?M*Vqaue7~zb>$MlUQ0i#m#Z^@%hE(?2 z+MRxsir}4dD9>uXSWvgeoW=F`S=D8Xl@}YyI-Fp4+N|Yizv(q$k5-f?T|Q|O{>^ig zB3#E1%=ltOS-fZEOQccs`QZ4^4YTR$!a?#a_i3lo7lR>L=556Fv1;9kN=qzo#ul44 zYNwu5xcRuww~QvQeZ8}_f4Z#DW;_^~IR8NmtOINIY9RKCIyUfy?j(nJ8_Jtos$z|k zHj&~PWVP85!gfwc!9jWeenb;vD$UV&@Z@CvT~Sc1N6qaIP?PO)H2(N%!)`{@>@AvW zA}6W41VYkdA;0);kN#7pDY0Y@LOgU9j}EmyBMh&OF>aaN5f?~`@G=jaCQDy{?bjiN zNx%Ec=H0fj3XZe*^8Jp`-NORn$wGsTBhVKmYJy&SLL2{N|K|k3*N# zR&Jgu7)7s$l=jXgDK zOv@}7Rem`04%E@%$~k}w?hR}{(ecUGd#Me{&bunzbKc%Chg@4TzD@*OCims?iLuhw7N;Pd(AO9;45)Hf{J z{N?XNp2@4Uc*W+%4uS~XegK??eATBfix%IGpAa(3=_rl&2;Q56aBta-@?SqtK(*`K zI>}SEXXZ>}P5=7S>v}|_`sjl3lW?cG53pkX8wT|4I-H}(&7zxDDUizXGI{|A@L-h1 z?w(hnPEk8H#`flbYCF5;n{ipESBLd2Kk`O~hU5*f;VU(FGKPzKjbe*ugF>X_f z$E){3UeTH;Wd4f8-?4XfLw{x-uQg)a$<+@e49(?n3MC$@%UL01n=hL5lM8TScXHEs zkNwmhiAa@OMo$gL^LYVwpWX1vW&6=~Rln1#dbg2(d%YnSh>cy$r@EnhFFIfEbW*MG zfUPEjKJoz~unZAmfZ-9nRH&h4)3FbL1F*h#gwo@CS)8*EHDytTAM5H{=`(lV;zP6v zjorm8f0#n+jVkQVxK6LlpZnV5nT-s79=6&xmfy^P)YVwh2=@M|hGeL(0P^>7GLUIf zQ&YjNqgT0=e2b>NJD70n-roik2vT*kL8VgG_4n<5PW_qa+R4t&zKY-wQ`3bFlvU&& z9{`#p4H%r=1%dO4q{g0N7Gu*yTu~9^cz;cpP24cw<-$r4uCM;-f^S^&!4EeebwXPo zVBGv-6jg$E0t+&MUwUKwk?Z* zd?W#7#FsL^v}&Q;B7N5qu6Y1;_^b{I72?Zcf8((8MD_?ga>MP|f3nKKF@~JZ%<7w| zcCH3$RiVNte!%4Cj0d^I9C(>y=p?y z&1*kOV7r>JiTgh}UAE$f{AgJpmY&$f(`OmkNe`}+#zFUBBB&ILNzSI;mY09UE+j_p z?1?&f6j(%UM)x^`aK?E$J)^}Y4tvW3xO|f2xEM6j6z%2LDB&V**0IBEJlDt8C!E6u z)M(j(k~KByCR8?B!!wL_aafMkTHuGC`}Q#SmQ}D>Y(}V+v0@h+3X~vV)93Cn zHXRTO;*H=$bhGb^o;kqWGLITPH&ty@{~PU0&0Tf(N!KN3iM4 zcz$1Q4Qd+@i(tv$37)PZ(lm5{Bl!p`SbUYyXDn{zv;5lKf_Fp)FiNjF(r%pL1}fjj zj`h|YJ+4V}2o`Pn0058(v@cZn|PYWDW0;W{@K(9l`sTtQpT)i8&*noJ< zSK6T)D##tv1$8ZJxS{l|a=cau_FcWNh&pSw%>+9{|N8NTqh?qR zsP^-Ybskh$yCiF6B9^&t(tOr25MfSNe5MmCpjuJbefH*{-hTtD3?Po6v*NH4R~py@w~G?^Q$h}g|P6A*PxS^@r;w*XyT{}Ee~^U)(5{nE7h z561w*yICw+f@=7a{h$sG1%Pfr+CoqBKYTFtY~InS{`)#ei}%~mkRWh=z8R#>&p%n4 zX{rNIs2>l~-@mp$Q^krOxo>Pj&n2t0aG8U*#3C*wWySsZ)jC3GQ^+})kWy|{*Sk5> z1gsXIKDhjRtv^y|-xVt_y4iSsMlR6B@b7l1o((5`1fZ+t=ujyUVwO>~qehA-`ub>W zjXr=C#7q}5F)}i;@1cAjvjiMwN4h_&@>%NTKu#4J2_Uar9Hos>8spytX;C~B`bP}q z&I~dos1M3AzMl9mBLEnJ_pRV_uC6Tp zWC?ke*&;q6kSmTy?BOhEK0If60r$8)F(GhC$pbr6LaPx5z$a58S_&UoX#yI&se*ix zm1^?0{&I^2>>wq>V^xb!s@Pt;H#YM|pUwc?4v!8*4Oj(_GFdl4{K&^>b&a^irT1fG z8`C(uIp7XuxKnNu^NU^QulpYty_OJ^q$l0po1PMHcu4BG0J{AS?@?(o*wrB4ne9oc zo@$|>WwUQmcY*nDg`vNGX9-y9t?lX)3ATPj&PYH!Y6-`X1&y~cSiV1Gj6j|}gm3() z0s9e6O@8PQ;nm!rsjtho_%7KvFLcK2HW5w@&vr1VcL49kLc&~JSGk;JG(wuh6Hk;~B{?Hj3tDaEn8#exCQsi0WJA+0_ z(!=^&_^Cf!-L0m`ZGczN=W~*r)BFC(%ZN?g!&Mr(f{L8HvxPg;+UVPki{0_%jI0VF z`@_0m$NtOJ)Lg8f*i^-5xL=di;@2z2s*q&=hS1Z-Kgb?He!!m0yX!5i^Cm9q{A6W* zclG4UYp$j7wzHc4)SJ4IOjjKB=KhdLC=|L2=vLpSmoyGDjeb5PqFHq(A5k(CcmR?N zSn1x2sNb3<^yW{iQu9V5-e;lC5DiPoR{J!@evQn$x9=YxY`E2~+J4Pal_8NxN+-6V zM-IqgNxzY4n}8XI+UKRf?Tw9&u6pfXR6Q9=s(%cS$Eqh^qCKw!I$zQFPyM;x70Z|S zY~^d}$fl^GqQxPy=#2L(LzK)PGWfr9$o~y8{`-A`yXRI$lvBkMpCo{%Toh1kL#+yp HM`8Z~tPeaL literal 0 HcmV?d00001 diff --git a/beginner_source/onnx/README.txt b/beginner_source/onnx/README.txt index f73ed11bc8f..5c9249c640e 100644 --- a/beginner_source/onnx/README.txt +++ b/beginner_source/onnx/README.txt @@ -8,3 +8,7 @@ ONNX 2. export_simple_model_to_onnx_tutorial.py Export a PyTorch model to ONNX https://pytorch.org/tutorials/beginner/onnx/export_simple_model_to_onnx_tutorial.html + +3. onnx_registry_tutorial.py + Introduction to ONNX + https://pytorch.org/tutorials/beginner/onnx/onnx_registry_tutorial.html \ No newline at end of file diff --git a/beginner_source/onnx/export_simple_model_to_onnx_tutorial.py b/beginner_source/onnx/export_simple_model_to_onnx_tutorial.py index fa09dc86abc..362b6ec8f28 100644 --- a/beginner_source/onnx/export_simple_model_to_onnx_tutorial.py +++ b/beginner_source/onnx/export_simple_model_to_onnx_tutorial.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """ `Introduction to ONNX `_ || -**Export a PyTorch model to ONNX** +**Export a PyTorch model to ONNX** || +`Introduction to ONNX Registry `_ Export a PyTorch model to ONNX ============================== diff --git a/beginner_source/onnx/intro_onnx.py b/beginner_source/onnx/intro_onnx.py index 05ad3090cc8..d86e917a5e5 100644 --- a/beginner_source/onnx/intro_onnx.py +++ b/beginner_source/onnx/intro_onnx.py @@ -1,6 +1,7 @@ """ **Introduction to ONNX** || -`Export a PyTorch model to ONNX `_ +`Export a PyTorch model to ONNX `_ || +`Introduction to ONNX Registry `_ Introduction to ONNX ==================== @@ -32,6 +33,8 @@ Dependencies ------------ +PyTorch 2.1.0 or newer is required. + The ONNX exporter depends on extra Python packages: - `ONNX `_ @@ -39,6 +42,12 @@ They can be installed through `pip `_: +.. note:: + This tutorial leverages `onnxscript `__ + to create custom ONNX operators. onnxscript is a Python library that allows users to + create custom ONNX operators in Python. It is a prerequisite learning material for + this tutorial. Please make sure you have read the onnxscript tutorial before proceeding. + .. code-block:: bash pip install --upgrade onnx onnxscript diff --git a/beginner_source/onnx/onnx_registry_tutorial.py b/beginner_source/onnx/onnx_registry_tutorial.py new file mode 100644 index 00000000000..8493dad3155 --- /dev/null +++ b/beginner_source/onnx/onnx_registry_tutorial.py @@ -0,0 +1,451 @@ +# -*- coding: utf-8 -*- + +""" +`Introduction to ONNX `_ || +`Export a PyTorch model to ONNX `_ || +**Introduction to ONNX Registry** + +Introduction to ONNX Registry +============================= + +**Authors:** Ti-Tai Wang (titaiwang@microsoft.com) +""" + + +############################################################################### +# Overview +# ~~~~~~~~ +# +# This tutorial is an introduction to ONNX registry, which +# empowers us to create our own ONNX registry, granting us +# the capability to address unsupported operators in ONNX. +# +# In this tutorial we will cover the following scenarios: +# +# * Unsupported ATen operators +# * Unsupported ATen operators with existing ONNX Runtime support +# * Unsupported PyTorch operators with no ONNX Runtime support +# + + +import torch +print(torch.__version__) +torch.manual_seed(191009) # set the seed for reproducibility + +import onnxscript # pip install onnxscript +print(onnxscript.__version__) + +# NOTE: opset18 is the only version of ONNX operators we are +# using in torch.onnx.dynamo_export for now. +from onnxscript import opset18 + +import onnxruntime # pip install onnxruntime +print(onnxruntime.__version__) + + +###################################################################### +# Unsupported ATen operators +# --------------------------------- +# +# ATen operators are implemented by PyTorch, and the ONNX exporter team must manually implement the +# conversion from ATen operators to ONNX operators through onnxscript. Although the ONNX exporter +# team has been making their best efforts to support as many ATen operators as possible, some ATen +# operators are still not supported. In this section, we will demonstrate how you can implement any +# unsupported ATen operators, which can contribute back to the project through PyTorch Github. +# +# If the model cannot be exported to ONNX, for instance, :class:`aten::add.Tensor` is not supported +# by ONNX The error message can be found, and is as follows (e.g. aten::add.Tensor): +# ``RuntimeErrorWithDiagnostic: Unsupported FX nodes: {'call_function': ['aten.add.Tensor']}. `` +# +# To support unsupported ATen operators, we need two things: +# 1. The unsupported ATen operator namespace, operator name, and the +# corresponding overload. (e.g. ::. - aten::add.Tensor), +# which can be found in the error message. +# 2. The implementation of the operator in `onnxscript `__. +# + + +# NOTE: `is_registered_op` is a method in ONNX registry that checks +# whether the operator is supported by ONNX. If the operator is not +# supported, it will return False. Otherwise, it will return True. +onnx_registry = torch.onnx.OnnxRegistry() +# aten::add.default and aten::add.Tensor are supported by ONNX +print(f"aten::add.default is supported by ONNX registry: \ + {onnx_registry.is_registered_op(namespace='aten', op_name='add', overload='default')}") +# aten::add.Tensor is the one invoked by torch.ops.aten.add +print(f"aten::add.Tensor is supported by ONNX registry: \ + {onnx_registry.is_registered_op(namespace='aten', op_name='add', overload='Tensor')}") + + +###################################################################### +# In this example, we will assume that aten::add.Tensor is not supported by the ONNX registry, +# and we will demonstrate how to support it. The ONNX registry allows user overrides for operator +# registration. In this case, we will override the registration of aten::add.Tensor with our +# implementation and verify it. However, this unsupported operator should return False when +# checked with :meth:`onnx_registry.is_registered_op`. +# + + +class Model(torch.nn.Module): + def forward(self, input_x, input_y): + # specifically call out aten::add + return torch.ops.aten.add(input_x, input_y) + +input_add_x = torch.randn(3, 4) +input_add_y = torch.randn(3, 4) +aten_add_model = Model() + + +# Let's create a onnxscript function to support aten::add.Tensor. +# This can be named anything, and shows later on Netron graph. +custom_aten = onnxscript.values.Opset(domain="custom.aten", version=1) + +# NOTE: The function signature must match the signature of the unsupported ATen operator. +# https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/native_functions.yaml +# NOTE: All attributes must be annotated with type hints. +@onnxscript.script(custom_aten) +def custom_aten_add(input_x, input_y, alpha: float = 1.0): + alpha = opset18.CastLike(alpha, input_y) + input_y = opset18.Mul(input_y, alpha) + return opset18.Add(input_x, input_y) + + +# Now we have both things we need to support unsupported ATen operators. +# Let's register the custom_aten_add function to ONNX registry, and +# export the model to ONNX again. +onnx_registry.register_op( + namespace="aten", op_name="add", overload="Tensor", function=custom_aten_add + ) +print(f"aten::add.Tensor is supported by ONNX registry: \ + {onnx_registry.is_registered_op(namespace='aten', op_name='add', overload='Tensor')}" + ) +export_options = torch.onnx.ExportOptions(onnx_registry=onnx_registry) +export_output = torch.onnx.dynamo_export( + aten_add_model, input_add_x, input_add_y, export_options=export_options + ) + +###################################################################### +# Make sure the model uses custom_aten_add instead of aten::add.Tensor +# The graph has one graph nodes for custom_aten_add, and inside +# custom_aten_add, there are four function nodes, one for each +# operator, and one for constant attribute. +# + +# graph node domain is the custom domain we registered +assert export_output.model_proto.graph.node[0].domain == "custom.aten" +assert len(export_output.model_proto.graph.node) == 1 +# graph node name is the function name +assert export_output.model_proto.graph.node[0].op_type == "custom_aten_add" +# function node domain is empty because we use standard ONNX operators +assert export_output.model_proto.functions[0].node[3].domain == "" +# function node name is the standard ONNX operator name +assert export_output.model_proto.functions[0].node[3].op_type == "Add" + + +###################################################################### +# custom_aten_add_model ONNX graph in Netron: +# +# .. image:: /_static/img/onnx/custom_aten_add_model.png +# :width: 70% +# :align: center +# +# Inside the custom_aten_add function: +# +# .. image:: /_static/img/onnx/custom_aten_add_function.png +# :width: 70% +# :align: center +# +# After checking the ONNX graph, we can use ONNX Runtime to run the model, +# + + +# Use ONNX Runtime to run the model, and compare the results with PyTorch +export_output.save("./custom_add_model.onnx") +ort_session = onnxruntime.InferenceSession( + "./custom_add_model.onnx", providers=['CPUExecutionProvider'] + ) + +def to_numpy(tensor): + return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() + +onnx_input = export_output.adapt_torch_inputs_to_onnx(input_add_x, input_add_y) +onnxruntime_input = {k.name: to_numpy(v) for k, v in zip(ort_session.get_inputs(), onnx_input)} +onnxruntime_outputs = ort_session.run(None, onnxruntime_input) + +torch_outputs = aten_add_model(input_add_x, input_add_y) +torch_outputs = export_output.adapt_torch_outputs_to_onnx(torch_outputs) + +assert len(torch_outputs) == len(onnxruntime_outputs) +for torch_output, onnxruntime_output in zip(torch_outputs, onnxruntime_outputs): + torch.testing.assert_close(torch_output, torch.tensor(onnxruntime_output)) + + +###################################################################### +# Unsupported ATen operators with existing ONNX Runtime support +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# In this case, the unsupported ATen operator is supported by ONNX Runtime but not +# supported by ONNX spec. This occurs because ONNX Runtime users can implement their +# custom operators, which ONNX Runtime supports. When the need arises, ONNX Runtime +# will contribute these custom operators to the ONNX spec. Therefore, in the ONNX registry, +# we only need to register the operator with the recognized namespace and operator name. +# +# In the following example, we would like to use the Gelu in ONNX Runtime, +# which is not the same Gelu in ONNX spec. Thus, we register the Gelu with +# the namespace "com.microsoft" and operator name "Gelu". +# + + +class CustomGelu(torch.nn.Module): + def forward(self, input_x): + return torch.ops.aten.gelu(input_x) + +# com.microsoft is an official ONNX Runtime namspace +custom_ort = onnxscript.values.Opset(domain="com.microsoft", version=1) + +# NOTE: The function signature must match the signature of the unsupported ATen operator. +# https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/native_functions.yaml +# NOTE: All attributes must be annotated with type hints. +@onnxscript.script(custom_ort) +def custom_aten_gelu(input_x, approximate: str = "none"): + # We know com.microsoft::Gelu is supported by ONNX Runtime + # It's only not supported by ONNX + return custom_ort.Gelu(input_x) + + +onnx_registry = torch.onnx.OnnxRegistry() +onnx_registry.register_op( + namespace="aten", op_name="gelu", overload="default", function=custom_aten_gelu) +export_options = torch.onnx.ExportOptions(onnx_registry=onnx_registry) + +aten_gelu_model = CustomGelu() +input_gelu_x = torch.randn(3, 3) + +export_output = torch.onnx.dynamo_export( + aten_gelu_model, input_gelu_x, export_options=export_options + ) + + +###################################################################### +# Make sure the model uses :func:`custom_aten_gelu`` instead of +# :class:`aten::gelu` The graph has one graph nodes for +# custom_aten_gelu, and inside custom_aten_gelu, there is a function +# node for Gelu with namespace "com.microsoft". +# + +# graph node domain is the custom domain we registered +assert export_output.model_proto.graph.node[0].domain == "com.microsoft" +# graph node name is the function name +assert export_output.model_proto.graph.node[0].op_type == "custom_aten_gelu" +# function node domain is the custom domain we registered +assert export_output.model_proto.functions[0].node[0].domain == "com.microsoft" +# function node name is the node name used in the function +assert export_output.model_proto.functions[0].node[0].op_type == "Gelu" + + +###################################################################### +# custom_aten_gelu_model ONNX graph in Netron: +# +# .. image:: /_static/img/onnx/custom_aten_gelu_model.png +# :width: 70% +# :align: center +# +# Inside the custom_aten_gelu function: +# +# .. image:: /_static/img/onnx/custom_aten_gelu_function.png +# +# After checking the ONNX graph, we can use ONNX Runtime to run the model, + + +export_output.save("./custom_gelu_model.onnx") +ort_session = onnxruntime.InferenceSession( + "./custom_gelu_model.onnx", providers=['CPUExecutionProvider'] + ) + +def to_numpy(tensor): + return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() + +onnx_input = export_output.adapt_torch_inputs_to_onnx(input_gelu_x) +onnxruntime_input = {k.name: to_numpy(v) for k, v in zip(ort_session.get_inputs(), onnx_input)} +onnxruntime_outputs = ort_session.run(None, onnxruntime_input) + +torch_outputs = aten_gelu_model(input_gelu_x) +torch_outputs = export_output.adapt_torch_outputs_to_onnx(torch_outputs) + +assert len(torch_outputs) == len(onnxruntime_outputs) +for torch_output, onnxruntime_output in zip(torch_outputs, onnxruntime_outputs): + torch.testing.assert_close(torch_output, torch.tensor(onnxruntime_output)) + + +###################################################################### +# Unsupported PyTorch operators with no ONNX Runtime support +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# In this case, the operator is not supported by any frameworks, and we +# would like to use it in ONNX graph. Therefore, we need to implement +# the operator in three places: +# +# 1. PyTorch FX graph +# 2. ONNX Registry +# 3. ONNX Runtime +# +# In the following example, we would like to use a custom operator +# that takes one tensor input, and returns an input. The operator adds +# the input to itself, and returns the rounded result. +# +# **Custom Ops Registration in PyTorch FX Graph (Beta)** +# +# Firstly, we need to implement the operator in PyTorch FX graph. +# This can be done by using torch._custom_op. +# + +# NOTE: This is a beta feature in PyTorch, and is subject to change. +from torch._custom_op import impl as custom_op + +@custom_op.custom_op("mylibrary::addandround_op") +def addandround_op(tensor_x: torch.Tensor) -> torch.Tensor: + ... + +@addandround_op.impl_abstract() +def addandround_op_impl_abstract(tensor_x): + return torch.empty_like(tensor_x) + +@addandround_op.impl("cpu") +def addandround_op_impl(tensor_x): + # add x to itself, and round the result + return torch.round(tensor_x + tensor_x) + +torch._dynamo.allow_in_graph(addandround_op) + +class CustomFoo(torch.nn.Module): + def forward(self, tensor_x): + return addandround_op(tensor_x) + +input_addandround_x = torch.randn(3) +custom_addandround_model = CustomFoo() + + +###################################################################### +# **Custom Ops Registration in ONNX Registry** +# +# For the step 2 and 3, we need to implement the operator in ONNX registry. +# In this example, we will implement the operator in ONNX registry +# with the namespace "test.customop" and operator name "CustomOpOne", +# and "CustomOpTwo". These two ops are registered and built in +# `cpu_ops.cc `__. +# + + +custom_opset = onnxscript.values.Opset(domain="test.customop", version=1) + +# NOTE: The function signature must match the signature of the unsupported ATen operator. +# https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/native_functions.yaml +# NOTE: All attributes must be annotated with type hints. +@onnxscript.script(custom_opset) +def custom_addandround(input_x): + # The same as opset18.Add(x, x) + add_x = custom_opset.CustomOpOne(input_x, input_x) + # The same as opset18.Round(x, x) + round_x = custom_opset.CustomOpTwo(add_x) + # Cast to FLOAT to match the ONNX type + return opset18.Cast(round_x, to=1) + + +onnx_registry = torch.onnx.OnnxRegistry() +onnx_registry.register_op( + namespace="mylibrary", op_name="addandround_op", overload="default", function=custom_addandround + ) + +export_options = torch.onnx.ExportOptions(onnx_registry=onnx_registry) +export_output = torch.onnx.dynamo_export( + custom_addandround_model, input_addandround_x, export_options=export_options + ) +export_output.save("./custom_addandround_model.onnx") + + +###################################################################### +# The exported model proto is accessible through export_output.model_proto. +# The graph has one graph nodes for custom_addandround, and inside custom_addandround, +# there are two function nodes, one for each operator. +# + +assert export_output.model_proto.graph.node[0].domain == "test.customop" +assert export_output.model_proto.graph.node[0].op_type == "custom_addandround" +assert export_output.model_proto.functions[0].node[0].domain == "test.customop" +assert export_output.model_proto.functions[0].node[0].op_type == "CustomOpOne" +assert export_output.model_proto.functions[0].node[1].domain == "test.customop" +assert export_output.model_proto.functions[0].node[1].op_type == "CustomOpTwo" + + +###################################################################### +# custom_addandround_model ONNX graph in Netron: +# +# .. image:: /_static/img/onnx/custom_addandround_model.png +# :width: 70% +# :align: center +# +# Inside the custom_addandround function: +# +# .. image:: /_static/img/onnx/custom_addandround_function.png +# + +###################################################################### +# **Custom Ops Registration in ONNX Runtime** +# +# To link your custom op library to ONNX Runtime, you need to +# compile your cpp code into a shared library, and link it to ONNX Runtime. +# Please follow the instructions below: +# +# 1. Implement your custom op in cpp by following +# `ONNX Runtime instructions <`https://github.com/microsoft/onnxruntime/blob/gh-pages/docs/reference/operators/add-custom-op.md>`__. +# 2. Download ONNX Runtime source distribution from +# `ONNX Runtime releases `__. +# 3. Compile and link your custom op library to ONNX Runtime, for example: +# +# .. code-block:: bash +# +# $ gcc -shared -o libcustom_op_library.so custom_op_library.cc -L /path/to/downloaded/ort/lib/ -lonnxruntime -fPIC +# +# 4. Run the model with ONNX Runtime python API +# +# .. code-block:: python +# +# ort_session_options = onnxruntime.SessionOptions() +# +# # NOTE: Link the custom op library to ONNX Runtime and replace the path +# # with the path to your custom op library +# ort_session_options.register_custom_ops_library( +# "/path/to/libcustom_op_library.so" +# ) +# ort_session = onnxruntime.InferenceSession( +# "./custom_addandround_model.onnx", providers=['CPUExecutionProvider'], sess_options=ort_session_options) +# +# def to_numpy(tensor): +# return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() +# +# onnx_input = export_output.adapt_torch_inputs_to_onnx(input_addandround_x) +# onnxruntime_input = {k.name: to_numpy(v) for k, v in zip(ort_session.get_inputs(), onnx_input)} +# onnxruntime_outputs = ort_session.run(None, onnxruntime_input) +# +# torch_outputs = custom_addandround_model(input_addandround_x) +# torch_outputs = export_output.adapt_torch_outputs_to_onnx(torch_outputs) +# +# assert len(torch_outputs) == len(onnxruntime_outputs) +# for torch_output, onnxruntime_output in zip(torch_outputs, onnxruntime_outputs): +# torch.testing.assert_close(torch_output, torch.tensor(onnxruntime_output)) +# +# +###################################################################### +# Further reading +# --------------- +# +# The list below refers to tutorials that ranges from basic examples to advanced scenarios, +# not necessarily in the order they are listed. +# Feel free to jump directly to specific topics of your interest or +# sit tight and have fun going through all of them to learn all there is about the ONNX exporter. +# +# .. include:: /beginner_source/onnx/onnx_toc.txt +# +# .. toctree:: +# :hidden: +# diff --git a/beginner_source/onnx/onnx_toc.txt b/beginner_source/onnx/onnx_toc.txt index 2386430ba7b..bd57025641b 100644 --- a/beginner_source/onnx/onnx_toc.txt +++ b/beginner_source/onnx/onnx_toc.txt @@ -1 +1,2 @@ -| 1. `Export a PyTorch model to ONNX `_ \ No newline at end of file +| 1. `Export a PyTorch model to ONNX `_ +| 2. `Introduction to ONNX registry `_ \ No newline at end of file diff --git a/index.rst b/index.rst index feee988b0e4..0d165be5a8c 100644 --- a/index.rst +++ b/index.rst @@ -281,6 +281,13 @@ What's new in PyTorch tutorials? :link: beginner/onnx/export_simple_model_to_onnx_tutorial.html :tags: Production,ONNX,Backends +.. customcarditem:: + :header: Introduction to ONNX Registry + :card_description: Demonstrate end-to-end how to address unsupported operators by using ONNX Registry. + :image: _static/img/thumbnails/cropped/Exporting-PyTorch-Models-to-ONNX-Graphs.png + :link: advanced/onnx_registry_tutorial.html + :tags: Production,ONNX,Backends + .. Reinforcement Learning .. customcarditem::