From 9fe7fc19a2833b90eec5671a6ff87f778d79375c Mon Sep 17 00:00:00 2001 From: Lakshmi Date: Fri, 10 Apr 2020 17:50:00 -0700 Subject: [PATCH] Implement DualRunning state to enable blue-green deploys (#186) * Working version 1 * Create setup for blue green deploys * [WIP] Setup status sub-resource for blue green deploys * Updates * fix bug * Fixes * Make running jobs calculation idempotent * Fix bugs * Reset running jobs in recovering phase * Make status index calculation simpler * Add container env and annotations * Update CRD to v1beta2 * Update CRD to v1beta2 * Fix CRD update issues * Fix lint * Merge master and restore v1beta1 to original version * Upgrade integ test to v1beta2 * Backward compatibility changes * Work around status subresource bug * Rename status array to VersionStatuses and add comment on k8s bug * Remove DesiredApplicationCount * Minor updates * Minor updates * Initialize counter * Handle edge case for jobId * Debug * Debug * fixes * Fix edge case * Fix unit tests * Debug logs * Fix overwriting of versionstatuses * Remove debug logs * Implement DualRunning state * Happy path:Add DualRunning and Teardown states * Add unit tests and make Teardown a bool * Minor fixes * Merge master * Revert CRD upgrade * Keep Status.ClusterStatus and Status.JobStatus unchanged for Dual mode * Remove unwarranted changes * Add version to deployment names * Account for SavepointDisabled * Add integration test and fix delete * Fix build * Handle delete when there are two running jobs * Handle delete for blue green deploys correctly * Fix ingress * Fix ingress URL and event name * Allow teardown to be a version instead of bool and force-cancel job in teardown * Redesign teardown feature * Disallow switching between deployment modes * Disallow switching between deployment modes * Disallow deployment mode change * Fixes and address all review comments * Add docs and update state machine diagram * Update links to state machine pngs --- deploy/crd.yaml | 2 +- docs/blue_green_state_machine.mmd | 46 ++ docs/blue_green_state_machine.png | Bin 0 -> 85502 bytes docs/crd.md | 11 +- ...ate_machine.mmd => dual_state_machine.mmd} | 2 +- docs/state_machine.md | 46 +- integ/blue_green_deployment_test.go | 147 +++++ integ/utils/utils.go | 51 +- pkg/apis/app/v1beta1/types.go | 26 +- pkg/controller/flink/client/api.go | 36 +- pkg/controller/flink/client/api_test.go | 2 +- pkg/controller/flink/client/entities.go | 4 +- pkg/controller/flink/client/mock/mock_api.go | 11 +- pkg/controller/flink/container_utils.go | 14 +- pkg/controller/flink/flink.go | 229 ++++++-- pkg/controller/flink/flink_test.go | 173 +++++- pkg/controller/flink/ingress.go | 16 +- pkg/controller/flink/ingress_test.go | 15 + .../flink/job_manager_controller.go | 23 +- .../flink/job_manager_controller_test.go | 29 +- pkg/controller/flink/mock/mock_flink.go | 98 +++- .../flink/task_manager_controller.go | 20 +- .../flink/task_manager_controller_test.go | 13 +- .../flinkapplication/flink_state_machine.go | 333 +++++++++-- .../flink_state_machine_test.go | 546 +++++++++++++++++- pkg/controller/k8/cluster.go | 10 +- pkg/controller/k8/mock/mock_k8.go | 6 +- 27 files changed, 1684 insertions(+), 225 deletions(-) create mode 100644 docs/blue_green_state_machine.mmd create mode 100644 docs/blue_green_state_machine.png rename docs/{state_machine.mmd => dual_state_machine.mmd} (92%) create mode 100644 integ/blue_green_deployment_test.go diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 4be01739..2d05d90d 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -85,7 +85,7 @@ spec: type: boolean deploymentMode: type: string - enum: [Dual] + enum: [Dual, BlueGreen] rpcPort: type: integer minimum: 1 diff --git a/docs/blue_green_state_machine.mmd b/docs/blue_green_state_machine.mmd new file mode 100644 index 00000000..47e1ae6d --- /dev/null +++ b/docs/blue_green_state_machine.mmd @@ -0,0 +1,46 @@ +%% This file can be compiled into blue_green_state_machine.png by installing mermaidjs (https://mermaidjs.github.io/) and running +%% mmdc -i blue_green_state_machine.mmd -o blue_green_state_machine.png -w 1732 -b transparent + +graph LR +New --> ClusterStarting + +subgraph Running +Running +DeployFailed +end + +subgraph Updating +Running --> Updating +Updating --> ClusterStarting +DeployFailed --> Updating + +ClusterStarting -- savepoint disabled --> SubmittingJob +ClusterStarting -- savepoint enabled --> Savepointing +ClusterStarting -- Create fails --> DeployFailed + +Savepointing --> SubmittingJob +Savepointing -- Savepoint fails --> Recovering + +Recovering --> SubmittingJob +Recovering -- No externalized checkpoint --> RollingBackJob + +SubmittingJob -- first deploy --> Running +SubmittingJob -- updating existing application --> DualRunning +SubmittingJob -- job start fails --> RollingBackJob +RollingBackJob --> DeployFailed + +DualRunning -- tearDownVersionHash set --> Running +DualRunning -- tear down fails --> DeployFailed +end + +linkStyle 4 stroke:#303030 +linkStyle 5 stroke:#303030 +linkStyle 6 stroke:#FF0000 +linkStyle 8 stroke:#FF0000 +linkStyle 10 stroke:#FF0000 +linkStyle 11 stroke:#303030 +linkStyle 12 stroke:#303030 +linkStyle 13 stroke:#FF0000 +linkStyle 14 stroke:#FF0000 +linkStyle 15 stroke:#303030 +linkStyle 16 stroke:#FF0000 \ No newline at end of file diff --git a/docs/blue_green_state_machine.png b/docs/blue_green_state_machine.png new file mode 100644 index 0000000000000000000000000000000000000000..b9d4cb1ab0981209ed762db81ae065c9b4c36ff2 GIT binary patch literal 85502 zcmcG0Wmr{P*zE=pknRp?K^mn?Ku}V;OS-$eOF&YP5-AbslJ0JhmhSGZJGaMkzVGjS zo_p61*=+Y-YsMSn9q*Wgy_1zdMIu6iKp?1Z--szdAn+9s2+Rl~Jb33hr#d49@)Yt` z?3I#h>h7G2R>Im!=h2XUH$1*S2iGe6@8qS4Ux~&wy)UeOn$*}ECa^N3P$qD-1U z-%XAp6By`CNC^!K%Q77iaooS9PEQYdf*x(SHF_o{(KT9TBuexW>E~YQ*RRa12Av{< zsXP+zhT}<02)J$8QHg$i)vP{uIZ+V%bFIRB#nkw7-`ISG9Eor0W1n7HR2~0PlsP$a!@jcarN*xo-**x&dD*VOPjVdUh9=fz6e3vdo^1Xd2svs&-g>T zA4voy{&{IApZ(?)1PRZv@7Og{WIuTbCi(SvAyUEWDo$!DiH+-RlOhJQc5S+V2b$45 zi)s-LGqY^+g`$E&nZVE<635?(dw;fn_h+c~Gz{CY4jbNGVc1Jb#X8A0P1;gB6Opbv zg{pQA4ojOOWBo~`5Z#62-KkO*ho+UTw#(ZSBUd+E3T{_?4&!0soa%Re2>MyeyRxa~>$FofQgafALVAq^*_pIiIb2d~Z*RBXQg~r) z&3w8&?pJ0cMReTwcd89p|1RG1`T6wAD}mmYfG)FJ_2b1q z+bXut6gi9qUBMVeY1%%y^6AvIqxg6wODiiFTRXef$!h*ea{(mbEb)8G-#^ndDm{5z z54rE}l_AfRd-)t5yzgplm)i-A2erfuSA&Ri$?7`?26#@YA|kMr3skc8Ub*xplz*s~ zl$NFy6!Z%W6l2znYMGsld*2cMF+DwMOT+b_-E;f$?cZUfrZMfPcC@$4fBPu0z9KJ= zPp|UhQ0C(3jA>}8jgH6bdROSCb9r;~qfyF0ip%ksT#M&niypIfc;niYjk4tQ)Gc32 zV0ATJ8gHfX`EJLr0@X)84-#&-N4}{I!hdar_g`Bnd#0Gn9fVXxe0;!rViOYw z*ZM}Lrlv6PKQ)6_+1c4(davBwc_CX{gocKOkcTLOzU~r!pIcY%0EDJ?hoP4yquG8_ z(_8KWf5sGpkl?+74SgBINJ2`Q#AZO9B|fm&9ixcDuqvyjMqQ}M_0@7ZHhDHp$R`Y4 z=oZ$+MfJN%VR!iuXB)67yOsG>V6SSG=E!JhkAK>)F%+oUWGWYkOK05sSk7J30kdS( zz31@rbNZ%Ihy~n3YN?g1!g?+gcvZ~DkD<|W>YUcUWhEsUJTAzVCv`<3YPF66+O-vH z2TgK^%MoqtCytxLu`yI03l|qs|5(qyfDG%yN6nhD>7y#Ejri))QZC+bsL8K#7?&r9 zhB{2m&MvJfzvy!y=Dcm0)O~2)oi-?`tZW${r|CO+`?iOghQ`3eB%-j8Nme!r(%#Xr zvb-!UE)H*8&f}n@te|j=s)%E}0X+7HdU?Wp10NQ7Mt8z<)np+bLeb#*57pHh+Ku+m zMRUWTrqwZA>13Oot$wKS8&MZq-8T#TC%pWQ?=k$UM1PPX_$fA45;!**nKhT!{K&{s z?&I;w<-UoD^WZ&)l`e8FyUpPz&bw0*`3mzSe0btuff&mFnJL*jhKfJ0s8zv!m)I=m zhvQ!uwgn5B&fnha9?s+qomkFP1n7ES1$_8`a(h?6>S!8$lP>yqozQXr#!jgF*WXo@ zS}51l)gAY947jc*8waodlYnAw+B!EO{{?>Ni)s14(fsebSD%rXp!5Cr>X}qB4f`j^ z{Q37yW3I^G&H3li$1ndr`tJ>5i}U|+w7++V@s#-+h5z1V)#C3=|M|WcwXWj-?GTJj z93gHcKkjDwbKYQcR*BQ~kXu^z*K_w9I->kLaPYGL4D~pE+EO$tipK7_rHx~}r~Vy& z9;AGP?eDXpKYo19CBgY$-}?VCU@(wX!HCGnjl2v_V$#2x27P$D`4l0k@Txu*NqyYW ziiu*4z4}bGO>BYJP`bd-)yb!~Qc_J4ThLGbxyVT9>Df{R)7IaEjNd~H7IHseYRZ*X zI!V;_P!>W-#bW3t$bJmFF{s^p>u`Vj8ragk!wvf<=GVlSn*ZLV{X6cn<}7g&jF|dN z5vl#UlZ`BH=Uog`B7VOfL)fPj6n=AcF7L9$A=cLWHN5lN<9&9Hj$hTw=^WB%IL{dn z(D$=+8pLbnx$@Qr?-E}}sKg{C#b2ozu0D1@C3vm$>otg#otB)B-j#eI*3o(WOyvh6 z_yM8Dcz9wxfFM2q$&9qLV(mY|V++>J8>m;;$ZiH*a zeu0#`Tv0^zSe4Re+h!KtFiprp0<)v!oNoT zh79Bb*2hRmqj7Oow=lrERR*D6bJ0ic16S^KGr<`B(rone>xhZf+Wy||?#0$1hw8b( zfq@`QD)*5h?M2j~@qER|Dyxhy&U@eEnfD6W|8p4XXg9a%cG!FI4UCw&Jey$}-R-#> zH&4&+_Lqk+MLLnRtR=X`^`qz#5)xS0*vVWrL@g~l^xCzI8$%MKU;J`dXX@Pq7CbpJ zN=o9C3REKSbzFmLoyZLi=7rQtmx;l5Hb+*>(p|sO+~1!i3wlu&6jD8E z3+!;q#ez>jULJlL-^Fri*JsFKA4`E+%`x9#Ms&7PzC>tr^vakVzKLv#D+ebhEw7U~ zYV&ZW03zCa57)wvbr2{o?q&Y-7PoFNosN*89N_Y95YxcKRVjUay0@~j?-!d#Dr^?Q z$i3BCzrN0R#A-DI=N~XR)^KA{Ve`P>)6?T?gvx+`fMEU$QSbKhLB7J2?f#0GxoXTvR^H)i+>m^gy~5=eMok0uKnh)cWbLGyYsF@JiTIiv>=fL z2n}bJ)n=1L zR1lxr(#It@QmDl63f0zBXe3ioJ+VfJX!pqmU1*B=u`I76@K*Yh$CSRWrnp5(kkR|J zxp|yz%f(QpbMx?&+pQ4EW(c+%G$AOK=uxD5Uq6Oy|1#?apv+Hof3q^-nfz|l`JY4$ zb&Z(R=gPf;^M>ro=O-u6OFb_3%f&Y^WBBqgHQwWrc~$U^jHq&ZUBf}Vz0a&`=b}hw zH>XPJGrX@cfyuzwo|&JLrj3n_A^Tola`z?_ym36my}!2L`oyfQx6p`TKJmj#qU%2& zjJe5DW2-ZK+G)scHuDPec^-%kd~|dskSJ2V_^YWHDg-EItJ@L;P&-erMMUYx=A-7V zu`Zh_S|+AQvoYG?@8gR7X?213cM|g@G9n_kq(()W)tHgQTqaW`^Nw4i!R$u8WA$!z ze7ygpPN>uUR?P%~>_E;&UaJ@!EwhwXvj~Gt;C5JlK}HtDK3Q(>u-e!2mmuGB-!!3F}&~GWD}fj^23su8j*3l8}DTwzV3nUNeHV_4r zmm@nn*X%feRpXHOcz5r?icO2Ml9%D#WL9M{;bRxhOx0h*8~tw}4gKBF-X3B3o7?#2 z^5~Ne%so6(aA06NL7J4<{s(c>uuLs2t#YsXYs&2=YZYS{E*$9xf#AA23AC|03lPu< z*_NH47JGG^@Z`xAfY|O+vl}v)|5LbLh`)K$o$2fA3-^WWuzbW#Y<2J(aNvCHx+qX< zaq)`h61{{?;&`1ibKF+XJ!32n>YJG@98VV+9jt6NQM7RrD9^MI_xO}Q2R{n1grEt6l748Pg^>|VeS}kwzVE)Dk%272;lSa zm7{yL)NbtvR6_l-T`9#KQD#I$tRrJi^A!Lc-?(Jx&ungE$}enf7)=jjCO# zJXtFcq-Y)^*AkMxrS?IdMlX)^ml`=d;u|iF90~iow(a2^O19A~mGqGe-lV=Ox$QGOM~1&Bq|{!O0sjMnc~VPF zRpfvUC{|-)VwMnYjU72Ldq{CLV)Md3aS+y;Th#t{ciOg7 z-B_L+o%f|a_JU`5IPP=K;cqcQbF~7m_hytM(Vo5kv5J#)PgBo@oNkUL!S(9sy?EZg zyN%GUOdbvQ6!#l378cg-!NJ3)9>ZfOe=*)8Y4~-c6)o_f#?MMR4*S3?T?|dM`Bi!$0%A#`D61oeb(;DP5{Ef>oA>zhi2{AF4NJ_|vh$UQ{v7xJpNKB>XD_uj4 zTi&`i@Pq*3=nd)56r|$usLIFx{D@XpAx=-n7<#@t5xwgEp z=fxq>;{GWSBRzkOZ*>U+_A;qudSb#EUD(~dHa7yZl%1iX{X3Hmf5JdYlksRa3;-*& z&Q@L6a=$1zIp;S`Yu2~7`=wX}I(icvlDJES00_@^IdIvWzhjyxoXvHrYUj0@h8fpr zqtk73DPPRHde>Mnk`ErZinqMHtlr>`t(fOM*c!Cpur8fF5@XHq@?}Df)Fi86H#Jzm zn7yhH^bEa?!3K!sLSxGPUDrZ%D2QOOO}jT6CWTP9SS*Q*jLap)DP?ySN5H7j1Qj_B z^TbAZP|$3{^ztbCaW2K*%Z#`a*IiCJf`a~HUmN9_5P=|j%E^FQA~DFXONEB zO+yUHjC<;e@PQcdP6w7xGYi;x=o?M|176QoHE`RnQY(K~T3&3)sxZUu{`^vReamey zt*$$s-XWRajT;RO&BgJ9PQ$!$SCjyt1#tlXVN$mjLO!@s+=N!{uz?OzU$A;P9;-vH zB&arHLc*RX3?wS_kzbM)X3@M)P%`a#wA7#cjLa=NaD?qxD~)&jn_?ajNX~GN3k}bA zZPcpwRdZ!LkmZJ2#k;U;Z?JNFtL1a&l(aS0WH_3g=~CgNpumod?>&^rBxU;h=e|{S29EaKc6@}*Njt8T+){ClC zI1}CzC)6?5059N5$upEbExF;nE}Se~Z1 zTSsT-(QgME$l}UaYGC@RgY6~6 z{9gH#{0BcqYbzza;(7~l=9w-qvHoO^L};<2TFmxCt7dxh4JAey`8Gf5fph?&f&Q$* zH;tZ6C8(hyBcN*_eW8^h`h9u@h&nVf{@(QgpAySyOYtbr090b{EQbw05T_ZNZp%K} zuSt&8oID*J8>3h1eq>`K*LY)%2TG$D3Q<~s?#%#io6JA05X_{_8zCp;h5vNcKzpdK z02FlfkDC$5KY*A|3*qMD8?Cl+X?*Zz?utrV2l4XJ;Z%oXwvzCC11YnQ**OBH!t27i^)752in-h#l3hY&G3W9lprvi`UU|Q zwK`{jyPUx{5Wa|`>HOvy(Z=qs*j(*Pm~X1Zv}?Ty@QZ0K@T{i8&y%ES8B{z2 z*ZL-75);Kc!j0(VzfQ^MMwFLRSz8xb&eg_kk9)apq`|mc?n9ZKgsGdMq0bZ)6i=G% z;(D^30F()37K%d<0i0ltq#GgWN~8I78HG__A}D|^fbYLh%tPy%n;YL$J7#?&>-R7a zaYw#_@Jjvp)0I}{bJ~?!fl4G> zvUFh3>%%F7v^Q_xW_sNi|E#IOB6Pb?>>Ait<6-l@?UGJ9WGmEo+1c3{JT=7~8ygF- z=~;0-2riaWB_CCF2w#O^L2F!;0Qj^1kt?vu?bk_9TTntur1>uC9%Nr#CSar`0&2I$V8sgFj0STa#~a~-^)>|DE7YP|sh9-Dq^7o)CV$VR%wK(0(vxwDmR)eT4<(zMu4I}@wLO%v=+ z+Z$j!VYxEI3R%H6MI2BZHr}lzv6C^ohuTj`S(8nV(6Jp9BB5_%6Y@z_J}Zmh>pKec z^&ua8p(Y8ptF;)*Y5SGsX-b#9W_iS26q^rz@;0se%755JvZvnO%5LY=Dj$fs0(cz! zrq;g}5zZ*seBy}-_n;Ykg>(Tr6E;?XuV<>a5Cl zB|P6RU&y-wLo(-LQU*fa_xtkuje&_7fJ9;tBojsY4B5v;7GfgQQ17}JR9N`=!`Y1_ z2}s-CIXhQLr%q+G1n908R^NBkyDborz9qx?l(jTa`tF7v@f&=X|I#@A(~z+T=t`X- zB7GJgN*@}lKHyU-o)5O3gCEP4c?xK1>sqg3oiwPiqY$wAA>qQhoea1DyF_m;(Qn{! zV<$n6{#<`PW2)npLnZa82;}jo(Htq(_w5#^J6us^M&uh83!>!$6Fzq?M6Hquy=;z~ z1r_FJ>_NykOZ)qhSI0(x!yZ7rfrnGtSo_3xI4UIy?SGBs3VqXk;Nf*46Uz`xGn@Fa zq}><+4S_>Ju$BFZ2OTbu!TbArL3k6a7?M%0SoE5N=P9H%v(B<^#Kaz1S-(pOIq+=P z`+0I@Nv#2Exm4T~1zg(@bcl=q@JJr}8M-Rz&fAaXf5g$;#Nf&MilGcD)e%(on8Ak=Q51yik8o1o1N(vA>cr2u%E9ySux<)%YZgCMx%ARD_eI zbNktm85NCmeWda6HyDjnoV$9TaB-=E&#^~46tE+AGnPZZ6qNsbom%=MgoJ9RBH(^{ z7ZiG-03f;p1OW&+E5{F>9e{>>xbTUi*KNcB1>;ldRo{`AcA`|okh~k=5Uwv55hXqX z{Dh|el?I@%YD8MNV;%8E5YQq&bkwHQK~ZiEZj~nt;~{u$$}Qj_D6!CZ09s>_7Kot` zj0}j47erZ8q?9tzsVw*u3Zvqn0T+6$R`z4{fvN`9DKe6Zk|zh;aR$)6Z)9Y~5DP6q z62##!i87wAw;|+MH&9SijEthb3YtdIWeNzF(d(0)YJ?TcKzV&&`I-E0<%0Pf1L^XW zapoEc2FAOVsW+6d*ig6{6!iMNfk8lZb>4{laV4+Q_j1=GrThCExCZygwc|B{EUK^( z=Utd4XI5)oX>tqta&tlR|CzFf72TpG`~qNuGhU_)Y>ZR{P^!EFJxJtkL_X-SVP2^jm({=rlXJ3thSQH3|A5|XwUQ~vBjFCoAnkHM;YzS=z=LO_iVP%?AFq9!ci9-9I zalB=U-)sQ$U!9G~rtoZdwRZLnrGKj@j{ILO03{_)woOULceB8WG>_XOh`)bs ziQS6x^;t^$c!8a`L~OnD;F}NgO7OGAR>T z9v9V9RZPD&-`MAkj_P-+zN5g1N(;W*(Z5!+bzu9i%zx9U?|&jp{qSKF5Ewq7yarox zl{ztuRe%d9w8%(Q8Yy=|(vI!vsVP?1!#?PeVp8RfmTx%JUztAVtqJ6@8HYMmVI9;Oi(?$>oIQJh&-4E*|QcYid)2~nVpBytGDjo zli3@M(>HB(?`yPDoNWm)zKcjvKY&({%5#2xg{<;pwZ24Ddd(`hyPKvi(9CvwxO>IT z%S%8iRpvlR0x#=Z$fs}VLvV>SwNvbjiP@vuZ;b8D6fR#DD~`IMqUA_*QyuuH)w1em zz?t%Gaz2HUhBPvYavsksVqfte7;PmX>%u{^j}W5Bvt{hmumI3r=4cCDEw@a6%Ls z5>nJY&;{RV53|M9xOfE$z+^rbl9P3>zQ8Btp!LMXJKDA(jpIlL@k`6};^3~wtmQZQ zFFascRLA)sF)>nAy#QNB4!zUX@tUcE;ut{TKS$)rkp9Mcz}UaNK6MKLEBPVn)b@>J za2Q8R9$|J)TPEt^K8CF``+r8+9q=^XN5a%-9xvv$XApEE~cJZ>aF4&xUQ5pjH3{YEew zY@Y(yJ{n1!wuy;}&1dKdu#b1IHnWx()k-|Pb$rwEzf*TK-Ggny6 z2!ID$zDQC5c7L3_S@4jkI2r!k0sr1>x78?=AAN)Qm1k?yO-&6jqs6@^&eu*#Vq~ye zTZv5^1}jWH%n}QpeE1tXz09A8rC3~4ZC(1VwO+Z{+VyJ0*3|r7n527uC*p@?{LFZ! z`alm#42p}3lj;SGw1};WU!#gZM;t#f#6ApVBeJ3s0r`#&WEBK=*G?VcIel|Au8+mN z;2x?;bPiw(ntmy}gT=CcNE%2SqMD5UdTsO8>CT~tKJiW~t))CPC;OPu@@h*HhOF3Q z&fqnZP=i2=EPi%asG4MvT4>}=^=7aA&CDHls8q@CXnvm^tFdCuote(gW0#Wwaer+e zAt|2q*t7e}t`~C+{BY8?{U}e=CI&T-13}2~0fxLP1rilQ{xG`^DD%}SHbvplQu{MS z>8=jj^nn(Jn6&R2SPI}w+U34DJ5AMf9@BgT8MJ#&QZD4%1D?`DypsC9ZbiKM*4E-` zQe|rG`N@c6V#cJ<`rebryOG`rl#~-)6L{e`6}LUeVQ;JHJMR@^(@-8RN`F3B5`WUG z>0Z(pWXzcy2xnEbb*Q`BWS0o3cPn(!o3B4mt7J&Z6!pL$P@60vi7no1EpwjGDAH7v z5<4ab#8v`GIr;KgWg~24?x0k{#(5F6)Rs3|IIG#8$`d}C69ESA{$m#DQKzQ2y4s-U;J^d?GNVq#};lQmE$B!g~#IPOWVN+GWbTZg0TB|T{3ebcJxZ4gqoM@kiNiHqH) zy(n(?74_Tc&EG-FHy0evel_9$T@AA@UKQ{O+NmTD4Noap+G$pQ5JW_VbR*lYt`=6y zNSBd}&F`OEgc`@2%0t=lgeY ztE+~9C<%(d z0ZqsIoAbi#Y-$4o1JMMAm!RVOMM}E!Im25cJR*WvCZ%fdyq=Uw4vax$`NW}~6B zh}Bt}rd-wH`=395&MOz)EM8+h$05+y!^7+e21rd(Uq2?EVa>=91qNE_^EkW!QAz4z z--g3%RG8@HXb|HxuY0e8;T;}q)=kStoka8tV8iI8tE<$4mwGQ`We2HgH72weREsFF z=>~tUu(t?RG|tCyRwtvKO0jR1+)P@7?;_1RSzeXzZ~7qcZH+Lh6qTXKZHew z9C7GPz-}JnkRT7%J>P_eG(QW)>-uvIx|vGk`YeO`vx&N2g|ndn2p4RYYBp8YbHZIw zbI06f^X?}Dn7}MH+?%GR3PAU3d3QJVF1SNLe@`_r_PZ^aj2wW5_DwjB)xbBW3xyJm z4An_en@nrG?nhBW)X(}3rgVPDO>grMkDN}MNZHxi8jgQA^1Q!7i6XlrCeQfpa;5+@ z2JaM19e$7IYC%*rC8dihmg(VRec8W}v(K8>;0? z?oXk&A;8dhK}i51%1l5PxOash<9P}+-0}=4vUVLrg+9QykT2-qQi@ z{QNNi>mtqdnMVXXYC2@Gi@eFGEqL7XxaYxccS?VKFs=9g&f}SS8Oo5*0~Vll+kk-+ z@SZBqRPF#yb|Ee`RqWX+v6UoT=N8-;DhZg0XJ`?q>N^(WT0-nKc|}|64VO< zQK_dJhQ-8>=9U&rU}xr&*3A~hs6F?ncFu4a$2H#MfCwbxb}748%do?6XFVyZc9939 zv6Q^S{=}HQqhq%9To+I|5{ik7FRmyibax{P-knkbt&{lCGL74QO%eZ@Qr;|?MQfmb zD!*I&Cmr{ITp7)i9=f=r)6?Z4A)k%QBdQA1ktplA>%}GckKewPGiulRMiNiHYxJa% zP37(l#l+568{z6><6vjM@7aQD{-itjex>uJR=sO_b8{jPHm0x}F)A{p^SgauX}k=) zySZ3gUY69;d+vTgmbB*TaS;s29x-?{lE{pO!i53>4`()?`z{gz_fW|M&#gHRjF%P@ zgPz{Y?Mo!Q6&avWeboEh%WUd!klDSq3k~Lp@E#!9BVk}*Ze(cVgg{&D7<7c*JGHiD zfhx6+VM5WBAc09Uwj&~s-Sbk({hVar;r;?|e*QH_hLFcOKR^&1tV>?!nuw=f2+Uf+ zW(HqU#EZg0`B`Pt{Dhn}Cn8x*2-0YMxJeS-ILfC!vu6yRUCS4)DDv=%85+XfTpZ|c z3|avL>A4++w!GDV;6um53XzQL@zr|D6{8yKC;hh0a@!?-(1;F)Zmjbj{O;Th9grg! zoUEUDUT-PsxNkWzD7(i$Miw-huQvmPPy!OJr&`^)2hj^x*z)Oqpbim@z}u_wyuxY= zxybas>y%FCPXR4yc7sl8Al7IG@-ncuiOkx~&3;)&htI@X{DY>e59FA28xe)Iv=~8K zL2fAhjzH+%$MdK|y76Y;0{2yw_u?!Lql2D2U0r0*H7*<0zfAJ-Mr9*1ayEj ztiuX}miL44D_(LLg0B=5D8bFp0lM)5N>c)XnKHp&*>5v!fl|hJ=>8*U*2`y#obNQc z?Ej9#q|%|Ipg{ADz#l3H+5nK)5CCQb?1|iT8LrvvIZxxYIiAT-`qG?>kn!C$IghO# z3k(cuXjrMV9|6uO;j33AHoAy5O?PBL$ePCf^zWMeIIYriv}(c`b#5OocwP9Q5ObzW zePU*JwUuK~`qtkabMVNJZ09p*1hQKH)&nYzAj`fV;J3pGO`GBDw-&(W+P^FJ5pY=} z_9a^N{$2;N63`qdfQIEK*uiWL>;ALFeD1qAA07m6tn9m)U;0U6(v>$b%)=2HPelN# zA<|GAiZh@Mv_Cec&FfQ~by(uy;GjR4S_%}#AkTzwSk27-`mxnEUA_Tr{{dz;&v_5Q zGjKHd8NoH#?Ikjb7q0llV-q@P2d6ttQrchphVk)4)LhA)Twe#0HS@cZ5iJ(k`({bm zrlrj8cEhNQCTZD+At9UjivHOEgy@ciCbF%qEk=zO8F)-Hz~$8;L@y8xTl{NI_XR>} zq#BpE@^nF%orhdrYYbL7?_GdGF%Bf2$4_6rgr=mp`1tlQxu)iC@^?Rh1%OSUU}yKA zD9q45S$CqQr5*nInhy!b7Ai#8nYe4tm9a1#-!_A|pLV-Dl>(vClP5>X7ZLa@J)mm= z`SLo#+4E|R5dswxmFPb`+kV+<$yu(~qJ>6!9brChzIHeV2$7iINGLzrfNtsVh~A~E zi$Jg*9ReVMbjtZ-AZFN|#Rr@`fnu(wWTA!=Xe?1^R^9fW?`hg-nkpA*tp#tvr@Zut z?Tk!$1WJCQ^WEvUz$Kvi9#DoVG}MvQPOryggv%If{_7d9Z_3~q0S6OX+1rFhFv@NPUJ;}Y z9q+41OWm&ZfgK0|s~*na;2>1p1iHLEAY!vigagf4IOS{peD!ipIib1Gft1O)#^G8V z)H-LwCKel`xkT%hs7+VA~LjNDQ;N8V#o3SWVo2m*273ZJj`nN1|8^{qybAc z+T_CphrJ9{KboFr26`3MA^0i}g;ekd$ zAr_GRTD6WazHK4shEajAed8k=gRfx?lV{d}1GPOtu}pAlvYZZoATbdFxsqO|o;roA zc;#3A3N#^rg|c7oH-jXyp!fbhh>_@BSv@`AReeZXrsL}fcZ*9(0@OY{?(^#sryY1^ zEp7=AEW^T{C>4H(>h4x`wnua3w8=rlJhPmwikU7K$`*z}@a-R>oAhBmJj$0e-8}D) zFxyu;TF&8iJAU>U3FnJ+viI^?-rRL<W0hIov;XAQ#IyGKnhvc^sOgrm z^fu;w`U>{ymX>j!X<=-S2byTKAlRsTmpTvKwa%euS-hU8L{_UR?zw6NlK>i>!`l-U zHjnd=7p8}B`}>Jb>>!)?rwiN|O$*Y~Kk;4ZLO+=@kSXKeSW`|tJd8`e0P!8oHzfrD zbkX=vq)njdq$6Sjzj(|Oj1MGNg4WjR_$5LaSRM&as!5KZ(V|{ESYg+QT?DCyR++y~-HL8~VaH)YYY#`_|Fr;*0GPC4e{| zDk@$8VmdUcQN-~yq5opNsrc@2>Cp?T8QRH%7HvEGr;ULQ{uW{)Ue>y%%A5fXR&n(0 zmYjgAdgQxzLH8|mH864nM$BGH5ltfloI@~9RG;)bpB~k;vo4C#mX^_%?|O-Iqk=_=7ZslUxI*eY55S95GG6lq zQvCHn(k_z>nV&l&Q`)ulP+v{r*q{QdVK$oma%b|P+os71EOF{uA3g}Epv6N474e;I zuYnK-6hyX(OvscB+lcRi(i7qR53>%Z!le)yt&U{Wk|b3-o|3Jvu8)oOV%Dkv z=r}p=X+6|pB2*X0t1}13-sQokPQltZkMS}9XLu{k_+tSHThMPp;7bWP{|xN>HH4C- zIbu&=6hmtc!9%4jFK%&dqdBjBq^D3pPj8MtMFr5w$@g~)>lX_F(~Vxwq5B{n2?5e- zSg==9~#Bx^| z_zVUgU!NE*$g&kyvwmo#wMOaH07ZQuvD>gBoM>`5^O!4#Hn6ltBqz7{no(3Tb#g);;Lz~H!3*o(sd9pD(N$J_ z7{tVFA1NB9s;m^(pOu0L%L?2fkxBCn*cB|2TaGLVAK@^p9LO`Z5)55Wwm4PYz3$_+ zSEqY?{@des$BP+a`&vgwp^wTMj;G)Vn|1EhK`Yq|K^d@Z9JM5e6xIJrBnpm2kbu?= z0ZS8qEGb{2Ucu;7n^o@$kD|NNF=G+YhxNmyI8Y?W&6sHe4gpfY*wHr z>{&F|*s@8g*w@^3R>#E0i@Cec$mDg`aNd2%$}&joQdCrYD=As8kJMsu1I*JXQ454_ zV?cv|Lp@AGf-bZ)G??V%psAS-=}+N`1kvYc?NjyrvH&x0d&8(maU3`av@~z@lKAbs5E$qRMaP4K`@xJ5HntJ2G_pe#=wb{E z+yzH?0-0`TWJ5Sm&8|D-=jSJH)VaT;?Q@v^+vI?A?oVO)0Xnr<$5+h)k?{BO$cHal zIy9{;g57+xp6HxkT#y^GC+i?Rc_N=r6B})c%2dj*v=rm&fv=pLAmNS1?!eq1SUA^U zE%#ZWr-nY^wppuB+T9%pI@?tD!bEmgpF43vv{Ys`Qo9QA#F~7}~dL1t}FF~5h@f&Ad$lNxMhlj`Ro{cxm zWZ^kLi#6B&bacSpyIX@!z|pwqfB>qd2eSS?mZq4z4UCP^Q+;VVJ6Bnll zj$YDV-ViqhYl4O+%?Zd0{=MREB_6F&AG7xKrQA>L~DGyqb#NOrD=blvWf8LGeb z)zQGHmP+&eWJr+__4C#v%rs(87DNF;MD!<5-VPX(W7b4N8-6?v>!}k%y^5SKJgeXy zN8qyZhH7n`-mCR*;eUvt0J^>Gytc-+heTDMz}C{SL}cXKf>(DM!*%63%?ygIn-!E? zXRXKr-!v*|O@@M3c6R~bd~~1DP=^-5;PHt2o|Fg<1~>;6d2)uHdDxK3^iaExBPQziX%al+5f;jod)anO*}X1m}q7?F_B zk9QkWX*r$YbAR+og6{0%sJr`FeSL0;@qo(dnUmQ_YZ|>`eRtx((*c5Lav_7^FJYDo z-WXn9+A~#F-PvzNx$P)F?$1e-nT=5bjeTcJ0Nv(<6`Y_TG69D%CqQfN?@f~;+d?Mw zt*lzE&t?h1yOYHWnU?Av;K)aeQju1o)$BQk=cVIJrA7N{PlV}cHYbpi04a)?r6q}G z)o$BBA?!-0n?kK4+hA%N(C zxhpOAsB^S>|DMun#uXeN^o<^Zy*?wBc=P7j(b1FLX$Ls;=&Lb*cvQ^`)!*X0h99xJ()!v1rW4>Rk8$p_!&y+)Bv$1siZM%ZE!4aJ~!S zoAIM6Q~BtgIF5`SioovApgirkI`QN8I1l_?!$inwi3JQ!!9{K1tIuUgpo$%h>8Ful zZdB7pIXX7@B8D&uqEeoU{sE$;r7_p)B8U|2E(bjhQ)TGEGC>&{ube$>zoRSSi)!(o zTBASQqAuYpRhNsul5%-#9L6Z4sYxu+W&g8|D@UVpMon|$Zpf*oy}EDK<5^6M{0E|! z&j%a=;ANH%np^<=^E5j5%v~uU~FZL5a-HwEf ztvm_}Phu3JUqK1a^GaQhG&pVMkv}e#PB+}J%6vYBPQ5GI#_;k`2TTb_#~bQ%Z7*#h zC+m@~q@*G~etc&=S34An*`Quw3j2w94lo~$)O2*^&U>7|j)%)lh&{bj4>v~?;nB}! z9ezLn|JD+YORQdIh^3<=K)_=UuUb6se%Hv=H9wC=p79u*7(WlVmqDT^qZJjUVo-6+ zc@=>7^QTgx;Y8kvM8a@y_w;17te>CXh-A(&yn=*ODC28m(=eOfyPN25Is)Yj0>EYh zFLSA>Ta%@}e^z~f`)++82=NlIWzPTZ@$w6{lA}Mr@phC7B84C_RhH!sT_JorI`r?V z#YtrX%*;^>-kL*GrH>TMEz|g19+PKipKgr_juZI#;q!RkzHr!xYWekoFE@AE&Y_^W z%!nAA4upSzB%_qP+4Q>+;$8^u_7ct^E& z`q$@s9~r!vn3%M*NEFCae{7Cwx|~Tx#qFo_-k@r6 z46T@o!`=+3_ssx;$;L7_4{o062<$f6+i!A^k4uiqkEx$6QGTaZfqYL^j~&LMM2-ne zyv&6|2zhVCWKn=j5s0x+fsmPRq5D|de~SB=X_ll8>i+jyr)Ovy3S5eaM|bv)j4c&r zZKe}H(DCq$aaeScx}udUx6{(3aw^KJs}0U~6WTjFGcROinz_o;Ijv^e2Gc?ms_>ap z(IE~eS}0_{gGvoHSnZbeDk|oEkQRFrG(ppV&2HI0-m20Hp$z+T0k;zkpc8xx z+9&8_{9hGv@Y=h&R+i*Rsi;KB1l*4^LmtZwJ=xsa%I&A_GZiYp!pp2=sJ&=_G=1Wc#Us= zLBKRnsi@kNdKi8Ai%kmekEPJJe_hcJOkEs=usW>UgHAFbw=HE?l$EfqF3Z~0=4Mj- zjfzT5SY)KZ#s0$BhjQ<`bLoqfx%SE7OvUb)wxlm#_Ds#q?a#LNl&r9^u^0FTJ0lx^ zsFlW)o4Dk*e;)A`bUj1_q{DQz5n4^fq;EFfGFK>c>ydEAd{3Pt5uI8~tX-Ek})jQ@9%O z3cMtno>^b%L{)yz&(6+E(=z^x2S4cfbUqL7QRTCzT2ChBnfQ{_MGB~V74lIOp5b82 zC+xkwKbA+LV5fRn(ar_Z=sQS$CQTrQh@p{-zy6#v0V`VXOW3T~cQrhW+6eI1hZaKI z<&RniTQVzIr;&+%U4Ap1C{SGjoM+h8iK^YQ*dqS#WI5n4R)EgEQ19XgUSXqj>E+ZQ$s`{7|SPPzkaPo3*{OZ8X|t%mUMGfGPycN1!$$aNZT9a zxCy0l=Xr2UvUz64KMWh+7KqjT{h8U#Mq!s%RzPL5ypa*|-o~cuYzM*ha8bV3#J{nT z41gJHc2sW5YPB2Q zR3RXHL}uthOa`+BKDj=4BLY=SmUf*u$Q6JBq#4X;`n=?=p{e<af)w|@cO03a-F7XPB4hMt^svR zW8AMM9z*#UoC80UXh7(Qz+b<;Du9Da^*T7hSgymxMMZe<>xuZH?~)oYm}j=ERQqgD zx$heVccwA9g2Z#gXE|_IsHl;OzEuqkUPmnCpD~t2--t+gasV{L#x)cc6;)L$iV2(x zPS_EJ1kE>J>We&jw;~$SJ)D3<>srdYJO)d3q@jSt8JJ50_*2U)4as*WV~JQOh};qR z`HxmQ2i5>;82f1_7aLbT(GfeA%am!ez?NJk1#}HF6*ODvzq}?vZp~NB!=9O$As{9; znsV-n3IJJuZM?vCf8MJZ%tw&HnIh1Ab-<#-rpYajUk9Y^^>vVZKqt3$8@>U zZ#B=`nuW=A7uG-zU7^MvIW>(Kkk*1CB4ivM=aRR#w^w%;W>BoM;KO1FjOzBj4jm~7 z*5-f$YzGjfJm>JaX8>K-763c97JNv+xf(#JY{n;xk}nUSFa?RUe59EB=|D$durhkO z$rdY{_PG?~Ezl6DuvE{$qOshXvNQec4N^KC5F-P#T3lWAi;F`@cwXIYD+FUaNYaue zA(yXNjm@NaPRGxWg&J9r$5&>=q1P2<1RZm^u}5kuKexv@Qf8*ySjG78u&|=y%m2gH zTL5Lbe$k_Zw1grlsS=V>(jf>aAYIakv~)K}sg%+kBHdB~(%m54AxMLOblv@(-@SMK zcjnHV8PA*pe((1_vG-nkt+nAI$o4oX}Cy>vjXJy4~ zYF=IZXbjUnnrzJ!fY6ZTc}71uHAcfC7yIDe!sD#FY` z9%^BJX`lUZQMDU$L&oQW8yS_MNjnzlw@RrdO84&BzOhIvXdR_rbQu9|jnovyFbK)) zk)+<1illTW0SQS&UY=d{4)hh88@vj*bk6hj>IXni-B!|g-2bmKDk=&K*XvW3vGsV# zg8bGvsv^^vk8wjIJL$@k&Yq`9wGwB4o;S$6osscLHMh>To?`;RJkuQE(elOLO*KAx z`Y9Ev#@Ba;n#4p$8rb!N3=PtQp>36IwE&1Cjer2U(eLV#g!uRu0G9DPBziwzF@GE* z0|~Uk8H?u`=WQdS%xIs+!L-K9ZxdZ+qOGkDyHbUJx82+?(F-&(Gy80)TjtZdJ_-Fh z2+}bdrlMm0=yP3hgP-zLd}uj1IOgJi|Naeu=VI~7EiI0{^$BOsvC_rk=ds6^L=>X9 z$fjxw(h4aUDn_<;ieC6mcD|fhVkbLp;g%D-Lsdm25Md!txPu>IfkZK#-qgc}&tLR! z{=`BYH&yt<>TsB%PSRu1URMr|K|b3>4Ili7dZOt3Y0VrY3ADDKB@uUvk+|-a&USa3 z6WWS0GIkx7-+B~qs<3iD?9}AHOeaNq&JfaB0@0ukxz!BV^Or(CSptTN^T_05L5~}A z(2fAow2MJV13c_ z!!8x1?4jyK=cw@F7=nVP_&i;86Q`M0dw7;d^A0HlofBo_tV^xuC}1ZI1YjLPZy!B| zDaLT6Ykst7tLMu@hKaFXrq*=r+xFUOqzq~zR#b2e%{lV zXX2gR@Iy&fc1x6@T#UzIeJ=i*av`TdO6uZ_qdU!qdJF%{aM|#$U+3#5G?^zF($W}| zDL3AL^Iz5xJm~$P;F}(@=tGP$kZYaRpg3me^ceR_K%8z(F0PnOs{*O2>WoydN+EF} zWkSAQ?4~EDh=QgeG`{OmXsDTa5P6#i$};@^J3_)YAVa59bM(~iNa3?Zw^*P!&rJUN zhkc}ZwX9ly4;!zgPtVrqM@trB(!4F&qaW+37UrH?8(3HXVZsDaWmryFbD0f2C+E(U zhc*HhPKd!=?GEhf7?ALl8n(&x4)IU%DNv z@TVst|DCbfN)R|A0i3#ny;05>q8MZ*?>fQ`-(DP}hTdQM7!raC;b~-hZ+j{;7lKs1 zM|GyLtP9e=e~yR}489NIQ^P5P?~=PmLgWCnlnRjcmnW0JYz|ppCq+EI)ePQe!IO!k zDfK?ILaW~m(_3CP|8Oh2zvK|ryH_r6PZgY=Q)QbD+Qla%fYX5jARg`84>yOfXh(q? z&jiTFS9tkd?$&cY9?#U8Uh}!2VXvz7*XFC=2|9YOEV)Ez!c?No`>_9Mr$zVk!0wZ% zLdK-BNcOVQc$CN(Tk^5*{t0vyiO7sGJv5LZETsvP(zY=m3Y!0pxlxq#oZ)3@s;Ey# zmaM7U ztVa9u=fFl8)0ZTIXi}q`#$%wcF<_^9vpth9b=Lc~XHWr=e$cZf{!@m(M?@v2v@GII zPL^)cW^-bd0S2!DGd#`dMcOlI>0tAbpsTp*rA*_&jPpBbn>$#P7#M;S6x*Z% z6*!#{X@+a^qOojqV9`+pc4;w*Jj{`-Gr@zKcf>utye^y-T`3Tj6d^YXo4uvBw&zC1F>#+iUjSDu?N7g;?Zz*D zy#~+gZmgm>c6})16T>MzGK;Rv8Cpv8>brcADa%Sr#kIAA|Nhm0({SGxNEgd1s;TeM ztVGSX!Gg`PGoEiaM5~f~C`-0cfn4Lm-j~=a6f51lt%EExbDw2ur|#~2@u{0+jZgmqKBYo*|2_`vl4qWtLR3^^ z9Gskpf&j^JYaH8VFAz~>MAB`=KQW|10t^*-(dR}xm9r|-2un`x7u)Nsj0`^j6M(xu zJUrYnKOg>XN|iyguiwGYI+ajV)CW+8IK%D#x3WH9*t;1>!7!hjCe)s}1i2sT;>HL` z!m;*G*9S;!Y+)O-HBb|Tb_^+6Xn7S>R94zLv4#{_zJBm96DVJ|1YM|1rYlzA3YD9y zRm;_xB4cHJE%g%OY9bv&YoKL5PX498YIlM5iM|hslkZbp!(C0l6Lw#;M>JEfN54#> z5xdXme0_N)3uKoFZf@>mq?cw`876>AECkfKNFYHzq1^xF(pB%L>p-p&DtrPidkNP1 z=H^{0s-W2MwS+*swEMaFwSUH2(oy?O=mr?Z`E ze!0}k>w+<^D4_7|l7^7nNLIb0*5~@u9i`aUXP>Fq_3NGZ2lkaSX1+V8#}*S&YpNPe zTJZ()q>dKFS^}ko!s{FPg3}7wd3oLG&vZ#eJOy$T^bK}rMgPBLbad6(p*9ecUQJh1 z%Ry?zsy&X;)FY#&0v#xRbBoYh=z1@Xpk18cbB*=YT36#>h2uBGdq|_cW@WWQj_7)@ za)!Zr7CXP@g-z`p8}Z4#cefuCH`3$=%59Aim7F_bwC-dFn;D z^73>l7VkgWBxPhE1J@2D;^YBABCQiAm;iPeU!F<=wEd;lsQ@r^C8yxW0R+p^`~T1= z1h1-g$2M?EG*E1jNTk>2$&+Q?hZpYBj4oq-mc>H5XUkBJj*Y3(Vx{U-!G`<{JFo>y z^M>g6J@Ic6ceR$2Y;Ud~+OKj-N;hnKIyE#TA(>4S3)a@=6Q)g-=jdzlfs>f^Tq4KY zEUOl~B$~iNVZY_87uU|os^B^s* zRmZ6n5EI_DrxvV>blmy_VtKsN3+mOA1ro&~Ez0pC?Pse!`E(ic!q4I#2Ir%Htgg2B z_#4tsN&N-i1g^M@%*@`|nt5;8#Z@3yTx_Im+2`frBUX zShU_cA7AEAqu6Zfouw)1?&N3UhSRAs;MP_v&HX7?cBx2f2Z=ByPDoR81|N8EqNcka zeS#g(*3y!kn`;K3fZg?_8r;e%=SMp;eu+FYTo$UL|BjB3z=wd?sH4wheW)9eFf1-g z10pCFL1m?Uo+9Gq_f;}ev*uSYU{xdEURuFlKql(V2F1PkNH#JQuV@J0G3rY8^K*AZ z;tA;c;<_dUfSfEWeO%j_`?V$kd3hX4O2dU(S?9uDhkt}E#_}WcL0sgGG+sC*K2wEz zb~soBb*|D+S3^LRAaYJgw=*~3LU!@hVT~R{009LWdy88hr0gYRfWawNTNBOB=^B&S zKu(>ep!1W-@W^CJ?1t+vlaABAs+%m1?a^8K0@0e&pDlMq0y?EJ4L+-4YYT}kh zTNgx);;!N!NihZ?-_Kao_MP89b; zX7*mKpbC(#{uDu84j$ol@H6A!>g@+!8t_XA*-KL4tMO)et6v-oV`B2jz#oiS6^}qU z4rOY3M#i&=;w$(hqvrSK0K3TcCmxMZ3PO9ZpR`EZ1&r+&fVPE%=*akExNw*anZca~ zwrebBha??y^;par18YmI7;^DXf`vV*>>>DU@1!&~3DO0e(zdlVn>1EemdNY-#%q?A zDW*JGA9?1qrHbO{G)lQcVh0AdKkE@poR?GIj|(+9Vj}56IN!eDP2}D4FS9d;R+Lzd ze1E*)i3>0`AZWBf%M2P+=}?wGHi}^~4vrWU2TE%D!Rmd)?s^Jz+ZX0=QJq9z%+E`h z52d3c$;QgP>H6FaRE_(9u|f&1Tx#$~_Gv>6l0=4Lx+3G>V=8g5J39sP1sTUz-Y9z9 z?$XBWYb*!~)UOI@;|2w>!tCrfs^We0k}hBH&0G3RwgIY@O6T@8AI= zApsbvF6kRMpP!l{Y8og;N=>DN7amcvcSoz5N<2gS>1dI5Ajnm})wN|4h-2>S@$P09 zdpPY2+#h&)x=ocA6nE6RAEDH{<4P(hI63{SbP_=V_Fdw4tKL|rm40m`jNtD` zVCoz3Q2N;6+S8!NwcZ22y+g`rwI=M9x5=Ph_j@#z{eRofXR36~XR5S&4r5~xzFt}X zC`D$IfZb|)y#4rKrEH}QOBCGI(sn1y%F4_;7?r=dCa$6vXp{iVpC6SBH;Vo7_DlGQ zh_nSg;G|3DDTkAilb1n9b9b&Tg^3E9Ee65zP3OpqI?8H?7Gexz2Y#_NW0X@GT<lw@WW ze9{JfWkn4Q4;w=L+Lw653|-O3S8Wl`BqaQ?i70XvoBcp|jz@L734)cwpTcaN8f94g z7>lu+unhGVk`D>R-@ZVEDFO-#nLup3O}yX1Z>_DHGbHL>zIbuELaqVzqciv} z1O?*c{ND2kFl?n?TGwQotE#rvW#HiOtF(;EdgtKG7NMLc=XOcI^iQhnO%~AP@Sj@i zfXNSztv!>%Qj2+%)W?q>L#cMUS9{m&?DFKG;|cDaFUS5Y82PZ^;8^a?*7$+Ri`yYF z4b37W@K6}wkMnc%8#hndHvGbToWh@9cNs_Jp)aS#xrG}j{w1Gyym9Nzm0-e=b)!zr zuM(duMaCywy=U`$*H`3ekdfXu`W4O9_@&?eKPM95>$egg$oFHlU&b{->+<#6x;|0kt|_i?M@rdt+Xc{zSknB>wL6 z#~*+P+`FFJMM7>YA0r|**mxx|UhmYXvB z`CUWlLkQs-sPxfM3=zie6!Vn7jS%aIj4JT!+ec<3fAlc2n$n zzkb#JshISBk}^=`@i$Lax^dq76F%YTXJb%-vxB@_&HE+wB5UY~Q3!Ee2_<0!M;yqw8Gy&4o(<&I`9g0ABh z8d1t3SDz5cvUa*JDP8h2HEHI#fN@K|+(^wWXbdq*4hJf7!9CBicYPHMjS!lV>9J=D z3K3J~J|GO{fJlakL`bLtDeqW|qf=4;M?!D8G?B#ikcp96CsSh9L@eh*`b0jCctDA1;6i#C}AU@sV&%IY(Xsx`};X zCAl;!%+TTG0W7mqk5#m|SO<&8V3E1c_4c*zkD!BL&~0Y;P>3`B*-e zx3`DcNH!b%BDeXkAkiD0jzFAe8{tKYjK4yMYaABvMZ95Awr8sSp=Mp*x(^=unt%Gv zC?O#xJ3ic>YW$v z1#(e8WqE6?h3~S6zyLo6{NkJ?7i1lf)u(!CLaV7&Y?Y6&6#IE1qRX9!D(Hmz7UV2&oCVu_8)cC$N_f~Uy2R75J2-cTW z;R2#b5PMFt{2q*gqjY%{K>$x#WW@oO-cihh%O`j4EIsBq7Lm05I~TojzIH71C^mRP z+|G(k77v@Yp`okG3=A9wKpP6)=}*=2`BWfjW>T{kmyOMY&hU{N8Tx-X0VW>?!=Rs^ z$dBlt1NrbdBu?-}m+41qF6nuXR8iPCMvG=Ss#03q$qReZr>tG%E@nAo(v$tm| zRAjVVFL(1mS9wv#pBMOxp`-Pd}ywv15n=L+6NUf7{Qd26sJ}+Js{OdR+h@!u0X%8{Ab1H)7-C&K+ z4W4n2YF|ZKY%DQQ0H7!7HMI5U(W3(>WhXusm#TIb!;8hja)V#uRlaNZ<%`W$vblwY z!J9~7bY*@_|_?8nKkS5FITIf4jqV;C` zh9yhfSw_8ubQb+t7I|=fgdX#Wy12Kb;?OHo(={_d6d1(rYKiB5d{pr~mV|=hQ)T7o z;N696yWb{u*Pb*G`DtCOWrJ{&{weYN5Q(m;B8IBzfkG<&eL)V6$7(IiM=YhbRHEJ; z&~#I-l79X?>mRhynkFazIz{%bqy)shOw79Du%bxU1l5Dcg$T~k`n=ce*w{agd-D{s zvWF3y7bgprj7nd*xlGw0Gyl8Xe#jEjhXP_!u*Ndxk~2Yur2qifAzq08sC#C`+j>v@gUv$yXO~z!AyoK4Jz2H6f=>faN^$t^`Li6SXnnBpL$PM0rO3VY&PyRF zSjP2ze)zasNCYOyr^=UVePsZ&^*nbuIO-Ev_k*@iAB2goYZWMLli=hRrQW8oMI}tp#d4x>#ePs z{74a0fxuLEgve-UXeb#)Zv|Bo+S&jbP{v#VS%??)!Sy(?1|IJr974eIpHWcs{DR^W zg6{k36I&#p#SwXVp~55!ph(ui*0j;l$sa11_~cE}l;X7Lc$W#jzBmX>-MF{;9@AS1M4qRT!`ab6rvHd0fB@w~xDqSb z>F##H4_9bKvf({(M^be}dCjT!VQ_;qUqEYvzCsV@c}&}MfJH~_h{gw5%zY|=+y1*F zUL}qyx{!)r4srAFKvC?MFcnPK#I<1H&sm(S*TN zo={FMbUtC*5-;V4QU!r1O+4@un1q4<_bFDEa`&Hp%e>bk{;)V)j%F@BaV0&FXn=r_qTE!W zZGJuhD0@`G*tDkdTyWF^(iW)x45odmsX-eUSS92CPCfOVjMHver&zbH?LFEEqf(3C z}j1rxQ$$%KX`X$P7orc(BX z2d36^!>z};(1d9%rODKiU&H^3!X_o4n$EWtR=c1ZrIc zcn-JbrVMOipZ)Z?SrVLSX)suM@DSdzxkc{TMOtfeU37tQoS{BRTu)zWeIy!A_VUKm z$11D3UsYDTAjBa3|0`V;T7y`j&}Sb^6I}(^LdgA|c#1$6pQ9Ns(9mJQs3|FRDJXlP zz#zt>`q$o`#e@U`YSudA*e%Q zJkPMRKTWl?e2Qfi%nq%7)9#7%?wzWWKP#(MX{FjubxaJdiAfC`hcu)y$Y;X^oO zvLXQ&bU4~X15EMs+leG5?f1K0Uc$3;J}q-s40=B?5vXT@x|QLtZ|z_zs0e*0I8NfR zkBG%D`SgCi|Ltc(sF^cRV4@Y{qT1iEIdrRM+W*m2;OR7q?~=@rqS%4Q^eyB$q-YuA z&*B-S;Jo-0@AmCzLPGIeKg22=FFs>Lda696`eAQAX%)R$YcAu%ZO1*4p!D?K#}S7s z-I5VhtV3?bH_xs@pdL!*_W>0tlX|g^Zx^Q^5c0Xc=L;^e^b4PeS)VC> z76k!Z{}X42Lm_BzL?vG~G&*n|Ti>(Yrvy&*nMdf{GsL( z1OCoCLO&eVkP#^U_Tf@@gaQEafj|0PsC3N|?Qwvs>|Lz~vM?^D zz3q`8wDdReV6oZ+=O29+63Ry|H9ZPG8G~bEKH?0iRCa`nxdRm<@a%_8Z}bPej_?)*JL!90@30jCW_ zj0Ue38mu;Es#*Q~;wH;lL55RbTG<~AZ)9^cPp<0S`6iwUkXys_k+ub~FT(Ro3~JPT zwbTp&nb~HheIA?XYfGo|zZxYLWtWNeHgqQiqh(h41I3pD9#sS=;vh;DRz5z}*!vJv zS=xU6qQJ+Wh<-la$@hPjXPb46{t~Z?^&69wrMKYV-MU4w1Rnc9f6UCxlsMWvLKGX$ z&UBBJzwsf(FsT9DMhE25;Otj3`%topOGwzd*d!jfPn76)ObbvWV$xVaoNDoQP>Swj zz*GS@eHDx+%HIEDVfQ|TXPncPIsqZ!t1l1Qf!KHtl)3bfT3ucq4`sj7{da^5LIGT#>zYs;lJ~ZN z(tJ^Jo}EhquJBXiLKHOY!!7Vb7xlrebFo27ez<;fs};!qs2xAg^-^NI=eRu?JXOwM54S$6w(~cKwFc^0QTTkM>q`eR z#P%#TAWsoEf)Pma)I`C6Qf=M7{m*Q2C@ctf4ihplAnk-w36lWs4!eD6Y3Wm7U;-kY z!u;?-xxZoBf%KTlKh99G{=sMo9u9+VWpW+C^8HnOSCe1#bRUmu!^gYWsbUBo=-PMB z;9JjBZC20Dkr0m#4XL{@ZfwqZ2ti>7a@ZPR07mYC%4+$s>~%+AA%bm$kKfwX#@+7# zWenhu8H$V_!qiw-L2Xz2o#*LOV)z;r70UV{nW{{P9~~NET;=)=2k+n3M5^e`KS^zE zjE9-qW@g{6l5ybwH_>P(a)?R+ts11*(271)x`IX=lyWqt@?$Tj>13~^#hN(O3n%~0 ziwsJszCGOW}v>Ov5@o&V0hoPt#-bW{YC$(wo*J2dO`EW>?X)cAW zw2(*ceV%A;Zhoesa(LU|E-5ML@#+H-@o&<=bw`}dXmnf=@7K)xhD>f^I&$x8TagyK zqt;+DSuofLLGKV$c7z<9J=wdZ%5FzY$>t-mhkuuh1RTu*lC;m*kG0UJ=o07}6d?3& z(3ktFR+hqLH-B=90QHnfwv19yyHe-)%5Ax_3JOAscRSN`j`m-}R2x~u{)N0A zaIpT`K?v?&VMI=$OeYORBDk4-0`%l?Eo~-2z=@25>1m7R>vpssDPPb4C#qwPV3C= zi~+8%;F9iz)O#7mtxQg8<_bpdsXsLuix7ZT)0d+ym!&lqPUEB^E0KdKlcKl8V&Otl zuPbJwpsfThkSLxfHm2K?4J-Z3H&TIJGT%*kfs7`#zW#Qu;53PM&o?Uz7tmDLTG3Cf zYL#<8Cnp2L*@ug>N2);m$ZRa13fQQoy|GufDfq}#RE#z!T(Z2c5Hq;0aDdhOKxLT2 z_HWumd1-jlRq)@a5GFSsYntd%HJ1Qr*UBMx%V3KAhYH1_6f@J3tNpSpQO*-8QDpoN zorHeU+tbcHduo(2f&qJf<^=c3?1Fxj4ZldA1qqVbk&xG-u#uZ`Apjf1n&+dcJiOt6fz}bCGFpx z7XPppDViUkE`_4h-!t7S$Ws|Wpsz9hIQd~)Bhb_$h3jbl7d1U;xlG*+EtA;$oq=?2 z@!`*JNWM)ER-W~xiGBhS&ac8LU&km1ghB)O$>5d(93gD*<$g~oNDPQ}av;uxQk}B~ z@Nk$}z_7PXH!_=as61Slm{uK@tmkRL@g-fKLb+O2bxDq4V zWaQ=XP_~m(QdXHfX&X5I_ndsHceO-)l`{W8!|+9#h!v9B70biSLJd*ohd^9`iz__r zzNy2O`V-vy$A%ZquccmN?w`S?zoRhN4G?n*v^ZhA7VdtAeb{SjTWU0#s|T1?=u@jr zhrj8fPMeOMy_V>m;bQZCq8nFn#hEcTQUD`J>RIfH>2oNh#7X5*-RL z3Hd-QT4j|cu0^VR^4EKWx9%yw_lu1kC(BY!NREzPIdyt)Py8n>n*Q@xB4}CUDNFwM zmeq@ALmO}S$BR?KW7MnxUtONAtlPxfidr4CY2O@EnfmD_I+9ajFjM6pN;X1bxi!0q zx04hLzANFB7{JxPJN4bTJ|t~-0pD=B4GkLYIG~6!Po);|BnJ)FSE(==YT;uL)d3I0 z|6p}h@VjDJ)quok$TonOQB&m_hJdLgaMFRP<2?xPU~lkq6Hf_~c2r(LA@2Oh=PR67 zAhL7JR8zk>SjhrqmP&~p5jnT*LmzKu1lDrO4q3ObwBg>JNsrqt4;FLm7kAMeI>m~j3Y`x&(nJ# zM=*u0Q~F(*&JzyWaHcd4kM!{ZfPjhIdEMhhIM6A128b0nGe5+_`iOWJPJ4QoP=~^> z!>EKRVW7D~v%FDU)kfIs{5=Sh6y)UI)ehQBF7}QUREwBz*EsmB4WumI+)X_lSm}-m z;V?Qcby#~-Wt}qEOd*`7E zm#3=>*py`K+BRUSjBLAl;6pmD1$j$Eir^I~S2n)9eES&XG%rkh`Y8lGL)qRw(^{+f zpK<;eH->E==JjbgX-9i|n0P^FVm>*9>kG3T?a%+w$)`0mMbi0j7|Zb0*e5ikHmgN~ zw|7sd5_G&|q4NiNTa^Y+a|DwJ>g1%SJ4PL$F;J}y{V#_U%TUY}RHbdZ&%4`_`Y_*J zGx!?WOE~v6*||$VZLvY@Z~vgPwKN)X-repP=8MvD=4iv2xjKC4HxTVi^PGSC92FTE z&+mW;lpVwSBiG|zm#1VZIc(shmjV2thH70XQK8`sV_R{db&Zs@SHy21uYiq7rMXzA$orP*LSrRG63#e zpRjp=ORk!!l)1Tyw-y3z!{$Hzhp-7zh22A&p)v)Y?~CK@piMkhNf0#w$rjvWxPY&_ zKAgz|l08iz>cNZ^z>n^$=Bpw!rz+LO%nzjp77_Y;!75}$-F?k`Ro!g4{jY|t zpW8!j0BQ;+=V9V79bkO-j%8!a@*XX129R<~O$YBl+R5dzOA7UzRGunOdH29GDii~s zYH3$5%-@pt zpCVcomPkN0vsjeCG2|sQo6elxlOXD?p&?&Dm0*5?5-xR^T>U z4EkSwzT8Q82kj!fIkh$4_?Cot##gS zGh-$oUx~Uw6;jM$IgSn98JF`<%gXGaJ(&QiXO&8e8oAD7UK)_3s6KRLIySfSquMPBrmwaodDKapHUzL|Y&JWf&NpRo~ z*3!y>YtjQM(9^9FO0sZV1A}yFV8}z9;K(+w)$bri0gm^In}3t_^k+H_6+=FJ3&yd=vB`xvfw*#4J>Ocv_1`xoB8F?1Orx zxMfUYR@T_puj<;Bc`1_~IGoRqqwTzXDCD#520T)DfS4I>e=s}vgNky1e_^d*Hy-7Z z_c-g02lb`m_aSSBHis!4=O7ZU-0xtpr+qzbdO`%ft`^&7*#o0&cT!HVec5Hn^~3@;QM#4rUzqNV$6TxImop zT&|rc305v!w!|+vytCka@et|eLZ1enBNP4()4?yvqCRN8Fa`m1a9gJrgYLT*1YaL1 zn@@BK;BmY5SksC*;l^iJ@#n@RBy1>wYZon2LI&NZT%aZ}sWP7CKPc4h?Idi5oe#hu&z@TWFG}uVwYK@`! z^3I#Ju5_0t2zxw0g5f1FH1(X@Qk~k%%WK}lMhL#~cgKwqd{aMMj0yz`=dEL8e5!9F z>CI7lxHl(_8DMwz8HB1_#yvXFE9@J}@UpeHUtDhAs;N4GYo}H8hQerfwtI3~%h1wN z5~ip?c)O4mqAod!E~PYkmx{P+`I7o)QS zCvG4R%^dlhZ@9p$v5komK`E)A3X8E1Ac*{KF}4R#k=)xAFX!D$EZ%AhBw)>51m4sT zR~?M0Mwete5j4hSSjEwRe zH^ecO7y%YBpZYEWTv!GHfpExzO2KN> `+Z-YytROe5{bh`;bUB9?Eb45jjkU{% zl#np={m#Grz%hpm@fy#1o!=d)eJNWCYXkmH$VWgTgXx(!{qNDzU?=*eEtrnxDuMCx z0$N2m-&)`P8l-u0DFSJ^xujLrS0XcRMsJn@si$}-^{>4%hbI`1va_kl_pyo;jFH64 zmLrq&N2daWI59L-4jAjN%l~Mz>HX~Jq*ln1jT=|ZQ~t8P&IhZ$oEd?B(3z&-vi$#W z0jPWT5Ayw`G9eLy+wg)Fnsf@YDpTK`VI(#dDthSef|8EmA7xb+Q<)flXXh;co~u>nsBXWMZ&Nt7Ipd?>eOEU%j17X#>sh1^snTTx*Q&5m*>qu~$q^WUN zu1**@-*(Nw?FROa*>)$^fKP`fKVo4~cDX*@N&1p_{?-GBy+pv*dHh!lGw;S&uQacE zoioo|4IuPAmCnM#!c1yRkW*he%x#So@FHtAUbDl8Wz9gblEfNC8_tHK8=N|AP}<0triwq?8wwqs!9vB z43Llf+nEXM>yv}QV?D!})5L*Eyq2bPz(xAa#>tEL4zqXr>7MF?EL|oAtv2yf_4de>}{c3U5u@g`e~;!p}7jHTUY0n z+StU4Yc$fGoIj5f<964eWeEnWfxf=@PFrt5T9E3rbv=Zp)t@+)BN-9|N0=Z?s`ciY zqzgEqxr$2*gLEb)^qN{)r7nASIwGj0>YQI8!zIXx7@1QWyse|tQ*P2<4LoH(F#bZi zJ~hIJjfw^ygunXY_w#l$2Bp_*T}m=MCjZ)*m}J*lkocOJg?A6(uOjR0vH;B&UKb1`lQz14G+Is^|Gc?=Uh7QxDMG;#Y#Cs#6CB9zFw^Lw&k`LY z4Y@*rR2>A4H4br3My7)n`f+SlQEMJR#p`qGYkpQo)xnVVQz00yKkXtJL9XwK=D$3i zL8*1xY66i0&`#(0cMFEifU$IMuGVSD%44+Z_i*D)fyQ9-__)A~9ZU`T*wu9ba(kuh z&^d<*`sdz56B9Y|DHg!$3iCs)`;wXpGlb))2Nnfd!Rqyulob6~zFJ#v!PEJ9C&U!3 zKYtXwyB`D)ltjXmVC6bzHK_Fi6my-WIk4mB#?}hLhT$h zZ!NI8SQL8_{oT*`Rppj}lUerFYuY<^l#p`AoE&H1*gx?&KA@mhx19QZ2gzkm$^@q( zG%ReY!HW*Ah)kWDpuR+&FG`u{e_Z<^&l>vHtu5sG7l}G;9QMpx52VtUge9IE29YzU zuEfMOK1XqJ5C{YKZbwh}reJ@ypIX}|V7}%0oi~@@Yf~4w*4G#C!q^4|@MPj>51L_( zfgAo5#sO35>G4OLd7;ik! zmPfHpO-9`~d^9MLlX34?aTs;H5fZk9+2&}qE_;x8*MUD4GpK?90m=eC+s8y6-9txq z$dm0iN6}!^A>y%9No|IjpiHC$yi?@n(joNoXD%=l_M{SWJxA%Lr|&>QNZ|m)>WYzY)`U--U@rBYR72! zYZ|iam6KJ|Q=4jT9Bq!h?fOi`qIIuncTTj_z)!TKKKl=dNi~h27&W@q4%QH??a$8pRZnVenFL!cbkvS3_dol$5Hu^UMJ(B-sAY z9TGoDy|IITz16kf+;@Jo1$8wwBC=3>amPR{yWRogM;%K#!>>LmXBF0XoDj&yd4pOj zY%Tw27}bC9_ddp7XjPyYb=>y;!$BVQC*g$l`OD1rmS+BiIl{;VI2GLFuRh3R<%f87 zvOUlEax?iH;E{yx6u~cd&FWFW+~)6LQpbIpik6A$O30pGLzWOqIviYSY3i`?3)j<2 z8no;4;&s=6Lx4R1Gmk~Z(#77y!JJE)N^J$J!=KZ`e%{{F(%gPnaYQ=Btbx7}eqZR6 z732mokc~LbG+a4j_b2fJOWhC#4}O9>zhkxc23^bNraleo;E~EL#lj-iv(-3$fd0W+ zo?7HI9p9#&2Sq`F8lF$H+FAr19X*$d?YhwY-5s+rAL4-Vk&)}|-y?|-sfJ4pc`ZiG zBy@C+)dS$_d^cAc3S+JGJa$+|H`y+)FBU*mu|MhnW|1Gkz7Ir1qB1f0!7(vtfMFW! zOc(i9z3PIC{6&AF)fvo2>`S{LgPacCpf8>lSV+Moje0UE0zB{($N*rZs~0(gZ$m`m z^)zB0`S`dxi`G9h_|6&XE{2<tn@tivg{Hh{61f^r82;H18rMOZ`6vWrZq&dY zf{6-E0Q%h_caJMAeE}C46mt?VK@&g|kQ?Jni9Ed0INF@H=7H-t0kVzH@uzPKnr_@v zG5%7yRm|__tKIMF>FL>rjAscC1vizG3T1TIH$x^18~rw93mR*WFK}oNzIe*2w^+?i zo?GO|<1lS)kTL0g8rYdxIz116J5xm$t$ez{y1z^%^uqM-;>-BN#8QLiZ=U8Eig(Sp z$=|=P0yl*S7=zHV(*2N8zrl8zoBq+GH-?6(tYR?K<{=lC>_nX~Jw2Ot!OzpvTZM%z zX?z;?_Sj&=IMq?>NG9I?NgmEwH=$7k+{C`4qZJgL2M_4!@qm4m!)8W7NEn=ufYs5V zAo4LG2{x46t5;8+JfVRJ(<>|YfDZE#Fth3D$jM39ckc>QF*CrU?}dy^$MiJz(6vK@ zk0`pf_7=nSOjlpuBi5Z~;a3NNjQNp~%HRbl)ERC*u?SP0U{{}mlA;r0lb)XSXt6Gl zgv5q3Yk(^c9bMMnwX}<=3Ud+?Hl0j!6&{`l>`xbl-FK5JUXGA7FZiU{? za56Gmpb)6?J&-){8;y_WgWcV^J68bHYLH!a0}clSsmnc1Xp!b!Gg-m&@{OhCt2j0Z zO#H@9=ma4_l}{^r`Q)K?-LoNYw)J1D!p?gl0a;mS_3mdN7FAVcB5J&@O`w(N08L%j z>3-|_R5_PT1Aj8i(NihXqJy5;0PLj?5u)us>_$L%3T_X3-&1a)6tW3Pe_;AHUY{?0 ztSkb967eFfogbcV48_IMz7~D)=AKta%197i%H0m--A{H~VD5R}V)M;ITH25LJ}DSt zH`himDq-P=oe?$Y>-%9VxvG8p_PorfvrR%oWQJVW144&ux<7W}DyKE(Hxyh|Te%{2 z{ShYvXJm86AUs!=@3oR`z zavS!aQsU!Xp>qO8Fu|^S_0{VWgVmEIgMS%^5_#FhHk1Ph&{*Kf78lF$QH#rUz}170 z&}eh~igyv5wm*4xGY7cR&HbI^Ujv4tG^Re zJ9|qfO)~(t20HhrXJFimYV53r936@th*>s}x_zCw&ShQvjUOo~DXEeljcaa>4;rIE z&n^}^IwDk5uiLogJ~@ye%iG(_RAEfRtXn61jE=T8Fy8x^s<#^kplIaq$Ve;r*VOFs zV;-i+vp@9(@Zcr3y18aFpLfCAgoIo>X6H+zSk@8!|2~LLniG#T{J? z_L6Bh!Uv=5P5}g8e}JFPo6+2cP@v>pT}VvA?zd?aP6Xt0Bukcgy4vP_O-)}Lv80?7 z{iTDPASxvA1op(qJ>qrQPb|M4yWD5aD(H)vsct0|aC`s|brxi+Q!Oksou!+@tQ-L~yRgG?MvfZ~U+FgmMg?ys-z??`+Z9=4<4w@)<>8OP|wRxs?|1V3e9&l{BGNCEM zWxMzs4#|fS8_TR-x7AtL^FWd(Aa;)6FhYm<eD;E^ zJeEqlY=2iht*}^-yC`r&ee%@(*bR0HX}4wXGbCIVPF|;{sbPx3B8M1NeGI0+MXqcFSM7%nNvtK=gZ<6cCU&0b5 zB>`{|44k+FgC^0K+ew23svXh>Z(*AcKCna zgnhNy-_dzjs4x3f9h5D%Q&V~N_sw5I&jB1#)N6(InYw!3KV+zcrKz_)eWa-^o;dGTemysbQ}H=i2rnFQnmdijwW5} zVuRBJ*{M@`HOMBu{P^)4(y09d`_`E{y!rXcvTF7$+c9{MON3E@&<{a>bf<@pRww|2 zeslVF5Dd8Q!9fMV>;H`pFn)?uF<5)HBcM5R+}v)dIE)2Xwed*Cj-`Qlu_JsVrP*Xu zWncOx>Y8mr6jzj9>IQ2)zAP-I?=PYJ1`xdkBqg8mE!#oDZ^;jHlnc~fb^aG)WGE`9 z!_w2;SoD4dJ@cnO=(t}*%9EdVSq@+37n25?!fMZ|+g1@CWH;dRm}jX(O8O(Tt?g}c z3Z)|}c744iM1#s-#yGTXT(I0w;=w;6XK$a(R9&@pgb4yui3j(7E#j(7CWdNOdh~AnnA`SWGU}!ZUk%!Sg&r%L! zEj$lk?0$<|bhkLSD5KJPP9F;Ng+e`Hl?~-Khsp9aMmP{}_m>~Lx{fN=Cg0tI&z1w= zw6xR%<|ur?qtsKWeFWL8-b$H_o7fX8F zkn7@>O!Ed${A|iK^UQ|c&(}Ya<3lJzO{z_@Xq5J3Hn{I@v24}W#c^?S-ySpLR+A7H zFJ6(D691s{B3-S8s8Y=Gc$>P$gEz9Ni39G1)FpsmfuQoG%m`z7Yes9szDs94^Sd!AAxM0jo-epgbC7D*i>)H2?5sphdvz&bF-i z3`Ug=dP()rH4Csj67(Jn9U!ac(R8ZglkXgBrpICr6pU?ZhDjajy?D`;=*g&!`2}Y~ zh2~Q<;66G!$VafHb1g z4Wb|*Axf8&bazQBjfB!I-Q9w8qkwdGck?|*fA78T9pgLyFl699d#}CLlk=GqD)MDe z`3-4kNCPd>u%oH1mry5zvdP;jUc`$!Jv~JfPhb$2O;hkK1hUX$3uB&EE;qeRM)noT zt}lm^3)E}BL1ICtRz)B@zg#-Z>#*@uW_Ffvq{@a0o--UuE_-18D=imDkbuY(vL`Y~ ztrN{lXDj1sb8z#^HU@o^$jJKM;hLQ_?2=8dV9-0zfL+4s7iy3|eX3wSQ}0RdH`ss+ z9w~0v;EAsSeZ)T-@#3$I@bMWe$y8o>+$_&)9}<)>^#mUF_;}7$BI>>+Fo3lGuXtpl z%J*eBKeTYF2GXwI^Gb%wG>l2o{Fm^jrtwYhuXG|9lVE2SIUXoJA;*Oda^slFSTYbO ztwu+aG9aGvy^e_qlujw2&3_+gMiBGPs&&rt9F%vQBi-Fkz!JSJkO<9Qem7D^bvc@1 za^Id7B}<{Nr-A$suf!B*@v+mpYxlk+f26}zMI{dtNj!LcZB%eA!*|WVfmqo?_zR>l z{?rd)Z|dt~rgaj?uo(`hf~qDAD_}vz6rL+yDA)ErodG3yT`izvK0Yl2ZTm9`BPJ+z zwvCJ|&vY?zfCkwhr^W1#rl%D>+c2}8BA5AHt(qRv345pEB&dp!wktmr_M-+llMYTU zz7wtwdy5lZzEe7v=Pw_tXc&@x-R@=Ez{xQ=yF1Sq9V6WNoYj@n+7z9*wabXh!sG|xf|5jJGc!joWO_pcv#+U?7~n$bSnqe%3ZWm-{r~C* zJ#i{P{&@%L2dt*!cfn&MTN{GyoQKtgfpj(UXF1pib8zq85~uAt&e@S~f$<+l99&!| zg8D>7*Y`V~~f`)DaICpRO z(K_Wu8WeOW5k=!gAQt)ctG{V2xx1Iw17YDF@U4bGga^Fl_4KMq1V5u7>;q6uy73?J z`t`pEo6b?$GGjxBJ|=wC-(|l($ce2ymZ&k2%x^la*Bnzf;b~k;n%2bM`1T{&urG-W z2*9ge0qU!FiFPk1*LCQ2`D>G{;_i_NHC| zd-m$vlc=4!DHv_>8CdX(N|eh)h^s(-;{mJrqlv1Moy#*mha)vALy`M%d7$NpUF@zk zoE*RHJ~D!qr(Wy*JjGl~m`LPktJQK#{RtF2q4H|AGiw8w4u8Lj{4sH17TKH!G(ubW zN&e(MhaQJgNv2Y1*)Z$2GaH2ZRnSGV7aTu9h6j&7b^4Ic4@Nn^Fi|+w@4b9{ zIxz>R>FDV(Ae`A>?#BZ=2|Irmn6VxXrn`umZ!4W&EWxa+DGJK->BYsYzndrr0F23s z)sNJC2=a-b41aM}2zMoLZj*qh42zC00hR*sW3rYMjtA0)n{TkM8Xn#uw0{uV^K;XQ zxOF(V3n}?@Xp4FTRw%EezT`m*R+Rs>P!&cHGH(05KF4KQztY3PVPQWM>aA6=&>Q6B zG#-jK!oPj<0o#+MnA;{!bs%4LM;7T{skQKZyD})6x6=IqN3f#xrre2R&QiB4!-&=)l@+)PFbvztq9t8m!M>*}KyCSg90 zfT~jQhu_v+UGK}JUxF52^Cw~rLZXtmO!-?yt`}2(y=K4qzvZ~64p=}oVur{R?Vsbr ztDw^O0D^Hpzcc)?+aE;o0bBvp2yE2Un*gB_dHy_oHfgv}M}K=aOHFg>7GMjTZ{i`_ zl7CP9sKNy+G(>K$h>of8I{SA!SWNYXp&v~0eqlnevmQX7OUlvF2TfV5`EUbMsgT;w z8ygiS`LnILl0$4ZMtwjn2?lK+SJ~|8EceCN>;>V27yNscg|CHffy=NkFKB9PWFB?_ z_W+q1Y_K&wThM*6*ifeP#f$4J<%Eh5LqXr#=EMshyf|U-6b?{0C@6X2_H~Q|aj#ad z8>MhXlhnl5`~@Y^%bybX+HT~$o~{6DO4$APu)NTqjt{T3wXY8e6Z7l!*2nBDDa+34$ooIO4 zhkgEh6U0W^F{x_0p=wITWl0ZdaRjp@;^MOr^UDUoK$rsh3V=b0gE_;^n`cix%!5F8 zcY;ywAW-rI-8u0+SaSnymJ4uQynE3nd#FIWFpmBEssbcK@Tevemga;-V1DR;>N+03 zsKIie{$`ZM5fkF_7NmWcyq0A6$Q_1>uOL_?yUWMG&{^G=1&27)ELSWPGO(im4lr;K zqjhzqM#S1z<*AKrpKa|G78NxN+o1_u9`it~D-netU%s{@fsMzHE%-Z&7$6m-)wa2y zo_zg@(c(duj=Y@Q^5BvUWNTl5_GV^F;1h6rB{+u zDjSAM+YQSy7m(ZO20&V|iAwdMIH;W-4h*}^$C&Z!Ox02Jjk|NTef#F&ZW#gGa!6=y|@~gh9fL)6kxo0}fC7M@vf> zOC2eUYyWj5hJ(!i%aOoa$bUeUKmws$2J}}Q_zxRA)PU6-qCjazrd$y9lF5`y%Pl>6 z{y$g%?8J^(5Ya%!$2VhSWPB+lrQ06H<$R)BYJ7Hnx`y>@)#r)P(DQPWkpc?{QzMtw z9zJMJ$}O`Q8L4kQ*meoE_=BlY1>WQ!Yk^0hV=ZMP3R~#3LI6p&_H=e)l10_(U z&IXU02+s~p<;L+E`&tHK#1W)hb#_*++b)Dm?2oG}kzQIx2521M0~dwx#^@-s1iL=Q z^!mCmD0w0S&1IVNUblX9j0#N6aDX&0Ox7b?ExE$*`y<$BtsoI*@asEQd^|36`e_FS z2Umu2@h7T}X6D;9VKfK_8(R_@J;dO(u^RSQZn8anI=7Y|RpfF;!mP^K1XH*$3yVML zcBNxFRo5QLRNtQ{Bo89aIG~vbr6BqD?^BE2uB3T+YaEUyj$qlzL3{=hPkPXJ2!^s3 zSZ8~Or?VGF0zC)4z$t(?<-*|!!WnbR%cA9`E>O8`7L@{r~Si? z%M+5<>)f?>P-ysd&k;RTQu%4(>#`^>JB5-e`ubR*P^-TDQewf3hrDm_I0dNGI!t=a6Q|gJ>4h!QDKo_u{}*Wlz-)q zFAytucBls7KS^ll3}`kaDHWFiFF3Zl-IR;Q(V5rv!NQUf7-@ZBu+j?ObfQW+ zoF>=uVy~^OP4kCWC=5!0ptR~}$x7GxiHB;fqF^i4#IwTfgPrkh6z5FT-ZK0q^T9I#! zP$Tgoco<*nqC~=Y{8hPh{A4xzI;_n%FR$Yz?lP?=ybdWMONIoT$pB4?&p+P)mR})X~$E zUKu!q$|WiTgHdd3Y+-rtB;iDVzdxuI&dtn7jg`f5Iok^$Roh*O&9yKY^i4mlvfc2` z%p?Q-f}VqTAe#Ic$5ReP+y|4BfeuS?_Yi9?;(N$H>;l>ebPfvUlzy~PHYI%+%FN^i z(2eBD>^CTaPke1(8UeM;gFeDf;ey%bhEK>_f)k{mq~+m}(mxn}h3j|zyVU5m>y-=t z&`=~ax- z4Cx7&WjMTf_Oqo$Q&-osv=kQ!q<;qj)oSb`<llo#~ni!2wO|8oWF#rOoX4>Nz8qM>)|tPGUyZ=4B0*|2ka z#)Xw5iuDa*?WtK%P!{F$V)@|3x9y!a*= zW94Q^aIjndPvdXe^b2DW{n#t&Kxt(q$MQM9QN+LQ&G``E)XP7ii#wV zBteW7(P=B`Uy}9=Pc$YwZs=K26 zh^%6yU=rX{+5@Rv)@#qgWcbO1vlV5Wz9aZQ350o>p(pn9vt2Q8S7*Jk0tO)ICuW~(b``w zG0~);M)G=gF5K_4B*0?;mnHh+$0mre9{7%x=m26$96-AcuaJcfq_9(MkYU9Iro|K$j8^$Lh%(OZr%M- zYN4~19mmb|Jtsr|d)omZ6%EF0S#Lko(sO#+kP)U|=8i$Ldg( zqpF-7_2MFl%(|J8$|ndEUrBYdvbNRhYYz?#w6(qHwUvT~hqLn?Ma3bmq39SGAq@*7 zq*EKEbO-&R8HhHWuPz(~oy=NVXt;)`sHmV5?od{Et9Hv|YASJfWL)K$UOv}G;WPuH z+71m!D$~!yfj@sfm=b$%<6mWlZOapXYZR`L9CL$&$4+5y*-!jZ@hlG@r&U_oocNm6pb!8c*6!klY zq~sg)AK5`H4oaUN!KU_Qzio9^`YeRlz{wTxUx0%98mPO2>js(@B7klI?`+yx#NgRO z9UYPV{dku1!qRRR#a_3joYtfAk`l+BYI|4k{DAV%A4`|Zv|dwL_zCf>Ql;gM4>lqJ zfl{32nj9CWE{V5$S0*VTkapEp5t=*#cC;3#el0nqzX55z!or=ZL$(0C2$H%Le{n7k0j8fd8Tm2B0s`vXV zXi&t72nHGHWZsoTZSBuXDGgpX`f&Z;t_Pa3uq+sRP97e5wH5}W%ku=#m*NGGU~|_i z5^@r;;Qs<+W;Mxr*MxyuCza;`%buB;{{PiOc*`MAHl9&<_b!#W0t-KQ7D`3nLs_3b zZE2jDD(L9WrH2|>b7LbOg5a8B1M9rD(oxc*uQ-GTrDsq#+wo7KYnUTK9>1Q|KG zT)w&s(BtP@KN5=sS|Cf>or((`KL68T8 zj#el5bC9tnH8eDY(U!T{S&7mQPxn&JRdx0BEccfMfbu^uGt)U?pUT(6ft>Y?kObab z6~VtSj@~fpfhd0%L~yH6Ac&3A-N^ZfsCO#njyC`BM&{(?txl*TkobR%KO$0@ZN@hk zRmlBs%RoYk5QWG+m~unSo}X8Mk_i>Yo6&|W$fWEYt7fJ(Gq4MCaxBeUju_R|OKpma zYHB_#%~4>5@&JfIBC0IOdZniHV^>?7GxfY9@3qgtGV?uge#T5U=jLv{eDw;W0gy9M zNL?tmZ?E0a|E#3^ZOL%s>Al^ikLVuIF>Hj;dt`)keZ2$f%S*p<&t?B)RoQvBouDI<>A#h-_x{;J!;6!(xq_0_s;?p@W?0W#Qd|u6 zXZ2gJpPH^`VrRi0ORWq%L%-BLUF@DBUOw?|V*zXs0??c(CzB_8_ylrr886ia&|BZ=Ts#?poTlT8gAJ9s+lV47Dd3JP(UnV|XfCJayP>h7t@4gDd)dDvG`KD{ zQR5H=1WA`sDQ#-t+NfNQAM!cXNkd)Z3$wT%@7{rl)Mt`|uUvahAcNixL74s7#;`6f zCnBVn2LR%tn=1PJt+V?I#8sGs>aX&RowQfdSN0I7`XsU zb3tio$Djpdxm8GvPAbi!z#LcUCJ+fs_WDL~q_?782n!R!dRC%oj{eb9R9uuXE-x*O zEc43`f%Ym1i+X3TfLTAfijSiMewnd}B%9;@`$@-aTpnBF#^Lf?B0l@j&U*AKZv(lZ z>nk!RPc;bhxq;@WitYM{1jq?6DVLF8y_3G7kjr!}7kzF*9r2@v#Y4s^+ zJrX*d6+zF*J^L+7f0>E?q{9+lPOgb11vDqMM+(%DV3Z+ve;*SO^MD@tO`?5v*upUB zMrEKb4ZBGjBv)OADIQEYosm$ct^by$*Ujf=8pqpCCxMCl>lL)Y(gH9oNf=#8grWWcZ3e65{4MC|T^A5A>}UAIf@rZzUxfVEVxMb? zgOm*9g%d6p)e)ZIp`^97YLL-eKCJxYlsQ&*b_dSS64lS)b?{MAKD8;%Mp(zkj}=Z8 zy$D!mZJL1broR@g5K5oOrP5pjh%9L9OA1l+;YHz(=Il*|+)k$RT-JA3wpQ za$66EU;$C|0D}pch9!m3R2{a=#Ar5&8NRV5*>0&B@h{lY5@nZ$QeiX@rD=D&A>KvLo^OA9`pY z%?N=#cHB2;g<>Y#(}=RW;G>(fH=#NH;|FDeRD_G(8#q?bt{>~lg1~UFAj#KM4{<<9 zNOpCL2&nCzJsW*mvAeqSM;)3ARho?a{QlK;_W@ro+aiAkbFmN6$uG) z80k(Oxb32`9lO(8I>obv1LHlRu+?Ul-^Bxc`{NJ4npKaecn&4ObfGElfF!HUxegP<;Y!#&pjJ4ywJs8Vd z+}7Kma+43`hlYmfM;bi(?|;0V64KjcUvpE!xW4m*(=JnyKjDx-Gs8oIz9PQK{#v5z zdZ`(|hK3)ZBxEenfbg=gE{PhVIGGAzVQ_ow*T7QUR1&m^FU2*q1>d&lkilg*$s zrVm9x$n-+P!n!;$D=`zZ-(LN^yx@rz5FogJe~xz;18Cag*M?b^9M&y@hknDbb~EfZ zBuu>ZbgLywFcAc1TP-s9#Yh1-ntPAwyz=u0W`D32nPm+K8WFFz1zTlJmtiFLl~Hst z^^xz7to0)C-DMESV-bBx^WO&R>B--u4)r!%8>&6uAEZH~z>SU4&`qEtfQbP_Z3cmT zIX_lvP`l_?`V?#K)#38|r}NoAOe_Tj=*ZytFd2Ex=8x?c2x=WR2} zFP&}nPpQ(ms@}Q}i2x+OarIY|UE`I5FiUWjp`OAzG8zdLIE$_XskVvWW2FaZ}&fHY_ETaAo zU=_M@RkXgauxwyx41vEsUN1&RbfNzCK(EP80~6xQC}ySzK*7R(NPa>0R>?a_mOCWR z`jHIizTO40pzhik!EJ>tc&Bz%%%d0izUm9Rv>;`;)uU- ze-hz;k5B93)}CTP?+XKPw;w?y9?0o*f$nN%c2=2gxaYIE*;(`eZAvv#%I;2yuRDho z8PD})RGW{Uo+7_K*wBssEr-lLM1(Cj$;qXGp*w;(q+4F&N9^+^Rjg z0sML(0&%*$rhj!>_Te`<;!mNB8$PUV6af$I2?b|j2F1m#dXBky1&`%rIz(#E0Ew)( zVr#M2Q>y?A6-U|Jzv*_yASFh+VZ-Z#gbWDugAC{Rq$Z;{z+9nKgx7-&U8f|C`W;AYzExVauBCrP2%;huWPtzDo*SCn zIm*O-5)U=_AOnf|^NPMMYq~B_)4vM1A{)tuYQme*;c{9fAKm~_liTkaq>(a!^KvUD*qp|*s&n)C1 z$A~Bu3IjfPF1K}bbZTm2%s0m$858lpvxPM)njlMXM-_!;lz5hsX@Oo zfC3%IL%9_c(C1cG$TIWux2~&pWf)=0Qn%wUCHeiGJ2qFS3ea9^_Q!TU{DXq%)h;UT zPci=Yyb?iRT(_I>Z-Xp4(&FO4OSlcu&|*B3#0U*MWMZpt2WcPQPh(A=n9CPHBlvvF zRoJg46q2!7C_}`?J?!iF49E+R{eB5OLEzAVxzGQuEcy(ZnzJ*qo_G}RyPe?K#0F>* zYcT?L=ibv-pf#gPS^jzG<%XKus)2n^Jp{tP8;~dzxhhghMYF~{59B4~+NDU7{8wle z_A`e0?@my8g+`v&-(?}F{By80rTM?E2eS>Jf#G~(hhODYwcs|0APS0Bn7pRK+|F89eJ=r`yj;I(uQu{=g z4kM&Lvfk7SSk(h$;jzi+N8>SIRq)K;I$NJ`N=@u=GWGEIOvm+9w#kceE~!#Ez@L zV=>!|dskXRLt_*VYlraf4Ud+-Mp6=4<9e^FyVFjqv9}mQ@m)US+rQg=s4T?KbbcJZ z!g7{+(iVzb$-i*1lHp-Ib=iv@XE*aB;drO@fWoAF<8eF5U)De14aAvfnwEh!>(X zo@wVY?KmVAY5!~uAh(gp>}6SX_;OH1EnY~#+3sV?mwkmSD_DKbgR_q0x;R~%Q#wVa zqtHkA$dfe#^~FK4R+Cw*xv$zo@c&QttdN8(z7bY{j|Pg|f9XPYNUiFO7}y z^?KtQpeJ;_p_>Cg+Kj*x<3N9JacZT1 zj%Ks&>^TG_(AY7vTuD>=`oxf^Sg*Fp>w11zXraG)Ewd)j+N)Z1Pa%Ulrz`#Ndp`Rkx@KVB+GY|>9I6bNl*qQSM zX{aZN7n)a{x0Dv8R|o`{o0Ur>&jUy+GgANj4;CPRE#3IalBz`CDdY{zV_D|T7LW)N z)eejm-(Mc7#a%Dirb@iU#e^(kQTt}TgyI#`e2B=JLG8~OqYwTs9(_-INX&d+r+5^X zq$%r-0C37Uo$n>c+$34F!~ z*svwpT+hwZrQ@1V@dFUc*I&7(#Jkg+I&Spq`;B(=Te00R=H;;0?LF?i$NJ3d(&unP zhYU{HluJ}gj?ztozC_CYep%j=`3qI24L&eupGY5TD9-THrJTYMt6lG^2FIaGC%Cr3D4(xM z%}%X1x17HjaVOJ_!E)uMCZ?4OSMTawQ9s|J1uP(yF>s)%n3DBO*iF5FBN6tcB(|V{ z6~?$F;1B?-)t4X&z9%1kx!1X{^zh+pg7kA2=xVd;1<*2fbxU#JHqH4jl#S)hG-4(! zMrpX}_9v480w)U8{Zhn(d(d}>Gvz;TZ?ncbEDC)X3eal{Y2C5cuH0#1fcTxZy`6!S z%ko;3`e}!v+PXe-BS>+J(8I{R|6bK(l1+*FqxTw*`-mHqW7aJzP4WBr#Iha| zm!$sCeQ!J<+$bK(zTrykiK!t1PY@e)pPE{3^JC*(#8C<4AbAZV4Vx8n%0Bq@DQ&W& z!Mu+{^XpE1t}gAEk(`b;?=$Do&b8ua$Ruof2W9b}%(sZ&i7?JxfTIK*H|uA{Bh9)% z4ls}^`^3!E(nGdQEHFMOAYi(K0TZ<^>y6zv4l|glGP`x`_?i4`9XRo(=ii{`l_T1_ z5VA9u@z|hG1Oy*q#?0z{ZrsPXNus#@_G-#?ti$Cf@FCmI6zEJm))b_Qciu%#PX15; zb6`Gr6iwrmpJ5qGJlR{B@lKA^MYGa@P#IxN+26Mzs8Xg@IOSemOkEw!$6D3IS_MV@ z&K}+wHQFN;aOF&SUi;$eGHPeNI2Z{uUG<=)kJUi@R<#T97%OsAOv+E@a?1&U+$h84 ztn7)QB{d}dqGyLDR$KjrDH#ne5Kl_awa~(x&Cm5eTVe5akudet7|GxzB}2k@#tSxj0h36_!<4sXr&j~5vDlpsW&5E5cehBW1N z^$2L{fzdU^V($6+XsPe=GC9nVetsxfH#6hAH9Pyc7Zd1i2a){z4I|H<&hu@B#F$ov zxn3}%Lw+_nx7QnGITz^imFGxAugm@$42y$d@!x(-UE1CZ=8I%squ&C4vm&BH-d%ML=sZE__E|P%bLMm{AE{yMD&B7L zgwAegLsTbV zl2^L+s-SPAA^%>5I^k=Dgs%6~$nKh~(Xhc+VKn&3&e_GnV5jOlX~xrk8-4MMe2ME& zD6HxlBF4z8-h7;q0&ZHtuU9~$qqtXkDKo=ouleTDVvjWU6A7o8A8)x%=EEya7Pa0P zRql3B0zHu0rm`$HiE~GlO}%;h!#FaYLnto$lNg7*8p(_Zl2TDmD6RrlSCvZO@5fN`@mn;DQ z$z*?ZVF*;1;jNTT@k5=>`hZ)$Ih9>5Fcoom6>0jFGZafpe8bLaT3zrV(I z^Gy*lUV9`693n4oGg7z4`rp(asIa)7S67uWD4Y5ThW%YO*VC9(E)(^8e+<{0Za29N zCVE7jZBeyC{(^iASS?ALUU_Ll9DpEI8%%I#*cWLw(yosbhC`@@&*x;zmlKV?y_R2( zyxq;=+7fhusndGtnO`rR4rX|L$_**1^XL8G@t?r&4@45`%j@B0UuG-6Vn$58Ot?lxjxrQ2>Oqhm>h$ZJ;3hsF2@ zZZ!H*G1KtyGyo$OR;}yWPzEaSwws|JR;n^HS4^*uZ;8m_@fDj+a7R3nbgjQABW`z zzOLQ*PAhVO^@`p8hLgo6-8e&DMf6gov)eZd_sScmCZb}5S4&5Jt%Y41pkdH zcH@axJYV%=%XtETWO+g0K=DlZI7B2RnlGB4l zim#Iyy<#|bM)2FnWaI3<*t|^WyjSa}_M+TWe{bCC{z`ueJ4>yCd6Gkd89Ixa=u8U812aSjBuebs&1kE?_onx9HGh5oc#}RzfM0*eI)Q)Uxjj@cyN_$1gbyr;`nK=dS^bi}`wgElb0Kf97aH9zUpN_ltys(twi8cm2e1sD7Oo^2XB^7 z9Fh*JT0I(8mES*l(b4UkXy=y~fwhjxy*Q<%iCy6DX{4`Y2W`JpzdhW1G-NM zzka;|Kr5l$q{D=)_?x9DJB3}KtmxueIzQ0wgw6rHGWI{mb$%kVI9qlO#Y%`RB&Yv@~T#~ zN6VO;R9G^rAeVz-wVV0%b?!E6u3C#-4sZtpTCJeH93HyEFqmH39%IJbMG+0-X;kSl z37xmi81LSndmG6liEOF&2kBS7Mid{=?|{2TI-c>3O;7^dRcd;k_Mf6J?6#n&q!5KiU#4!rmidN z&Dy)tRdz&jM*McX9drq>RrSFyJD-_+Fuh$5mhUco7Dh&#Efzd3WL^2jKw2oO0n~8$ zR-2XjK&g2RjCLC&!LOGs0GbG%85>L*@Gf=z@YVq4!rNTtcbDvvq+(q4e`ishK=KF? z7G|5t=WE*&c7{Ou5#&o1-v)7l`9cIApl^{&jfS-k*H6&vCZRfddQknvaB%WH=SEKf)(DSA(m5>U03t93jOJzcirwCcmERcbl(rFD+8@KAkl@9 z0WL>JL>3K?EYlh3n}5Y_I^G#;+2Jtmu=E0wI=jt)?>t*Ic)=OC9Fz6}OR`PSodcZ{uZm=ZUoT_Kw7n#LF8>SU>)+`RB%extJV(@l@>sPR0>S(UNZL7cay?| zJEzH5ndVTAT{QoNNnW)n9c`>R36I?-{kDx%JnuJVHEzq_Z=gT)6P@>H90w>8WaI`c zA*2ZfrxDf?CoRsS`i;!*O8NI+l|Gkt4pdlZngH(VZ4RIvHXP}*u9OUO%)dS zz*$R+coa~&K9x-<>|E$z+AoC&5@Bc|S7&wHm%QV4PIi7&KSjcsp;0fn@~d4WoJJpb zZ&=`2ZmX1?H%4hNcXkh_cZ|6;dO#vJyx`NhG}K+%FXVu>KfH*aPXcck>qpYP zZa@;#^6(D@1km;#bC-_l5%q)J(9a+PP%Q;d3%lifoOEkuPbA8j$7?3S7*RGX<)P8{ zIssV9)M*Olx(1<^U!d*)WL!6I+}O1KBC+I&U-tk)zDbuOv42ro1q9JOsV|Ew7r&{O z7~X8Earit^&5?1}+ThFKT7EC4<#R*0=@Ud&+m}=hfQSc>aT0PmD;$vNH{Sq<*4*qi zm_3Z%+&s^??rNa0o8V^JtLAaxx%302ZI?T;i~WrEFwosGf%EC2_)7U{31$0Wm>=P%fx)-+Y|E|Z z;s_rFhFtV&_Ig_Px7^4mcpMEz_a7hvA2`?#z?Q&4C2sPWcs`g;+n*`#1lm|%$)8gD z`*TG=I-uAJ!U^j<2Q`|lrOvsMfnqG+jEZ{wjQ8;z$iS0kA4jJ^5fwli*VJB|EQb zzF-`DvMz8|u1{14L(&Q%uG;cJus|I$@Ke4OGCbYisZ6-V;u`L9b_f*?8s77KySotd zbG=pn;|XDhY`JM1$`X_6@v4>*gu6-Oz2%dRPa&$Z+AIc93{0yf=2=)Y-t|~m?cJ4( z{9?FGfLnR8`S?;+-HhrKgIXHUBKCAdBtS2knfwXe2!&pgPrkq3wG1K8Q(qS|ME(pU zD8PtBf<#6pX?66}*!~XagKJIIUA!~0s<#siRr=&e>5xB`+V0P&ey(S#@}p+B`>Yme z`|K9+u`)d(q6E^W7jOqHH)YPHB&9gp1RI2@xuB+V>!TzyD#oos7P1P|y9C!BRIBEt ztLK;J^)4}}f5yDgo<^oD2LwUStU4aia#5rV0LLxET&s-!ctsK@wl99|0T1}52SRpM zNSqKy5MpwKmjVtzfmu2ecNMb@8DnGYzX7U=SYQ@^>UY_8yNRdn@I-y}R}&y`{u$EQ z?H-=1yNv7kY?@-V)@?kFr}N>r*`1Fm#$Ar)o*3Ek1Fw$i^l|d@^rc!~w)9`Ux{$kq zAe!EE111@3A{4@=kQ-3($6EjmGyg|F14asdLd-c*YQ(V=H%ygZ&H;R!N}I{sz5Hja zG0!ffd*jQ^4}QDpFZE3L*8U<4T?LJZN6jtq()sEWu|d=kue5&VhybO8(>peXD+lij zWB-5v-W+pT*iI*;q7Xs$~=5yIRDe9EQ^t&JWi!gd%VdhDOk0pZ;I@;TurMt*vOx92kFO? zrB7wk`+5tSVJCfOyu|#jlT^@e-Y$3_I}ng%gC2}VCbGqqqVw0D;3R%iso4HO#ZtY# zFsPdM=_}h)+jWJUQr=SoSyYlNDkG-y@!e|{7EBcY68c*{nf(rCK|Wity9rAinhWU9 z4VPRBOQE(QtnrNaS=OodEj=LSi@hHt!EZi|1l6R-6?BpYh&UkI@Ma!zeW7)K5L-TM zsYtg|5e#e_Fn54t7X3Cg6O@J__MtokNO2hj`tx2YxnFz4p{om>EcIF=1=qV`j!?j5 zT5ty`AVbZQ4k`_;Yk^kW5_!;S-J9|Ke6mrV8|$=52IPk`UAF+EDA>e-0|CIk?Uw~0#_Ql${3OU-U%M!IOoM7E@K{RxX#lD zj^XY{F8ovSy0sZEY*HdU{~egaKi8#OzbokVx(2INA8sGjYiwB)fk1le?>x2ahO%?9 z$n%tv&yF*esj@ZVN$d{ONpNb;sf6G&qD_ z6WLhk+0gEFFwAvWyF8*}V2FW56Nl&orY+vEWfQU$4FAx!9L0;QO#G>_Wof136g1bZbH*b2-+hB?>0bv3p7*&92O#I{eqYs7wb4_ZY)zI8&S|35Yw35 zkVJ>zUk-x*E=2JEXBU$;A?G8BAWQ<_^B*&oPfZ3#M~9VOBe-!RbfEHhb`RU65bAo~ zD)p}Strj0Z5;%Q7F9v?&TRgP8-P8*?Fos?iY4+1WhI~7BTM}~LY2{A$LlP__;?Vj; zn{UHwOjS}J{fH#cnKD_)$YPKTCTlXAUefzUsh`q)oD0O%_taA1Xqh_i)a!a**<{8W-x&QHc1@=*c+$2&>)2ZKI#% zyupQnC{4bH)f4h`6cznm8sA$E`gV1i=f`JwD&uORh)-(Ki&v@;vkR~}AKyhp2nvrN znvePUISQyL-%0ZpCJkE5FAERY_RSMz%F(56BzjP}*9wX!^hSwE-;jdUBSfs;tLH6x zrMJ;+w?&?r(_rdUy zCH(S51e@C>2APQu<%H=CUV68;jVNsB z8@QbBxNV;dknx?G{*8`c)rmeVbWqtJO~}A6%_=5LGv;7Tu2VU0KkVgY_k^^V@U~Oc zphYSg&O1vzLvL*&al*b|!dindR-fNND3(|{yoUQThc)zr$bYYxYvD_HY|u<}u1fjN zFZq0ENK65veB_|T-{ld-g7Rem`O~kHwbh{^eH|;D;M9K1{+xoCQ~-jb8}_v(aY6Uf z!msU@y-xqRiF3Y|9Ja4?_L*c?Ox_6Kk^5pCD^uN zxhbVsBiOMEdnZK!Y*V*&e%ZcGMMESuJDn^sNLd@?%4T&Z_n_Jn;RJ*;f18Wok-bL& zo_clM8Z+*+B?YQNP6J-4<}=twp1?Rgqw!18f^)zw^+*!w)~^3R&jB7Dcu@M@OYb{ z;LfsI%i21cA_@iepE#V7GJ*}u$~yLz3TPRPMJ>2e|MrI>*}Z4R$GM~3eG-x?5ZzPl z?GGN!S*{2?{rg;*v3fnuQ5R?U=EG{k+!SJ>sd8fwSAi;gs!Dkf?|z>NeAjMah^4AH z_6yV^_e1^!v;0!q+|C8LF`-cH0TtU!0tB0Ix|EDL_jB(*wN5PfeEWBfChV+APpe?k zA56*z7)0bqmvQsGao;rMdEkq`M@SinW#9KDhCX;uqCMZn1ABUb_6FyaaNJ{ceKK#N z5*##S0k_<^2;%@a`Ki&c&n$0ULpwegW{P#TJTT-GA6S|RtG88^-^0B!CR>u zgL_1h<|{1(>)R6caH`PS@Uc$oJ6mEw?Fz4PS_WpM zWu(OkZ)95I-D%UCZ!5osff{hf;UN$Rjjz0r69y0@eYc^1gDSQus>gt8gJ9o}EBBxV z@e^BG)D1?jgc(a0Y%eF?_~S3|85oyL<(3u7{?^$u$*y&>yTZ2M5F9KpIn2+}~A4J}XH^M7|{Zne0+8Ws^@qrA6hSR5Yy7A>ylKDFGR z_j(wrgXtS}kJ;Ht`RvW(+;Ua9*gIEV$#E+wIY}z}pRZ3(RU;=v%?!OHDgr9^@8%0T zC(cv0%Y8FTo`20U^=t_q1^27fzN99BxE|R_ZtG?9^5M0M{bm1ruR1$-!j zX~*VvV&|+$;K+9@?AVjo^Tt1gh>Ndhg;j2cu||I;?Xh}Fxi^K_`kqP!YNCiQ-X4$j zGHWH{5n(3tuNZlHVVrHx!FwI@&d$yZI$eW;0tvn4!z;Xh_ZGF7MfC)GZt0c8%aAEN^GXW|(wKze4^ZAl?2v1q+Ax+dzn{`tGoUzSq!v#H6YNE* znG07B<>BRZrreJ0U6M}``git9X6NQGm5fw20PUcj0i|n&=aj+v-`LWtOUpU|zggeD z!aEy%PuV_z;=CEt+q{eMS$>;8MDqSE1fPj9+7N#v=k4< zgkFvg#El|8VmtP{2N1c78f|zQfiTjlHRX-a4w`%Sv;s;G=@Wq`ZdFEl>BiC>#^)}m z`}A{ZpOnQJ$X*moV_|qsA@{?bu^7`t`OPPNjQqloKRkBFeCto~QZ%E1&2nE-INRbR zDkk1ODhgcU@8{8{Qd{b65xAkl!Z~K(-yihG* zHeDr#!6)Q@umJ8-Eu^IsL|j-9r>H6!>A1&bE9RaytuQ7n9yq21vRUPp;-^C&--duG@fh2%?M0)>NCSW_ z#dFcr$MD`OMwNLUk>VV}$eWZUcd!B_#(=w&;MQYRny;~W()z@OGKdi9wUA_oN&wqp zbWHk0l8CSR&?rM7+c5Wk`*!{XaE5Op2Tj$3y05hUjT@P-cqx0+8rE8(8DO@M-wv)T zvG+(ss8f(+8kL)*=&aylB->)wz@&I;72?tnY&000=|jCAAno31FrKD?{R&&!(jdjh(}i4H z%i$i&aORr7%Zv4h@$2?^G~rf;6dmx z?*}G*mqaI*PY+8Om}M(a4S->HvpNou#>2<*0c@R#oYLBljnlb*7l{$x_ARr36CBdW zu&~{%L5EaJcCIxICKShrna3{$MX)pGs}J!sDcpz)>)2W#Y_LQ_7h1IecqYecRFs&y^L*4EaV>wB4*nU6h_ z2V&AXoQS+69sq582`%Zw-TmflJH zfX3MeNC&4q&VEwjMIOi%@+l+3(gD#LVekk$2ZtSU)Nl>-pRZyWiNHs2U}k$JdEQPJ z^`I+w{c(iQ;pR-y+Zk}5!>J4v6%4yHGi<{nhZbunDD(p>Q2(h`*+<~a6xOf8yM`fB zJCz2F!{BePU4n2CC9^n`Ke1}ZD8$H?ytHU)&t_BwuiTslHx2L(Ag|nA9#La3F?AA6 zbVdz(V0c5+&X}?JPxIW<24X(0xjc!hpxXptN4`p~z< zyL8mX(KAvpsqE{XCH$0v%y0@HGOu2Rp`?Z&JUuko3s1(>i>{knHx!j zMtZA_joD5$bQc|tA=2><;Xpm8ZKq153l`(Hb+|SN3FP!Bv%R{%lfBA5SP3+Z&}> z$92~nN9F=zf}$vrJ4&ct4&{xt*Qe=ozFPiI!~GRmIL!+bhAlYW4Tm8Ar6S16F~h^c z0=c5p&8l+L3+bEwh}v|?pL5pWOc$;DT(6?IL1r-r-_JPuKs%X?p!t4_6C5KDIaJ z+1Q&WiAr(8B9haGq+Q z)D?f9|18C88|a7e&gGV5YRW34B}3h6TVd<0O|!Zg&$0Vy%M|tLXbt7Vr#}xBjxevP z-qsK@mVTl?and>6%3l44+8{ufgPXp5HrPbVJh$!T-~HML0A$Q(zHt(oCN8?$M5TiLM)piu78iDW+X8|0UZ zZ_H?OsNw0~w+;z8zY-AgnPlg=cMIQ6lwl72*!QV+PQwC@C>QI8txF>jG!-r`sLuMP z%M|y|^iUdF*~l^d+B13i4wo}KW(syFA-4q2Q)~NF(z-9D)H|KolpWLJmXXVeeenvb zn`o$E7{qO~200}RF9cqxZ2rbH(X+mjx^Y4E2^DJTR~(Uttw;i7q2bCZD=#lkY>8HY zkWULaTxlPho12}+EOup4!-uePRA6CVuwSAuPU&gjf6WkjuG;ikhs1acs$IeZsq3L8 zBSSTUQ+?=KTXU_0O=b2u0XwdcnK_(h{l9MZ4Qa$k!dWlY=M*%C0fm%^2G5j(ipT)R zsxduvv)04olXm1es~+|I$tKJ_+nDH+ORghfUXR$n6X4-#SpX{G-N4fk{b(loBOGHK z2)fU%J2_8sHA666nwpxJ5cepJ9=7M7^(6r7+X7wtAn1=$(_BnoE7YOw)_ z_;ZFsfL}HKVCJ&W6R`KQDJ5LS;A;Q9b@1i&_4Qm&oIoy2x7&J=XYQdqWCfM5YcXzb z9UgmtxXRJKlemSKuJP#x?r-%r2Rb;Z^u4i~^dA7M!DY@La{0i;2rDI%4bR_)SqEqtg@V_Jv{Ra9s-M*OMJ-=1R=ZbbslIJ?S#YRbQv?mChNS$ z(*?zFO-)mrYM1v-OiOm0Yx>ibnQdM|bbN;$d$0e`#lJUg#RzGA)Jw4$U*hl;9FJ0A z#QmcOJ}%S+A|gHfBlpRAuUd)vKZ^7+a_0^rw{Bok)<4Q9PkaQ$(g zg}}YR*MxgNsQMeOgkCH_1>>M#ub-dE@p(YRvoF!op2U~m)Aa^1o#y2 zx)?q%-Qre11xW&gmi-On2lNr3_Pj`w)tHr(F@)!;bl<-=$9ddGEF_2QlcAy_Ck0y+ zr6%i>G}0fXHt9)p9@>v+%OK|om-UDL&yB|@7o+VJo&27YlVkDc&!4IU!W$}z@51wv zOYnc|UlX0!sg}tx>h2@n2Ooo21vOk6{;0Fz9SPk$q?3Oan4caD;om-FD-^Y0I$QB6 z3Wzxbtd35O4`t6p7$X`Kw-8Fx65_)sXk9+}V|O)_Sp9swNs1C~?;)8L=D6#KEgq7n zH8ZasTI|-_BF>D2WqFUdsz!nDicygxS2O|Mx=AHq96mJr_ZU)>Q&G8mEOEf+v;!!t z>daRihq!^)P|?K9JPNW(?qMM0@{RS5^>h+R#EE4@YmcmGa*dLMe60)vdnu;Bpj!MG zbL9phDD??M{8kK5?TZlKPq&KSdGh3m*D{oQSFgA5=G7k1=aqltNtG3c5kzqsrAu}A z6jscTi$|L*o8Sl9i?5@bWVcRyApieK^!neBfK=<%tG$4FM{o_z5v3n45Yoy6k=W%v z20p%RCh%>-zhNXhGWRHa)R#2Fx`38?#Xh@iDmkYMJp_qvSQbY9$9`q{mh zugqSss&^XOP_^3gYc9@@({+yoa;C<*9&Rkl!%o1Nj5p!BsfTv^m^Tz4z5s|-(bs;O zyel?eWbXa(_cSzrc>lhS;5P|AwxvC|m~u{=uSAKo%vSNwVT`G$sB~;6e2^2;I4lI@f!q<->`#k}(4rc9>3R!PWxjN+O&eYIlJFF= z{bW1tL~gaw-VI$Sv?fVo5s=kG^MoFmBBSVg4{f>@KGTupp&Qc zOzhhpHUnoE#y}9)G{k(UGutEz%5Nm6Bt+BfmS&-W41_q+9lsx4#V;@LnP))nX5;ND z6LX}=bxo4VDmgQdO`0PZH-{iU5kQt(v5l&OI(@5Pk-Tg7;}SdQJdOzKq4N+3gmNod zoofqh#-H{k-ceT<1cAyhAMx1;Wlq-ck_;AW+ zHboQ+zP5R80x3TBEoB7-1>plq`iM9$OtRbP&Y_P+Syr5~*_Y1sV2$?{u~L?^_s&0@ zQB=Zx!&oRc2oA503jW%Je|dj|8sTJ|W#ot010y26 zljxeDq892^#iF^U89phwMV=g6HZF7gd8vPPm_z&C1q6t6Q}8ruj=-Eg_clB5>gAjO zof0mzeiYeA5RlOzMgiSdRjrWgPPw=_hrn2my7f! z=5C3ORISHn2Q%GW@HF+Ezdv^J-JmOw7DWb}+*XLMuC{W28C+Tn2k;=`%v0*OK;rmYthxT9pS$2Y!?V_wmE zd+V3ey&2Nt4S7Zru{=T0j!l|XmjL=elzA$Y3=nt%P^NLY)YByOE)%!i@3v77T5co3 z^>y9ER|XWAI6Kcyj)Y#0?kTLCWD&PmPkDxRF|qUk&8 zyX-g&H(sD}C^4ae?;9#=NDKy-*43Bh>D-#@;^{?Cc32h`-DrCS^|s<(68s=F!b?B) zUfs&oYMni?^~NS}f^v=y*1Ufb5B)A#rCf7PhFz(kcoADkuR;qeRNWHY>-iLvQ9h{u zG-?Kn8ZRaVtDYeV(mq1D?jkYEOCVL8r06nP+}&^@!NV;;U)LNOd zfJFWNFLN9bG<7pRDTQ<3?b#W8PTns1=SN_Hzh+&p<0YYg^&&*nZ{nYm72f#F^j6gx`>39A(w{g{6BGvxTRHgD+!tx+=q6asami${!97(C0=d8uI9m$R z5Cl~8Qd5II@`Qn>($aTw@5k{oQ5x2MLAl3&_+aA)N1vq+L%uBd$;s#wl?7h~P{4wGIyT#bxo)LJN#CRXbx3zSTV zGbP^DGmz2CTPYFuwcJUNqQYOoLanKdG~7T+hE#GPAC~el()*w8?1Mb{8`SVR1nwt* z-Yp|;A1*`wkk*jq;pH`3=;IZqR=3po`ZU1n<>VA`{4~xEI?bl}5NJ*(aL1mZV5-ls z1g2Ro1iojN>?InS|hY=5gXz2VQ4X^r>6(DrOFUIshT#Y%v&nC6oP23(zwHf zvGI+oWu%3{sD|F#$!<5f{Z$`B^brFieM+zq+a>-VqL#+3ASd@5dkZQ<&tfNH`7__- z{JGm0(DztrdVH{bn9y@$Dp`86gE%ov`WMj0ql3pBMP<0a1hEXRDUAZ*u2pXBS-NKft+6cpeM3=BAm(8zs#N{2zf7XR?I zZ6x~q?8qGENA_Yx1qCgfC9A5j7a}(YBVlcgIigrwG^FbR%J&*)5H!bq(0WHPaS_`4rt(X@!DeRhflQGa zpyehp{C!s+2m}T^Lt(Rm38?s&kYG&iGfcbe`hs!>{Pn*`J5=828iSUs8SDFo(6hMo z%#_Ftn&bwd&qd5$smmEADS7s-S4)bNdFD&e>q`wU<7Vyt!DX=sCS=&v3Sb!>Vi<0y z0+Pu{X~+RlEe!~B=r23kPrW;p2*H^8xWVRF_Wu@t|Yf1=b#|5ak|{Tf|Mos!+~1vz${bVaB$dOSj5>+0a( z5QFODrqKF%dSQ^~%rOjINPRx=>0wKBd;QIB-HV}yGI#DYj?kD zv*ofD3V^2vkqUQ4)G&qFvh&s7fUzQ1XvYl>4vrp*O-I8niKN~PJ|~&94iNtGD|@n> z6iX42k)>bZy&u;T>x*06vnJ$!kG0pXA+5SpF0=`;_d*ok>A<3>)`trP}VXm7M9B8dGgFyyiUBW2D zxAn*1Y7&57sVYf_vF%k9;N*&=-kU^6K6u?Fi)GWRb-i4mW{#6CjKEX!>^Oh@=#96* z!HMUOCU+0@5Y$ml155+QIK?JJG1U{RNlPHXS#9-7HmGkbPevreHuLP6`hf=T4lVlks6BV+1?Y zHSGh)wS4!iO$pfD{|wIC0VMnKg=@~*uxmmMlwv2MAJc)CX=EB&Nb0y-()N!}&?WjH zBy4>z?Ozd0CWMuH^XfHF2g#YfVc>qNKs8vdz*kBc7EoqsYzBS*$)k_2@BJc3E~KS( z*&=)N{D1N5OJOKJ)Vn%q#~5nZbtk#VS6Ei0uU_eDf~YwTK@O(FdmaT=)7<-fEuWW@ zRas#%GBwN~P&_+2FK)pamX++h2H-FoQMvL>0DSoQ`FR-Ujx9P=xx8?~4YgV4gBp?I z=vF%Gzn(T0@@oG!IrK&De>$VQxVYFJv53r>|6t!xE=FNi)kjCrDI+?oHuc($!$^g?-eXma@_NeQf; z)sWPyAI~$6eqNoQZt%99*tK6dbfaUQb?w=%h$VlzWPBCV)VaTQTY?YoRdn}U(ERT6 z$jDJ{2(`=%3e{g|u|;AoWd6-wma$M##r3k0b?z1x7Ip^qfJ3c)?bO3wk<+-i4o*oOx#eC91-VW>d z{P$vs8u~yT0^SOviDnt_2LIlQ6;OsV$MS7!(#mF@|k? zzmv3u45Q%ruPQ#J-t~H(Rah7t!)1v_Sh>k>-=42pj8jQ_2|6zCUAxgnZ+m-x#@__d zOe8TWDe3b#UTEfCYuoyNgi`(Q~t!c)E3_(Nf+e;7RtGZc~{%HnmorQS9Qp!oc|8yZsh z>tlFKN-4pcK(4f608groxMNsMjEt%+5Igb_Hme3F+N@B$(sH=?c?#V3oMHjCAmON% z&#&x?&&khTQ|luzf`ufq$)5?Rq2y>$BlTYzH~dX~y@X|0JMXALVKtEY1*^jD0v7{B zC$S)GbYAlYhK6-q;4aPZgnUGboX(ee4#*sGa2S}Q(dXInhjN2wSV308P8NXb{kLCl z5)yQH3xMv=A+jG7iT_7QZ|qB@rKJI$?s5F)u`fUtnDx=E%CMI5xh=%r6Yy4>>RCv3 z8Wg@nYwUA5w!!NosvwOE3c7Dz{Lj{n(qG{TJ+zV7juWi-w)XDQxOVB@?po@gH7t9q z)P!aZC3Wd9^;f=+3|rz&zvaXXj5FSkCybD{3MDowRCkb;IUe4%Y3)*T{ty(TGdwciU!XF%eZ(4!dHbW&resA? z1=}@TpYq~B!J%qag-F*oMMbhTbKR@?Y8#m{@~G81qHH44{N!icF-2peTaJk{Q3(m% zDN;-b#B;C@m7k#VYoSW4YmW7Vh4EQitsBnX!6HK-)h;7s#dv^tKrZD&nDY8mz$!5{ zHBh%>78Lf&U#EYjTP8c1&Q-pA&+Gl27Y}nao7Pjx8MKmz_lMPtD1x;}tnD_h-Msl8 zMuRULYgYXEQ+AU%(&yY_4oqIJAy7KARwGY?EWyj&!0 zL3&yA+86Y$)7-V!Pwwk2{unHD!pdDc;M@m7J&YF{KTX8L@91AL{9`G=9QpOD6i|{* zFJ#RQA}4jGW$yQHiQSb1^>Qn$laoQG{n-^k5s`1Hty(#{G-~uic2)%@2U|~}%RE)O z=hjefqc7_Z=%2yF-r_~-3{$=h@HoxINS0R$^SB>rMR%9pl_d_H`u&}ZjIH9J{L`g^ z7lz{wuP(kEN1y+4o-j|rc#d*sS>%C@kW2>}*ktvcRb?2r{iUA^c^m0GJkXw5s_Ts} zOPdD=eOJarkPgrVda=``pGBS}L6glkaP8_i=eQpaM=O5H4z4QbH7F#V4xVuqbQ6ZQ z&rbtotJhz-sByv$wp~sTvW>j$_C}w7g!}ooi!k*tDEJ0_^W{NoR7*qQOk*qG1>eVU z8>Ld1*FVvmMT&X&YPrfDv?wSLG$MossOP zeMR#kiZR?#z`w!myk$3Bve@3A%0$6!L=R&d=wbSU>>^5e{VR#3)m)bbaOGN7+Bt>Z z3#&~r>2)+6-Ft}qgl61x6pG4Cno9E6Ar~eDV%hw?set|JgM~xDaQ*PF@OpvT_~!A@ zLUOv*!^_W)Dny+28I-eyvceprx%8;J?;|67!kJg6mX=KIZd|G$OYxeV#uDe?aMhmX z{82#JlW=-5Jc07ccV7Z6G_#S0<&oV;EA>l;01`BVAw|$;^yOpde;KK3&&2vkE26 zj0J62O=S}tFz$f7pycrg$0g=C2pTCl_3?3W{i1KXl}En0SkXT=w6b816X(lJWT(&c^ z4Cs*PfJbC5w#QmXCgHLDdb!AI;8o57G9Zr?(sR4q)Okx>62$bfNEm5vybzv{>z?58 z!EsXmLpVRCdu#V{bgE+DP0SL+L*rx?Kk4&p1!(#{r^e3apEK8sPKgMUl90F$NZ>t1 zQPnRRGWY=QlK%9<^n-er6iq(5)Ko+$-f9cL&Vin4HsW6I5pe^GWm2H?5OTa59v6pq zxI=>mq@ZJH@CVmKL{0-UmzJn)Y}h>Qb;1HnRG&T57!oc}XWOsUd#(+%nky@LcB|v> zqI4JXO5)`k3Ju$(Wqao$)DpjTcQ^lr+Sn*QU@kt|=+sk;;}3i>$U;RUE1gH~BJ#VB zdhYbeUisaWf&=hdQoPmTwLt_f|PU)a!7yu`c&J6tfkGPwl2}8 z91T1tMM|Bx_|-W7#-7D-`oUx6!E?Jq~jc*W`*`tOis&j6{UE& ziYY-NB7rBl{tPS&8fcwS1z;t31qQ9f~+zxiT5)OF= zGKgAxdxzljZE9pnyiTq^U1(zsc}{)dkf^cH=<_6QkJ1BZ5^>=C%poaxQejrrizzG$d|qDT<1Y%5OPu z&fjY8@Avw2Ukb*-&jaazw5&&S+C6WDt;(`2=xG|r=)E}CMHx`%VULJ3DUi0+ZJZnq z0FM>GIo}09mxlWowm)DdesX%Az&l^G=Q@RPuouU2?np=-#$^ldknwB6&itSW7=)em z{}_=mqjPb&g-FY=?^KR*l~9M%D=3QQY@cDVGm$e33$L@Y@c_+X(Cxy2K2aQ0-dI;8 zd(}Oq+cGbg2;t^QBE;zt%a?inlj#Fde z5PA1L6UmdD=A_`mwXvyh!L}gs+QR=)5tU{CVrpTb3$TFoojmo*pEB}Z#_(`RR%OpZ zIDtG_^$lSVqSpF81(7L{hkjw;u)w)h!~1$j*To;^;9NnzNg-MM`<5@AN5w{T%hUIv zxYi@NBOrF!&b6i9u$%0@1@!4q3(5Q;EiH9sX68Z3!sXkPSCLajZ-WT~PwL(VjWyJJ zxdjE&(|_pgNjUsHU~*o}WxN--Dh#3Mp4mKMn5;H9vWoS4HbJ()>@r?H5mhM~>g?{! zzuwcgAOXY;u60frl@rc7Z}C@#FNL1YZ2iH%8-n!D5&c@$X+V@j1x&}7j4{E6c2G0t zZRa-l&A>;=Tz{X%jH-e4Iq8LL4i;0Hb2_|I*`JI}+wVgc5HnP#Nx@WgLpyxrY`}eA zf`zrRE6$J&;kYY%=~BD>Wc^cUJfc~t=6MYr&@ISK!^wi``_ajc-Dm%|*;m!ciPW_S zX+ej&Ol6rR%y4Eq5)4 z-i>Yumv}kVc{bbZPZKP>4l^8QVAq7Q%}#5*0rDpF@b`P90Bu*RCk{|;SXaZXz1^6_$^1Tg0g$z5@BC_ z=T~ou)5x#wr#jU7?52>f3@k3*A|_;3sRUh-EiWh6*xZb(tW5OJ2EHZL0)6Cu@W=5D zz7uU&JF?al&4B1^E@Y<)Ic0T6k}8 z8bYVv+no~^*XuIY0L z&OxDy_G5#orzKSX_TbxmQtx#Jh6rGvI13{X&vI{12gttS{V!W0@lLq?L-V|E4xIQ; z%)3T{@oS-BOB`76v>MT6@-5I~Z9!MF2#yL-!bYYv&M~bLCLNUophGe7pThWE7%qtUb%sj zTR>J!OhN=s)2Hx8|8UWwe)O~@fYagdoJAEDHx;7t47L{p-H*72&u$*aV{$FY<8}z_ zALOvF+--4@{+lFb#z?fm0@i)+fUZg#DNOVV@1+wcH`b$43RXRjFPb5VZARYmFSWE_GdeGBND17A*H!7fWS&J-P z4Gam4)2I;+E-z0&gHiV|FplPq)Z!&l)i`W|h0lCNqnxUJm0r+%;)hFOY`wEop;Bf1 zV&OR`3d`#CrEPF{K%v1j`bbL(GWy~IeNgS60*0K@U?Z-2-Xee>*9p9 zOXq6V`uKcLm-H6)@j3TdD=Hl78ON?d&lL`+02w2ri=m<03d@yHhWzX6yKt47PiNMu ztwh*~JA#0!)MmAY{j*B?hck&9N|hxC#oP!A%I3PQ!)w%i{e};n!}PkN}Ig( zn<6oq;ACN7nq8t$5AdU_rShDac~pUw2k*tEUH&>SkO$1lT`Q}xO+9-UM!*lVZyD_U z=w&{=enJ@)ZWk^var7Q&EZ{(ky*gUIsmS;!Q|A=A|9?SPoOC-Re3ad#KbD(ma7+rHVI|?rLl6k*%*z5Qt#eR zx0&_tis2y>7oSk+egFQw^!CE-ba$P$W!#JRXI1Ht%$Jw6_tK-Z~Uh~KA7X1b2&T34984zG0E+H;?sn&$N2pG{GeT0C+dWCs!Fl0va=r_;pgdmZC<|0 zIOhn;K2yD)XM!!y6HR<+N!8;0hxDWZ?wfquykKsMUjDq2`067`ulT8;anbT%k~#*s z-PWx%zuss=|0i5&pH&_y1HZ_;=&rT^i|QILD3!Nw+qS2tu?Q9#k0vgV6{E%aG%cu` zNoG3`m|Zh7r!F?#O1iZSVRsl~3h#5`3+h8SN4pc=6-DJ_Z0aq0>v{%eW>PiI?iDh5 zFd5dOs(V^oPjAM6#WaEoCR+U39%?nPv}_HeynY@x)66u{<5WzpUK7>}-DT&-+OU}+ zVzr2FQ%J<|2L%P6Qy(E1{uz_0zRv%j#nXNQecawS%E<~*r9dV$f;e9E^bFHG7Z)`( z@S+;7Kim)D3-Sd0X(8f+edpE1z@zWc@$-*!7i0#rbb);VAO2epFpwpV(yJ+U*#y(x z|Kw~>cZGLNTyHW<3bX^jHm9z~$0BTBPoITkl#O}Zjv`(8ePTzutS3CAg0#y7_-Yi9 z0c8+dW8(spsr7LWE{a|Ib#XBJkOKO_k!!}^xLZ>~UY;etdLs?7`j7?+KGWYv3|M%Y zO2Md-k|*Y(dVN_$BaTweZ|uc#xqy$}f2RPloL)7$jgw3OQgaeC=&ATwXlQ8>$_g3*+@8$Gox4YD zdCt71llk{3$^f6jFEDCnfsxB^LzHFadT3-p_=AG&G{91KyyX9@Df&}0NG?7I>q!`) zpg~kw+RT_elhLy#f4;t~dmwL3`dy@phE!DNGjXU&!~eWSrGq{Nri%N=2PIB3p)P0; zM$2fA29IiJgHnPpG7(J9OwSb1Oz;N>6JlpXeM^4*gjrre!DOb5`3RU>MYmPeoOj(R ziYk^k)Z^b|MM7`wYYhDp$sttfh4Z*Wg!kZEX$U3 ziPyF1Urk>7yEq40c)A{zo;%&w=+ECi8I2BPXoC@zKnM9CLy2Ieu8syRLn-|dd07}7 z#-j4O!SS#4(&t3Srun=FI<~>G;C2Ew2eerRz~6=70N^D#Nhx}Wx*y$`?|JsHBr6J$ z*%|Du0c8atYkDkC&62E3#41GOX=2;%fMg#dt6FaGTaX7Yo41k23I>1>qvXU)O5dWlnmq)lJ)H+?lO z0JUKEmbNtUbw9yW>oPJCK#S`Rtmujn$Wj%gWc4~gDgy=H%O;c77>h+UX7r&bO z*=p!J+D4%R@&V9uym9x&FzZR-?{hO06!P*uLH+0!jEkt*q{a#l58{y9!y$l9Mc|TP zbDQ=R2}pC{!EwM95#bphkBw!fx8WHsx5m4E*Hy(}-ebMl(P`?Pd-cxnd0v!w!Oz&>Z&T28F3-AiLre}Cb%o=1)^d(~uL|F>>Ed6p-D zb-xbxjH}|+@@G^|Q~E**nR;x9TnC$vaGh5^itmtcms_9 zqn<~NCx^~3CaSStS)6@(K-o{HY*BPa-2{+X`3jqrq2>5zI<7~AEkTEYFnfa%4z`6g z5)yuX$O*&RxBmyN^+wc6vm7ZM+wlpY;+xFk;K5z42KbPiIBzVZQ_68h?m>`0lhVpHsCke@Oi z)iE&txq1DrB22n$0QL|D$OsrseU%i@=EBP<>O>M^ZQalJHy-0t@q5*sXv)H%N?=m% z_q50E0ea)WDi=;&T>>ZAC77OuvDQMBy1wz^ZTn0s!ZZOAKa-gMINJiKSNwnxN1QQC zuhtOlGHWEPKqw3rUk4$_0NtnO<1^~rJ_84sDU$t=%5$!pZVt$Xd)!vC^9knxQ%48i z3rq%+ad)ijMC~f4>nq;wSTP@XI*?bqHEdNv0~BA%^>Ddy^yeeI9*-BfW@=_$W=(Ip zdChh%GKSGXO_0&TcOf+R=xEInO+dxMK0nT8sI7HGwvg6+4{LPHyW)7~P`%hx2nvoG zChGVcCyqTTg!1jgwMv)L?u8vva!rv`S6?B%)c+1@+O=oIvsr6a+S{42Y;-tXc21G| z!DDU+9|UwJ3NSJG4e3&W8IXE0_@W(J;?V_NcG$deJbd6Mc-Eicl3TdAI52QK)oiF1 z?R)M>`V5bur#FcI%#SMh63j3^F(HVrl-|%&gIYEY?aL*p zUb}(seu)v^E5D^_s`u%b>yIi7rRJszodQE#wrFg_i8te8$&X3a*54HKLVPvH=IJQ@ zrd7}DO+t$HdBH*Ntov7IXiYaMM7Kn@XwGA{HkS8J2GHsMvQe+WJ=rJ!@TlIPBJWmn zgSRXILN}$}DBj@P-m(`f3l~EK2zVdNSEltmbnZp4MC|R1gjoFE&H!f;2?lM#F5uzK zWM#MvycM<BSP^ifp$a-uDL(Ja& zEIMu7`jg5TXYRDoe&X{6d!vgdWUcKwjT7*}X%y(=j#W86T^aS2mMKY5NxuLXVK_jX zmo5*+I!+`sCwb(hs^yoN4rXydL;@d{1VgWTzGq&~RLvAk@;Je?oj>{Ne|e(7_&(cg zr;AGV6Fyq!-?WTM@mLRj9|KhRbTBJu?{H|+kM1_m=vo4|{fXVmD93Jz(`wB-i0TC# zMW7OQtSLfw7g$Eg+UzA^@Eee*7;P`y2bSN9!gi)g%foyd$4)R=Tn{*Cg8mFlv^F%* zcJAxld_$DfHq)-h_2N6A*i+Zx;?eyG2lUeus;XO6PGJ2kqfK{S{0)^&Dv8qetHS*QVfGhRbC)i1ls zu5#4XrHIUYu$}LLX;sRDS%aCnV?j}FTVuBR{uDnS^d=;wj*J0REXuW~HA zlkYMmeF4pO?kgJS^KVwrHj0a%(+x1SdW8)p(d}0PtF?z*z&vf-eZ_2IRrz*IGu)HY zL4aW+D@|^3@gtbcLM14qp~Ylultw~I+Tn@bw6<|z*}2Z9 zQ+b|_@t+<5JcX(cpa zRt4HeHA^)S${K*QFW?;Z!DbUtK#ahu+Q(EsfOQ8TY=udH?!9~KtJ*t9d}dD=ZoA8_ z?3^A?V8U1`mLzv!Do^k=F$*?bM|?p|C5rHa%+GdI%v#7w&uova3}jHjdHLkHS{B1K zEn?hD>zAZg;yT2!X0$dd#`CFCb3{gQr z1mokEfij+b&Dqy?aDKB_d~$!@?;=T2IB@CTEPGo5N)W;yn^v%H@m*a-SUuP1Rbqhv z+7J4rd}DZI4Qd-RY?0u`g85-?%A;FmUTxzXmLAib(fc*l6?rY=gs#;&dNR)n)kiMt zTJZl!DaUAXToJ1e)ZnV#d#N6(ByRUx@Xj6jA4MkQO@4JfhpR+0-bA0~4%a?E1k8gd zL~O^dFp=l^H-Yl<(H|rhd1>D-XlkkHOpHfAYgibxHewp8pXfZmegF`Sps?`d>|*juXHTn2ZA}*Ovn9P-`cqpmE1EyCJkB}QKRH-jf8{)TjG!U?I$Kcu zt_o_6&2_B;gx*npl6RqC;7Zi0Gk45nsSWDZ=@E@wv?#g5JUi~pvW}O<*a|~|rj}9$ z>8hKPxVXo`)}K~rQ~L^H_*pF`R4kEnxfDKz!M zDFN{6uk`m9 zKFyNz7;%xeayujZozZhYllKvqA9+w@sIYwnCO*BsXGBpNkREsnL$_x;nwMf&pg4E0+`$15oXvuSRL93P6q_(n{$ygUvVZn`!bwTbql zh{Ajr0@uUh0|r+TuHsr@g4}#v{(PuE+8?OY!ui#&on? z=(?w&m>)rKE}+&lI4unmo*N43!jRccauS;3+Yu~}ISN_^8@%6DpE&DjmMql6 z93n->eSTY8yc`h#C9nO$L_z{g>}m!1j(d#CC-1?( zu^rNR2n@szfKmD0?n;fdt?hi6atqosvyUp7my6@I&DTP<$`9(}x@z}~Qm>P0-D$km z-9vn-q8*BjQ``>ccr8XRdxkLy6k0*|hsV+5SU#8Yo)C`yfn7K`;e#=(&xgxAh&{>m zmb{Ym_-c!r*zjU}1x;Xoz&fkl-A4YOV?{k3bM&>))+(+@1ME@*Z}9^=;s+^Y30D&he< zy54r(vC!U(dUtdn;$iAI*jzOskOkf=;Wy7%Q_wW@dLA;t}V2SArs|` zmodGPSe7?f@HF4R9xaVpjZj*EA^a765_-szBuR`|zp@bKMR)Ay_KF!Ff6^i~Zm4B=3) zq0{?g@95}Q^)gSPx6J+c2xA_{?2<)iWHaCkzZMog#0?D$cvsk_-c#E@V(;&fh&l}rmFNTYQ5stlOKz2PsO+|g?=b?ST=S)wr8-N4`xMO@j={y zGW>seer=V$3*MIGlu_B?ki`!xG&K2EXsAHzwv{*IzPSR)f{(xI`@r|*39=g^?zSQ6(qXBPz;>V{= zpY~g*#a`%iOip+>Lab+*G9>VA)@8JfogH!!wYC3sfpYaEinaEo^n;eJ`IoszOb^tU z&fRLS%eSVu!6^A{&}4b^H_VHTJ`@BfKSYe*(_%Tvc)E+u-!<5LWNL4-H$9MgpA2Rt z!CpRGs%PF z*WP+6uf7fpX|EF|D|8CQ+k7ZcJd0uFGQ zTIj>qpG78kf!iBdirS;jK=gWVtWr|-^MiN!b%{MNNd^e7MfBYG5+K@sSoo0~9bMA( zc-sUD3_p0|kl31-M=->uFkmf>IHExX#=k0#IIO(v8EBGA>_GDWhpU43$s9^h=$aTKV?r@z;IBcGN0)a7DRSh;r?~ zl4xS1N}P0JvHFFYQC|);9%IEcZ6W2O<3+$Hm_UmXN_5AT0Q;uf$5h$%&E8YYYDW}dx$lVVx18ar}izdg3R|M z)J#=Q+=c$34bXwJQ0|nQ^0yY^4P@!N0x88IN1;$&PFgpc1I|nvJ3}(r_pOprnkBb> zKd&o(4HHy>tfmQ~_O)wv_pUAMHOpkp9JO+DZzfurycy|`+_!l!&a>)bg%)1N?l@w; zDb8v6d}TmeC&6i}-{twG^{cH!TYSD5*x?HuIni*L&^`WFHc9c*0wi( zzpm5R#=!wNEn&WTV&mW-L0J1&DWdH*vD*+8kGAUSMo@~oGcq$Xp__$Td`lBT{t7%R z7nj4)!tF&rfJEzb9EV`v zNEy48o(NPoyJF=&*n9rdL)_6}XMbv-xhR!`G*fA&`*UTvoIQQ>K{ei}_6B)+M-*oM zg^TKkN2MkWjS+{lL$0-|30V<9=K)!!G`%|7y**Wf!^%bzlgobXMvYCW+Jzx8-Ns#W z)rSRzh0L5g`E`+8_6xGGI{l@~HO(FN>y^5}!NI=4HCvTYdAJURUo^XbsCkAMDy$$G zlP!@B{UvavrLk)kzy9%q2IBg2e^aibf>0^^mG*ride6K`IRr;3bW%!$+R;MciJ^5m z;13M0cXp)-<1%OdUvJ+XPW2nVe?-GBGh|do2-(>pdC1Xh3wKB{_ zFISRbC%h!1=RBJseLg8NE_LRtA978a(Fu(wq(jir98rj`xBm3*7WRYX&tEP440W8L zWJVY*i2WXU@w5ExrP>OA&@u;NV&}IWGu}mtjL|-NoR& z-h0WY)Fn7ONJ?gcK9SFDvIw6!9cK`cq09EYWXi_VBVxY-MZk9m3nxTvyHr$oxou56 zIoOg7W=`k7W+v_7K% z3dI+IsWa;;{`q+UNWI#V`~p4y>@^S9`R(WuCm*x#5$HTDmVlIk(-{20Nyg3L%D$Hu zyC}B?Z-&l?BT%2?0^<(w4x+TV(JSFA=(gQ-9Hk}?E)Ke7f_epJHkDhG5D(k(HpgKn zGILYIUwdSH&B0~p9W`scV>oA`uc=7|>hYD^pINQ~cI)3!OmHeH|(E;(KN?cHdNmiVLFaPYR5vcU06~QT~osh z18NjQ`69|bF5A_k2J+zO-^PT zAMy9RD-xUIc3#kyc&x4>C^M4<^5e~k=)=|n%xd~wQv173yh%xv(}NX~zzab%j59Wk zQScE|3*o#0QY@MYn$Wzesctt^m+sOWMhqf2o1HVv6c9T2zc7>PR4~gHu5`( z76QK$p?Cv4bTy$0-YSZU$LaZWN8v6o7%FH15gRJh0@~;C%a;_0jm04l1ZV;|(?Ote zQHvEeJ+p&f6j(D1W-hWAEN6Z3VuuJ;3s`UV<@N09SEo&-ym!xm2PC;WR5|cysrq>D zr;2C;W7Z}ZPqFOD-@B&Mbm$5|2kYuoXh{j zo4eK8?{sdn@6g=Sq$n4oV|F3Jaukmd#Vkp6$KF$9HxvymEn8W@b#}&9`z1^i;r3fU z4{Kw-_fup;)(P3uIRrRi5T{E!qkvEH_W;}n#C{`yCW!=h1V~qOttqRF{4390-|vx` zOG{tm5Ec;fCwMVXem3?Icn6MC8tr`&GP}5Z_f^$Iw*v=4JC9eV^1;@M_<6%6Qs`>d zdU&fNuUvc@s>Nk(10dGl2jL9%F-+Htjp@7qlvUp!1e9cD&iqKxiI0**54V})Q-6tw z+(XIkR-D+sL!Bv6o%C+_7h+_6q~-aN2fGC`)gb+7|*JM-oJ9pau+aY*U$CJA5a>W3pX%PuM6&zKv1yvrmkTvgz`7uE0eAU)>rN9auJWqL6EvvWk?x6J^-A0uv6 zD6dQ4eR7lo^s}`1y=oe_Jb&15)_8BL#?R#0#HQFw`VyP^&$&msM)krk9HY)}sVfZx zR1kD?_?#B8jq3)JWJU-`df();N%Kz`j#_VK@^(~-QOM!Zj<3614|h(HMIdC^_SMpdyg!509O8GZZ7uk zNYprYbYx^R1epCk2dMvt$dL?%-kwY0cjP60IpVIb6UI9;zew#rVp}R7wjUe(Wfku= z;bapoQ0l_JDPIvTx?9He%mzT;SwnGrT(?A>aF{oDjfFNb&Cn*W^ z2-qzLwrxQXtyFjQ)WwepgnV$h-g9Jp{PNy*lABvZ*~}^+M69g^t6KB{6F1rCibg{U zCrP-0C42Y{Eh~F_d2Q_*x~VSpev~}S_e!2P@g(aOc4Efw_wl%#1L;tv6wGTy-{JGX zkrQk?uWy>fQai=P$Ri6m?|t^-J%wI-6_Jjj6>2q>b|11IyYOV03UpYpPyV!t$y}Zh8slqiO}~k@!&TQgrYGnMskrF z=EQO~-x!v0mZ=?#WPQow(h7lKWKubf&_A-@Ka zf;v={lhOH)URal|&SzauZn<943@OsP8-BrRBt!k;Ws;k>$O3Qs#l%#qB_}f&e>%7y z`!>l%%B+B?pk&RozAne~^JDp+SvOe}s-&}w4)Hm zy5{(Wfm25XMwyRM+kHL2qVdaexE=O-_P$DqE{Vw71!rpLB9()bbMn7M1f_F1Iv@+b zt6bwj*`G_MdPYa(q^fof8NgXDrNVfTr*tVZtw7pP{dG`}wYSe)_xzkSGE9IC?<0|b zpW5-RkWYvU>a)>W#P6&$hK>n6Hcl!mjK8c_7Mjkfjv;*SWvHkq25~Q~WF!-3%X!`# z;5MuhUYwOVc)Ezc#6cn${6wO8_=(Kwp)B%@`0f4orWe|15SEs}H+22?>(w!uN8@{I zbaQ67`TwkH`h(PiC;Ir)=93H0Z{B3QK3(ey(&Fbqzuny%Uq%A#d`MqEjh3A& z8hs6RKA!IIDNh^N&UV=l5FNMdD#Etct%!kI@~GDi_ijoe6z=TY=JL!>ica1Ej){<2 zQ2<==f~=pj(vz;7>p^jfDS&6#yC-hWb7Vq#h!}OazAh*>u zH5)c%wEtXPds(ziTIJ9ah(1O^N}iXK`%dF{gOkSPkCBm)*F3guA+R^Jmb(H1RT6W; z3l(SzV6C63A-{!*kH7Xo6HCL=7kQXGiWoB7v9%fT2sXJRfMpK{)aNTq{;Tcvv4Fsb zilJdb^tjF+~tY#)7oW|mdT@hM0k!u(@!`ke|UQFFm_7v${*X<$xO`TF&1 zufUz7N81$W)`0T8qL|zo6sjVzfs{K>g)RuDorrnpk+GMl3TJ= z6IH(6^?cV8kQekbxX`mQGCNO)-v8M(DK!*@nt6&GPjgVYX3L`upcEntw2q84G)Djv z-D^Vn3APO4I4f!*0vF}H{cE6%U*J|94I|DsB4U~35D{R61S|KqZ*9!?QGdn`EpUmu zoS^4_TfZ_bEzH5%H*RP)xB@(m+C@rSxR!Vbs52tINaWCv?$3KmI&q$vIGC*^P+Li;VgdzB19uCH9lO z2D!ZjWX>vZU0#&*%gJUI;EYik3Ac0PJxV<2wG1VldOYrlB6aHFF}$LZ*Fe}NQ@5Xo?6t}8wB z&e{*5D_38(9<0|g8k2mMS=T2h?ZVVNx(MOZoxZPRUf%&GfaH<+sV^MnFJK zQ?SXt1`x<(D64GD&2h5qw6rS^K?4!`B7%-B8G8JfSoZ(}<7>QP8&wG~@y&vWLkN~Ms&f_2T8#y1@!2G0TouAm0ikofEU0XKEnBSA z0u(dR6|JFbu%!?XCDB)Ij%MzP7VqB*Wy66O)+22l#;~XjyFu?A_AOJ;;Rg8N5cHON ztot^A9r0$<_eh^c_MyBnE2u_@4tiJ-H#C?MK}sb>&lhnP4r@<=G0f)?JI{MiYD8C7 z`yOuWE}SN2^LccH5vA+rvHpqdiS;mz>mmY($(dhdPBSJhYw-F1NecOcxh|OSj2rLK}#h_nsRffK_hK= z`0nWF>l2K)mwK69%X0;-qF&pFY%9c{@agfrDU>GIGE!39o5C&5_!3_~a=YoHbS0ha zimG+;z3sWaHBzp{rIH*z9QLL^Atl9M9qAE6&u=!Bkn8eu9y>&^o70_E z=YYN<)p25F%baLq78g%AhA4C}h{CB7Hszv^0*6n?%a>k1GtuI&(5blm2cqfPTteOK z;H~zT{6Byo9}^H>LX+^*j|xv30Q44t!9(sNvmh0Zfq`!j34UPNmbgG#Dfiobu32F% z+L45Hf>P-D1%1u*rY7Z-RKd|)pLq0bH4)eszRLdjnwQL_2k7>6RPQ!miczdGnsV+9 zFEqPfi!bKekB|XYLTje{i$8v9cQy0PPbscxH`w9o9LJT?0TXi@5XO*_RY&mZY{w)F zV1ai8%#Ezw($dJ6xq-M#C5}_$#&H)NAYo{k@T1nJW0v8Ql#I^GvPMQz5kpHT2$lp! zmZ4Y(EJ~3T78ne?>s>bfoJeV}?3)E|^|NgcC7Lx`l%f|uL#!~@yG{u`NV5h^X9O>; zDr`aM=i#rXUbS2(vpZE_Ccq)*B{@)L+!$~It&@94V1I8BC1&SMj2&=2`y;jaEGX^G z_Z=AhD)xics4o#Q+c*>mT8eO=Or|>|-4;q-aB{o;j$0zcd?y-l*9*+V{Ep(ec+(jL zjQSP(J~g}jN-9GDLxmXe(ZQ?PMvSlG;xxwUaH4MZ##HR`O}Ff9IN@G`7^_rgy9_`F zkAqnv%OKHzzVAms@VwF=$KLeu-kv&g#-+)eMZuf7Rd)3ht90T0j+7HkaoDF{zm`ck zaxNMML_|D$$*vGzSjhUd9&5HSS1&g?LIJQ%;*PN~1Gu4=Mll0n+=zDcb$!G3_G^0C*Fc2_g?4Me=QS7kSc<6$3FKP44~ zl|Pv4z7Jz}U+aQzS#u}7n`@WTzj20^6-Ai2gU;Gwdx5!m zixf{F3^}xwu17Y-2(v#1LD?XXj0NWeQ+-pbHRQRDn@c0RQta;IZ18^?)^iV@%?R8s zegl3D)SNHV&`6Kx79RqOPJ4j&>0^?Sh=rQv>aZW0hE^wzOXCjD=_EO0?-IXY7VJ`V zRXN4yA6l&=;DP)oH;97P64qU$#5V) zv-UgHb@oS}jEV#=PKHU3jJyXW)w}b(=;CpV^x?PahajWqU`j!iO#)TGucuSf-Ff1G z*pZp!6Z*YTQsCqq0u;mTYCyc4l=C11$DgJ*`v{@>(hZaUfJMY*=C+8)MtE3WC>-yt z1iPQVMonAx+SL-Yla!=pa8+q--3G?&|IlC-yE&rI9-tx^+GHxx+2P@-Y z5kfwKPLt=kxh4*S0A1;sSrs*d(`QZDOG--W*XI;AtW`+K$xS;mbg9@|l8}*xfa-O> zUF1AzU-9;jH|JF6xujZGxx2`($43pY=Dm{1*O_GfSohU>XcKY@l`n~*N^gIe>9osx zwDo!{=f}r;aveX=;cxdT>G|Fm+|7El<%oE+(l7<64#Z=4!q;I+4B>%aKsv4%B0M(A z-zFwj3%)lh`c8mgOAb8>B|PMBXLW$O^}9lq z%$d>i0CU!OIAenNWTPRIV>I74b@)EMU-UWr-$(Dg)ZM)9ZjJQoNvA*!cx)_G!Ss8C zuKey2E$scHidfruqLU&i>^E+NZDr;I9OGe&9Da`0n_6L_=OLirfyA33JT9&=?M67L zJmOcsV3S*VJ7OCZ|G(9;4?Qm~X82m?nq)ih35XIPcn4mP?s_|)zIfGO>E#ZM+~rP# zGEk>IqU^s9qqes$;`C`!Ur?En5_M3P#i>N1Oybo z2o4iW(JEDV7=%ZxUBC}1ykfEabxbjsxziE?LxArPKZdaCImI*-_*LvA-JAyx_Kkja z-%wbyKFGZBPNR4tDu<8?fx#q*-FJ5ja&l9Re(IwE0=ntYO7eKAkS!k5zg|vY+galG zZielm`wgUVU%ZqJQslTkY~kC;s@`w}9P6pkCD?1BQJV^?&(B ClusterStarting diff --git a/docs/state_machine.md b/docs/state_machine.md index ca92e269..4d6c9c65 100644 --- a/docs/state_machine.md +++ b/docs/state_machine.md @@ -6,9 +6,10 @@ with the underlying Kubernetes resources, it takes the necessary actions to upda Typically this will involve traversing the state machine. The final desired state is `Running`, which indicates that a healthy Flink cluster has been started and the Flink job has been successfully submitted. -The full state machine looks like this: -![Flink operator state machine](state_machine.png) - +The state machine for a `Dual` deployment mode (default) looks like this: +![Flink operator state machine for Dual deployment mode](dual_state_machine.png) +The state machine for a `BlueGreen` deployment mode looks like this: +![Flink operator state machine for BlueGreen deployment mode](blue_green_state_machine.png) # States ### New / Updating @@ -17,52 +18,73 @@ The full state machine looks like this: created, and we transition to the ClusterStarting phase to monitor. The deployment objects created by the operator are labelled and annotated as indicated in the custom resource. The operator also sets the corresponding environment variables and arguments for the containers to start up the Flink application from the image. - +#### BlueGreen deployment mode +Along with the annotations and labels in the custom resources, the deployment objects are suffixed with the application +version name, that is either `blue` or `green`. The version name is also injected into the container environment. +Additionally, the external URLs for each of the versions is also suffixed with the color. ### ClusterStarting In this state, the operator monitors the Flink cluster created in the New state. Once it successfully starts, we check if the spec has `savepointDisabled` field set to true. If yes, we transition to `Cancelling` state else to `Savepointing`. If we are unable to start the cluster for some reason (an invalid image, bad configuration, not enough Kubernetes resources, etc.), we transition to the `DeployFailed` state. - +#### BlueGreen deployment mode +In this mode, once the new cluster is started, we transition into the `Savepointing`/`SubmittingJob` mode based on the `savepointDisabled` +flag. There is no job cancellation involved in the update process during a BlueGreen deployment. ### Cancelling In this state, the operator attempts to cancel the running job (if existing) and transition to `SubmittingJob` state. If it fails, we transition to `RollingBack`. - +#### BlueGreen deployment mode +This state is not reached during a BlueGreen deployment ### Savepointing In the `Savepointing` state, the operator attempts to cancel the existing job with a [savepoint](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/state/savepoints.html) (if this is the first deploy for the FlinkApplication and there is no existing job, we transition straight to `SubmittingJob`). The operator monitors the savepoint process until it succeeds or fails. If savepointing succeeds, we move to the `SubmittingJob` phase. If it fails, we move to the `Recovering` phase to attempt to recover from an externalized checkpoint. - +#### BlueGreen deployment mode +In this state, during a BlueGreen deployment, the currently running Flink job is savepointed (without cancellation). ### Recovering If savepointing fails, the operator will look for an [externalized checkpoint](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/state/checkpoints.html#resuming-from-a-retained-checkpoint) and attempt to use that for recovery. If one is not availble, the application transitions to the `DeployFailed` state. Otherwise, it transitions to the `SubmittingJob` state. - +#### BlueGreen deployment mode +There is no change in behavior for this state during a BlueGreen deployment. ### SubmittingJob In this state, the operator waits until the JobManager is ready, then attempts to submit the Flink job to the cluster. If we are updating an existing job or the user has specified a savepoint to restore from, that will be used. Once the job is successfully running the application transitions to the `Running` state. If the job submission fails we transition to the `RollingBack` state. - +#### BlueGreen deployment mode +During a BlueGreen deployment, the operator submits a job to the newly created cluster (with a version that's different from the +originally running Flink application version). ### RollingBack This state is reached when, in the middle of a deploy, the old job has been canceled but the new job did not come up successfully. In that case we will attempt to roll back by resubmitting the old job on the old cluster, after which we transition to the `DeployFailed` state. - +#### BlueGreen deployment mode +In the BlueGreen deployment mode, the operator does not attempt to resubmit the old job (as we never cancel it in the first place). +We transition directly to the `DeployFailed` state. ### Running The `Running` state indicates that the FlinkApplication custom resource has reached the desired state, and the job is running in the Flink cluster. In this state the operator continuously checks if the resource has been modified and monitors the health of the Flink cluster and job. - +#### BlueGreen deployment mode +There is no change in behavior for this state during a BlueGreen deployment. ### DeployFailed The `DeployFailed` state operates exactly like the `Running` state. It exists to inform the user that an attempted update has failed, i.e., that the FlinkApplication status does not currently match the desired spec. In this state, the user should look at the Flink logs and Kubernetes events to determine what went wrong. The user can then perform a new deploy by updating the FlinkApplication. - +#### BlueGreen deployment mode +There is no change in behavior for this state during a BlueGreen deployment. ### Deleting This state indicates that the FlinkApplication resource has been deleted. The operator will clean up the job according to the DeleteMode configured. Once all clean up steps have been performed the FlinkApplication will be deleted. +#### BlueGreen deployment mode +In this mode, if there are two application versions running, both versions are deleted (as per the `DeleteMode` configuration). +### DualRunning +This state is only ever reached when the FlinkApplication is deployed with the BlueGreen deployment mode. In this state, +there are two application versions running — `blue` and `green`. Once a user is ready to tear down one of the versions, they +set a `tearDownVersionHash`. If this is set, the operator then tears down the application version corresponding to +the `tearDownVersionHash`. Once the teardown is complete, we transition back to the `Running` state. diff --git a/integ/blue_green_deployment_test.go b/integ/blue_green_deployment_test.go new file mode 100644 index 00000000..414b5606 --- /dev/null +++ b/integ/blue_green_deployment_test.go @@ -0,0 +1,147 @@ +package integ + +import ( + "time" + + "github.com/lyft/flinkk8soperator/pkg/apis/app/v1beta1" + "github.com/prometheus/common/log" + . "gopkg.in/check.v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func WaitForUpdate(c *C, s *IntegSuite, name string, updateFn func(app *v1beta1.FlinkApplication), phase v1beta1.FlinkApplicationPhase, failurePhase v1beta1.FlinkApplicationPhase) *v1beta1.FlinkApplication { + + // update with new image. + app, err := s.Util.Update(name, updateFn) + c.Assert(err, IsNil) + + for { + // keep trying until the new job is launched + newApp, err := s.Util.GetFlinkApplication(name) + c.Assert(err, IsNil) + if newApp.Status.VersionStatuses[s.Util.GetCurrentStatusIndex(app)].JobStatus.JobID != "" { + break + } + time.Sleep(100 * time.Millisecond) + } + + c.Assert(s.Util.WaitForPhase(name, phase, failurePhase), IsNil) + c.Assert(s.Util.WaitForAllTasksRunning(name), IsNil) + + newApp, _ := s.Util.GetFlinkApplication(name) + return newApp +} + +func (s *IntegSuite) TestUpdateWithBlueGreenDeploymentMode(c *C) { + + testName := "bluegreenupdate" + const finalizer = "bluegreen.finalizers.test.com" + + // start a simple app + config, err := s.Util.ReadFlinkApplication("test_app.yaml") + c.Assert(err, IsNil, Commentf("Failed to read test app yaml")) + + config.Name = testName + "job" + config.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + config.ObjectMeta.Labels["integTest"] = testName + config.Finalizers = append(config.Finalizers, finalizer) + + c.Assert(s.Util.CreateFlinkApplication(config), IsNil, + Commentf("Failed to create flink application")) + + c.Assert(s.Util.WaitForPhase(config.Name, v1beta1.FlinkApplicationRunning, v1beta1.FlinkApplicationDeployFailed), IsNil) + c.Assert(s.Util.WaitForAllTasksRunning(config.Name), IsNil) + + pods, err := s.Util.KubeClient.CoreV1().Pods(s.Util.Namespace.Name). + List(v1.ListOptions{LabelSelector: "integTest=" + testName}) + c.Assert(err, IsNil) + c.Assert(len(pods.Items), Equals, 3) + for _, pod := range pods.Items { + c.Assert(pod.Spec.Containers[0].Image, Equals, config.Spec.Image) + } + + // test updating the app with a new image + newApp := WaitForUpdate(c, s, config.Name, func(app *v1beta1.FlinkApplication) { + app.Spec.Image = NewImage + }, v1beta1.FlinkApplicationDualRunning, v1beta1.FlinkApplicationDeployFailed) + + c.Assert(newApp.Spec.Image, Equals, NewImage) + c.Assert(newApp.Status.SavepointPath, NotNil) + + pods, err = s.Util.KubeClient.CoreV1().Pods(s.Util.Namespace.Name). + List(v1.ListOptions{LabelSelector: "integTest=" + testName}) + c.Assert(err, IsNil) + // We have 2 applications running + c.Assert(len(pods.Items), Equals, 6) + c.Assert(s.Util.WaitForPhase(config.Name, v1beta1.FlinkApplicationDualRunning, v1beta1.FlinkApplicationDeployFailed), IsNil) + c.Assert(s.Util.GetJobID(newApp), NotNil) + c.Assert(newApp.Status.UpdatingVersion, Equals, v1beta1.BlueFlinkApplication) + c.Assert(newApp.Status.DeployVersion, Equals, v1beta1.GreenFlinkApplication) + + // TearDownVersionHash + teardownVersion := newApp.Status.DeployVersion + hashToTeardown := newApp.Status.DeployHash + oldHash := newApp.Status.DeployHash + log.Infof("Tearing down version %s", teardownVersion) + newApp = WaitForUpdate(c, s, config.Name, func(app *v1beta1.FlinkApplication) { + app.Spec.TearDownVersionHash = hashToTeardown + }, v1beta1.FlinkApplicationRunning, v1beta1.FlinkApplicationDeployFailed) + + // wait for the old cluster to be cleaned up + for { + pods, err := s.Util.KubeClient.CoreV1().Pods(s.Util.Namespace.Name). + List(v1.ListOptions{LabelSelector: "flink-app-hash=" + oldHash}) + c.Assert(err, IsNil) + if len(pods.Items) == 0 { + break + } + time.Sleep(100 * time.Millisecond) + } + + c.Assert(s.Util.WaitForPhase(config.Name, v1beta1.FlinkApplicationRunning, v1beta1.FlinkApplicationDeployFailed), IsNil) + c.Assert(newApp.Status.TeardownHash, NotNil) + c.Assert(newApp.Status.DeployVersion, Equals, v1beta1.BlueFlinkApplication) + c.Assert(newApp.Status.VersionStatuses[0].JobStatus.JobID, NotNil) + c.Assert(newApp.Status.VersionStatuses[1].JobStatus, Equals, v1beta1.FlinkJobStatus{}) + + pods, err = s.Util.KubeClient.CoreV1().Pods(s.Util.Namespace.Name). + List(v1.ListOptions{LabelSelector: "flink-app-hash=" + oldHash}) + for _, pod := range pods.Items { + log.Infof("Pod name %s", pod.Name) + c.Assert(pod.Labels["flink-application-version"], Not(Equals), teardownVersion) + } + + c.Assert(err, IsNil) + c.Assert(len(pods.Items), Equals, 0) + + // cleanup + c.Assert(s.Util.FlinkApps().Delete(newApp.Name, &v1.DeleteOptions{}), IsNil) + var app *v1beta1.FlinkApplication + for { + app, err = s.Util.GetFlinkApplication(config.Name) + c.Assert(err, IsNil) + if len(app.Finalizers) == 1 && app.Finalizers[0] == finalizer { + break + } + time.Sleep(100 * time.Millisecond) + } + + job := s.Util.GetJobOverview(app) + c.Assert(job["status"], Equals, "CANCELED") + c.Assert(app.Status.SavepointPath, NotNil) + + // delete our finalizer + app.Finalizers = []string{} + _, err = s.Util.FlinkApps().Update(app) + c.Assert(err, IsNil) + + for { + pods, err := s.Util.KubeClient.CoreV1().Pods(s.Util.Namespace.Name). + List(v1.ListOptions{LabelSelector: "integTest=" + testName}) + c.Assert(err, IsNil) + if len(pods.Items) == 0 { + break + } + } + log.Info("All pods torn down") +} diff --git a/integ/utils/utils.go b/integ/utils/utils.go index fb632179..d32a5674 100644 --- a/integ/utils/utils.go +++ b/integ/utils/utils.go @@ -386,7 +386,10 @@ func (f *TestUtil) FlinkAPIGet(app *flinkapp.FlinkApplication, endpoint string) url := fmt.Sprintf("http://localhost:8001/api/v1/namespaces/%s/"+ "services/%s:8081/proxy/%s", f.Namespace.Name, app.Name, endpoint) + if flinkapp.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + url = f.getURLForBlueGreenDeployment(app, endpoint) + } resp, err := resty.SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)).R().Get(url) if err != nil { return nil, err @@ -405,6 +408,17 @@ func (f *TestUtil) FlinkAPIGet(app *flinkapp.FlinkApplication, endpoint string) return result, nil } +func (f *TestUtil) getURLForBlueGreenDeployment(app *flinkapp.FlinkApplication, endpoint string) string { + versionSuffix := string(app.Status.UpdatingVersion) + if versionSuffix == "" { + versionSuffix = string(app.Status.DeployVersion) + } + return fmt.Sprintf("http://localhost:8001/api/v1/namespaces/%s/"+ + "services/%s-%s:8081/proxy/%s", + f.Namespace.Name, app.Name, versionSuffix, endpoint) + +} + func (f *TestUtil) FlinkAPIPatch(app *flinkapp.FlinkApplication, endpoint string) (interface{}, error) { url := fmt.Sprintf("http://localhost:8001/api/v1/namespaces/%s/"+ @@ -453,7 +467,7 @@ func (f *TestUtil) WaitForAllTasksRunning(name string) error { return err } - endpoint := fmt.Sprintf("jobs/%s", flinkApp.Status.JobStatus.JobID) + endpoint := fmt.Sprintf("jobs/%s", f.GetJobID(flinkApp)) for { res, err := f.FlinkAPIGet(flinkApp, endpoint) if err != nil { @@ -514,9 +528,42 @@ func (f *TestUtil) GetJobOverview(app *flinkapp.FlinkApplication) map[string]int jobList := jobMap["jobs"].([]interface{}) for _, j := range jobList { job := j.(map[string]interface{}) - if job["id"] == app.Status.JobStatus.JobID { + if job["id"] == f.GetJobID(app) { return job } } return nil } + +func (f *TestUtil) Min(x, y int32) int32 { + if x < y { + return x + } + return y +} + +func (f *TestUtil) GetJobID(app *flinkapp.FlinkApplication) string { + if flinkapp.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + return app.Status.VersionStatuses[f.GetCurrentStatusIndex(app)].JobStatus.JobID + } + + return app.Status.JobStatus.JobID +} + +func (f *TestUtil) GetCurrentStatusIndex(app *flinkapp.FlinkApplication) int32 { + if flinkapp.IsRunningPhase(app.Status.Phase) || app.Status.DeployHash == "" || + app.Status.Phase == flinkapp.FlinkApplicationSavepointing || app.Status.Phase == flinkapp.FlinkApplicationDeleting { + return 0 + } + + if app.Status.Phase == flinkapp.FlinkApplicationDualRunning { + return 1 + } + + // activeJobs and maxRunningJobs would be different once a TearDownVersionHash has happened and + // the app has moved back to a Running state. + activeJobs := int32(len(app.Status.VersionStatuses)) + maxRunningJobs := flinkapp.GetMaxRunningJobs(app.Spec.DeploymentMode) + index := f.Min(activeJobs, maxRunningJobs) - 1 + return index +} diff --git a/pkg/apis/app/v1beta1/types.go b/pkg/apis/app/v1beta1/types.go index 14e47f37..8b646b1b 100644 --- a/pkg/apis/app/v1beta1/types.go +++ b/pkg/apis/app/v1beta1/types.go @@ -58,6 +58,7 @@ type FlinkApplicationSpec struct { AllowNonRestoredState bool `json:"allowNonRestoredState,omitempty"` ForceRollback bool `json:"forceRollback"` MaxCheckpointRestoreAgeSeconds *int32 `json:"maxCheckpointRestoreAgeSeconds,omitempty"` + TearDownVersionHash string `json:"tearDownVersionHash,omitempty"` } type FlinkConfig map[string]interface{} @@ -168,30 +169,34 @@ type FlinkJobStatus struct { } type FlinkApplicationStatus struct { - Phase FlinkApplicationPhase `json:"phase"` - StartedAt *metav1.Time `json:"startedAt,omitempty"` - LastUpdatedAt *metav1.Time `json:"lastUpdatedAt,omitempty"` - Reason string `json:"reason,omitempty"` - DeployVersion string `json:"deployVersion,omitempty"` - UpdatingVersion string `json:"updatingVersion,omitempty"` - // To ensure backward compatibility, repeat ClusterStatus and JobStatus + Phase FlinkApplicationPhase `json:"phase"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + LastUpdatedAt *metav1.Time `json:"lastUpdatedAt,omitempty"` + Reason string `json:"reason,omitempty"` + DeployVersion FlinkApplicationVersion `json:"deployVersion,omitempty"` + UpdatingVersion FlinkApplicationVersion `json:"updatingVersion,omitempty"` ClusterStatus FlinkClusterStatus `json:"clusterStatus,omitempty"` JobStatus FlinkJobStatus `json:"jobStatus,omitempty"` VersionStatuses []FlinkApplicationVersionStatus `json:"versionStatuses,omitempty"` FailedDeployHash string `json:"failedDeployHash,omitempty"` RollbackHash string `json:"rollbackHash,omitempty"` DeployHash string `json:"deployHash"` + UpdatingHash string `json:"updatingHash,omitempty"` + TeardownHash string `json:"teardownHash,omitempty"` SavepointTriggerID string `json:"savepointTriggerId,omitempty"` SavepointPath string `json:"savepointPath,omitempty"` RetryCount int32 `json:"retryCount,omitempty"` LastSeenError *FlinkApplicationError `json:"lastSeenError,omitempty"` + // We store deployment mode in the status to prevent incompatible migrations from + // Dual --> BlueGreen and BlueGreen --> Dual + DeploymentMode DeploymentMode `json:"deploymentMode,omitempty"` } type FlinkApplicationVersion string const ( - BlueFlinkApplication FlinkApplicationVersion = "Blue" - GreenFlinkApplication FlinkApplicationVersion = "Green" + BlueFlinkApplication FlinkApplicationVersion = "blue" + GreenFlinkApplication FlinkApplicationVersion = "green" ) type FlinkApplicationVersionStatus struct { @@ -245,7 +250,6 @@ const ( FlinkApplicationRollingBackJob FlinkApplicationPhase = "RollingBackJob" FlinkApplicationDeployFailed FlinkApplicationPhase = "DeployFailed" FlinkApplicationDualRunning FlinkApplicationPhase = "DualRunning" - FlinkApplicationTeardown FlinkApplicationPhase = "Teardown" ) var FlinkApplicationPhases = []FlinkApplicationPhase{ @@ -261,7 +265,6 @@ var FlinkApplicationPhases = []FlinkApplicationPhase{ FlinkApplicationDeployFailed, FlinkApplicationRollingBackJob, FlinkApplicationDualRunning, - FlinkApplicationTeardown, } func IsRunningPhase(phase FlinkApplicationPhase) bool { @@ -351,4 +354,5 @@ const ( GetTaskManagers FlinkMethod = "GetTaskManagers" GetCheckpointCounts FlinkMethod = "GetCheckpointCounts" GetJobOverview FlinkMethod = "GetJobOverview" + SavepointJob FlinkMethod = "SavepointJob" ) diff --git a/pkg/controller/flink/client/api.go b/pkg/controller/flink/client/api.go index 76048d0b..5084f768 100644 --- a/pkg/controller/flink/client/api.go +++ b/pkg/controller/flink/client/api.go @@ -48,6 +48,7 @@ const jobSubmissionException = "org.apache.flink.runtime.client.JobSubmissionExc type FlinkAPIInterface interface { CancelJobWithSavepoint(ctx context.Context, url string, jobID string) (string, error) + SavepointJob(ctx context.Context, url string, jobID string) (string, error) ForceCancelJob(ctx context.Context, url string, jobID string) error SubmitJob(ctx context.Context, url string, jarID string, submitJobRequest SubmitJobRequest) (*SubmitJobResponse, error) CheckSavepointStatus(ctx context.Context, url string, jobID, triggerID string) (*SavepointResponse, error) @@ -82,6 +83,8 @@ type flinkJobManagerClientMetrics struct { getClusterFailureCounter labeled.Counter getCheckpointsSuccessCounter labeled.Counter getCheckpointsFailureCounter labeled.Counter + savepointJobSuccessCounter labeled.Counter + savepointJobFailureCounter labeled.Counter } func newFlinkJobManagerClientMetrics(scope promutils.Scope) *flinkJobManagerClientMetrics { @@ -104,6 +107,8 @@ func newFlinkJobManagerClientMetrics(scope promutils.Scope) *flinkJobManagerClie getClusterFailureCounter: labeled.NewCounter("get_cluster_failure", "Get cluster overview failed", flinkJmClientScope), getCheckpointsSuccessCounter: labeled.NewCounter("get_checkpoints_success", "Get checkpoint request succeeded", flinkJmClientScope), getCheckpointsFailureCounter: labeled.NewCounter("get_checkpoints_failed", "Get checkpoint request failed", flinkJmClientScope), + savepointJobSuccessCounter: labeled.NewCounter("savepoint_job_success", "Savepoint job request succeeded", flinkJmClientScope), + savepointJobFailureCounter: labeled.NewCounter("savepoint_job_failed", "Savepoint job request failed", flinkJmClientScope), } } @@ -181,7 +186,7 @@ func (c *FlinkJobManagerClient) CancelJobWithSavepoint(ctx context.Context, url path := fmt.Sprintf(savepointURL, jobID) url = url + path - cancelJobRequest := CancelJobRequest{ + cancelJobRequest := SavepointJobRequest{ CancelJob: true, } response, err := c.executeRequest(ctx, httpPost, url, cancelJobRequest) @@ -194,7 +199,7 @@ func (c *FlinkJobManagerClient) CancelJobWithSavepoint(ctx context.Context, url logger.Errorf(ctx, fmt.Sprintf("Cancel job failed with response %v", response)) return "", GetRetryableError(err, v1beta1.CancelJobWithSavepoint, response.Status(), 5) } - var cancelJobResponse CancelJobResponse + var cancelJobResponse SavepointJobResponse if err = json.Unmarshal(response.Body(), &cancelJobResponse); err != nil { logger.Errorf(ctx, "Unable to Unmarshal cancelJobResponse %v, err: %v", response, err) return "", GetRetryableError(err, v1beta1.CancelJobWithSavepoint, JSONUnmarshalError, 5) @@ -227,7 +232,6 @@ func (c *FlinkJobManagerClient) ForceCancelJob(ctx context.Context, url string, func (c *FlinkJobManagerClient) SubmitJob(ctx context.Context, url string, jarID string, submitJobRequest SubmitJobRequest) (*SubmitJobResponse, error) { path := fmt.Sprintf(submitJobURL, jarID) url = url + path - response, err := c.executeRequest(ctx, httpPost, url, submitJobRequest) if err != nil { c.metrics.submitJobFailureCounter.Inc(ctx) @@ -385,6 +389,32 @@ func (c *FlinkJobManagerClient) GetJobOverview(ctx context.Context, url string, return &jobOverviewResponse, nil } +func (c *FlinkJobManagerClient) SavepointJob(ctx context.Context, url string, jobID string) (string, error) { + path := fmt.Sprintf(savepointURL, jobID) + + url = url + path + savepointJobRequest := SavepointJobRequest{ + CancelJob: false, + } + response, err := c.executeRequest(ctx, httpPost, url, savepointJobRequest) + if err != nil { + c.metrics.savepointJobFailureCounter.Inc(ctx) + return "", GetRetryableError(err, v1beta1.CancelJobWithSavepoint, GlobalFailure, 5) + } + if response != nil && !response.IsSuccess() { + c.metrics.cancelJobFailureCounter.Inc(ctx) + logger.Errorf(ctx, fmt.Sprintf("Savepointing job failed with response %v", response)) + return "", GetRetryableError(err, v1beta1.SavepointJob, response.Status(), 5) + } + var savepointJobResponse SavepointJobResponse + if err = json.Unmarshal(response.Body(), &savepointJobResponse); err != nil { + logger.Errorf(ctx, "Unable to Unmarshal savepointJobResponse %v, err: %v", response, err) + return "", GetRetryableError(err, v1beta1.SavepointJob, JSONUnmarshalError, 5) + } + c.metrics.savepointJobSuccessCounter.Inc(ctx) + return savepointJobResponse.TriggerID, nil +} + func NewFlinkJobManagerClient(config config.RuntimeConfig) FlinkAPIInterface { metrics := newFlinkJobManagerClientMetrics(config.MetricsScope) return &FlinkJobManagerClient{ diff --git a/pkg/controller/flink/client/api_test.go b/pkg/controller/flink/client/api_test.go index f2fd582e..f7e13930 100644 --- a/pkg/controller/flink/client/api_test.go +++ b/pkg/controller/flink/client/api_test.go @@ -405,7 +405,7 @@ func TestCancelJobHappyCase(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() ctx := context.Background() - response := CancelJobResponse{ + response := SavepointJobResponse{ TriggerID: "133", } responder, _ := httpmock.NewJsonResponder(203, response) diff --git a/pkg/controller/flink/client/entities.go b/pkg/controller/flink/client/entities.go index afcb87fa..ffbdb8f3 100644 --- a/pkg/controller/flink/client/entities.go +++ b/pkg/controller/flink/client/entities.go @@ -31,7 +31,7 @@ const ( Reconciling JobState = "RECONCILING" ) -type CancelJobRequest struct { +type SavepointJobRequest struct { CancelJob bool `json:"cancel-job"` TargetDirectory string `json:"target-directory,omitempty"` } @@ -63,7 +63,7 @@ type FailureCause struct { StackTrace string `json:"stack-trace"` } -type CancelJobResponse struct { +type SavepointJobResponse struct { TriggerID string `json:"request-id"` } diff --git a/pkg/controller/flink/client/mock/mock_api.go b/pkg/controller/flink/client/mock/mock_api.go index 67b7b773..873d96a8 100644 --- a/pkg/controller/flink/client/mock/mock_api.go +++ b/pkg/controller/flink/client/mock/mock_api.go @@ -17,7 +17,7 @@ type GetJobConfigFunc func(ctx context.Context, url string, jobID string) (*clie type GetTaskManagersFunc func(ctx context.Context, url string) (*client.TaskManagersResponse, error) type GetCheckpointCountsFunc func(ctx context.Context, url string, jobID string) (*client.CheckpointResponse, error) type GetJobOverviewFunc func(ctx context.Context, url string, jobID string) (*client.FlinkJobOverview, error) - +type SavepointJobFunc func(ctx context.Context, url string, jobID string) (string, error) type JobManagerClient struct { CancelJobWithSavepointFunc CancelJobWithSavepointFunc ForceCancelJobFunc ForceCancelJobFunc @@ -30,6 +30,7 @@ type JobManagerClient struct { GetTaskManagersFunc GetTaskManagersFunc GetCheckpointCountsFunc GetCheckpointCountsFunc GetJobOverviewFunc GetJobOverviewFunc + SavepointJobFunc SavepointJobFunc } func (m *JobManagerClient) SubmitJob(ctx context.Context, url string, jarID string, submitJobRequest client.SubmitJobRequest) (*client.SubmitJobResponse, error) { @@ -108,3 +109,11 @@ func (m *JobManagerClient) GetJobOverview(ctx context.Context, url string, jobID } return nil, nil } + +func (m *JobManagerClient) SavepointJob(ctx context.Context, url string, jobID string) (string, error) { + if m.SavepointJobFunc != nil { + return m.SavepointJobFunc(ctx, url, jobID) + } + + return "", nil +} diff --git a/pkg/controller/flink/container_utils.go b/pkg/controller/flink/container_utils.go index e760b6f9..64b22051 100644 --- a/pkg/controller/flink/container_utils.go +++ b/pkg/controller/flink/container_utils.go @@ -47,7 +47,11 @@ func getFlinkContainerName(containerName string) string { } func getCommonAppLabels(app *v1beta1.FlinkApplication) map[string]string { - return k8.GetAppLabel(app.Name) + labels := common.DuplicateMap(k8.GetAppLabel(app.Name)) + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { + labels[FlinkApplicationVersion] = string(app.Status.UpdatingVersion) + } + return labels } func getCommonAnnotations(app *v1beta1.FlinkApplication) map[string]string { @@ -58,8 +62,8 @@ func getCommonAnnotations(app *v1beta1.FlinkApplication) map[string]string { if app.Spec.RestartNonce != "" { annotations[RestartNonce] = app.Spec.RestartNonce } - if v1beta1.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { - annotations[FlinkApplicationVersion] = app.Status.UpdatingVersion + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { + annotations[FlinkApplicationVersion] = string(app.Status.UpdatingVersion) } return annotations } @@ -227,14 +231,14 @@ func InjectOperatorCustomizedConfig(deployment *appsv1.Deployment, app *v1beta1. // Injects labels and environment variables required for blue green deploys func GetDeploySpecificEnv(app *v1beta1.FlinkApplication) []v1.EnvVar { - if !v1beta1.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + if !v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { return []v1.EnvVar{} } return []v1.EnvVar{ { Name: FlinkApplicationVersionEnv, - Value: app.Status.UpdatingVersion, + Value: string(app.Status.UpdatingVersion), }, } diff --git a/pkg/controller/flink/flink.go b/pkg/controller/flink/flink.go index e8f36ed6..9bb8418f 100644 --- a/pkg/controller/flink/flink.go +++ b/pkg/controller/flink/flink.go @@ -26,6 +26,8 @@ import ( ) const proxyURL = "http://localhost:%d/api/v1/namespaces/%s/services/%s:8081/proxy" +const proxyVersionURL = "http://localhost:%d/api/v1/namespaces/%s/services/%s-%s:8081/proxy" +const externalVersionURL = "%s-%s" const port = 8081 const indexOffset = 1 @@ -46,10 +48,10 @@ type ControllerInterface interface { CreateCluster(ctx context.Context, application *v1beta1.FlinkApplication) error // Cancels the running/active jobs in the Cluster for the Application after savepoint is created - CancelWithSavepoint(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, error) + Savepoint(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (string, error) // Force cancels the running/active job without taking a savepoint - ForceCancel(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error + ForceCancel(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error // Starts the Job in the Flink Cluster StartFlinkJob(ctx context.Context, application *v1beta1.FlinkApplication, hash string, @@ -58,7 +60,7 @@ type ControllerInterface interface { // Savepoint creation is asynchronous. // Polls the status of the Savepoint, using the triggerID - GetSavepointStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) + GetSavepointStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) // Check if the Flink Kubernetes Cluster is Ready. // Checks if all the pods of task and job managers are ready. @@ -111,6 +113,20 @@ type ControllerInterface interface { // Update clusterStatus on the latest VersionStatuses UpdateLatestClusterStatus(ctx context.Context, app *v1beta1.FlinkApplication, jobStatus v1beta1.FlinkClusterStatus) + + // Update Version and Hash for application + UpdateLatestVersionAndHash(application *v1beta1.FlinkApplication, version v1beta1.FlinkApplicationVersion, hash string) + + // Delete Resources with Hash + DeleteResourcesForAppWithHash(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error + // Delete status for torn down cluster/job + DeleteStatusPostTeardown(ctx context.Context, application *v1beta1.FlinkApplication, hash string) + // Get job given hash + GetJobToDeleteForApplication(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) + // Get hash given the version + GetVersionAndJobIDForHash(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, string, error) + // Get version and hash after teardown is complete + GetVersionAndHashPostTeardown(ctx context.Context, application *v1beta1.FlinkApplication) (v1beta1.FlinkApplicationVersion, string) } func NewController(k8sCluster k8.ClusterInterface, eventRecorder record.EventRecorder, config controllerConfig.RuntimeConfig) ControllerInterface { @@ -160,28 +176,34 @@ func (f *Controller) getURLFromApp(application *v1beta1.FlinkApplication, hash s return fmt.Sprintf("http://%s.%s:%d", service, application.Namespace, port) } -func (f *Controller) getClusterOverviewURL(app *v1beta1.FlinkApplication) string { - externalURL := f.getExternalURLFromApp(app) +func (f *Controller) getClusterOverviewURL(app *v1beta1.FlinkApplication, version string) string { + externalURL := f.getExternalURLFromApp(app, version) if externalURL != "" { return fmt.Sprintf(externalURL + client.WebUIAnchor + client.GetClusterOverviewURL) } return "" } -func (f *Controller) getJobOverviewURL(ctx context.Context, app *v1beta1.FlinkApplication) string { - externalURL := f.getExternalURLFromApp(app) +func (f *Controller) getJobOverviewURL(app *v1beta1.FlinkApplication, version string, jobID string) string { + externalURL := f.getExternalURLFromApp(app, version) if externalURL != "" { - return fmt.Sprintf(externalURL+client.WebUIAnchor+client.GetJobsOverviewURL, f.GetLatestJobID(ctx, app)) + return fmt.Sprintf(externalURL+client.WebUIAnchor+client.GetJobsOverviewURL, jobID) } return "" } -func (f *Controller) getExternalURLFromApp(application *v1beta1.FlinkApplication) string { +func (f *Controller) getExternalURLFromApp(application *v1beta1.FlinkApplication, version string) string { cfg := controllerConfig.GetConfig() // Local environment if cfg.UseProxy { + if version != "" { + return fmt.Sprintf(proxyVersionURL, cfg.ProxyPort.Port, application.Namespace, application.Name, version) + } return fmt.Sprintf(proxyURL, cfg.ProxyPort.Port, application.Namespace, application.Name) } + if version != "" { + return GetFlinkUIIngressURL(fmt.Sprintf(externalVersionURL, application.Name, version)) + } return GetFlinkUIIngressURL(application.Name) } @@ -235,29 +257,14 @@ func (f *Controller) GetJobForApplication(ctx context.Context, application *v1be return jobResponse, nil } -// The operator for now assumes and is intended to run single application per Flink Cluster. -// Once we move to run multiple applications, this has to be removed/updated -func (f *Controller) getJobIDForApplication(ctx context.Context, application *v1beta1.FlinkApplication) (string, error) { - if f.GetLatestJobID(ctx, application) != "" { - return f.GetLatestJobID(ctx, application), nil - } - - return "", errors.New("active job id not available") -} - -func (f *Controller) CancelWithSavepoint(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, error) { - jobID, err := f.getJobIDForApplication(ctx, application) - if err != nil { - return "", err +func (f *Controller) Savepoint(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (string, error) { + if isCancel { + return f.flinkClient.CancelJobWithSavepoint(ctx, f.getURLFromApp(application, hash), jobID) } - return f.flinkClient.CancelJobWithSavepoint(ctx, f.getURLFromApp(application, hash), jobID) + return f.flinkClient.SavepointJob(ctx, f.getURLFromApp(application, hash), jobID) } -func (f *Controller) ForceCancel(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { - jobID, err := f.getJobIDForApplication(ctx, application) - if err != nil { - return err - } +func (f *Controller) ForceCancel(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error { return f.flinkClient.ForceCancelJob(ctx, f.getURLFromApp(application, hash), jobID) } @@ -311,11 +318,7 @@ func (f *Controller) StartFlinkJob(ctx context.Context, application *v1beta1.Fli return response.JobID, nil } -func (f *Controller) GetSavepointStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { - jobID, err := f.getJobIDForApplication(ctx, application) - if err != nil { - return nil, err - } +func (f *Controller) GetSavepointStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { return f.flinkClient.CheckSavepointStatus(ctx, f.getURLFromApp(application, hash), jobID, application.Status.SavepointTriggerID) } @@ -380,9 +383,10 @@ func listToFlinkDeployment(ds []v1.Deployment, hash string) *common.FlinkDeploym func getCurrentHash(app *v1beta1.FlinkApplication) string { appHash := HashForApplication(app) - if appHash == app.Status.FailedDeployHash { + if appHash == app.Status.FailedDeployHash || appHash == app.Status.TeardownHash { return app.Status.DeployHash } + return appHash } @@ -400,7 +404,7 @@ func (f *Controller) GetCurrentDeploymentsForApp(ctx context.Context, applicatio } cur := listToFlinkDeployment(deployments.Items, curHash) - if cur != nil && application.Status.FailedDeployHash == "" && + if cur != nil && application.Status.FailedDeployHash == "" && application.Status.TeardownHash == "" && (!f.deploymentMatches(ctx, cur.Jobmanager, application, curHash) || !f.deploymentMatches(ctx, cur.Taskmanager, application, curHash)) { // we had a hash collision (i.e., the previous application has the same hash as the new one) // this is *very* unlikely to occur (1/2^32) @@ -499,14 +503,20 @@ func isCheckpointOldToRecover(checkpointTime int64, maxCheckpointRecoveryAgeSec } func (f *Controller) LogEvent(ctx context.Context, app *v1beta1.FlinkApplication, eventType string, reason string, message string) { + // Augment message with version for blue-green deployments + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { + version := app.Status.UpdatingVersion + message = fmt.Sprintf("%s for version %s", message, version) + } + f.eventRecorder.Event(app, eventType, reason, message) logger.Infof(ctx, "Logged %s event: %s: %s", eventType, reason, message) } // Gets and updates the cluster status func (f *Controller) CompareAndUpdateClusterStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (bool, error) { - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { - return f.compareAndUpdateBlueGreenClusterStatus(ctx, application, hash) + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + return f.compareAndUpdateBlueGreenClusterStatus(ctx, application) } // Error retrieving cluster / taskmanagers overview (after startup/readiness) --> Red // If there is an error this loop will return with Health set to Red @@ -518,7 +528,7 @@ func (f *Controller) CompareAndUpdateClusterStatus(ctx context.Context, applicat return false, err } - application.Status.ClusterStatus.ClusterOverviewURL = f.getClusterOverviewURL(application) + application.Status.ClusterStatus.ClusterOverviewURL = f.getClusterOverviewURL(application, "") application.Status.ClusterStatus.NumberOfTaskManagers = deployment.Taskmanager.Status.AvailableReplicas // Get Cluster overview response, err := f.flinkClient.GetClusterOverview(ctx, f.getURLFromApp(application, hash)) @@ -562,8 +572,8 @@ func getHealthyTaskManagerCount(response *client.TaskManagersResponse) int32 { } func (f *Controller) CompareAndUpdateJobStatus(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (bool, error) { - if v1beta1.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { - return f.compareAndUpdateBlueGreenJobStatus(ctx, app, hash) + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { + return f.compareAndUpdateBlueGreenJobStatus(ctx, app) } if app.Status.JobStatus.LastFailingTime == nil { initTime := metav1.NewTime(time.Time{}) @@ -582,7 +592,7 @@ func (f *Controller) CompareAndUpdateJobStatus(ctx context.Context, app *v1beta1 } // Job status - app.Status.JobStatus.JobOverviewURL = f.getJobOverviewURL(ctx, app) + app.Status.JobStatus.JobOverviewURL = f.getJobOverviewURL(app, "", app.Status.JobStatus.JobID) app.Status.JobStatus.State = v1beta1.JobState(jobResponse.State) jobStartTime := metav1.NewTime(time.Unix(jobResponse.StartTime/1000, 0)) app.Status.JobStatus.StartTime = &jobStartTime @@ -666,7 +676,7 @@ func getCurrentStatusIndex(app *v1beta1.FlinkApplication) int32 { return 1 } - // activeJobs and maxRunningJobs would be different once a Teardown has happened and + // activeJobs and maxRunningJobs would be different once a TearDownVersionHash has happened and // the app has moved back to a Running state. activeJobs := int32(len(app.Status.VersionStatuses)) maxRunningJobs := v1beta1.GetMaxRunningJobs(app.Spec.DeploymentMode) @@ -682,14 +692,14 @@ func Min(x, y int32) int32 { } func (f *Controller) GetLatestClusterStatus(ctx context.Context, application *v1beta1.FlinkApplication) v1beta1.FlinkClusterStatus { - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { return application.Status.VersionStatuses[getCurrentStatusIndex(application)].ClusterStatus } return application.Status.ClusterStatus } func (f *Controller) GetLatestJobStatus(ctx context.Context, application *v1beta1.FlinkApplication) v1beta1.FlinkJobStatus { - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { return application.Status.VersionStatuses[getCurrentStatusIndex(application)].JobStatus } return application.Status.JobStatus @@ -697,7 +707,7 @@ func (f *Controller) GetLatestJobStatus(ctx context.Context, application *v1beta } func (f *Controller) UpdateLatestJobStatus(ctx context.Context, app *v1beta1.FlinkApplication, jobStatus v1beta1.FlinkJobStatus) { - if v1beta1.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { app.Status.VersionStatuses[getCurrentStatusIndex(app)].JobStatus = jobStatus return } @@ -705,7 +715,7 @@ func (f *Controller) UpdateLatestJobStatus(ctx context.Context, app *v1beta1.Fli } func (f *Controller) UpdateLatestClusterStatus(ctx context.Context, app *v1beta1.FlinkApplication, clusterStatus v1beta1.FlinkClusterStatus) { - if v1beta1.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { app.Status.VersionStatuses[getCurrentStatusIndex(app)].ClusterStatus = clusterStatus return } @@ -713,26 +723,26 @@ func (f *Controller) UpdateLatestClusterStatus(ctx context.Context, app *v1beta1 } func (f *Controller) GetLatestJobID(ctx context.Context, application *v1beta1.FlinkApplication) string { - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { return application.Status.VersionStatuses[getCurrentStatusIndex(application)].JobStatus.JobID } return application.Status.JobStatus.JobID } func (f *Controller) UpdateLatestJobID(ctx context.Context, app *v1beta1.FlinkApplication, jobID string) { - if v1beta1.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { app.Status.VersionStatuses[getCurrentStatusIndex(app)].JobStatus.JobID = jobID } app.Status.JobStatus.JobID = jobID } -func (f *Controller) compareAndUpdateBlueGreenClusterStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (bool, error) { +func (f *Controller) compareAndUpdateBlueGreenClusterStatus(ctx context.Context, application *v1beta1.FlinkApplication) (bool, error) { isEqual := false for currIndex := range application.Status.VersionStatuses { if application.Status.VersionStatuses[currIndex].VersionHash == "" { continue } - + hash := application.Status.VersionStatuses[currIndex].VersionHash oldClusterStatus := application.Status.VersionStatuses[currIndex].ClusterStatus application.Status.VersionStatuses[currIndex].ClusterStatus.Health = v1beta1.Red @@ -741,7 +751,8 @@ func (f *Controller) compareAndUpdateBlueGreenClusterStatus(ctx context.Context, return false, err } - application.Status.VersionStatuses[currIndex].ClusterStatus.ClusterOverviewURL = f.getClusterOverviewURL(application) + version := string(application.Status.VersionStatuses[currIndex].Version) + application.Status.VersionStatuses[currIndex].ClusterStatus.ClusterOverviewURL = f.getClusterOverviewURL(application, version) application.Status.VersionStatuses[currIndex].ClusterStatus.NumberOfTaskManagers = deployment.Taskmanager.Status.AvailableReplicas // Get Cluster overview response, err := f.flinkClient.GetClusterOverview(ctx, f.getURLFromApp(application, hash)) @@ -773,14 +784,14 @@ func (f *Controller) compareAndUpdateBlueGreenClusterStatus(ctx context.Context, return isEqual, nil } -func (f *Controller) compareAndUpdateBlueGreenJobStatus(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (bool, error) { +func (f *Controller) compareAndUpdateBlueGreenJobStatus(ctx context.Context, app *v1beta1.FlinkApplication) (bool, error) { isEqual := false var err error for statusIndex := range app.Status.VersionStatuses { if app.Status.VersionStatuses[statusIndex].JobStatus.JobID == "" { continue } - + hash := app.Status.VersionStatuses[statusIndex].VersionHash if app.Status.VersionStatuses[statusIndex].JobStatus.LastFailingTime == nil { initTime := metav1.NewTime(time.Time{}) app.Status.VersionStatuses[statusIndex].JobStatus.LastFailingTime = &initTime @@ -797,7 +808,8 @@ func (f *Controller) compareAndUpdateBlueGreenJobStatus(ctx context.Context, app } // Job status - app.Status.VersionStatuses[statusIndex].JobStatus.JobOverviewURL = f.getJobOverviewURL(ctx, app) + version := string(app.Status.VersionStatuses[statusIndex].Version) + app.Status.VersionStatuses[statusIndex].JobStatus.JobOverviewURL = f.getJobOverviewURL(app, version, app.Status.VersionStatuses[statusIndex].JobStatus.JobID) app.Status.VersionStatuses[statusIndex].JobStatus.State = v1beta1.JobState(jobResponse.State) jobStartTime := metav1.NewTime(time.Unix(jobResponse.StartTime/1000, 0)) app.Status.VersionStatuses[statusIndex].JobStatus.StartTime = &jobStartTime @@ -866,3 +878,110 @@ func (f *Controller) compareAndUpdateBlueGreenJobStatus(ctx context.Context, app } return isEqual, err } + +func (f *Controller) UpdateLatestVersionAndHash(application *v1beta1.FlinkApplication, version v1beta1.FlinkApplicationVersion, hash string) { + currIndex := getCurrentStatusIndex(application) + application.Status.VersionStatuses[currIndex].Version = version + application.Status.VersionStatuses[currIndex].VersionHash = hash + application.Status.UpdatingHash = hash +} + +func (f *Controller) DeleteResourcesForAppWithHash(ctx context.Context, app *v1beta1.FlinkApplication, hash string) error { + appLabel := k8.GetAppLabel(app.Name) + deployments, err := f.k8Cluster.GetDeploymentsWithLabel(ctx, app.Namespace, appLabel) + if err != nil { + return err + } + + oldObjects := make([]metav1.Object, 0) + + for _, d := range deployments.Items { + if d.Labels[FlinkAppHash] == hash && + // verify that this deployment matches the jobmanager or taskmanager naming format + (d.Name == fmt.Sprintf(JobManagerVersionNameFormat, app.Name, d.Labels[FlinkAppHash], d.Labels[FlinkApplicationVersion]) || + d.Name == fmt.Sprintf(TaskManagerVersionNameFormat, app.Name, d.Labels[FlinkAppHash], d.Labels[FlinkApplicationVersion])) { + oldObjects = append(oldObjects, d.DeepCopy()) + } + } + + services, err := f.k8Cluster.GetServicesWithLabel(ctx, app.Namespace, appLabel) + + if err != nil { + return err + } + + for _, d := range services.Items { + if d.Labels[FlinkAppHash] == hash || d.Spec.Selector[FlinkAppHash] == hash { + oldObjects = append(oldObjects, d.DeepCopy()) + } + } + + deletedHashes := make(map[string]bool) + + for _, resource := range oldObjects { + err := f.k8Cluster.DeleteK8Object(ctx, resource.(runtime.Object)) + if err != nil { + f.metrics.deleteResourceFailedCounter.Inc(ctx) + return err + } + f.metrics.deleteResourceSuccessCounter.Inc(ctx) + deletedHashes[resource.GetLabels()[FlinkAppHash]] = true + } + + for k := range deletedHashes { + f.LogEvent(ctx, app, corev1.EventTypeNormal, "ToreDownCluster", + fmt.Sprintf("Deleted old cluster with hash %s", k)) + } + return nil +} + +func (f *Controller) DeleteStatusPostTeardown(ctx context.Context, application *v1beta1.FlinkApplication, hash string) { + var indexToDelete int + for index, status := range application.Status.VersionStatuses { + if status.VersionHash == hash { + indexToDelete = index + } + } + application.Status.VersionStatuses[0] = application.Status.VersionStatuses[indexOffset-indexToDelete] + application.Status.VersionStatuses[1] = v1beta1.FlinkApplicationVersionStatus{} +} + +func (f *Controller) GetJobToDeleteForApplication(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) { + jobID := "" + for _, status := range app.Status.VersionStatuses { + if status.VersionHash == hash { + jobID = status.JobStatus.JobID + } + } + if jobID == "" { + return nil, nil + } + + jobResponse, err := f.flinkClient.GetJobOverview(ctx, f.getURLFromApp(app, hash), jobID) + if err != nil { + return nil, err + } + + return jobResponse, nil +} + +func (f *Controller) GetVersionAndJobIDForHash(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (string, string, error) { + version := "" + jobID := "" + for _, status := range app.Status.VersionStatuses { + if status.VersionHash == hash { + version = string(status.Version) + jobID = status.JobStatus.JobID + } + } + if hash == "" || jobID == "" { + return "", "", errors.New("could not find jobID and hash for application") + } + + return version, jobID, nil +} + +func (f *Controller) GetVersionAndHashPostTeardown(ctx context.Context, application *v1beta1.FlinkApplication) (v1beta1.FlinkApplicationVersion, string) { + versionStatus := application.Status.VersionStatuses[0] + return versionStatus.Version, versionStatus.VersionHash +} diff --git a/pkg/controller/flink/flink_test.go b/pkg/controller/flink/flink_test.go index 57e4c575..09b1dfc7 100644 --- a/pkg/controller/flink/flink_test.go +++ b/pkg/controller/flink/flink_test.go @@ -250,7 +250,7 @@ func TestFlinkGetSavepointStatus(t *testing.T) { }, }, nil } - status, err := flinkControllerForTest.GetSavepointStatus(context.Background(), &flinkApp, "hash") + status, err := flinkControllerForTest.GetSavepointStatus(context.Background(), &flinkApp, "hash", testJobID) assert.Nil(t, err) assert.NotNil(t, status) @@ -267,7 +267,7 @@ func TestFlinkGetSavepointStatusErr(t *testing.T) { assert.Equal(t, jobID, testJobID) return nil, errors.New("Savepoint error") } - status, err := flinkControllerForTest.GetSavepointStatus(context.Background(), &flinkApp, "hash") + status, err := flinkControllerForTest.GetSavepointStatus(context.Background(), &flinkApp, "hash", testJobID) assert.Nil(t, status) assert.NotNil(t, err) @@ -518,12 +518,12 @@ func TestCancelWithSavepoint(t *testing.T) { assert.Equal(t, jobID, testJobID) return "t1", nil } - triggerID, err := flinkControllerForTest.CancelWithSavepoint(context.Background(), &flinkApp, "hash") + triggerID, err := flinkControllerForTest.Savepoint(context.Background(), &flinkApp, "hash", true, testJobID) assert.Nil(t, err) assert.Equal(t, triggerID, "t1") } -func TestCancelWithSavepointErr(t *testing.T) { +func TestSavepointErr(t *testing.T) { flinkControllerForTest := getTestFlinkController() flinkApp := getFlinkTestApp() @@ -531,7 +531,7 @@ func TestCancelWithSavepointErr(t *testing.T) { mockJmClient.CancelJobWithSavepointFunc = func(ctx context.Context, url string, jobID string) (string, error) { return "", errors.New("cancel error") } - triggerID, err := flinkControllerForTest.CancelWithSavepoint(context.Background(), &flinkApp, "hash") + triggerID, err := flinkControllerForTest.Savepoint(context.Background(), &flinkApp, "hash", true, testJobID) assert.EqualError(t, err, "cancel error") assert.Empty(t, triggerID) } @@ -949,3 +949,166 @@ func TestMaxCheckpointRestoreAge(t *testing.T) { // Test valid checkpoint that can be recovered. Recovery age is 10 minutes assert.False(t, isCheckpointOldToRecover(time.Now().Unix()-100, 600)) } + +func TestGetCurrentStatusIndex(t *testing.T) { + app := getFlinkTestApp() + // Dual deployment should always return 0 + app.Status.Phase = v1beta1.FlinkApplicationRunning + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationUpdating + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationClusterStarting + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationSavepointing + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRecovering + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRollingBackJob + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationSubmittingJob + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + + // Tests for bluegreen deployment mode + app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + + // First deploy should always return 0 + app.Status.Phase = v1beta1.FlinkApplicationRunning + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationUpdating + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationClusterStarting + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationSavepointing + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRecovering + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRollingBackJob + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationSubmittingJob + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + + // Subsequent deploys return 0 when in Running or Savepointing phase + app.Status.DeployHash = "hash" + statuses := make([]v1beta1.FlinkApplicationVersionStatus, 2) + app.Status.VersionStatuses = statuses + app.Status.Phase = v1beta1.FlinkApplicationSavepointing + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRunning + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + // Else return 1 + app.Status.Phase = v1beta1.FlinkApplicationUpdating + assert.Equal(t, int32(1), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationClusterStarting + assert.Equal(t, int32(1), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRecovering + assert.Equal(t, int32(1), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationRollingBackJob + assert.Equal(t, int32(1), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationSubmittingJob + assert.Equal(t, int32(1), getCurrentStatusIndex(&app)) + app.Status.Phase = v1beta1.FlinkApplicationDualRunning + assert.Equal(t, int32(1), getCurrentStatusIndex(&app)) + + // Once teardown has happened and the one of the two apps have been deleted + app.Status.VersionStatuses = make([]v1beta1.FlinkApplicationVersionStatus, 1) + app.Status.Phase = v1beta1.FlinkApplicationRunning + assert.Equal(t, int32(0), getCurrentStatusIndex(&app)) + +} + +func TestDeleteStatusPostTeardown(t *testing.T) { + controller := getTestFlinkController() + app := getFlinkTestApp() + app.Status.VersionStatuses = []v1beta1.FlinkApplicationVersionStatus{ + { + Version: v1beta1.BlueFlinkApplication, + VersionHash: "blue-hash", + ClusterStatus: v1beta1.FlinkClusterStatus{ + ClusterOverviewURL: "blue-overview", + }, + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "blue-job-id", + }, + }, + { + Version: v1beta1.GreenFlinkApplication, + VersionHash: "green-hash", + ClusterStatus: v1beta1.FlinkClusterStatus{ + ClusterOverviewURL: "green-overview", + }, + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "green-job-id", + }, + }, + } + expectedStatus := app.Status.VersionStatuses[1] + controller.DeleteStatusPostTeardown(context.Background(), &app, "blue-hash") + assert.Equal(t, expectedStatus, app.Status.VersionStatuses[0]) + assert.Empty(t, app.Status.VersionStatuses[1]) +} + +func TestDeleteResourcesForAppWithHash(t *testing.T) { + flinkControllerForTest := getTestFlinkController() + app := getFlinkTestApp() + app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + app.Status.DeploymentMode = v1beta1.DeploymentModeBlueGreen + app.Status.UpdatingVersion = testVersion + jmDeployment := FetchTaskMangerDeploymentCreateObj(&app, "oldhash") + tmDeployment := FetchJobMangerDeploymentCreateObj(&app, "oldhash") + service := FetchJobManagerServiceCreateObj(&app, "oldhash") + service.Labels[FlinkAppHash] = "oldhash" + service.Name = VersionedJobManagerServiceName(&app, "oldhash") + genericService := FetchJobManagerServiceCreateObj(&app, "oldhash") + genericService.Name = app.Name + + mockK8Cluster := flinkControllerForTest.k8Cluster.(*k8mock.K8Cluster) + + mockK8Cluster.GetDeploymentsWithLabelFunc = func(ctx context.Context, namespace string, labelMap map[string]string) (*v1.DeploymentList, error) { + curJobmanager := FetchJobMangerDeploymentCreateObj(&app, testAppHash) + curTaskmanager := FetchTaskMangerDeploymentCreateObj(&app, testAppHash) + return &v1.DeploymentList{ + Items: []v1.Deployment{ + *jmDeployment, + *tmDeployment, + *curJobmanager, + *curTaskmanager, + }, + }, nil + } + + mockK8Cluster.GetServicesWithLabelFunc = func(ctx context.Context, namespace string, labelMap map[string]string) (*corev1.ServiceList, error) { + curService := FetchJobManagerServiceCreateObj(&app, testAppHash) + curService.Labels[FlinkAppHash] = testAppHash + curService.Name = VersionedJobManagerServiceName(&app, testAppHash) + + generic := FetchJobManagerServiceCreateObj(&app, testAppHash) + return &corev1.ServiceList{ + Items: []corev1.Service{ + *service, + *curService, + *generic, + }, + }, nil + } + + ctr := 0 + mockK8Cluster.DeleteK8ObjectFunc = func(ctx context.Context, object runtime.Object) error { + ctr++ + switch ctr { + case 1: + assert.Equal(t, jmDeployment, object) + case 2: + assert.Equal(t, tmDeployment, object) + case 3: + assert.Equal(t, service, object) + case 4: + assert.Equal(t, genericService, object) + + } + return nil + } + + err := flinkControllerForTest.DeleteResourcesForAppWithHash(context.Background(), &app, "oldhash") + assert.Equal(t, 3, ctr) + assert.Nil(t, err) +} diff --git a/pkg/controller/flink/ingress.go b/pkg/controller/flink/ingress.go index e45e4614..ccb5c0c1 100644 --- a/pkg/controller/flink/ingress.go +++ b/pkg/controller/flink/ingress.go @@ -1,6 +1,7 @@ package flink import ( + "fmt" "regexp" flinkapp "github.com/lyft/flinkk8soperator/pkg/apis/app/v1beta1" @@ -12,6 +13,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +const AppIngressName = "%s-%s" + var inputRegex = regexp.MustCompile(`{{[$]jobCluster}}`) func ReplaceJobURL(value string, input string) string { @@ -27,7 +30,7 @@ func FetchJobManagerIngressCreateObj(app *flinkapp.FlinkApplication) *v1beta1.In podLabels = common.CopyMap(podLabels, k8.GetAppLabel(app.Name)) ingressMeta := v1.ObjectMeta{ - Name: app.Name, + Name: getJobManagerServiceName(app), Labels: podLabels, Namespace: app.Namespace, OwnerReferences: []v1.OwnerReference{ @@ -36,7 +39,7 @@ func FetchJobManagerIngressCreateObj(app *flinkapp.FlinkApplication) *v1beta1.In } backend := v1beta1.IngressBackend{ - ServiceName: app.Name, + ServiceName: getJobManagerServiceName(app), ServicePort: intstr.IntOrString{ Type: intstr.Int, IntVal: getUIPort(app), @@ -45,7 +48,7 @@ func FetchJobManagerIngressCreateObj(app *flinkapp.FlinkApplication) *v1beta1.In ingressSpec := v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{{ - Host: GetFlinkUIIngressURL(app.Name), + Host: GetFlinkUIIngressURL(getIngressName(app)), IngressRuleValue: v1beta1.IngressRuleValue{ HTTP: &v1beta1.HTTPIngressRuleValue{ Paths: []v1beta1.HTTPIngressPath{{ @@ -65,3 +68,10 @@ func FetchJobManagerIngressCreateObj(app *flinkapp.FlinkApplication) *v1beta1.In } } + +func getIngressName(app *flinkapp.FlinkApplication) string { + if flinkapp.IsBlueGreenDeploymentMode(app.Spec.DeploymentMode) { + return fmt.Sprintf(AppIngressName, app.Name, string(app.Status.UpdatingVersion)) + } + return app.Name +} diff --git a/pkg/controller/flink/ingress_test.go b/pkg/controller/flink/ingress_test.go index 3272dfcc..95f1c42f 100644 --- a/pkg/controller/flink/ingress_test.go +++ b/pkg/controller/flink/ingress_test.go @@ -3,6 +3,8 @@ package flink import ( "testing" + "github.com/lyft/flinkk8soperator/pkg/apis/app/v1beta1" + config2 "github.com/lyft/flinkk8soperator/pkg/controller/config" "github.com/stretchr/testify/assert" ) @@ -25,3 +27,16 @@ func TestGetFlinkUIIngressURL(t *testing.T) { "ABC.lyft.xyz", GetFlinkUIIngressURL("ABC")) } + +func TestGetFlinkUIIngressURLBlueGreenDeployment(t *testing.T) { + err := initTestConfigForIngress() + assert.Nil(t, err) + app := v1beta1.FlinkApplication{} + app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + app.Name = "ABC" + app.Status.UpdatingVersion = v1beta1.GreenFlinkApplication + assert.Equal(t, "ABC-green", getIngressName(&app)) + assert.Equal(t, + "ABC-green.lyft.xyz", + GetFlinkUIIngressURL(getIngressName(&app))) +} diff --git a/pkg/controller/flink/job_manager_controller.go b/pkg/controller/flink/job_manager_controller.go index 279c0a44..9f41a935 100644 --- a/pkg/controller/flink/job_manager_controller.go +++ b/pkg/controller/flink/job_manager_controller.go @@ -21,8 +21,10 @@ import ( const ( JobManagerNameFormat = "%s-%s-jm" + JobManagerVersionNameFormat = "%s-%s-%s-jm" JobManagerPodNameFormat = "%s-%s-jm-pod" - JobManagerVersionPodNameFormat = "%s-%s-jm-%s-pod" + JobManagerServiceName = "%s" + JobManagerVersionServiceName = "%s-%s" JobManagerContainerName = "jobmanager" JobManagerArg = "jobmanager" JobManagerReadinessPath = "/overview" @@ -170,20 +172,20 @@ var JobManagerDefaultResources = coreV1.ResourceRequirements{ func getJobManagerPodName(application *v1beta1.FlinkApplication, hash string) string { applicationName := application.Name - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { - applicationVersion := application.Status.UpdatingVersion - return fmt.Sprintf(JobManagerVersionPodNameFormat, applicationName, hash, applicationVersion) - } return fmt.Sprintf(JobManagerPodNameFormat, applicationName, hash) } func getJobManagerName(application *v1beta1.FlinkApplication, hash string) string { applicationName := application.Name + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + applicationVersion := application.Status.UpdatingVersion + return fmt.Sprintf(JobManagerVersionNameFormat, applicationName, hash, applicationVersion) + } return fmt.Sprintf(JobManagerNameFormat, applicationName, hash) } func FetchJobManagerServiceCreateObj(app *v1beta1.FlinkApplication, hash string) *coreV1.Service { - jmServiceName := app.Name + jmServiceName := getJobManagerServiceName(app) serviceLabels := getCommonAppLabels(app) serviceLabels[FlinkAppHash] = hash serviceLabels[FlinkDeploymentType] = FlinkDeploymentTypeJobmanager @@ -208,6 +210,15 @@ func FetchJobManagerServiceCreateObj(app *v1beta1.FlinkApplication, hash string) } } +func getJobManagerServiceName(app *v1beta1.FlinkApplication) string { + serviceName := app.Name + versionName := app.Status.UpdatingVersion + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { + return fmt.Sprintf(JobManagerVersionServiceName, serviceName, versionName) + } + return serviceName +} + func getJobManagerServicePorts(app *v1beta1.FlinkApplication) []coreV1.ServicePort { ports := getJobManagerPorts(app) servicePorts := make([]coreV1.ServicePort, 0, len(ports)) diff --git a/pkg/controller/flink/job_manager_controller_test.go b/pkg/controller/flink/job_manager_controller_test.go index fe5376ae..c33b9ad5 100644 --- a/pkg/controller/flink/job_manager_controller_test.go +++ b/pkg/controller/flink/job_manager_controller_test.go @@ -3,7 +3,7 @@ package flink import ( "testing" - flinkapp "github.com/lyft/flinkk8soperator/pkg/apis/app/v1beta1" + v1beta12 "github.com/lyft/flinkk8soperator/pkg/apis/app/v1beta1" "github.com/lyft/flinkk8soperator/pkg/controller/config" @@ -44,11 +44,12 @@ func TestGetJobManagerPodName(t *testing.T) { assert.Equal(t, "app-name-"+testAppHash+"-jm-pod", getJobManagerPodName(&app, testAppHash)) } -func TestGetJobManagerPodNameWithVersion(t *testing.T) { +func TestGetJobManagerNameWithVersion(t *testing.T) { app := getFlinkTestApp() - app.Spec.DeploymentMode = flinkapp.DeploymentModeBlueGreen + app.Spec.DeploymentMode = v1beta12.DeploymentModeBlueGreen + app.Status.DeploymentMode = v1beta12.DeploymentModeBlueGreen app.Status.UpdatingVersion = testVersion - assert.Equal(t, "app-name-"+testAppHash+"-jm-"+testVersion+"-pod", getJobManagerPodName(&app, testAppHash)) + assert.Equal(t, "app-name-"+testAppHash+"-"+testVersion+"-jm", getJobManagerName(&app, testAppHash)) } func TestJobManagerCreateSuccess(t *testing.T) { @@ -313,7 +314,8 @@ func TestJobManagerCreateSuccessWithVersion(t *testing.T) { app.Spec.JarName = testJarName app.Spec.EntryClass = testEntryClass app.Spec.ProgramArgs = testProgramArgs - app.Spec.DeploymentMode = flinkapp.DeploymentModeBlueGreen + app.Spec.DeploymentMode = v1beta12.DeploymentModeBlueGreen + app.Status.DeploymentMode = v1beta12.DeploymentModeBlueGreen app.Status.UpdatingVersion = testVersion annotations := map[string]string{ "key": "annotation", @@ -321,11 +323,12 @@ func TestJobManagerCreateSuccessWithVersion(t *testing.T) { "flink-job-properties": "jarName: " + testJarName + "\nparallelism: 8\nentryClass:" + testEntryClass + "\nprogramArgs:\"" + testProgramArgs + "\"", } app.Annotations = annotations - hash := "f0bd1679" + hash := "5cb5943e" expectedLabels := map[string]string{ - "flink-app": "app-name", - "flink-app-hash": hash, - "flink-deployment-type": "jobmanager", + "flink-app": "app-name", + "flink-app-hash": hash, + "flink-deployment-type": "jobmanager", + "flink-application-version": testVersion, } ctr := 0 mockK8Cluster := testController.k8Cluster.(*k8mock.K8Cluster) @@ -358,21 +361,21 @@ func TestJobManagerCreateSuccessWithVersion(t *testing.T) { "FLINK_APPLICATION_VERSION").Value) case 2: service := object.(*coreV1.Service) - assert.Equal(t, app.Name, service.Name) + assert.Equal(t, app.Name+"-"+testVersion, service.Name) assert.Equal(t, app.Namespace, service.Namespace) - assert.Equal(t, map[string]string{"flink-app": "app-name", "flink-app-hash": hash, "flink-deployment-type": "jobmanager"}, service.Spec.Selector) + assert.Equal(t, map[string]string{"flink-app": "app-name", "flink-app-hash": hash, "flink-deployment-type": "jobmanager", "flink-application-version": testVersion}, service.Spec.Selector) case 3: service := object.(*coreV1.Service) assert.Equal(t, app.Name+"-"+hash, service.Name) assert.Equal(t, "app-name", service.OwnerReferences[0].Name) assert.Equal(t, app.Namespace, service.Namespace) - assert.Equal(t, map[string]string{"flink-app": "app-name", "flink-app-hash": hash, "flink-deployment-type": "jobmanager"}, service.Spec.Selector) + assert.Equal(t, map[string]string{"flink-app": "app-name", "flink-app-hash": hash, "flink-application-version": testVersion, "flink-deployment-type": "jobmanager"}, service.Spec.Selector) case 4: labels := map[string]string{ "flink-app": "app-name", } ingress := object.(*v1beta1.Ingress) - assert.Equal(t, app.Name, ingress.Name) + assert.Equal(t, app.Name+"-"+testVersion, ingress.Name) assert.Equal(t, app.Namespace, ingress.Namespace) assert.Equal(t, labels, ingress.Labels) } diff --git a/pkg/controller/flink/mock/mock_flink.go b/pkg/controller/flink/mock/mock_flink.go index 15b64c91..830bcfa2 100644 --- a/pkg/controller/flink/mock/mock_flink.go +++ b/pkg/controller/flink/mock/mock_flink.go @@ -11,11 +11,11 @@ import ( type CreateClusterFunc func(ctx context.Context, application *v1beta1.FlinkApplication) error type DeleteOldResourcesForApp func(ctx context.Context, application *v1beta1.FlinkApplication) error -type CancelWithSavepointFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, error) -type ForceCancelFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error +type SavepointFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (string, error) +type ForceCancelFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error type StartFlinkJobFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jarName string, parallelism int32, entryClass string, programArgs string, allowNonRestoredState bool, savepointPath string) (string, error) -type GetSavepointStatusFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) +type GetSavepointStatusFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) type IsClusterReadyFunc func(ctx context.Context, application *v1beta1.FlinkApplication) (bool, error) type IsServiceReadyFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (bool, error) type GetJobsForApplicationFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) ([]client.FlinkJob, error) @@ -30,11 +30,16 @@ type GetLatestJobIDFunc func(ctx context.Context, app *v1beta1.FlinkApplication) type UpdateLatestJobIDFunc func(ctx context.Context, app *v1beta1.FlinkApplication, jobID string) type UpdateLatestJobStatusFunc func(ctx context.Context, app *v1beta1.FlinkApplication, jobStatus v1beta1.FlinkJobStatus) type UpdateLatestClusterStatusFunc func(ctx context.Context, app *v1beta1.FlinkApplication, clusterStatus v1beta1.FlinkClusterStatus) - +type UpdateLatestVersionAndHashFunc func(application *v1beta1.FlinkApplication, version v1beta1.FlinkApplicationVersion, hash string) +type DeleteResourcesForAppWithHashFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error +type DeleteStatusPostTeardownFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) +type GetJobToDeleteForApplicationFunc func(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) +type GetVersionAndJobIDForHashFunc func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, string, error) +type GetVersionAndHashPostTeardownFunc func(ctx context.Context, application *v1beta1.FlinkApplication) (v1beta1.FlinkApplicationVersion, string) type FlinkController struct { CreateClusterFunc CreateClusterFunc DeleteOldResourcesForAppFunc DeleteOldResourcesForApp - CancelWithSavepointFunc CancelWithSavepointFunc + SavepointFunc SavepointFunc ForceCancelFunc ForceCancelFunc StartFlinkJobFunc StartFlinkJobFunc GetSavepointStatusFunc GetSavepointStatusFunc @@ -53,6 +58,12 @@ type FlinkController struct { UpdateLatestJobIDFunc UpdateLatestJobIDFunc UpdateLatestJobStatusFunc UpdateLatestJobStatusFunc UpdateLatestClusterStatusFunc UpdateLatestClusterStatusFunc + UpdateLatestVersionAndHashFunc UpdateLatestVersionAndHashFunc + DeleteResourcesForAppWithHashFunc DeleteResourcesForAppWithHashFunc + DeleteStatusPostTeardownFunc DeleteStatusPostTeardownFunc + GetJobToDeleteForApplicationFunc GetJobToDeleteForApplicationFunc + GetVersionAndJobIDForHashFunc GetVersionAndJobIDForHashFunc + GetVersionAndHashPostTeardownFunc GetVersionAndHashPostTeardownFunc } func (m *FlinkController) GetCurrentDeploymentsForApp(ctx context.Context, application *v1beta1.FlinkApplication) (*common.FlinkDeployment, error) { @@ -76,16 +87,16 @@ func (m *FlinkController) CreateCluster(ctx context.Context, application *v1beta return nil } -func (m *FlinkController) CancelWithSavepoint(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, error) { - if m.CancelWithSavepointFunc != nil { - return m.CancelWithSavepointFunc(ctx, application, hash) +func (m *FlinkController) Savepoint(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (string, error) { + if m.SavepointFunc != nil { + return m.SavepointFunc(ctx, application, hash, isCancel, jobID) } return "", nil } -func (m *FlinkController) ForceCancel(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { +func (m *FlinkController) ForceCancel(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error { if m.ForceCancelFunc != nil { - return m.ForceCancelFunc(ctx, application, hash) + return m.ForceCancelFunc(ctx, application, hash, jobID) } return nil } @@ -98,9 +109,9 @@ func (m *FlinkController) StartFlinkJob(ctx context.Context, application *v1beta return "", nil } -func (m *FlinkController) GetSavepointStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { +func (m *FlinkController) GetSavepointStatus(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { if m.GetSavepointStatusFunc != nil { - return m.GetSavepointStatusFunc(ctx, application, hash) + return m.GetSavepointStatusFunc(ctx, application, hash, jobID) } return nil, nil } @@ -173,27 +184,27 @@ func (m *FlinkController) GetLatestClusterStatus(ctx context.Context, applicatio if m.GetLatestClusterStatusFunc != nil { return m.GetLatestClusterStatusFunc(ctx, application) } - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { return application.Status.VersionStatuses[getCurrentStatusIndex(application)].ClusterStatus } return application.Status.ClusterStatus } func (m *FlinkController) GetLatestJobStatus(ctx context.Context, application *v1beta1.FlinkApplication) v1beta1.FlinkJobStatus { - if m.GetLatestClusterStatusFunc != nil { + if m.GetLatestJobStatusFunc != nil { return m.GetLatestJobStatusFunc(ctx, application) } - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { return application.Status.VersionStatuses[getCurrentStatusIndex(application)].JobStatus } return application.Status.JobStatus } func (m *FlinkController) GetLatestJobID(ctx context.Context, application *v1beta1.FlinkApplication) string { - if m.GetLatestClusterStatusFunc != nil { + if m.GetLatestJobIDFunc != nil { return m.GetLatestJobIDFunc(ctx, application) } - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { return application.Status.VersionStatuses[getCurrentStatusIndex(application)].JobStatus.JobID } return application.Status.JobStatus.JobID @@ -203,7 +214,7 @@ func (m *FlinkController) UpdateLatestJobID(ctx context.Context, application *v1 if m.UpdateLatestJobIDFunc != nil { m.UpdateLatestJobIDFunc(ctx, application, jobID) } - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { application.Status.VersionStatuses[getCurrentStatusIndex(application)].JobStatus.JobID = jobID return } @@ -214,7 +225,7 @@ func (m *FlinkController) UpdateLatestJobStatus(ctx context.Context, application if m.UpdateLatestJobStatusFunc != nil { m.UpdateLatestJobStatusFunc(ctx, application, jobStatus) } - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { application.Status.VersionStatuses[getCurrentStatusIndex(application)].JobStatus = jobStatus return } @@ -225,13 +236,60 @@ func (m *FlinkController) UpdateLatestClusterStatus(ctx context.Context, applica if m.UpdateLatestClusterStatusFunc != nil { m.UpdateLatestClusterStatusFunc(ctx, application, clusterStatus) } - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { application.Status.VersionStatuses[getCurrentStatusIndex(application)].ClusterStatus = clusterStatus return } application.Status.ClusterStatus = clusterStatus } +func (m *FlinkController) UpdateLatestVersionAndHash(application *v1beta1.FlinkApplication, version v1beta1.FlinkApplicationVersion, hash string) { + if m.UpdateLatestVersionAndHashFunc != nil { + m.UpdateLatestVersionAndHashFunc(application, version, hash) + } + currIndex := getCurrentStatusIndex(application) + application.Status.VersionStatuses[currIndex].Version = version + application.Status.VersionStatuses[currIndex].VersionHash = hash + application.Status.UpdatingHash = hash + +} + +func (m *FlinkController) DeleteResourcesForAppWithHash(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { + if m.DeleteResourcesForAppWithHashFunc != nil { + return m.DeleteResourcesForAppWithHashFunc(ctx, application, hash) + } + return nil +} + +func (m *FlinkController) DeleteStatusPostTeardown(ctx context.Context, application *v1beta1.FlinkApplication, hash string) { + if m.DeleteStatusPostTeardownFunc != nil { + m.DeleteStatusPostTeardownFunc(ctx, application, hash) + } + application.Status.VersionStatuses[0] = application.Status.VersionStatuses[1] + application.Status.VersionStatuses[1] = v1beta1.FlinkApplicationVersionStatus{} +} + +func (m *FlinkController) GetJobToDeleteForApplication(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) { + if m.GetJobToDeleteForApplicationFunc != nil { + return m.GetJobToDeleteForApplicationFunc(ctx, app, hash) + } + return nil, nil +} + +func (m *FlinkController) GetVersionAndJobIDForHash(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, string, error) { + if m.GetVersionAndJobIDForHashFunc != nil { + return m.GetVersionAndJobIDForHashFunc(ctx, application, hash) + } + return "", "", nil +} + +func (m *FlinkController) GetVersionAndHashPostTeardown(ctx context.Context, application *v1beta1.FlinkApplication) (v1beta1.FlinkApplicationVersion, string) { + if m.GetVersionAndHashPostTeardownFunc != nil { + return m.GetVersionAndHashPostTeardownFunc(ctx, application) + } + return application.Status.VersionStatuses[0].Version, application.Status.VersionStatuses[0].VersionHash +} + func getCurrentStatusIndex(app *v1beta1.FlinkApplication) int32 { desiredCount := v1beta1.GetMaxRunningJobs(app.Spec.DeploymentMode) if v1beta1.IsRunningPhase(app.Status.Phase) { diff --git a/pkg/controller/flink/task_manager_controller.go b/pkg/controller/flink/task_manager_controller.go index 3b0d9d30..36f2085f 100644 --- a/pkg/controller/flink/task_manager_controller.go +++ b/pkg/controller/flink/task_manager_controller.go @@ -20,12 +20,12 @@ import ( ) const ( - TaskManagerNameFormat = "%s-%s-tm" - TaskManagerPodNameFormat = "%s-%s-tm-pod" - TaskManagerVersionPodNameFormat = "%s-%s-tm-%s-pod" - TaskManagerContainerName = "taskmanager" - TaskManagerArg = "taskmanager" - TaskManagerHostnameEnvVar = "TASKMANAGER_HOSTNAME" + TaskManagerNameFormat = "%s-%s-tm" + TaskManagerVersionNameFormat = "%s-%s-%s-tm" + TaskManagerPodNameFormat = "%s-%s-tm-pod" + TaskManagerContainerName = "taskmanager" + TaskManagerArg = "taskmanager" + TaskManagerHostnameEnvVar = "TASKMANAGER_HOSTNAME" ) type TaskManagerControllerInterface interface { @@ -143,15 +143,15 @@ func FetchTaskManagerContainerObj(application *v1beta1.FlinkApplication) *coreV1 func getTaskManagerPodName(application *v1beta1.FlinkApplication, hash string) string { applicationName := application.Name - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { - applicationVersion := application.Status.UpdatingVersion - return fmt.Sprintf(TaskManagerVersionPodNameFormat, applicationName, hash, applicationVersion) - } return fmt.Sprintf(TaskManagerPodNameFormat, applicationName, hash) } func getTaskManagerName(application *v1beta1.FlinkApplication, hash string) string { applicationName := application.Name + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + applicationVersion := application.Status.UpdatingVersion + return fmt.Sprintf(TaskManagerVersionNameFormat, applicationName, hash, applicationVersion) + } return fmt.Sprintf(TaskManagerNameFormat, applicationName, hash) } diff --git a/pkg/controller/flink/task_manager_controller_test.go b/pkg/controller/flink/task_manager_controller_test.go index 9f3edca6..4200383d 100644 --- a/pkg/controller/flink/task_manager_controller_test.go +++ b/pkg/controller/flink/task_manager_controller_test.go @@ -53,8 +53,9 @@ func TestGetTaskManagerPodName(t *testing.T) { func TestGetTaskManagerPodNameWithVersion(t *testing.T) { app := getFlinkTestApp() app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + app.Status.DeploymentMode = v1beta1.DeploymentModeBlueGreen app.Status.UpdatingVersion = testVersion - assert.Equal(t, "app-name-"+testAppHash+"-tm-"+testVersion+"-pod", getTaskManagerPodName(&app, testAppHash)) + assert.Equal(t, "app-name-"+testAppHash+"-"+testVersion+"-tm", getTaskManagerName(&app, testAppHash)) } func TestTaskManagerCreateSuccess(t *testing.T) { @@ -238,6 +239,7 @@ func TestTaskManagerCreateSuccessWithVersion(t *testing.T) { app.Spec.EntryClass = testEntryClass app.Spec.ProgramArgs = testProgramArgs app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + app.Status.DeploymentMode = v1beta1.DeploymentModeBlueGreen app.Status.UpdatingVersion = testVersion annotations := map[string]string{ "key": "annotation", @@ -245,13 +247,14 @@ func TestTaskManagerCreateSuccessWithVersion(t *testing.T) { "flink-job-properties": "jarName: test.jar\nparallelism: 8\nentryClass:com.test.MainClass\nprogramArgs:\"--test\"", } - hash := "f0bd1679" + hash := "5cb5943e" app.Annotations = annotations expectedLabels := map[string]string{ - "flink-app": "app-name", - "flink-app-hash": hash, - "flink-deployment-type": "taskmanager", + "flink-app": "app-name", + "flink-app-hash": hash, + "flink-deployment-type": "taskmanager", + "flink-application-version": testVersion, } mockK8Cluster := testController.k8Cluster.(*k8mock.K8Cluster) mockK8Cluster.CreateK8ObjectFunc = func(ctx context.Context, object runtime.Object) error { diff --git a/pkg/controller/flinkapplication/flink_state_machine.go b/pkg/controller/flinkapplication/flink_state_machine.go index 8ecd201d..2a7d79eb 100644 --- a/pkg/controller/flinkapplication/flink_state_machine.go +++ b/pkg/controller/flinkapplication/flink_state_machine.go @@ -154,7 +154,7 @@ func (s *FlinkStateMachine) handle(ctx context.Context, application *v1beta1.Fli updateLastSeenError := false appPhase := application.Status.Phase // initialize application status array if it's not yet been initialized - s.initializeAppStatusIfEmpty(ctx, application) + s.initializeAppStatusIfEmpty(application) if !application.ObjectMeta.DeletionTimestamp.IsZero() && appPhase != v1beta1.FlinkApplicationDeleting { s.updateApplicationPhase(application, v1beta1.FlinkApplicationDeleting) @@ -186,6 +186,9 @@ func (s *FlinkStateMachine) handle(ctx context.Context, application *v1beta1.Fli updateApplication, appErr = s.handleRollingBack(ctx, application) case v1beta1.FlinkApplicationDeleting: updateApplication, appErr = s.handleApplicationDeleting(ctx, application) + case v1beta1.FlinkApplicationDualRunning: + updateApplication, appErr = s.handleDualRunning(ctx, application) + } if !v1beta1.IsRunningPhase(appPhase) { @@ -237,7 +240,16 @@ func (s *FlinkStateMachine) handleNewOrUpdating(ctx context.Context, application fmt.Sprintf("Failed to create Flink Cluster: %s", reason)) return s.deployFailed(application) } - + // Update version if blue/green deploy + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + application.Status.UpdatingVersion = getUpdatingVersion(application) + // First deploy both versions are the same + if application.Status.DeployHash == "" { + application.Status.DeployVersion = application.Status.UpdatingVersion + } + // Reset teardown hash if set + application.Status.TeardownHash = "" + } // Create the Flink cluster err := s.flinkController.CreateCluster(ctx, application) if err != nil { @@ -285,36 +297,35 @@ func (s *FlinkStateMachine) handleClusterStarting(ctx context.Context, applicati return statusUnchanged, nil } + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + // Update hashes + s.flinkController.UpdateLatestVersionAndHash(application, application.Status.UpdatingVersion, flink.HashForApplication(application)) + + } + logger.Infof(ctx, "Flink cluster has started successfully") // TODO: in single mode move to submitting job - if application.Spec.SavepointDisabled { + if application.Spec.SavepointDisabled && !v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { s.updateApplicationPhase(application, v1beta1.FlinkApplicationCancelling) + } else if application.Spec.SavepointDisabled && v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + // Blue Green deployment and no savepoint required implies, we directly transition to submitting job + s.updateApplicationPhase(application, v1beta1.FlinkApplicationSubmittingJob) } else { s.updateApplicationPhase(application, v1beta1.FlinkApplicationSavepointing) } return statusChanged, nil } -func (s *FlinkStateMachine) initializeAppStatusIfEmpty(ctx context.Context, application *v1beta1.FlinkApplication) { - // initialize the app status array to include 2 status elements in case of blue green deploys - // else use a one element array - if v1beta1.IsBlueGreenDeploymentMode(application.Spec.DeploymentMode) { - application.Status.VersionStatuses = make([]v1beta1.FlinkApplicationVersionStatus, v1beta1.GetMaxRunningJobs(application.Spec.DeploymentMode)) - - // If an application is moving from a Dual to BlueGreen deployment mode, - // We pre-populate the version statuses array with the current Job and Cluster Status - // And reset top-level ClusterStatus and JobStatus to empty structs - // as they'll no longer get updated - if application.Status.JobStatus != (v1beta1.FlinkJobStatus{}) { - s.flinkController.UpdateLatestJobStatus(ctx, application, application.Status.JobStatus) - application.Status.JobStatus = v1beta1.FlinkJobStatus{} - } - - if application.Status.ClusterStatus != (v1beta1.FlinkClusterStatus{}) { - s.flinkController.UpdateLatestClusterStatus(ctx, application, application.Status.ClusterStatus) - application.Status.ClusterStatus = v1beta1.FlinkClusterStatus{} +func (s *FlinkStateMachine) initializeAppStatusIfEmpty(application *v1beta1.FlinkApplication) { + if v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + if len(application.Status.VersionStatuses) == 0 { + application.Status.VersionStatuses = make([]v1beta1.FlinkApplicationVersionStatus, v1beta1.GetMaxRunningJobs(application.Spec.DeploymentMode)) } } + // Set the deployment mode if it's never been set + if application.Status.DeploymentMode == "" { + application.Status.DeploymentMode = application.Spec.DeploymentMode + } } func (s *FlinkStateMachine) handleApplicationSavepointing(ctx context.Context, application *v1beta1.FlinkApplication) (bool, error) { @@ -331,24 +342,29 @@ func (s *FlinkStateMachine) handleApplicationSavepointing(ctx context.Context, a s.updateApplicationPhase(application, v1beta1.FlinkApplicationRecovering) return statusChanged, nil } - + cancelFlag := getCancelFlag(application) // we haven't started savepointing yet; do so now // TODO: figure out the idempotence of this if application.Status.SavepointTriggerID == "" { - triggerID, err := s.flinkController.CancelWithSavepoint(ctx, application, application.Status.DeployHash) + triggerID, err := s.flinkController.Savepoint(ctx, application, application.Status.DeployHash, cancelFlag, s.flinkController.GetLatestJobID(ctx, application)) if err != nil { return statusUnchanged, err } + if cancelFlag { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "CancellingJob", + fmt.Sprintf("Cancelling job %s with a final savepoint", s.flinkController.GetLatestJobID(ctx, application))) + } else { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "SavepointingJob", + fmt.Sprintf("Savepointing job %s with a final savepoint", s.flinkController.GetLatestJobID(ctx, application))) - s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "CancellingJob", - fmt.Sprintf("Cancelling job %s with a final savepoint", s.flinkController.GetLatestJobID(ctx, application))) + } application.Status.SavepointTriggerID = triggerID return statusChanged, nil } // check the savepoints in progress - savepointStatusResponse, err := s.flinkController.GetSavepointStatus(ctx, application, application.Status.DeployHash) + savepointStatusResponse, err := s.flinkController.GetSavepointStatus(ctx, application, application.Status.DeployHash, s.flinkController.GetLatestJobID(ctx, application)) if err != nil { return statusUnchanged, err } @@ -364,11 +380,21 @@ func (s *FlinkStateMachine) handleApplicationSavepointing(ctx context.Context, a s.updateApplicationPhase(application, v1beta1.FlinkApplicationRecovering) return statusChanged, nil } else if savepointStatusResponse.SavepointStatus.Status == client.SavePointCompleted { - s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "CanceledJob", - fmt.Sprintf("Canceled job with savepoint %s", - savepointStatusResponse.Operation.Location)) + if cancelFlag { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "CanceledJob", + fmt.Sprintf("Canceled job with savepoint %s", + savepointStatusResponse.Operation.Location)) + } else { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "SavepointCompleted", + fmt.Sprintf("Completed savepoint at %s", + savepointStatusResponse.Operation.Location)) + } + application.Status.SavepointPath = savepointStatusResponse.Operation.Location - s.flinkController.UpdateLatestJobID(ctx, application, "") + // We haven't cancelled the job in this case, so don't reset job ID + if !v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + s.flinkController.UpdateLatestJobID(ctx, application, "") + } s.updateApplicationPhase(application, v1beta1.FlinkApplicationSubmittingJob) return statusChanged, nil } @@ -400,7 +426,7 @@ func (s *FlinkStateMachine) handleApplicationCancelling(ctx context.Context, app if job != nil && job.State != client.Canceled && job.State != client.Failed { - err := s.flinkController.ForceCancel(ctx, application, application.Status.DeployHash) + err := s.flinkController.ForceCancel(ctx, application, application.Status.DeployHash, s.flinkController.GetLatestJobID(ctx, application)) if err != nil { return statusUnchanged, err } @@ -499,7 +525,7 @@ func (s *FlinkStateMachine) submitJobIfNeeded(ctx context.Context, app *v1beta1. } func (s *FlinkStateMachine) updateGenericService(ctx context.Context, app *v1beta1.FlinkApplication, newHash string) error { - service, err := s.k8Cluster.GetService(ctx, app.Namespace, app.Name) + service, err := s.k8Cluster.GetService(ctx, app.Namespace, app.Name, string(app.Status.UpdatingVersion)) if err != nil { return err } @@ -591,10 +617,7 @@ func (s *FlinkStateMachine) handleSubmittingJob(ctx context.Context, app *v1beta } if job.State == client.Running && allVerticesStarted { - // Update the application status with the running job info - app.Status.DeployHash = hash - app.Status.SavepointPath = "" - app.Status.SavepointTriggerID = "" + // Update job status jobStatus := s.flinkController.GetLatestJobStatus(ctx, app) jobStatus.JarName = app.Spec.JarName jobStatus.Parallelism = app.Spec.Parallelism @@ -602,6 +625,14 @@ func (s *FlinkStateMachine) handleSubmittingJob(ctx context.Context, app *v1beta jobStatus.ProgramArgs = app.Spec.ProgramArgs jobStatus.AllowNonRestoredState = app.Spec.AllowNonRestoredState s.flinkController.UpdateLatestJobStatus(ctx, app, jobStatus) + // Update the application status with the running job info + app.Status.SavepointPath = "" + app.Status.SavepointTriggerID = "" + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) && app.Status.DeployHash != "" { + s.updateApplicationPhase(app, v1beta1.FlinkApplicationDualRunning) + return statusChanged, nil + } + app.Status.DeployHash = hash s.updateApplicationPhase(app, v1beta1.FlinkApplicationRunning) return statusChanged, nil } @@ -623,6 +654,11 @@ func (s *FlinkStateMachine) handleRollingBack(ctx context.Context, app *v1beta1. s.flinkController.LogEvent(ctx, app, corev1.EventTypeWarning, "DeployFailed", fmt.Sprintf("Deployment %s failed, rolling back", flink.HashForApplication(app))) + // In the case of blue green deploys, we don't try to submit a new job + // and instead transition to a deploy failed state + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) && app.Status.DeployHash != "" { + return s.deployFailed(app) + } // TODO: handle single mode // TODO: it's possible that a job is successfully running in the new cluster at this point -- should cancel it @@ -679,6 +715,11 @@ func (s *FlinkStateMachine) handleApplicationRunning(ctx context.Context, applic // If the application has changed (i.e., there are no current deployments), and we haven't already failed trying to // do the update, move to the cluster starting phase to create the new cluster if cur == nil { + if s.isIncompatibleDeploymentModeChange(application) { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeWarning, "UnsupportedChange", + fmt.Sprintf("Changing deployment mode from %s to %s is unsupported", application.Status.DeploymentMode, application.Spec.DeploymentMode)) + return s.deployFailed(application) + } logger.Infof(ctx, "Application resource has changed. Moving to Updating") // TODO: handle single mode s.updateApplicationPhase(application, v1beta1.FlinkApplicationUpdating) @@ -697,8 +738,16 @@ func (s *FlinkStateMachine) handleApplicationRunning(ctx context.Context, applic logger.Debugf(ctx, "Application running with job %v", job.JobID) } - // If there are old resources left-over from a previous version, clean them up - err = s.flinkController.DeleteOldResourcesForApp(ctx, application) + // For blue-green deploys, specify the hash to be deleted + if application.Status.FailedDeployHash != "" && v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + err = s.flinkController.DeleteResourcesForAppWithHash(ctx, application, application.Status.FailedDeployHash) + // Delete status object for the failed hash + s.flinkController.DeleteStatusPostTeardown(ctx, application, application.Status.FailedDeployHash) + } else if !v1beta1.IsBlueGreenDeploymentMode(application.Status.DeploymentMode) { + // If there are old resources left-over from a previous version, clean them up + err = s.flinkController.DeleteOldResourcesForApp(ctx, application) + } + if err != nil { logger.Warn(ctx, "Failed to clean up old resources: %v", err) } @@ -769,7 +818,9 @@ func (s *FlinkStateMachine) handleApplicationDeleting(ctx context.Context, app * if app.Spec.DeleteMode == v1beta1.DeleteModeNone || app.Status.DeployHash == "" { return s.clearFinalizers(ctx, app) } - + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) { + return s.deleteBlueGreenApplication(ctx, app) + } job, err := s.flinkController.GetJobForApplication(ctx, app, app.Status.DeployHash) if err != nil { return statusUnchanged, err @@ -790,7 +841,7 @@ func (s *FlinkStateMachine) handleApplicationDeleting(ctx context.Context, app * } logger.Infof(ctx, "Force-cancelling job without a savepoint") - return statusUnchanged, s.flinkController.ForceCancel(ctx, app, app.Status.DeployHash) + return statusUnchanged, s.flinkController.ForceCancel(ctx, app, app.Status.DeployHash, s.flinkController.GetLatestJobID(ctx, app)) case v1beta1.DeleteModeSavepoint, "": if app.Status.SavepointPath != "" { // we've already created the savepoint, now just waiting for the job to be cancelled @@ -803,7 +854,7 @@ func (s *FlinkStateMachine) handleApplicationDeleting(ctx context.Context, app * if app.Status.SavepointTriggerID == "" { // delete with savepoint - triggerID, err := s.flinkController.CancelWithSavepoint(ctx, app, app.Status.DeployHash) + triggerID, err := s.flinkController.Savepoint(ctx, app, app.Status.DeployHash, getCancelFlag(app), s.flinkController.GetLatestJobID(ctx, app)) if err != nil { return statusUnchanged, err } @@ -812,7 +863,7 @@ func (s *FlinkStateMachine) handleApplicationDeleting(ctx context.Context, app * app.Status.SavepointTriggerID = triggerID } else { // we've already started savepointing; check the status - status, err := s.flinkController.GetSavepointStatus(ctx, app, app.Status.DeployHash) + status, err := s.flinkController.GetSavepointStatus(ctx, app, app.Status.DeployHash, s.flinkController.GetLatestJobID(ctx, app)) if err != nil { return statusUnchanged, err } @@ -867,6 +918,204 @@ func (s *FlinkStateMachine) compareAndUpdateError(application *v1beta1.FlinkAppl } +func getUpdatingVersion(application *v1beta1.FlinkApplication) v1beta1.FlinkApplicationVersion { + if getDeployedVersion(application) == v1beta1.BlueFlinkApplication { + return v1beta1.GreenFlinkApplication + } + + return v1beta1.BlueFlinkApplication +} + +func getDeployedVersion(application *v1beta1.FlinkApplication) v1beta1.FlinkApplicationVersion { + // First deploy, set the version to Blue + if application.Status.DeployVersion == "" { + application.Status.DeployVersion = v1beta1.BlueFlinkApplication + } + return application.Status.DeployVersion +} + +func getCancelFlag(app *v1beta1.FlinkApplication) bool { + if v1beta1.IsBlueGreenDeploymentMode(app.Status.DeploymentMode) && app.Status.Phase != v1beta1.FlinkApplicationDeleting { + return false + } + return true +} + +// Two applications are running in this phase. This phase is only ever reached when the +// DeploymentMode is set to BlueGreen +func (s *FlinkStateMachine) handleDualRunning(ctx context.Context, application *v1beta1.FlinkApplication) (bool, error) { + if application.Spec.TearDownVersionHash != "" { + return s.teardownApplicationVersion(ctx, application) + } + + // Update status of the cluster + hasClusterStatusChanged, clusterErr := s.flinkController.CompareAndUpdateClusterStatus(ctx, application, application.Status.DeployHash) + if clusterErr != nil { + logger.Errorf(ctx, "Updating cluster status failed with %v", clusterErr) + } + + // Update status of jobs on the cluster + hasJobStatusChanged, jobsErr := s.flinkController.CompareAndUpdateJobStatus(ctx, application, application.Status.DeployHash) + if jobsErr != nil { + logger.Errorf(ctx, "Updating jobs status failed with %v", jobsErr) + } + + // Update k8s object if either job or cluster status has changed + if hasJobStatusChanged || hasClusterStatusChanged { + return statusChanged, nil + } + return statusUnchanged, nil +} + +func (s *FlinkStateMachine) teardownApplicationVersion(ctx context.Context, application *v1beta1.FlinkApplication) (bool, error) { + versionHashToTeardown := application.Spec.TearDownVersionHash + versionToTeardown, jobID, err := s.flinkController.GetVersionAndJobIDForHash(ctx, application, versionHashToTeardown) + + if err != nil { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeWarning, "TeardownFailed", + fmt.Sprintf("Failed to find application version %v", + versionHashToTeardown)) + return statusUnchanged, nil + } + + s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "TeardownInitated", + fmt.Sprintf("Tearing down application with hash %s and version %v", versionHashToTeardown, + versionToTeardown)) + // Force-cancel job first + s.flinkController.LogEvent(ctx, application, corev1.EventTypeNormal, "ForceCanceling", + fmt.Sprintf("Force-canceling application with version %v and hash %s", + versionToTeardown, versionHashToTeardown)) + + err = s.flinkController.ForceCancel(ctx, application, versionHashToTeardown, jobID) + if err != nil { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeWarning, "TeardownFailed", + fmt.Sprintf("Failed to force-cancel application version %v and hash %s; will attempt to tear down cluster immediately: %s", + versionToTeardown, versionHashToTeardown, err)) + return s.deployFailed(application) + } + + // Delete all resources associated with the teardown version + err = s.flinkController.DeleteResourcesForAppWithHash(ctx, application, versionHashToTeardown) + if err != nil { + s.flinkController.LogEvent(ctx, application, corev1.EventTypeWarning, "TeardownFailed", + fmt.Sprintf("Failed to teardown application with hash %s and version %v, manual intervention needed: %s", versionHashToTeardown, + versionToTeardown, err)) + return s.deployFailed(application) + } + s.flinkController.LogEvent(ctx, application, corev1.EventTypeWarning, "TeardownCompleted", + fmt.Sprintf("Tore down application with hash %s and version %v", versionHashToTeardown, + versionToTeardown)) + + s.flinkController.DeleteStatusPostTeardown(ctx, application, versionHashToTeardown) + versionPostTeardown, versionHashPostTeardown := s.flinkController.GetVersionAndHashPostTeardown(ctx, application) + application.Status.DeployVersion = versionPostTeardown + application.Status.UpdatingVersion = "" + application.Status.DeployHash = versionHashPostTeardown + application.Status.UpdatingHash = "" + application.Status.TeardownHash = flink.HashForApplication(application) + s.updateApplicationPhase(application, v1beta1.FlinkApplicationRunning) + return statusChanged, nil +} + +func (s *FlinkStateMachine) deleteBlueGreenApplication(ctx context.Context, app *v1beta1.FlinkApplication) (bool, error) { + // Cancel deployed job + deployedJob, err := s.flinkController.GetJobToDeleteForApplication(ctx, app, app.Status.DeployHash) + if err != nil { + return statusUnchanged, nil + } + if !jobFinished(deployedJob) { + isFinished, err := s.cancelAndDeleteJob(ctx, app, deployedJob, app.Status.DeployHash) + if err != nil { + return statusUnchanged, nil + } + return isFinished, nil + } + + deploySavepointPath := app.Status.SavepointPath + // Cancel Updating job + updatingJob, err := s.flinkController.GetJobToDeleteForApplication(ctx, app, app.Status.UpdatingHash) + if err != nil { + return statusUnchanged, nil + } + if !jobFinished(updatingJob) { + if app.Status.SavepointPath == deploySavepointPath { + app.Status.SavepointPath = "" + } + isFinished, err := s.cancelAndDeleteJob(ctx, app, updatingJob, app.Status.UpdatingHash) + if err != nil { + return statusUnchanged, nil + } + return isFinished, nil + } + + if jobFinished(deployedJob) && jobFinished(updatingJob) { + return s.clearFinalizers(ctx, app) + } + return statusUnchanged, nil +} + +func (s *FlinkStateMachine) cancelAndDeleteJob(ctx context.Context, app *v1beta1.FlinkApplication, job *client.FlinkJobOverview, hash string) (bool, error) { + switch app.Spec.DeleteMode { + case v1beta1.DeleteModeForceCancel: + if job.State == client.Cancelling { + // we've already cancelled the job, waiting for it to finish + return statusUnchanged, nil + } else if jobFinished(job) { + return statusUnchanged, nil + } + + logger.Infof(ctx, "Force-cancelling job without a savepoint") + return statusUnchanged, s.flinkController.ForceCancel(ctx, app, hash, s.flinkController.GetLatestJobID(ctx, app)) + case v1beta1.DeleteModeSavepoint, "": + if app.Status.SavepointPath != "" { + return statusChanged, nil + } + + if app.Status.SavepointTriggerID == "" { + // delete with savepoint + triggerID, err := s.flinkController.Savepoint(ctx, app, hash, getCancelFlag(app), job.JobID) + if err != nil { + return statusUnchanged, err + } + s.flinkController.LogEvent(ctx, app, corev1.EventTypeNormal, "CancellingJob", + fmt.Sprintf("Cancelling job with savepoint %v", triggerID)) + app.Status.SavepointTriggerID = triggerID + } else { + // we've already started savepointing; check the status + status, err := s.flinkController.GetSavepointStatus(ctx, app, hash, job.JobID) + if err != nil { + return statusUnchanged, err + } + + if status.Operation.Location == "" && status.SavepointStatus.Status != client.SavePointInProgress { + // savepointing failed + s.flinkController.LogEvent(ctx, app, corev1.EventTypeWarning, "SavepointFailed", + fmt.Sprintf("Failed to take savepoint %v", status.Operation.FailureCause)) + // clear the trigger id so that we can try again + app.Status.SavepointTriggerID = "" + return true, client.GetRetryableError(errors.New("failed to take savepoint"), + v1beta1.CancelJobWithSavepoint, "500", math.MaxInt32) + } else if status.SavepointStatus.Status == client.SavePointCompleted { + // we're done, clean up + s.flinkController.LogEvent(ctx, app, corev1.EventTypeNormal, "CanceledJob", + fmt.Sprintf("Cancelled job with savepoint '%s'", status.Operation.Location)) + app.Status.SavepointPath = status.Operation.Location + app.Status.SavepointTriggerID = "" + } + } + + return statusChanged, nil + default: + logger.Errorf(ctx, "Unsupported DeleteMode %s", app.Spec.DeleteMode) + } + + return statusUnchanged, nil +} + +func (s *FlinkStateMachine) isIncompatibleDeploymentModeChange(application *v1beta1.FlinkApplication) bool { + return application.Spec.DeploymentMode != application.Status.DeploymentMode +} + func createRetryHandler() client.RetryHandlerInterface { return client.NewRetryHandler(config.GetConfig().BaseBackoffDuration.Duration, config.GetConfig().MaxErrDuration.Duration, config.GetConfig().MaxBackoffDuration.Duration) diff --git a/pkg/controller/flinkapplication/flink_state_machine_test.go b/pkg/controller/flinkapplication/flink_state_machine_test.go index fc15c62c..99389ff1 100644 --- a/pkg/controller/flinkapplication/flink_state_machine_test.go +++ b/pkg/controller/flinkapplication/flink_state_machine_test.go @@ -149,7 +149,7 @@ func TestHandleApplicationCancel(t *testing.T) { }, nil } - mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (e error) { + mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (e error) { assert.Equal(t, "old-hash", hash) cancelInvoked = true @@ -187,7 +187,7 @@ func TestHandleApplicationCancelFailedWithMaxRetries(t *testing.T) { updateInvoked := false stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { + mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error { // given we maxed out on retries, we should never have come here assert.False(t, true) return nil @@ -256,7 +256,7 @@ func TestHandleApplicationSavepointingInitialDeploy(t *testing.T) { stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.CancelWithSavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (s string, e error) { + mockFlinkController.SavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (s string, e error) { // should not be called assert.False(t, true) return "", nil @@ -291,14 +291,14 @@ func TestHandleApplicationSavepointingDual(t *testing.T) { stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.CancelWithSavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (s string, e error) { + mockFlinkController.SavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (s string, e error) { assert.Equal(t, "old-hash", hash) cancelInvoked = true return "trigger", nil } - mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { assert.Equal(t, "old-hash", hash) return &client.SavepointResponse{ SavepointStatus: client.SavepointStatusResponse{ @@ -340,7 +340,7 @@ func TestHandleApplicationSavepointingFailed(t *testing.T) { updateInvoked := false stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { return &client.SavepointResponse{ SavepointStatus: client.SavepointStatusResponse{ Status: client.SavePointCompleted, @@ -470,7 +470,7 @@ func TestSubmittingToRunning(t *testing.T) { mockK8Cluster := stateMachineForTest.k8Cluster.(*k8mock.K8Cluster) getServiceCount := 0 - mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string) (*v1.Service, error) { + mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string, version string) (*v1.Service, error) { assert.Equal(t, "flink", namespace) assert.Equal(t, "test-app", name) @@ -655,7 +655,7 @@ func TestRollingBack(t *testing.T) { mockK8Cluster := stateMachineForTest.k8Cluster.(*k8mock.K8Cluster) getServiceCount := 0 - mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string) (*v1.Service, error) { + mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string, version string) (*v1.Service, error) { assert.Equal(t, "flink", namespace) assert.Equal(t, "test-app", name) @@ -797,7 +797,7 @@ func TestDeleteWithSavepoint(t *testing.T) { savepointPath := "s3:///path/to/savepoint" mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.CancelWithSavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, error) { + mockFlinkController.SavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (string, error) { return triggerID, nil } @@ -837,7 +837,7 @@ func TestDeleteWithSavepoint(t *testing.T) { assert.NoError(t, err) savepointStatusCount := 0 - mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { savepointStatusCount++ if savepointStatusCount == 1 { @@ -972,7 +972,7 @@ func TestDeleteWithForceCancel(t *testing.T) { } cancelled := false - mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { + mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error { cancelled = true return nil } @@ -1038,7 +1038,7 @@ func TestDeleteModeNone(t *testing.T) { } cancelled := false - mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { + mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error { cancelled = true return nil } @@ -1097,7 +1097,7 @@ func TestRollbackWithRetryableError(t *testing.T) { retryableErr := client.GetRetryableError(errors.New("blah"), "GetClusterOverview", "FAILED", 3) stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.CancelWithSavepointFunc = func(ctx context.Context, app *v1beta1.FlinkApplication, hash string) (savepoint string, err error) { + mockFlinkController.SavepointFunc = func(ctx context.Context, app *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (savepoint string, err error) { return "", retryableErr } @@ -1198,7 +1198,7 @@ func TestRollbackWithFailFastError(t *testing.T) { mockK8Cluster := stateMachineForTest.k8Cluster.(*k8mock.K8Cluster) appHash := flink.HashForApplication(&app) getServiceCount := 0 - mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string) (*v1.Service, error) { + mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string, version string) (*v1.Service, error) { hash := "old-hash-retry-err" if getServiceCount > 0 { hash = appHash @@ -1351,7 +1351,7 @@ func TestForceRollback(t *testing.T) { mockK8Cluster := stateMachineForTest.k8Cluster.(*k8mock.K8Cluster) getServiceCount := 0 - mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string) (*v1.Service, error) { + mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string, version string) (*v1.Service, error) { hash := oldHash if getServiceCount > 0 { hash = oldHash @@ -1452,7 +1452,7 @@ func TestCheckSavepointStatusFailing(t *testing.T) { stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { return nil, retryableErr.(*v1beta1.FlinkApplicationError) } @@ -1506,10 +1506,10 @@ func TestDeleteWhenCheckSavepointStatusFailing(t *testing.T) { stateMachineForTest := getTestStateMachine() mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) - mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.SavepointResponse, error) { + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { return nil, retryableErr.(*v1beta1.FlinkApplicationError) } - mockFlinkController.CancelWithSavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (s string, e error) { + mockFlinkController.SavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (s string, e error) { return "triggerId", nil } mockRetryHandler := stateMachineForTest.retryHandler.(*mock.RetryHandler) @@ -1536,7 +1536,7 @@ func TestDeleteWhenCheckSavepointStatusFailing(t *testing.T) { }, nil } - mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { + mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) error { return nil } err = stateMachineForTest.Handle(context.Background(), &app) @@ -1546,3 +1546,511 @@ func TestDeleteWhenCheckSavepointStatusFailing(t *testing.T) { assert.Nil(t, app.GetFinalizers()) } + +func TestRunningToDualRunning(t *testing.T) { + deployHash := "appHash" + updatingHash := "2845d780" + triggerID := "trigger" + savepointPath := "savepointPath" + app := v1beta1.FlinkApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1beta1.FlinkApplicationSpec{ + JarName: "job.jar", + Parallelism: 5, + EntryClass: "com.my.Class", + ProgramArgs: "--test", + DeploymentMode: v1beta1.DeploymentModeBlueGreen, + }, + Status: v1beta1.FlinkApplicationStatus{ + Phase: v1beta1.FlinkApplicationRunning, + DeployHash: deployHash, + DeployVersion: v1beta1.GreenFlinkApplication, + VersionStatuses: []v1beta1.FlinkApplicationVersionStatus{ + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "jobId", + State: v1beta1.Running, + }, + VersionHash: deployHash, + Version: v1beta1.GreenFlinkApplication, + }, + {}, + }, + }, + } + + stateMachineForTest := getTestStateMachine() + mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) + + mockFlinkController.GetJobForApplicationFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) { + return &client.FlinkJobOverview{ + JobID: "jobID2", + State: client.Running, + }, nil + + } + + mockFlinkController.IsClusterReadyFunc = func(ctx context.Context, application *v1beta1.FlinkApplication) (b bool, err error) { + return true, nil + } + + mockFlinkController.IsServiceReadyFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (b bool, err error) { + return true, nil + } + + // Handle Running and move to Updating + err := stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationUpdating, app.Status.Phase) + assert.Equal(t, 2, len(app.Status.VersionStatuses)) + assert.Equal(t, "", app.Status.UpdatingHash) + + // Handle Updating and move to ClusterStarting + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Equal(t, v1beta1.FlinkApplicationClusterStarting, app.Status.Phase) + assert.Equal(t, v1beta1.BlueFlinkApplication, app.Status.UpdatingVersion) + assert.Nil(t, err) + + // Handle ClusterStarting and move to Savepointing + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Equal(t, v1beta1.FlinkApplicationSavepointing, app.Status.Phase) + assert.Equal(t, updatingHash, app.Status.VersionStatuses[1].VersionHash) + assert.Equal(t, v1beta1.BlueFlinkApplication, app.Status.VersionStatuses[1].Version) + assert.Equal(t, updatingHash, app.Status.UpdatingHash) + assert.Nil(t, err) + + // Handle Savepointing and move to SubmittingJob + mockFlinkController.SavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (s string, err error) { + assert.False(t, isCancel) + return triggerID, nil + } + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (response *client.SavepointResponse, err error) { + return &client.SavepointResponse{ + SavepointStatus: client.SavepointStatusResponse{ + Status: client.SavePointCompleted, + }, + Operation: client.SavepointOperationResponse{ + Location: savepointPath, + FailureCause: client.FailureCause{}, + }, + }, nil + } + assert.Equal(t, app.Status.SavepointTriggerID, triggerID) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, app.Status.SavepointPath, savepointPath) + assert.Equal(t, v1beta1.FlinkApplicationSubmittingJob, app.Status.Phase) + + // Handle SubmittingJob and move to DualRunning + mockK8Cluster := stateMachineForTest.k8Cluster.(*k8mock.K8Cluster) + + mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string, version string) (*v1.Service, error) { + + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "flink-app-hash": updatingHash, + }, + }, + }, nil + } + + mockFlinkController.StartFlinkJobFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jarName string, parallelism int32, entryClass string, programArgs string, allowNonRestoredState bool, savepointPath string) (s string, err error) { + return "jobID2", nil + } + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, "jobID2", app.Status.VersionStatuses[1].JobStatus.JobID) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, "jobID2", app.Status.VersionStatuses[1].JobStatus.JobID) + assert.Equal(t, app.Spec.JarName, app.Status.VersionStatuses[1].JobStatus.JarName) + assert.Equal(t, app.Spec.Parallelism, app.Status.VersionStatuses[1].JobStatus.Parallelism) + assert.Equal(t, app.Spec.EntryClass, app.Status.VersionStatuses[1].JobStatus.EntryClass) + assert.Equal(t, app.Spec.ProgramArgs, app.Status.VersionStatuses[1].JobStatus.ProgramArgs) + + assert.Equal(t, v1beta1.FlinkApplicationDualRunning, app.Status.Phase) + assert.Equal(t, deployHash, app.Status.DeployHash) + assert.Equal(t, updatingHash, app.Status.UpdatingHash) + assert.Equal(t, v1beta1.GreenFlinkApplication, app.Status.DeployVersion) + assert.Equal(t, v1beta1.BlueFlinkApplication, app.Status.UpdatingVersion) + +} + +func TestDualRunningToRunning(t *testing.T) { + deployHash := "appHash" + updatingHash := "2845d780" + teardownHash := "9dc7d91b" + + app := v1beta1.FlinkApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1beta1.FlinkApplicationSpec{ + JarName: "job.jar", + Parallelism: 5, + EntryClass: "com.my.Class", + ProgramArgs: "--test", + DeploymentMode: v1beta1.DeploymentModeBlueGreen, + }, + Status: v1beta1.FlinkApplicationStatus{ + Phase: v1beta1.FlinkApplicationDualRunning, + DeployHash: deployHash, + UpdatingHash: updatingHash, + DeployVersion: v1beta1.GreenFlinkApplication, + UpdatingVersion: v1beta1.BlueFlinkApplication, + VersionStatuses: []v1beta1.FlinkApplicationVersionStatus{ + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "jobId", + State: v1beta1.Running, + }, + VersionHash: deployHash, + Version: v1beta1.GreenFlinkApplication, + }, + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "jobId2", + State: v1beta1.Running, + }, + VersionHash: updatingHash, + Version: v1beta1.BlueFlinkApplication, + }, + }, + }, + } + + stateMachineForTest := getTestStateMachine() + mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) + mockFlinkController.DeleteResourcesForAppWithHashFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) error { + assert.Equal(t, deployHash, hash) + return nil + } + mockFlinkController.GetVersionAndJobIDForHashFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (string, string, error) { + assert.Equal(t, deployHash, hash) + return string(v1beta1.GreenFlinkApplication), "jobId", nil + } + mockFlinkController.ForceCancelFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (e error) { + assert.Equal(t, "jobId", jobID) + return nil + } + app.Spec.TearDownVersionHash = deployHash + expectedVersionStatus := app.Status.VersionStatuses[1] + // Handle DualRunning and move to Running + err := stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationRunning, app.Status.Phase) + assert.Empty(t, app.Status.VersionStatuses[1]) + assert.Equal(t, teardownHash, app.Status.TeardownHash) + assert.Equal(t, expectedVersionStatus, app.Status.VersionStatuses[0]) + assert.Equal(t, "", app.Status.UpdatingHash) + assert.Equal(t, updatingHash, app.Status.DeployHash) + assert.Equal(t, "", string(app.Status.UpdatingVersion)) + assert.Equal(t, v1beta1.BlueFlinkApplication, app.Status.DeployVersion) +} + +func TestBlueGreenUpdateWithError(t *testing.T) { + deployHash := "deployHash" + updatingHash := "updateHash" + + app := v1beta1.FlinkApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1beta1.FlinkApplicationSpec{ + JarName: "job.jar", + Parallelism: 5, + EntryClass: "com.my.Class", + ProgramArgs: "--test", + DeploymentMode: v1beta1.DeploymentModeBlueGreen, + }, + Status: v1beta1.FlinkApplicationStatus{ + Phase: v1beta1.FlinkApplicationSubmittingJob, + DeployHash: deployHash, + DeployVersion: v1beta1.GreenFlinkApplication, + VersionStatuses: []v1beta1.FlinkApplicationVersionStatus{ + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "jobId", + State: v1beta1.Running, + }, + VersionHash: deployHash, + Version: v1beta1.GreenFlinkApplication, + }, + {}, + }, + }, + } + + stateMachineForTest := getTestStateMachine() + mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) + mockFlinkController.StartFlinkJobFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jarName string, parallelism int32, entryClass string, programArgs string, allowNonRestoredState bool, savepointPath string) (s string, err error) { + return "", client.GetNonRetryableError(errors.New("bad submission"), "SubmitJob", "500") + + } + mockK8Cluster := stateMachineForTest.k8Cluster.(*k8mock.K8Cluster) + mockK8Cluster.GetServiceFunc = func(ctx context.Context, namespace string, name string, version string) (*v1.Service, error) { + + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "flink-app-hash": updatingHash, + }, + }, + }, nil + } + + err := stateMachineForTest.Handle(context.Background(), &app) + assert.NotNil(t, err) + assert.NotNil(t, app.Status.LastSeenError) + assert.Equal(t, v1beta1.FlinkApplicationSubmittingJob, app.Status.Phase) + + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationRollingBackJob, app.Status.Phase) + assert.Equal(t, "", app.Status.VersionStatuses[1].JobStatus.JobID) + + // We should have moved to DeployFailed without affecting the existing job + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationDeployFailed, app.Status.Phase) + assert.Equal(t, "jobId", app.Status.VersionStatuses[0].JobStatus.JobID) + +} + +func TestBlueGreenDeployWithSavepointDisabled(t *testing.T) { + deployHash := "appHashTest" + app := v1beta1.FlinkApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1beta1.FlinkApplicationSpec{ + JarName: "job.jar", + Parallelism: 5, + EntryClass: "com.my.Class", + ProgramArgs: "--test", + DeploymentMode: v1beta1.DeploymentModeBlueGreen, + SavepointDisabled: true, + }, + Status: v1beta1.FlinkApplicationStatus{ + Phase: v1beta1.FlinkApplicationClusterStarting, + DeployHash: deployHash, + DeployVersion: v1beta1.GreenFlinkApplication, + VersionStatuses: []v1beta1.FlinkApplicationVersionStatus{ + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "jobId", + State: v1beta1.Running, + }, + VersionHash: deployHash, + Version: v1beta1.GreenFlinkApplication, + }, + {}, + }, + }, + } + stateMachineForTest := getTestStateMachine() + mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) + + mockFlinkController.GetJobForApplicationFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) { + return &client.FlinkJobOverview{ + JobID: "jobID2", + State: client.Running, + }, nil + + } + + mockFlinkController.IsClusterReadyFunc = func(ctx context.Context, application *v1beta1.FlinkApplication) (b bool, err error) { + return true, nil + } + + mockFlinkController.IsServiceReadyFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (b bool, err error) { + return true, nil + } + err := stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationSubmittingJob, app.Status.Phase) +} + +func TestDeleteBlueGreenDeployment(t *testing.T) { + deployHash := "deployHashDelete" + updateHash := "updateHashDelete" + app := v1beta1.FlinkApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1beta1.FlinkApplicationSpec{ + JarName: "job.jar", + Parallelism: 5, + EntryClass: "com.my.Class", + ProgramArgs: "--test", + DeploymentMode: v1beta1.DeploymentModeBlueGreen, + SavepointDisabled: true, + }, + Status: v1beta1.FlinkApplicationStatus{ + Phase: v1beta1.FlinkApplicationDeleting, + DeployHash: deployHash, + DeployVersion: v1beta1.GreenFlinkApplication, + UpdatingHash: updateHash, + UpdatingVersion: v1beta1.BlueFlinkApplication, + VersionStatuses: []v1beta1.FlinkApplicationVersionStatus{ + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "greenId", + State: v1beta1.Running, + }, + VersionHash: deployHash, + Version: v1beta1.GreenFlinkApplication, + }, + { + JobStatus: v1beta1.FlinkJobStatus{ + JobID: "blueId", + State: v1beta1.Running, + }, + VersionHash: deployHash, + Version: v1beta1.BlueFlinkApplication, + }, + }, + }, + } + + stateMachineForTest := getTestStateMachine() + mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) + + jobCtr1 := 0 + jobCtr2 := 0 + mockFlinkController.GetJobToDeleteForApplicationFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string) (*client.FlinkJobOverview, error) { + if hash == deployHash { + jobCtr1++ + if jobCtr1 <= 2 { + return &client.FlinkJobOverview{ + JobID: "greenId", + State: client.Running, + }, nil + } + return &client.FlinkJobOverview{ + JobID: "greenId", + State: client.Canceled, + }, nil + } + + jobCtr2++ + if jobCtr2 <= 2 { + return &client.FlinkJobOverview{ + JobID: "blueId", + State: client.Running, + }, nil + } + + return &client.FlinkJobOverview{ + JobID: "blueId", + State: client.Canceled, + }, nil + + } + triggerID := "t1" + savepointCtr := 0 + mockFlinkController.SavepointFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, isCancel bool, jobID string) (string, error) { + return triggerID, nil + } + + mockFlinkController.GetSavepointStatusFunc = func(ctx context.Context, application *v1beta1.FlinkApplication, hash string, jobID string) (*client.SavepointResponse, error) { + if savepointCtr == 0 { + assert.Equal(t, deployHash, hash) + assert.Equal(t, "greenId", jobID) + } else { + assert.Equal(t, updateHash, hash) + assert.Equal(t, "blueId", jobID) + } + savepointCtr++ + return &client.SavepointResponse{ + SavepointStatus: client.SavepointStatusResponse{ + Status: client.SavePointCompleted, + }, + Operation: client.SavepointOperationResponse{ + Location: testSavepointLocation + hash, + }, + }, nil + } + // Go through deletes until both applications are deleted + err := stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, 2, savepointCtr) + assert.Empty(t, app.Finalizers) + assert.Equal(t, testSavepointLocation+updateHash, app.Status.SavepointPath) +} + +func TestIncompatibleDeploymentModeSwitch(t *testing.T) { + app := v1beta1.FlinkApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "flink", + }, + Spec: v1beta1.FlinkApplicationSpec{ + JarName: "job.jar", + Parallelism: 5, + EntryClass: "com.my.Class", + ProgramArgs: "--test", + DeploymentMode: v1beta1.DeploymentModeDual, + }, + } + + app.Status.Phase = v1beta1.FlinkApplicationRunning + stateMachineForTest := getTestStateMachine() + mockFlinkController := stateMachineForTest.flinkController.(*mock.FlinkController) + mockFlinkController.GetCurrentDeploymentsForAppFunc = func(ctx context.Context, application *v1beta1.FlinkApplication) (*common.FlinkDeployment, error) { + return nil, nil + } + + err := stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.DeploymentModeDual, app.Status.DeploymentMode) + + // Try to switch from Dual to BlueGreen + app.Status.Phase = v1beta1.FlinkApplicationRunning + app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationDeployFailed, app.Status.Phase) + + app.Spec.DeploymentMode = v1beta1.DeploymentModeBlueGreen + app.Status.Phase = v1beta1.FlinkApplicationRunning + app.Status.DeploymentMode = "" + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.DeploymentModeBlueGreen, app.Status.DeploymentMode) + + // Try to switch from BlueGreen to Dual + app.Status.Phase = v1beta1.FlinkApplicationRunning + app.Spec.DeploymentMode = v1beta1.DeploymentModeDual + err = stateMachineForTest.Handle(context.Background(), &app) + assert.Nil(t, err) + assert.Equal(t, v1beta1.FlinkApplicationDeployFailed, app.Status.Phase) +} diff --git a/pkg/controller/k8/cluster.go b/pkg/controller/k8/cluster.go index 1da90a1c..3f4fe249 100644 --- a/pkg/controller/k8/cluster.go +++ b/pkg/controller/k8/cluster.go @@ -32,7 +32,7 @@ type ClusterInterface interface { GetDeploymentsWithLabel(ctx context.Context, namespace string, labelMap map[string]string) (*v1.DeploymentList, error) // Tries to fetch the value from the controller runtime manager cache, if it does not exist, call API server - GetService(ctx context.Context, namespace string, name string) (*coreV1.Service, error) + GetService(ctx context.Context, namespace string, name string, version string) (*coreV1.Service, error) GetServicesWithLabel(ctx context.Context, namespace string, labelMap map[string]string) (*coreV1.ServiceList, error) CreateK8Object(ctx context.Context, object runtime.Object) error @@ -90,7 +90,11 @@ type k8ClusterMetrics struct { getDeploymentFailure labeled.Counter } -func (k *Cluster) GetService(ctx context.Context, namespace string, name string) (*coreV1.Service, error) { +func (k *Cluster) GetService(ctx context.Context, namespace string, name string, version string) (*coreV1.Service, error) { + serviceName := name + if version != "" { + serviceName = name + "-" + version + } service := &coreV1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: coreV1.SchemeGroupVersion.String(), @@ -98,7 +102,7 @@ func (k *Cluster) GetService(ctx context.Context, namespace string, name string) }, } key := types.NamespacedName{ - Name: name, + Name: serviceName, Namespace: namespace, } err := k.cache.Get(ctx, key, service) diff --git a/pkg/controller/k8/mock/mock_k8.go b/pkg/controller/k8/mock/mock_k8.go index 9e3c79c7..de383800 100644 --- a/pkg/controller/k8/mock/mock_k8.go +++ b/pkg/controller/k8/mock/mock_k8.go @@ -10,7 +10,7 @@ import ( type GetDeploymentsWithLabelFunc func(ctx context.Context, namespace string, labelMap map[string]string) (*v1.DeploymentList, error) type CreateK8ObjectFunc func(ctx context.Context, object runtime.Object) error -type GetServiceFunc func(ctx context.Context, namespace string, name string) (*corev1.Service, error) +type GetServiceFunc func(ctx context.Context, namespace string, name string, version string) (*corev1.Service, error) type GetServiceWithLabelFunc func(ctx context.Context, namespace string, labelMap map[string]string) (*corev1.ServiceList, error) type UpdateK8ObjectFunc func(ctx context.Context, object runtime.Object) error type UpdateStatusFunc func(ctx context.Context, object runtime.Object) error @@ -40,9 +40,9 @@ func (m *K8Cluster) GetServicesWithLabel(ctx context.Context, namespace string, return nil, nil } -func (m *K8Cluster) GetService(ctx context.Context, namespace string, name string) (*corev1.Service, error) { +func (m *K8Cluster) GetService(ctx context.Context, namespace string, name string, version string) (*corev1.Service, error) { if m.GetServiceFunc != nil { - return m.GetServiceFunc(ctx, namespace, name) + return m.GetServiceFunc(ctx, namespace, name, version) } return nil, nil }