From 2c1b6c8baa85c9e74f00a711d1a2761b141dd9f9 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 16 Jul 2025 15:18:18 +0200 Subject: [PATCH 1/4] fix(demo/trino-taxi-data): Use new listener service --- demos/trino-taxi-data/setup-superset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/trino-taxi-data/setup-superset.yaml b/demos/trino-taxi-data/setup-superset.yaml index 1174c2a4..74b86137 100644 --- a/demos/trino-taxi-data/setup-superset.yaml +++ b/demos/trino-taxi-data/setup-superset.yaml @@ -39,7 +39,7 @@ data: import logging import requests - base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088 + base_url = "http://superset-node:8088" # You can change this while running locally, but remember not to commit the change. superset_username = open("/superset-credentials/adminUser.username").read() superset_password = open("/superset-credentials/adminUser.password").read() trino_username = "admin" From fc1a99182513209926ae2456c45b656cafcd6c2d Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 16 Jul 2025 15:18:56 +0200 Subject: [PATCH 2/4] fix(demo/trino-taxi-data): Use updated superset config See: https://github.com/stackabletech/demos/pull/251 --- demos/trino-taxi-data/superset-assets.zip | Bin 8561 -> 9602 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/demos/trino-taxi-data/superset-assets.zip b/demos/trino-taxi-data/superset-assets.zip index c0a686b55cbc2b39dab1679605f516eaed80785a..9dc7a395477940b53aa4b4d2a7b9e4eedba25ed1 100644 GIT binary patch literal 9602 zcmd^FTW=e=6;}4r2cbY)V1f2&!J-ezb}_c@js+B`VyljuSaxM4n{3e%)C@;rJQrmy zvX!F1{)M7XMgKtG`wR9@^s(>#1MOSU?~okbBHJ;NU1Vz+GBk(e;kl7K{LY*_`tYMq zO6K#^(%-)R{(pas2l{@FuSipwrCeT2;w0m>O08MhskR2yYNOd~Y=<%vp2)=3OoYKy zCaH>}Hmhz`wkqXpHj!;+#+Ne{$}|(}Dce>E}+@2woVh4N0dwp9Hb+3c9?= z645R*rl(5zR3%w10uHkTk{lVt35=knVXW{VpNK?+nEelb#84THR3r-;2#mRL6iGMZ zA{nK`oj&lm@@K`whBnT!2~Wwcn1miubk@%zE-Sl!1MV8aagBzH zUzMgaba#i%s^_*`zgb4GCCWwkI23pIX&0m$=TU|*fn$fXXpGVKj-T`f_kR8v8f4xJV|8aB6hW15%GG}nCL$jNu;6?&p@;t3X?!` z^V77zR8Dk&_Y#!6aN;JyZ{=%Dz$i+&bt&6WTjgmN@ z3?aE)!NxD-^EqkRGA{N)c0QI7Bj>YUFsI5gdMrd{5*v^<1D3r91vVLQGRwv>LZ4!9 z&6EqvU1FOo&anM_$E>pu`Wv?o7M_C5zeAy8CH}@NWB1kUdYjlaaM@}a93FK~2c4sn zd#%sd7l(s~j415O<6al$es_z^uG)9Ca;*NyLvpM)z2M(EI-Yj=dk;JP_gU>?M|x{r z_m7`Ec>G>_c(J|Q0%eOdNz7vZT*|YzR{5|u=pJ<6>sVgaIgF!he2bKxNs-(Jb?q_8 z12MV{>K3QcEsN%HoZqHsz7>&-o6!HIo@>a+9QV2bxU!0-v^)IZC*@L!zV&O-6n88V zU@5)ld>}3quWB464U}74>3h!^wQPV(T$yCbSGe)g)mG=zmgEXTRk#PJQ3U`cblX(D zvbW+;U0{&L7lB6M8f5_!_*Z!2*tNI?1_ESSqj8{eh8jhC*RM*snUu?nbb7!CJ`KzA0hdx~wt#g)9u4e|!EzA5YLv%c0_tZkGCs3a1V%?K*Q ze0HPN*Pr=qU(0Eq=5hyeggg5&^p1@9fB@nNrZNo!4Z#ALiZwD3isVct-Bo-D;L0)? z`W48aCZ7TjYO^L-nr`%rKD7!_3OI)`hl$ZBks}Q)b)Fuel^{dHkqWTzDt8*_a6=5( z2F=JqE{p3jHEz0c5q%i~v9~iKZvab(Jh!sdooEC{wzM7^t(gnA2msw{9oQk_6=(@H z8#rf?#1=FTXCj6H3X^r|vkHhU=}|yb4pW| zcPR=1{JspjOE8_)zy=P&7@fYw&?(Tmn}^V)D4J#U8Wcjx-!Ry!HS2Yt8SgyWmSTs( zZxL!s4ev#?t@-HEs1wzl`%!B*s5S%MRO8x? zHeal|E%JbU#3onSx=+uE2Y{FMnfcbZAkj{2_Y5rDG0);S$P}tv1rf!bK!pl8apIv6b29OT@79@z0;uHkNPvkMiYd9a|;$$*sEbUlT>XHffY^jkRmpao~ z2~dej8RopPA)x6vj7G3cFSgMOb_@Evn4v2`4pWgE2t7oW%Rs5MxggT4xMe934E7ZI3`3HtIg_U_E`r!4+QyqDh-qcW1Uc zvIpCJQ%0I13q%1f4f4@yM*5~n>LecqDjhFsamG50R{eg}^MzA$>qDne^>-Xm5z?vp z^@>;9^@pPFmghc&I|;WPI7h>z@pSJuYUkKwRPI;>9*S5tRW%9AdRL6rd73T=({G)k zAKHzA2xF;6V{A{&;`Gsbi8O{o^m3vq|OI$?SRD{8D!@Ig?fo&-Dh+X}P~IFtfCrtuM+be>Yxnt}-|3y6 zbo$*M34mPqP2FAy{sonZ^;;z*Z;C4=)|*a8sW6QTmc2KvUcazVakNSw72z?*e{6ZW zyh~d@GGc*^-{U&8i}4#+(n;s>f{O~n94|o>gY*SVFCW@TeKpDxePguZ;LQ*mghiEn ztp~=gUCmD(-TB~?4=;c323w&1e1(@@ZuAz;I^*@XXAIj?>8m?`zLs%;A1uF1v(A70 zeTxPEuRpj>aC>TOdi!F1nsp#zG0hKP#H-&@+6YUmPq2OfEhf15^tuV=r_6Q7(aEEa RzekKe#^1l;+3C;d>3^v&B-#J~ literal 8561 zcmbtZby$?!*BwF-q)R%b1qTUX=x*s&V1Pk}j-g9hWB{oFrCY$EO9TW->5`NbP(VRY zLdtLO{<6Zl_hjsQ|XKhVY3`{b>*|=vOumA1he;@RyZ?Kyi1m^}uuqKNxhtMbu49fUl2- z6!kCwfM8qz;KuL3=z_hVW>#PrnAaQZVBdsLO-L0c>e@ObYv@j+JU>I-RNVZ;IHcuTg;QxNwepzP39UhaD z->8+rcHLNqc?8OQ)bawE-J{K>ATWV`C*E3$3!?Yz>9`c_XlERI)*=%caSSb`5dxn| zacuk6k{5?9hN(7hf<$@hN?P%KNKGkLu(?=9=#3B$ckOt9Y@~sRk12Zf7*yg~%q~3vD+RIz_q8u_@w=d`9 z=?n|tEx0SYUt;*kcl$n_t)^1_9qDBvyvRaoc2`W#^0$mp4`e0gzP7Q`)CaIRMjyl) ze0hIsaGUAtp}Wc@q23NOiW~Rsui#-QP<9c?zYZg|z|07wh7z_doLYd;jj3hoTNY4Fi6nJ@bU&vLJs zHsUFZI5nrVR~>=Y79@0C>CoPsw|Hj1K?$)TT&Jiv%aT~D=5l~mzK`0xIXB_7g@Kk_0G@*rIHt=J}qRq}c$U^79(qw-VC!^%(-9XrEl_ksD z$J}$;1GW7FMIsGc;C=N9Ew{<1PAm$UIllT`x-Z|eULAu}5rNaJwG-DzRS`8dd%BMF z)&e9(-PTiTTPFua9*^V=E9Bl0=QWQ3IOK2L=fZil<;H_syN+>^bOqsN?3|QOWt4gl zlX$&9X@X7l(OY+>TIv4dW#$5RHcyD@@j;P>BUXWme*0Q=(~Z=$bs6NNudAYO(ua?* z|9Qi6{K8I!fi1vpkiYKt5LC)(-k*-5MFRl(QSa-&fuidQb#(f+)o(vgc5DOU$?t>0 zco*mIG)Y{*_W4#SM(s0eB&g^^a0W55pVbH+pfKtWRV6?`ArSBe?+pR|>q6JLfj|%t z$d3DcN_E!d-JylTQ`hx;=06^*j03iCy5MXZyh$GDG>_Obj`1pms zQ3;~k>o@}>YCPhG*%J~xnXa^)eIyWpUO~F-!i8(WTFUw&vHK&~gy(SUwst@_k|0>P z%6RiV0W_Z@u4c$QLriwUe1l{HTAiDI(DA|`!zPFYAB#FQHXhRpsZvWkl3n4w^M)Oq zAKwEs8K>O&xF#-Zfi9GcfTjm5U+W&OEpCoAZbV%1o1Y-c0u)V%LpU^osZuRVTw@jg1ct}1-w z&JxdRRlb%s$z&FP4xYSrL`jdiJnqi&u0J`GZMoBQMgciByHE}sN~MS`6AxIAXpE5> zQ2IhnOiG8nnTYW+OG^S5hftAA`Cao|YoON<7V_lsrUMa{&p2I%&R|5!f_G(odYV2W z-L5YIUn%Nxxah>KkOnvR2cYV{EO>zMe8$`sJp&d|u7ahFo$ItmM1kg>)q!-lc#3{C zISY;B(`iCI)4Y9t-;-t^2}hlfrv34vRX#6s!^(2I*Mw+OibGP8_%o*>@mxZFbW!W+ zE$dGf89MAU=NzZBX;%+(Q~Q||5@s1r85B2C(fgl@mX(xUbFx2aZt(SDLcI2N@$pc- zr{%lj&B7s+9(pXYY&n}!Mr|`Ch+1t)t zpr3qzBT7HVy;ia?hbO5m&#&)pj8ay`2o}cE^)qX?= ztvPzsu-P9w=v*)bHZ8DXpy=<45=|n%N`GA^7}(wn<_dL2kzWA#jrSsWtVm~TcDU{2P+PMGrNr#9rp>NgUW=P3-;yv^9w&QDh_%bo5pfDr5$YEFY*>}F zot0AO>ikll{;4FMT-f`1f+nd{Jo9W%_ZO7aG@~DDbm@^LQ-fyh)Vc7L5PQ?17(Oz+ zDlzI{c+P!m#Z?%!x=9rrP9Y{tYA@SSK>n6KRA#AupWEob&DX(}nqc;2`lT9*#9qAm*iyS6#2t zoc~mIS&i8v(GofdjhJ)3Ig77g%V59*(Z0bY{ItXC#mUCnEDz+pSZ97oVZ>l8ce7u> zf$M&Xz>xie*bcRicamfuIP_EKnfA5Qj@p}A7!SxL-CwI#O05mG4XAp&ny*w_y`>f8 z7q4Dd856K7P7Y*xns*@4WN2`lwSfr$ zfPRlzP)AEA2M8*!iu^$>-5G6EUiBX|IKb|kj3f=2S6K|Aiobp#14I|iYo{n*3+KI( zJsP;B;&E?u343t;y=~bYNYb(tXj&#o463NdELv)k()7}Ei9q~?no!;+9t{4CqHHA6 zT3Mz;-%VR}rDc-q%Qh)rX_8Q0^6>?=a_JJTm$@yPhF5bZvxGAhhjo?1gR^3cKj3$n ze_~~dxqjhZ+Z>gOt%EU%TJiEegB*>lXxZbqO{w=KGLySQUFFS$;R{wYrA|S(K(FU+ z?LD5bjN4@Q>K5S3mciTJ7sL*Yn5F2h_~SK0d^ zod&{h*^pIGkK!iv;w@i-J?uLCdg>aUfXzZ#geM)3H^jw zvt(TVr^Ca&Be+iU1(INBU|!;WTrr#nx4SYCsqbzK)GBzgU;qtxI2w$QH!5A_&l?5C z9y34dw%+asC}I&>R2bh&yEY;NJYES?nyTd0uINiPu$PA_^iKVhHdSA$$9_SP`vw*O zAp5K2mPW}S@Eypv#A#;b?h1yXG!csB!rxdP_JevjjKX|6b4L#fdvV{6$s zJcin&oUe1NtR}A1St6LLeXb(Us<^qm2;@dcDwi!Tg;P8bZgYRpm>t7CHM4VLi7Q%c znZ`DaH741-)wx{G8An_lpT}Wbm83_Ai`|ZwmNJ!9G5||WWyOMv+2+;KQ<5H{R42wp zhb9Hw*LqS8d8&QAJf4r#fZX%JQ0Xk7c6My29Bu?dvFXQI}i8}a~ z#~MT<9|08+cmvjS$HbSk@Z|Ba?@@%ST>E(2fcS_D7&)31x%wq)f{7<|$ueulM^CWk zB8OCE*(8Zk^8Nip36fsJyZzB5=&kXOE&~UT@mrz77#?D=`0v@PIq=91hz_?;D6%Ix z&3dk$c9K$dO^B75^t5R4YD)A_6cV>65lymMnm%!T(2toJ|*Z%%O0xc2;r-CvDLd3<`&V32~yP%~9vHghWuaIh30SnYov!@DW+V>4jZ&v5~gNXjzQy9 zTN6aifm=TXTVC$UUN5%R?@=9p=hWbPJRC|(?;B9>WgOp87txzqh;Yc*pyV~_O3z!E4z7vBVdr3Iwzat z*2#q*%ZB^jwFxbh7)iqZowD&wiu_qNZhYHNNqQ(nB23hLWbm&3c1sZXOl7XgtW^*| zc^a8MY9Xz;th5?VViaGAq&S-QR1&{R0}a}3{piQv=Izg4s|{o%=V6NWP_UtiykA7Z zhP|C$ufTHaKB@DP91aB>AUZrYG*Xbjk>giJ`7#VwWL?x#sX9>kctGC6?&X~b8nU8#q2fSTrP(9j~EuR|dXPZR)Y)o6Bnt~7Q> zZ*^Ms#{!CxIxD38QUu~~X0X{pih@W|?FB14qHsDuzCY8oAankUrQ=sPHg$0a216k=q=d0(^m~k^9PLTor*Q5@SLu z@cQWv$yoHv5o%8cWscp>8+M#Ic{|ZR+IEpy>FR4ygo6JhMvB^0fSuws+M*&kJ)%MW z(wOJV{ck=c^Y(o$7_jtZ3V1=sV*Mp2z@#rZZxP~>*Y6Owz27~`S||XiM;Olce=vHD zSGj0@k74b>G}TVjR1l@FMw<2rjz89`I3+FI_b{=xLCT$G@k!I#T0w1HXEcX_$vM#o zJqDSBXVM=V?-h$)HCsBO@66R%h`Lt%X7a+#kK{G`F_bPcD& z+nr1B+gT**fi5Mv@8sU$tWhqlZg~@84P0vxcPIsF6v`|#t~TrRE9#2r)83p!)MImVo-$vwyIL`PZowD2th~SMB;G zwZnoprglV|AI|o%Ee7J!Ld_OFSq)eUW%7E3n{5YcR6&|kb|8(7y(B7kV1i!=|M~3A z{+}iA%nCSjga5_$<@t4xZ}!bk*6-wpzV-u@4b+VT0Lc6Q_|_@8ga^hg;m9`yPmv~kn1(ZGs)Oh}T9H71^_6xz7~<2{m3 zQX$2n&|=vl_;9yfEIEtCnk2Gp3JVq$_vNn4HktXycAg8A;|?Y9I{5wdP!-b!yyBeV zd=Bef2cvtO&(n3?@e>+)@U%6>2CA61l4ZZt+wnlWn_GwKy*Spp3g33EygKq+Yc*FF z4YY`!Nz{iQ?W9khnvUnG=Uu{j$B~A<{N)m6$@}|S@z{JdNI{QG+vl|LpqZBw4++6~ zuVbZLae4N97xIL>bbYc`mKINEcG}t|N|dq7aZ7jmH*Lo1itGloK=P7|8LfMw#-h&# z6Hf90Z;9lcCHiOcud7__Zb?gjHeq#g^zlx$hQ^?$+2*{=8`_{|v0}1t9CDniUA?V4 zL@d*Kho?K2HVkYh^82$Q(xp|87=|QuW_cv8(I-3IwN2Qh;^o+8AkA+)(#+l1Y3OS> zGQfz;eV(W$$Fji*k8)wPmUnk!iK2pUSm+AA(p3rFe%up){$oM9q{EQKjFNiksFs4n z|A$w9a7tg*X;kX0C>@LkFLT{Y%UebuaG8fGk`gy;8%1YI*Zs>yZ_;F+dXC2?N`(+D zX?Ax|2a1+$rB*azLc;p)b{g4^!xyCa&@dyhASsL^@EKb~ylhw6l4p(GKwM&{ocV&O zi1v8XO%cyvFsd$LDO<|S-aMPCjcJ~DSNX3!ZbCAE}b5LId{zDok zJA=|pzlJ!$5TD>$$mGy0Q;^9hQt-oOMY=_v8ABjjts68@vD|6H;2070sY;{w%a^XU zVM!THuZ<<_+DtrFFRLui0Q10xM^4wuG$hiRj)$&F*EpqFkg(}80%`Zd5<`fFUbma! z&6gN;XnmO2i=38C2!%pUAC10Tnq9K^vXrL3jSFca`O1?y>7L9XNll9%zp8B{V>LWm zr;wAFz=u}_W{Rn@03}WF;jtT;CUzLAgn~`4%%e`xXy}p{|J+qWQSjS1kv{wSuO8b! zW1V%`&c}(g80zxhv3}^e{Q-M!BK_QTJBPg+6!JT;Klb4MZ5_@U2Iu2Mng``D{MX#S zw-3Hg{qGtG=csps+n%kF!DWi*f$l8u&Tj&u66X0p&@4F(8VuDD(Eu2m9m1bROpn?`Mvk zNxbuMBE3oeUvd5-`2INuok_m)aU#u6@ynp+0`Nxy|11H|q3;F-{4Vt03c~aCIp^`u k(%^f%amwF(l)uLNCKNPPFQ7IZ06>I#VWX~eY0iHAAAHEzW&i*H From 4fbb12a54449189cf07e2736ae66beffea8a34df Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 16 Jul 2025 16:17:33 +0200 Subject: [PATCH 3/4] chore: Bump Trino worker by 2GB RAM --- .../demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc | 2 +- docs/modules/demos/pages/trino-taxi-data.adoc | 2 +- stacks/trino-superset-s3/trino.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc index 4406aaa1..2c2d95d0 100644 --- a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc +++ b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc @@ -23,7 +23,7 @@ This demo should not be run alongside other demos. To run this demo, your system needs at least: * 10 {k8s-cpu}[cpu units] (core/hyperthread) -* 32GiB memory +* 34GiB memory * 35GiB disk storage == Overview diff --git a/docs/modules/demos/pages/trino-taxi-data.adoc b/docs/modules/demos/pages/trino-taxi-data.adoc index 96c48a4d..a4ec1de2 100644 --- a/docs/modules/demos/pages/trino-taxi-data.adoc +++ b/docs/modules/demos/pages/trino-taxi-data.adoc @@ -27,7 +27,7 @@ This demo should not be run alongside other demos. To run this demo, your system needs at least: * 7 {k8s-cpu}[cpu units] (core/hyperthread) -* 16GiB memory +* 18GiB memory * 28GiB disk storage == Overview diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index ab5a4bcc..99fca784 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -29,7 +29,7 @@ spec: min: "1" max: "4" memory: - limit: '4Gi' + limit: '6Gi' roleGroups: default: replicas: 1 From 6d2a408315dd9dab13bc427d56eb6cfb47e674da Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 16 Jul 2025 17:07:42 +0200 Subject: [PATCH 4/4] fix(demo/trino-taxi-data): Disable ANALYZE step See: https://github.com/trinodb/trino/issues/26214 --- .../create-table-in-trino.yaml | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/demos/trino-taxi-data/create-table-in-trino.yaml b/demos/trino-taxi-data/create-table-in-trino.yaml index de3fc97e..f040b453 100644 --- a/demos/trino-taxi-data/create-table-in-trino.yaml +++ b/demos/trino-taxi-data/create-table-in-trino.yaml @@ -59,29 +59,32 @@ data: run_query(connection, "CREATE SCHEMA IF NOT EXISTS hive.demo WITH (location = 's3a://demo/')") run_query(connection, """ - CREATE TABLE IF NOT EXISTS hive.demo.ny_taxi_data_raw ( - VendorID BIGINT, - tpep_pickup_datetime TIMESTAMP, - tpep_dropoff_datetime TIMESTAMP, - passenger_count DOUBLE, - trip_distance DOUBLE, - payment_type BIGINT, - Fare_amount DOUBLE, - Tip_amount DOUBLE, - Total_amount DOUBLE - ) WITH ( - external_location = 's3a://demo/ny-taxi-data/raw/', - format = 'parquet' - ) + CREATE TABLE IF NOT EXISTS hive.demo.ny_taxi_data_raw ( + VendorID BIGINT, + tpep_pickup_datetime TIMESTAMP, + tpep_dropoff_datetime TIMESTAMP, + passenger_count DOUBLE, + trip_distance DOUBLE, + payment_type BIGINT, + Fare_amount DOUBLE, + Tip_amount DOUBLE, + Total_amount DOUBLE + ) WITH ( + external_location = 's3a://demo/ny-taxi-data/raw/', + format = 'parquet' + ) """) loaded_rows = run_query(connection, "SELECT COUNT(*) FROM hive.demo.ny_taxi_data_raw")[0][0] print(f"Loaded {loaded_rows} rows") assert loaded_rows > 0 - print("Analyzing table ny_taxi_data_raw") - analyze_rows = run_query(connection, """ANALYZE hive.demo.ny_taxi_data_raw""")[0][0] - assert analyze_rows == loaded_rows + # Disabling ANALYZE for now, as it seems to fail when run twice (which can happen if Trino OOMs) + # See issue: https://github.com/trinodb/trino/issues/26214 + # print("Analyzing table ny_taxi_data_raw") + # analyze_rows = run_query(connection, """ANALYZE hive.demo.ny_taxi_data_raw""")[0][0] + # assert analyze_rows == loaded_rows + stats = run_query(connection, """show stats for hive.demo.ny_taxi_data_raw""") print("Produced the following stats:") print(*stats, sep="\n")