From 34f1ae3d7937831ed90efe6bfceb501c36d09094 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 13:19:47 +0200 Subject: [PATCH 1/9] wip --- .../create-spark-anomaly-detection-job.yaml | 16 +++++++++++++--- .../setup-superset.yaml | 9 +++++++-- .../superset-assets.zip | Bin 4279 -> 4665 bytes stacks/trino-superset-s3/superset.yaml | 6 +++--- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 9170ee66..5d746d2c 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -9,11 +9,21 @@ spec: initContainers: - name: wait-for-testdata image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data"] + command: + [ + "bash", + "-c", + "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data", + ] containers: - name: create-spark-anomaly-detection-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml"] + command: + [ + "bash", + "-c", + "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml", + ] volumeMounts: - name: manifest mountPath: /tmp/manifest @@ -89,7 +99,7 @@ data: spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions spark.sql.catalog.prediction: org.apache.iceberg.spark.SparkCatalog spark.sql.catalog.prediction.type: hive - spark.sql.catalog.prediction.uri: thrift://hive-iceberg:9083 + spark.sql.catalog.prediction.uri: thrift://hive-iceberg-metastore:9083 # AFAIK This can be removed in 24.3 #spark.driver.userClassPathFirst: "false" #spark.executor.userClassPathFirst: "false" diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml index 58596353..e5f9e9a0 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml @@ -9,7 +9,12 @@ spec: containers: - name: setup-superset image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"] + command: + [ + "bash", + "-c", + "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py", + ] volumeMounts: - name: script mountPath: /tmp/script @@ -39,7 +44,7 @@ data: import logging import requests - base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088 + base_url = "http://superset-node-default-headless:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default-headless:8088 superset_username = open("/superset-credentials/adminUser.username").read() superset_password = open("/superset-credentials/adminUser.password").read() trino_username = "admin" diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip b/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip index 2139e878a51ce65d74df01f0f7c4301333db473d..5ccd11bbf984d37774fce01dc09ac2549b464e2a 100644 GIT binary patch literal 4665 zcmbVQc|6o>7aviUC?pXvldKbF#ul2i^0fj;2-?%K}+6GjBsScaR$!tQrLKusEkw0u#&JMyP>}2k>dMpR|MWWORMd>Si~;= zo`Zo}!N6!9!e`b)CQEARo_I|1FwuG6RI52ZyCSUBD_3-n+M}Nvk&}PdZ8tHV3!&(o z%NY1H8kTab;Zi4W_cRK?4-4^|G{7nj^MW#A`}aKTI-|$ppdZ3(xzu5C>VC?k^=a@* zL^YgcNN+y&yk>pm?Lx2@%gU$IukMFPI#YLyQw*0QAEg%w$)5(bmYim3N_ZV>_nT6s zP_5z>2x%NU*y8$?SO3{A=$HXHDCmBIc3~^?AWqV)uyuObJrl^avq=BY2cF=?in#iC ztDt<7Y|^p1gP0)TbPq>hHf6_aOYw*IxKusC#l{2=hM44?<5EKyDvFFDF?rYSfkbe= zfAu1!^7;AZ#i+Rk{*ae#ADg2Jv}_a>FNqu+QMJXUz*EXdkPu~h1Tj8OvX-$U%JrF= zm}i!`{sUo@cr$-3h%r%N3aJ+OZN3j?qC6=;A*IjrqO(Z;DY%_s1og{N)$M#G?-Yb^ zIX%Ul@VbjIMcX#3hPN0&)qS_%m($;<)cWr4?&|D_H2m}gG%a|hwdbtvozX9nj|wl1 z-l4w^6iM)JJEkAN)NjI^Vf~+LeHCp;;+egyCEO9;+pvQ%7lxG525X8JYT7YZ zpN- zDsw(og}3FdqqP1&<4u#+2xQ&+ir*L*0Q#GSAUz zCmpWaB@gms8*tW;w0@^KqIzKI7MILD?&tNV3ud$RC;V*G8o!@AXtsruK#?oSEzmx?k=4o|isO*>6Gt96n>{Dw3bofU z*NQ(L`m#M|Sn}GfsfC;S+@i|jQ;$Dyq!z~VT#y|S-xq{$yl@|YMoMhrX~RIAO7sLyWNy|#fPVk z|g|fFGuiy=qd>+F;dGz97KCElML$%GZ*~89BT+exHd%c*o zeOKniaF2u3h5WHBXp4!nKGW>1-vz&%tcV2XTE3M-Msab}zKzHsm46~#QyXCX=KX?A zA$X)j@Vea)R@HmDed#Wx074cw)1`8botmc)g~ElI0?bT$I>+MDFN;S2*<9ke4QChG zWU%b!vcarr+Tn3Qmg>o&OW#dT8o_;%Ij%V9;;$T?xo5E=XsnlsO12-eG9T|$Sd@;_ zFqlMSV)vYVYh2@x*GXv&u-Hw`@agby-~4f z;)QE28Fb1)WsLo$L%)0&G@)Fn|J`LRO0XLvi!gX1k`RJ(u+yFjj^_7@)BpvOF2lng z)8FcI74|1i9GHrlz!|~6YoujGNoDn2sDQ^5geFa?hqOJh>7+wWGkti{5--mt&e&d^ zM<=AkZ@D~`F8(!gu@XudCO7$$c_C-HD=4SBJ}+SE%UwxAdBR%qeUfFBBNRPIQ!N+o zhm-aFD>Jr9_P+^P4|do1793n~EUGGln_1lWebO?0Hjg?0%&{AJqzJb97`vRiMECQA zBf6z%uQorM9K7V#m8_lEU#fm|JD&BQTa9U%i+3*optxOCE*N(_nm}A5!Jvg9oDa(W z;A7bwTS2AR4dCHLbr(b&3qrQiM7r3@Kf}^zX2y>{yy)8(ql#U@>?IP;!2_5Zts`IF zMAgGIT*^I+hje&m_*j$7>};~bi%$3{bKZ+TP>unb@;Ex2gT09kjikVGK-LnSg|GRv znLk@Ij>Z~^XprszO|xTP6$fXtXzf67H!RTc@maC4AwVVj%poprNSf0mj1B>3HRor z{JyLdcPCPjuj3h19z1QYep+X4BQQ8Fz3jvow(Rtxbfyhg z_9cJ(qpIq~vdlS{ALuIilenc%V1y6V*;ZRLks3x_h<$@aC40-BQ9$nw#b9M`s_CLo zweo2uHz@cejB4rNTuHQm)a|onJhuk9Cvo0Y%Dkc{nUCfMu1o~?RQC&x1>S2dQyZM2 zPL<^^7+EB%FxLeFo?ckHHY_wkt}YNjU`(M@kM_2@!L+^kOn^lbQI04gYK@p)Xgq;d zQk8&EDTow=h3M;nRs#NzXCb;^Y5YZDJ(Pf=tAm9k6e77cS|~x}mE<5&atg;}pq_tPhC$Hjw|< zKdiUmAz`0W$?HwwV*OPlHe~3nf$!Y4njLr3J^?!cfP5MR0PtV)+jFV_ delta 2924 zcmZWr2UJtb77c-50znDA_l_7ON|7cay@e8tfG9+IGk`#_1Vm~?!5}4|G${fRL8ODy zq>D&zqEa59QiAmMf&YE@{=2j8nl-a$?{B69CKzK$45LA?0RdnD03Zl(d3O6O z2M_>oqX7W407TOhLd4PIqWC8kMqbJ18EclMSErd_^2c9Rz(%6TxiR(%FRW3{L%bbr z?K%mfMXXbA{n;18?Q_iF5je#1=MGuwCKMMt`lGH~Yn@g*6G5gP7i(9m!y>}i8a0}R zgw^Tq%x6?jj|!lsGHknY18Qc|Djj>ZG@Exnx%CEAQsdFRe4Q7VTS9`5L zECAst+9t1WRDic<>`MK*36^uYYS399$FbguZS^^K!pFL{l&z6KNJJclh;S5fTjo_mt=4?VgVxA}wF+J@(;2aDB1tz4kAbjZ{h9Y>2(Tc=iz=O#>luqmN9fqt;t596D}3dB#0IeJVP2J+P4;k z;4erwj$!j?UvyBCmu_{Lm}kTaWo9OcR?|#WN2JCVY<}6er^Eqq$t-@U*Y&=2;4QB9 zl+@InkMB(>8fwQiufl`hn6?Ipv#|0SIy$BjX%fh+w{34@@n?<`VzaP_mXj@3d!d5L z&jYaf^%v#m(V|iQ{;oSGzD41Rp8eW=+$9;q0vHI1$cIQ0n<$uev>BRzJ)26_N9fA% zyHH@K*49=I(ocLMzzGD9Sm4RO&tj|qeB~|M^@A9Fb#rmm!Ga?Z|>5jaAzz z?v-Y`STnx!NVvg`Yog`0O(&(t3Ut8C$_=yHQYZR5Te{xbU`iCzt+>0Y&yb}2@INkE zUqckzwP_SjlUt+Z;?LEF#9U4%_*31Ofps99T-#w?5-SBrJ1U-P2Vac4o@V2N?N#Lr z>vmsq;*(w7u|m49`jmvqjL%wyFY>=7QuO+jyg=3;zft#-lYcoiCeQsq>xA)?X-m== zI9`2s%2!vX-5HUF4Oy z%eQst3lg|nxk;b^F;>chH~H;gcPEMR?-AUh5NAvlAi-w+J-L81tZeWFdHKdXmnRp$ zl{z#tw+yybxEcb?%o`$^}V)sNkV_xkl^ zzO~?O_gwH*Lk(T*jvQr3-I-JxPq=FKv{CialgGQJuFmRSQty{2P3Hax_fFiag?k>L zZ03vJ^Jx0eWX0gb-iLvhxC9hENflKbH~82+$b_BF&Ey)Qii?gF@90m}*}(fj#Z~g1 zvdvzb1;bO*$!`%4gy`a(wzDRnj^c_P`cdXOb7#Dm)M3qbB@90OA}h9ESl(RWd~v68 zIcqx4kHVS(3h&ofDZtLI={-K)H<78|y%)M+)DrwH!XKtgziGds^e#Meh44R)4vi}yxXuV zzI!f43F<1H6L0Q*&3d`h^4EmfSpy$U#8Vs3&dB zx`?iIyoBvUBxu$&LFM2f@v7|3bB74igGz(K zHFL{f1va{hdfFSEx6EKS?$fx8uorVRinyOktz+JAnG|zPI!kAB<~pzuibJvvmy=d# zP_p(&$6aNQSFeeEU(zTpYS&P1;1>RLvsrx5!&|))BnIwpQxSR}`4EvB9qT;47rWfT zvzX!p$c;Wm^ext`@fXT-L7|vD0H#ziAOcSFajFv%QYj zo3Os2og{gJCL54vR$yr0t7I70f+e@eC7R&UhYcSo<@{$AlT0R`@n%yn-RX~bKZ zZTAkL*$G>%4i2QiQOy&5bc-#XPR?j^j@9_7D{Q1LR(M#6M{deXj5tn1lrb-ol2heht zikQO41LfP#*nWS+E~v<1r{wRR{hFxGAVOTAp+AWvjBca;6JmxBSuhg$lp2N zff}R45eC@N*aaTg`9IYD?f6HGpx}$t2L#xW|Nn0x2Yz?_&w Date: Wed, 16 Jul 2025 13:28:05 +0200 Subject: [PATCH 2/9] fix assert zip content --- .../setup-superset.yaml | 2 +- .../superset-assets.zip | Bin 4665 -> 3893 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml index e5f9e9a0..74eebbd7 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml @@ -13,7 +13,7 @@ spec: [ "bash", "-c", - "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py", + "curl -o superset-assets.zip https://github.com/stackabletech/demos/raw/refs/heads/spark-anomaly-detection/fix-superset-trino-https/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py", ] volumeMounts: - name: script diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip b/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip index 5ccd11bbf984d37774fce01dc09ac2549b464e2a..3e8ec795dee1348c891a497ca04573e2210b849b 100644 GIT binary patch delta 437 zcmdm~vQH zVPfF|38k{DfQ{I^n%#$yg%czs$*Ii34`Re|YO+X!l&|D$Wn$q42?g*ZTVdLz0+a#) zfrh`1AUZiCv8be2pNl~O#A9G&kYMOA@Aq+;EFd5|c?PeLkSGI8i$X&~qbg9%jl7YrujG^lTJ0dkiz0Oj j7@E1MC5b7CC5e-31+_(3ffh0_Z~@^1pg-(E;m-g7%o$6I delta 1087 zcma)*F-yZh7>3Up+i21dijzX9XGy2hQ+{uT+9(H_ zx=I^3##M&~I5swx(<}8fN9#EIzR(9vH(iTVrm#3>^0h7QKbHt=h6`_csa4H*8<^+z zPr^|gP4y<+YdZ6#UQUwn4XVSlcEB7diCNDSWL6g%`Y!88Ni?On)K+Ym=vzTeilWUa zU~R`?93F+yKs?fe&7#S;Oc2#WGy!_W?z$}{QTwX?r&j#B-_5W_CVNzfrjYC$?NQ*k!^l1c!Ml^CgN66h2C*&Kfi5*S= From 4792fe0ebd9080cd3ddeeb3f5678cf117a9ef7f5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 13:41:08 +0200 Subject: [PATCH 3/9] fix zip --- .../superset-assets.zip | Bin 3893 -> 4665 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip b/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip index 3e8ec795dee1348c891a497ca04573e2210b849b..7be91ae3de2199d6d1e788cf2b89863bdb7dbfdf 100644 GIT binary patch delta 1142 zcmdlgw^L>ETXrcWIR*xX#Ny)AlH&N(ih}&2l6WHnBO^ltW5Wda1$+|q=2Ci^TAe%N5)@041K6xXL zgQW&sBgi&Qc0#tnlMc|6h7OpC1{TgBXXFz!ZE`%X0{iafxIM1lZcl#5t81Z-;+(r2 z#5f0(Sz$3|po?OF5+@-8CM)py8(6|Mft;O7SW_ckw1GB?CJ`QjHofDsH zVbb9OvQiRD5|a{(Q;YRsW^piBr+5N9*99UY+lXo!^px3GDMP7nS~$3 zh~v~`kp{`F Date: Wed, 16 Jul 2025 13:47:44 +0200 Subject: [PATCH 4/9] fix zip 2 --- .../superset-assets.zip | Bin 4665 -> 9602 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip b/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip index 7be91ae3de2199d6d1e788cf2b89863bdb7dbfdf..9dc7a395477940b53aa4b4d2a7b9e4eedba25ed1 100644 GIT binary patch literal 9602 zcmd^FTW=e=6;}4r2cbY)V1f2&!J-ezb}_c@js+B`VyljuSaxM4n{3e%)C@;rJQrmy zvX!F1{)M7XMgKtG`wR9@^s(>#1MOSU?~okbBHJ;NU1Vz+GBk(e;kl7K{LY*_`tYMq zO6K#^(%-)R{(pas2l{@FuSipwrCeT2;w0m>O08MhskR2yYNOd~Y=<%vp2)=3OoYKy zCaH>}Hmhz`wkqXpHj!;+#+Ne{$}|(}Dce>E}+@2woVh4N0dwp9Hb+3c9?= z645R*rl(5zR3%w10uHkTk{lVt35=knVXW{VpNK?+nEelb#84THR3r-;2#mRL6iGMZ zA{nK`oj&lm@@K`whBnT!2~Wwcn1miubk@%zE-Sl!1MV8aagBzH zUzMgaba#i%s^_*`zgb4GCCWwkI23pIX&0m$=TU|*fn$fXXpGVKj-T`f_kR8v8f4xJV|8aB6hW15%GG}nCL$jNu;6?&p@;t3X?!` z^V77zR8Dk&_Y#!6aN;JyZ{=%Dz$i+&bt&6WTjgmN@ z3?aE)!NxD-^EqkRGA{N)c0QI7Bj>YUFsI5gdMrd{5*v^<1D3r91vVLQGRwv>LZ4!9 z&6EqvU1FOo&anM_$E>pu`Wv?o7M_C5zeAy8CH}@NWB1kUdYjlaaM@}a93FK~2c4sn zd#%sd7l(s~j415O<6al$es_z^uG)9Ca;*NyLvpM)z2M(EI-Yj=dk;JP_gU>?M|x{r z_m7`Ec>G>_c(J|Q0%eOdNz7vZT*|YzR{5|u=pJ<6>sVgaIgF!he2bKxNs-(Jb?q_8 z12MV{>K3QcEsN%HoZqHsz7>&-o6!HIo@>a+9QV2bxU!0-v^)IZC*@L!zV&O-6n88V zU@5)ld>}3quWB464U}74>3h!^wQPV(T$yCbSGe)g)mG=zmgEXTRk#PJQ3U`cblX(D zvbW+;U0{&L7lB6M8f5_!_*Z!2*tNI?1_ESSqj8{eh8jhC*RM*snUu?nbb7!CJ`KzA0hdx~wt#g)9u4e|!EzA5YLv%c0_tZkGCs3a1V%?K*Q ze0HPN*Pr=qU(0Eq=5hyeggg5&^p1@9fB@nNrZNo!4Z#ALiZwD3isVct-Bo-D;L0)? z`W48aCZ7TjYO^L-nr`%rKD7!_3OI)`hl$ZBks}Q)b)Fuel^{dHkqWTzDt8*_a6=5( z2F=JqE{p3jHEz0c5q%i~v9~iKZvab(Jh!sdooEC{wzM7^t(gnA2msw{9oQk_6=(@H z8#rf?#1=FTXCj6H3X^r|vkHhU=}|yb4pW| zcPR=1{JspjOE8_)zy=P&7@fYw&?(Tmn}^V)D4J#U8Wcjx-!Ry!HS2Yt8SgyWmSTs( zZxL!s4ev#?t@-HEs1wzl`%!B*s5S%MRO8x? zHeal|E%JbU#3onSx=+uE2Y{FMnfcbZAkj{2_Y5rDG0);S$P}tv1rf!bK!pl8apIv6b29OT@79@z0;uHkNPvkMiYd9a|;$$*sEbUlT>XHffY^jkRmpao~ z2~dej8RopPA)x6vj7G3cFSgMOb_@Evn4v2`4pWgE2t7oW%Rs5MxggT4xMe934E7ZI3`3HtIg_U_E`r!4+QyqDh-qcW1Uc zvIpCJQ%0I13q%1f4f4@yM*5~n>LecqDjhFsamG50R{eg}^MzA$>qDne^>-Xm5z?vp z^@>;9^@pPFmghc&I|;WPI7h>z@pSJuYUkKwRPI;>9*S5tRW%9AdRL6rd73T=({G)k zAKHzA2xF;6V{A{&;`Gsbi8O{o^m3vq|OI$?SRD{8D!@Ig?fo&-Dh+X}P~IFtfCrtuM+be>Yxnt}-|3y6 zbo$*M34mPqP2FAy{sonZ^;;z*Z;C4=)|*a8sW6QTmc2KvUcazVakNSw72z?*e{6ZW zyh~d@GGc*^-{U&8i}4#+(n;s>f{O~n94|o>gY*SVFCW@TeKpDxePguZ;LQ*mghiEn ztp~=gUCmD(-TB~?4=;c323w&1e1(@@ZuAz;I^*@XXAIj?>8m?`zLs%;A1uF1v(A70 zeTxPEuRpj>aC>TOdi!F1nsp#zG0hKP#H-&@+6YUmPq2OfEhf15^tuV=r_6Q7(aEEa RzekKe#^1l;+3C;d>3^v&B-#J~ literal 4665 zcmbVQc|6o>7aviUC|e?8CMisq8C%F!SqCG=nr$$Hn8D0sSGGd-^$ICV)(}aylvF}Z z3{8Z}5?M2r=z7&Vbnkt6@2$Ace4hFJ{+M&V=bY#JoMjB*^bFj9wPj|XWV-(H$Cn`O zi6RitM1n2a$HN^@w1q)nFepSCiiE--azYCV1YjY46M9&MAwEzRY~P-Too97f9N-~*=F9D7r|+dsSe^l| zM%Ex$26gA+&TBM8-6{fmv8*nfd3`TL!kM~bjH161^*EzgSndp{rSuF-Q{tOAyWbS6 zgzFS8Lr7!Tfo9h)eDLSHprd-^prCt+T173)12_q{qL!%@_bedy&SLn%kG#Q)mGKP; z7C{9@*`(w32QWdvscw$IY|4%|=3)=;a;tcPOAHAf46!LY$0P?aR1_IQV)Cxr1BvAP z@cLzJ)r<3w7Nchy1wvl6etHyLsA(m?7<%Nuu!=P{6_Hv_f`lm1BZ&!l5_OF2(XP)` zMLlnr!taZq#2yLMff$qICv8;&zs~i-jFct>DWr@!KJ*Qee=2TgI6>`FbWIz7>3ex$ zTuygM2crHWOu@R(vvX{uzasEtg?SOpIiDa&x?jLg)`akaX%~72j9+}GA+NZYTj7q7&&Q+ zzbc=GLE}U^zbH#*9?7tVrFd3;@ly_cB)rm_mI!<0n`zt_T3Aq2uDa*^!KuVi?@^>7 zlqX4Po+|V5OPY0OfYoe?Ot`{+4=320Twq0j7x;ck?9c=L*}S57L>chf{w`>H8ZV;2 zI2JMP_Ix?>1+0l+^IY>C;K%tw?3b6_5k+vecSqqJ|37EAWCh4_?EnB=82|u{?Wi&K zK;d0&Q8;%j%FWl-5luup5MA7H>schlxDS`1#vV8Z_L=A6Xk>WZz~S)36scX2(OoKD zHMNHpK5L#9)8vbvJbEo-hQnuryhzV82^gtpqz;n3>0~RKrMgivvS;$J6f7}|xQnxJ zM57E^Soj`&r+Tr$z&OS(xdPv6w2Phyu!Cw@_CHF#IcJ51+dP24mDQCXAU_hbHCuL>+xdN z@?D)1#XSj96ZXflpv}f@e5Tl0zX>g!s*D2Xn!l5^jppX4dl#8Qs#qXhRqbc|>iv>U zK6toP=$hRiR>ga&ZTSwR5JDC+(V=pVo}Qx*gCazj0!)m$J4WL(E{R0~*<2EM^k){? zq_OO#GQq6rS`qO<=4vTn%ioMo86bR8I4(Qr;4dGWzH7EBWT=~kO0gfbFdgfVUzCbh z*PB3QVfWa)GpzN;Yp1pZnC&KK`n3B#%<)ogfV%5V|5d5}UMNMkz&awz9z{TJqNDVA24W!uwME!&)qpwU+UTkG&{ktUz^DSH-t+d z=ZzHHI@zFvGl{?aX|7kHMP-+?R~plaJ$WPWDwT+Z!H#h-n{={s%JbbO;q>WEev7}G zy>!hbgHAgrkFvjV=#vYFCRQl+y}zVM33g*-5dn`!5khbdc3P9cF#=vu>Y!lKB}Bv% z`utv35r5+N{>kWZoB`sSdisrM$s4^FDiN`TVabzfA+1lXI_PYtm_9yjPLN|0V{EI* zqZ8H?FkhL>5c?9fSOujFk(>Oa=yT{cpmS16{Sgg$L#xi>u2KCT7=vpD@pu$)olIbL@s6D}XIN#jWHn)BQN% z9Nkj1*P5T79K6KVm8_rGU#h-$JDv@nTMTKLi*GLgps-z4E*N(_nm}A9!GM`Qf*;C$ z|5N!aTVa*xb>N{zH5X(&3sR=aNUFrbKhxZ2dfHDQqWJ5QLFKMs_EK@@;C{^Ymf@wh z(G4(lmkJNVL2ce?e%53YJFD!7;*)+#oOcuUS73m~yp9g%U~gl>qA0K&kfnG>(Hnj( z=FgUlBXI^t)JeC2#@TVNOM`v}Ac!@ZY;a=?nLU-jfro@=(ff=f2 z3}yo1e3JxW@f-X==4EQqFx)xQvnzyGkGcP*dF{SVIlbf_PuCt;8X}5p?jJ`GcYjpe z*V@hl(7}+zx*v;>&*^;WdfHNF)qkw|n#E^fb?;ZYOrnxb9~m5U8}|8{byq#dow-$ym{$^R*boxk&YSML_)bDpZFsn)f zNx8$+vDUBDf*02$sEf|{nv;+3btO@4-y|?7-+$It^Q_*~N^oFIYQ>2$e8uTy*>o$e ze5qjUlZwj4@~l~yALt5sLCoAIFw%$WY^`-Pi5gCwk9&(nrFhGnl}GOm!(e4@s_LLn zb#m!O*D3gAj7r(SY-x<3RXCb;^Y5YZDJrsdQ*9Hp-C`4j?v`~b|Dat}5W#x~{$U(pm zh#~}Xl*JHEw?mcwM}x{?npta$J!n;Z?fG*PSpdMb9|7n9yZ`_r%(xKWOUu8T*U#`o z02}d6&;);u_s6KSJ=lL5i2j5H1_cFQ+$!wm@o19~xAg?$#tOIPyVxe*-Do}6&wby} zwrzs`W!S&_zqz^F=>MJVK>w`O+ho?ih=}N>h=?)%IwF7e0-KPx5Rkuj3>yRDz76F6 z^$#0us1LPi(D>Z9Ej}FG+1z~3wJ_Tc}i zUT=&&cDwK!tM?{TeoDX}<>ZZ~aI^j@5}Pvg*1&h}TFZ{RX`g_d06+l^0s#0gUEXxV From db1b9596af1e7c2619718968ff5088c980e08ca3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 13:54:28 +0200 Subject: [PATCH 5/9] use main branch zip --- demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml index 74eebbd7..8fd93e44 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml @@ -13,7 +13,7 @@ spec: [ "bash", "-c", - "curl -o superset-assets.zip https://github.com/stackabletech/demos/raw/refs/heads/spark-anomaly-detection/fix-superset-trino-https/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py", + "curl -L -o superset-assets.zip sets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py", ] volumeMounts: - name: script From 67ba6e2ef4c27ea7e461b04ce3b189db9972b18b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 13:59:49 +0200 Subject: [PATCH 6/9] set version back to 4.1.2 --- stacks/trino-superset-s3/superset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/trino-superset-s3/superset.yaml b/stacks/trino-superset-s3/superset.yaml index fafe7d8e..ed2e3647 100644 --- a/stacks/trino-superset-s3/superset.yaml +++ b/stacks/trino-superset-s3/superset.yaml @@ -5,7 +5,7 @@ metadata: name: superset spec: image: - productVersion: 4.0.2 + productVersion: 4.1.2 clusterConfig: credentialsSecret: superset-credentials mapboxSecret: superset-mapbox-api-key From e1c175e238a6f13e79a4c32d80ffc1726a244c50 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 14:29:02 +0200 Subject: [PATCH 7/9] Apply suggestions from code review Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- .../create-spark-anomaly-detection-job.yaml | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 5d746d2c..928037f2 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -10,20 +10,24 @@ spec: - name: wait-for-testdata image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: - [ - "bash", - "-c", - "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data", - ] + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for job load-ny-taxi-data to finish' + kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data containers: - name: create-spark-anomaly-detection-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: - [ - "bash", - "-c", - "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml", - ] + - bash + - -euo + - pipefail + - -c + - | + echo 'Submitting Spark job' + kubectl apply -f /tmp/manifest/spark-ad-job.yaml volumeMounts: - name: manifest mountPath: /tmp/manifest From 14c88bec40854fb68fc57d1833040e1234b9bbbc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 14:30:08 +0200 Subject: [PATCH 8/9] Update demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- .../setup-superset.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml index 8fd93e44..73e3152e 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml @@ -10,11 +10,13 @@ spec: - name: setup-superset image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: - [ - "bash", - "-c", - "curl -L -o superset-assets.zip sets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py", - ] + - bash + - -euo + - pipefail + - -c + - | + curl -L -o superset-assets.zip sets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script From c14189ba30afcc524a4978e4f8bd4204b7037c9e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 16 Jul 2025 14:31:06 +0200 Subject: [PATCH 9/9] use listener service instead of headless --- .../setup-superset.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml index 73e3152e..2655e98f 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml @@ -10,11 +10,11 @@ spec: - name: setup-superset image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: - - bash - - -euo - - pipefail - - -c - - | + - bash + - -euo + - pipefail + - -c + - | curl -L -o superset-assets.zip sets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip python -u /tmp/script/script.py volumeMounts: @@ -46,7 +46,7 @@ data: import logging import requests - base_url = "http://superset-node-default-headless:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default-headless:8088 + base_url = "http://superset-node:8088" # For local testing / developing replace it, afterwards change back to http://superset-node:8088 superset_username = open("/superset-credentials/adminUser.username").read() superset_password = open("/superset-credentials/adminUser.password").read() trino_username = "admin"