From a0c2d75ecf0593ddbda880b3ba4fd4ee61ea6b16 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 14:13:43 -0500 Subject: [PATCH 01/27] add `sambah-image` to Makefile --- test/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index ea1a3898..ce27f6e0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -57,6 +57,10 @@ net2cog-image: Dockerfile net2cog/environment.yaml docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile \ --build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog . +sambah-image: Dockerfile sambah/environment.yaml + docker build -t ghcr.io/nasa/regression-tests-sambah:latest -f ./Dockerfile \ + --build-arg notebook=SAMBAH_Regression.ipynb --build-arg sub_dir=sambah . + images: harmony-image \ harmony-regression-image \ hga-image \ @@ -70,4 +74,5 @@ images: harmony-image \ trajectory-subsetter-image \ variable-subsetter-image \ geoloco-image \ - net2cog-image + net2cog-image \ + sambah-image From ce98d439ef195f7a9baa6df3635e83eb756e1bd4 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 14:14:44 -0500 Subject: [PATCH 02/27] add "sambah" image to list in run_notebooks.sh --- test/run_notebooks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_notebooks.sh b/test/run_notebooks.sh index 62c1b55a..1656356d 100755 --- a/test/run_notebooks.sh +++ b/test/run_notebooks.sh @@ -15,7 +15,7 @@ echo "Running regression tests" # Specify the test images to run, by default all built by the Makefile. If # the script is invoked with a list of images, only run those. -all_images=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name) +all_images=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name sambah) specified_images=() # Parse command line arguments while [[ $# -gt 0 ]]; do From 5b425975e1a0ff65a62370c8b1c9888fa6859898 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 14:15:13 -0500 Subject: [PATCH 03/27] add "sambah" image to list in test-in-bamboo.sh --- script/test-in-bamboo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test-in-bamboo.sh b/script/test-in-bamboo.sh index 2fa432d7..804d2ed1 100755 --- a/script/test-in-bamboo.sh +++ b/script/test-in-bamboo.sh @@ -42,7 +42,7 @@ echo "harmony host url: ${harmony_host_url}" ## e.g. if REGRESSION_TESTS_N2Z_IMAGE environment was set, the value would be used instead of the default. image_names=() -all_tests=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name) +all_tests=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name sambah) for image in "${all_tests[@]}"; do image_names+=($(image_name "$image" true)) done From 68011d4f156de381bf0fa0c5282d799d65cde36c Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 14:16:07 -0500 Subject: [PATCH 04/27] add "sambah" image to build-all-images.yml GitHub Actions workflow --- .github/workflows/build-all-images.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-all-images.yml b/.github/workflows/build-all-images.yml index 8659b779..51ca4d45 100644 --- a/.github/workflows/build-all-images.yml +++ b/.github/workflows/build-all-images.yml @@ -58,6 +58,9 @@ jobs: - image: "net2cog" notebook: "net2cog_Regression.ipynb" + - + image: "sambah" + notebook: "SAMBAH_Regression.ipynb" uses: ./.github/workflows/build-target-image.yml with: From 0831cde6d45bf3bab20fef05d873bce2ad7de86b Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 15:53:32 -0500 Subject: [PATCH 05/27] merge commit from different branch --- .../VCF5KYR_1991001_001_2018224205008.png | Bin 10181 -> 0 bytes ...5KYR_1991001_001_2018224205008.png.aux.xml | 4 -- .../VCF5KYR_1991001_001_2018224205008.png | Bin 9527 -> 0 bytes ...5KYR_1991001_001_2018224205008.png.aux.xml | 4 -- test/sambah/utilities.py | 65 ++++++++++++++++++ 5 files changed, 65 insertions(+), 8 deletions(-) delete mode 100644 test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png delete mode 100644 test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml delete mode 100644 test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png delete mode 100644 test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml create mode 100644 test/sambah/utilities.py diff --git a/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png b/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png deleted file mode 100644 index 406a8aebbf1ee1f99257bb88c17a8338777be596..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10181 zcmd^l1y7w_)Aa=hcb5ajwK&Dy-QBf#ad&s8KyfWr9E!U;94PKB?(X*9Prje=tz^&6 zPG)8$la=gDvZ7U#q*0N;NB{t!%F0Nn{dfQV-$4NV7m={<|BE0%ML|Onbc+Cgh=H_^ zj=6)3xPy(ejf}R93A+s@*hIqGL`UDmLf8a@Hc3fW;fPle30F~Z*3hxmu&`IK(N+l% zSBYWQNMTpVL2DG03kal(NTl5 zj&Ty6W*V7w43=>Wk$w!70)k2k!61U*kc{JlC&3&;a9sWHtV2lLJ&24WC`^NBjD1*S z!(hrjVq6Fj`V2YNBst0~CBh^F%;6<8{MC(!2D_}HA;k1ik^$KA$ z3y~D7;H0ZiluMChN-=CwLAJ>-_K7eyi3pByFm`bWZc!kwaFAO#ykiWUQzU|M4vaw_ ztU(TfNhX|DE{a7OtVI%%K`I7!Cn`rH0ci_8V!I%0y9i>vB&bUS)Gh{UkpeZyf$9`M zRVqN27|<>WG|K|@N6ZT`g3`@lb4)=Qrf~n&WgEf$*Pv7zRICkKXaFkIfy>p0uT%q- zYk0|2W3)Bpd2 zBOL+*06+n5YD(%K|9=Bee}Jb305U*ULR7;m=PVad1AAq$ToP5!>Vu~`)l%9nG_ zBxIn2{6&0VkKcSA9V}EgG^{oBFVwa#VS=#$K5|C3SzA(kz>)0g(|zQT%adHA%P#rY z$<*&+$B2jzgu_bhRoH{Rw%fnHb8m#Bq+cKJvbj%9S*fxT=*0CJHA}Ui_~3&g4_x~! zfb-w>R>s~!ui&8DOs(`WlwQim#9E5Uo4=8+uu~C`xm5gC*R%Cpw$RZCA?}^)SyLxj zPCNjw@c`gEW0!$9LD`kV_6yryoxr@B*5pm^CR;0k|4Qup^zs|@><}E*+d7qxg!*?C z{3q+rQ1tK00;sy$u0X+~Q-wp1y_VF`6C0zNE5)#LKmQARqu4n^TGA!QKcg9(92nWp z#q0SVE_^pP9E7~1t&!HYB1@3WA8tyY@yAK#Yq_eZir{#OX?{_!D2*7QXDD?1)l0 z$cNb4pyblb;-S2wx<4fQ~$X+46b4jPRU45jjp7ul0{6c@Gpe$+Yoc}DLVt|0(Z)V(qjGFwCMUbiH* zt_~ze5j5#pQtkBt%_G0ee_$zaj=Q^SSFK`OVP$Mvr2~_b-O!fE_M9~9;_81OI%|*- zhwvZyc(d#{sqil=wN4XbS9tJ=_3Wk%UPPl;5z)8yHU}dJ@5cowgzXj>Ow8$6nl0G~ zo*_eoP+PEe{%4;!x*?F29|cWj7PnWZ(_2->+je&O+xTDc&v#QX?9b#_DbDMKD|QYV zC#H|QdQHt_zZDF-pvaDlTIZFPavZd#u{DqR_M5BN_Wpg%xB0*|P=Xn`>zQTK{RVqU zv*qR8=54>(iHfHbjh-{{BB*WK3`0FgHkhZUrGkg6Emn)ng-u#ssgG&th^hClg{1wa zpW1!I-cdtevaQDW`iDqSW4QmN_Q@#-v=jz=!pC2=Vu3?Mpp)&izT_#bJ9d0WlKT6lwuYT!}2E+VN{=@R$D6ojdu~0txLg=w$~{WMWWK{e#HSvA=mrg^KKX zJ14-mlE>XNoBa^m`ZoD_wMbtBPJ|AC+-uEaP%_wwiQ5fmgJ=cqeHnU=j|0Q_8I4Qy z&06LTE2(KGqU(i+(|J2g45#8dzgKcaV!lOH68mdw+7JbrHp@xBK;PQzghM!+Kw<+F za*hiWRmFGKBBH-Ntqo;0+%T_#wmb={L&48siZK1W{X8<)`>y6`FEnozgLd(V>e-|< ze>n#!fpwG~Z=Cbl+3`Bc@Izq=x{TK_gd+Rj#!i~cOmd2f2G;gsV%6BQc-Ur#8Y!gR z&CMS538qmVMU_iFRvVhlrg}n_$eh@V%sNFjwO@=d{Db`o`o$nVIPp<2l&fHW^HV)B zssP`**Je^Fb#ZATYYe=Blvc!u8T)L+m9@?h<{UVCm!~Q2@0rQ&cbvjEz$cv9VyN8c zIco))L|sD_eNt%)xJYYQ7;N+81_Dc_u2QYa>&xn$wAtC0<4Q)e39|aioe$pNOgOeUemx!>%rmmpImWgB?^U z+H#hA$MjKEwGa4vcNx?AtGywp{JB9t97j)AQdi?!UGr2(e5~JtE{&gQBbjZ!R+8@I zY;;nKJhmyW#`Kzv5&ns~hPKL~tAb}90~6VEPz@jsBY?G`=Km=0_R(qi6ce%)>N)psXT5Z`b-tpRDg*n|Q`>7a zQebDd-|q)5z#_)d3*qr3Z62}qMgsv$$qxiksHOwHUASrz0LMZ_lpI|YgCvIH_Pq>2 z&EVm8QPuuI$|m1G_un77fq}x`-zWM^UQQxE(v6dhT`BX`^zU}=Tof) z?bgERr)<+qTIqaZ_DIf0CHvaEzZ|A?m>`K6K07OU z{uCT6j}p(illA=UkWJeThh%}B8$q_fiiy;JeH0F9#DqCJD0~~b6Yyaxj_4CFzjdN% z!L#Vck}JDu4G*7Jip7S#BX2QBu2CPixpU`i32=eNWkt7r^lJXTO0p$<4QZY}eG} z9Om~K)?#r!pQoO@@4_kM@1rg@KJ?}>(a>=7k4{QfX6b$=*y;j&auhwfxfwc(@S@~5 zl(67S3_Z-;7Zm!a!b=QG;p(2Ax}J!MmuT=SvYv)WiQ2CnRpEaU>7@Li@7blZCJ$Z> z@5g*L?iR)1cs4Ttw-jV%qZBZ*#5<{DLKsY=%ha@zi|$Ll+ug4i}+nd^ij@ z)P6SsiDHeSTZ&Yp^~3@M{4@bzmryP4y6A)Cla5g+posD*O8E3n5(wlusa5|DWdbak-?p<#lt|?uI4w3tT)vC(&hZF>Jj>B z0qd{{I0cC=BKT)=hFWZ2gg!6xtPZ#B7o*kGSQ!W|{&mVIQIf%Ez!1&9BbCme69L^+ z{6!GV*OI`Dd~{xMelzz$;e(pQ0gnrMB&jxx+jlpSU?H32{F+EaV}>93b$mR&!}yvL zV|%>EI&uLsw*OD!>nHoW1pg?)AN>UfEXdbeJG(MpI9v{h=Ita#>u!ZjU)5Eel-Ko*tZe|Z@sPMxXa)R_Ch#>`d&W0@zk0WoEYdW-eX1?Jd zTqnWKd)j?v>OzHEvZ|Cl1ARiokdUh2yR%OVrX50?1XDsZlm7Km-mY<8*TO;O3hhpaZ3MKU{OK4;2u=v9HMwdg zoHt2^#~glE#l`M6{yh7cdJ1bwca85``BMy~{`)%sQoIY+Gm6`lsgR z&5)d?@o$`3i9ZkirKhvZs7!AFGhz1)su=&N`68E-0Z$Ih%}iHuR5ED28F- zuAzS=kebDSj_)fjo|N?J#79fa7VgOyJN=5|+D4Z+y#w1z2D9dbR0GGR%73hSaLGza zqIY54F%Dn6*Ep%k0`k0fYw*P5<$tjKoVhDHPGvQOvy7SbWT&8Z`5`Oqiy1e9r){T4 zM~!28U&PsfQtTCWbU}v|t!(f$?qe0{B1SF0za4rJHuvxgjFf~INyQIe8x<;QfhhUd z=%NcL`y`e0JLsw_7kT2%Gb~;P+8c_tHr_93If&SVR%7RIV>6^z-#M;~w=vz`Wc~i( z)9p=MmdPB!l}@S^W3-<*Rs{Z~7|z4LQqo$kJr<7{%0U1%dknf~=zz|C_}2F|d}5m~ z=x7@cn-`FgyTDWRUFT5rzRty! znws$b9oH^lFuOQ6PFMVouAUEqzpU9XMQ3xUE@4%O=?*)=yA@6^HyMkE%#aVf9j%yz z7m8~nNQMd-8LOUbI3XzmRGFRJ*QX`aQ(W=}OW@PCkKDIG?h>=@-Y9M9YCeE2JLWZ2 z6i>J+0C#;_F7CdTWV@=Kr|0hrtM=YN>JRd5AAyF7(kBUJ(xo)uzn#B=b))6};M#Yz_ z#`ub=Z@3gT5F(1OdVB0(=UZI7@m4t#DHRY%iG>(QN{EM;SLUPR>(vpq|AmT~E>xe~ z?O-VGG^n%vC6h3Zh%JKpuP8s1JipQD(s)#6U7>YQX|L3gEyhEtrY)KMuMmCtAEL*} z+Wm`|?-;^m_0hXR3f$njp4$dPghJ^}B+1uB4|}!o6#^W%B{?rd$Ai;?m+_Udv5eL# z8V2G`CsElkRXe9OY%&}eM0MzRFXolXA=qB6R}qJi+87itzbf$_S@1MYJc1F8?{*Dv zVHsky6D3236GPS_LuReGFfH&xoyHtrRvwq~pO&7#(yV}fax@~IpSN2sTnTl6P|~GD z_!*c(RB{Bqk23#b>Rkp@e&=)pL4p|gkF8O@>#XFn1%`peX{BF zLiR`ehZj9UoM>|O*N3DPFaU2ScH%`zQ1bfFJ5($7zg$R3_P2e>|jsM`O_m$WTbL?ZVu&5%? zd!gugf+uggW28sy!Mo=@PNHzgxHh9pR&=lzdtF!`i(EBHw)a6 zvPQ__i8U@qw8TRxF)@ihdHc&i?3StZOYYX?;pRZ3(k8lbV(;hIsY+3?eTyl>D;+`N z^NIdqwKs8gb}cAy4zafvO))1o3!@^d8PTPA*)Po3^}ZnG6DvqX~dyx23MHvM#* zuXQ_{BVO^D)p5z2=b1QyozC+wvu;cRcJG^Akv7XECV0fT-F4 zw4lAdmaI-m{T;M9sJ?q0y4EdA*Nuf!Jjs_D(Q*1jwBzM|<{RQDhDTl$i$w%QJuA>w zAlu0uLgWoFF;&zV@0FgTw0*cl3G*y-2oIIaqs-n2y6{st6qdyE{8+FR_Y8{7LwCVe z&w2BnPttKJS6G2H8^eei&MZs>WcxkbQdu@*jJOb_;3NhpW-GVG)==X@FSyXrEq$)C zjaNqUITBu!7ol^SQ_PBS+7Z9<<=T&Tx8iZl9|!#&`K19v;Vk%IG${bb0(K|FMT_>7 ztV1Tl@g1czPE9{5Y1%q?uZ+}Y+2~mfcv+qayZA-voQ9&ImBx;24dkGIn>v?q;U-u+ z(+U`RQAPYUSch!{jmjJXp@0I;YRSJSaK$JKb)1d&(36Va*V1O?D?)j6&I zD@t658x$WT6=7szUSVFTA!_Ns?)@%L&M3E4eGDr^K>bvSWca)PE|Y4Oo@6Wm)(VHM zC)@PjRrO%SR_tJ(o?d;v>&&n{U!r0cl1QKtUFEljqK%-R#L}FHe3Xi^q9|#T&y4D? zm&0VV%aTP`XTDC#SOr^Mq>j>UDSzg2T4I-WzM1M5ym9{GTqqNnsOF}4WvGoVqvRVw zKTRq_99+z?-A=qTD0T8xgm;`4{-0f%N>whTmoEues&fgx^D!xamriwrG&l`vO(T@g?ZbC;BBT`+tDcB?M*wj z)+c?j)kB1Z1ancgZ~&PfaFl)uksW9D{#pNd@zbqlhrY}NzZaJ@gmw9jJtRQ0#^d0r zc>jg67KUPZh{P4k8TmL@rZTVbU@zGmL5%3U2efHb2`v^tGO60`YYuR zr-gkkxIR&p@lEjxE%{sLxjEnS{`ftJs(bkONK3?%5x^2I6?GAijuDeg7) zb83 z9nIFo-rlq?9ujZ2+8}|La~(9)g0&^)Ssvw+0yvzx^TB9lSh<3}JkG>zs8#>iV`H*P zRI}t0tDHAGT258E5W9VFJ>esrtO-emmwV~7;Y1CMhe6fpUg)>n8h zR9ZXS+C4fkF`ZUR1RsFo=+mC2xpyq`;iLv9_qG;`HW`V5Z$@Ihq zwtbx_W$NP;5B=k#TtpT1@bn#;4VJd1BXZ#B0B&fUTp2lEO#Aq!kkDLpnllTfZu0d< z6nv2GB~dNB#OFyjEy$Mqrg#VWnIkb;WA;tPKn16k*BFcc2C*vzvS`zkxeQotr$(5wZ+Rd1Q$x6nxMGY;LdyIQcyD8YVLMP`t{2+N1wNX;7%_xq7c5Wul;t<0%6?qK}z{ zvjSuc-%{LMi(c|xIq)GNTRgeM&VP?X&u8BrwHFH^HEGu)&!4A_`?3ZqCwplYCCIcMoc;Xrg;Ub%k!=#OLrtl)GlpFKrk2gzYh9d{$T2sAGX`YHwff$t$q!P&mhwGlQj8&2DNo@pJ60 zMmZR+dYd3}ZR)s2$J}JQZb?kEq+au4ZBNIMXcEqfR3RVqTfj&cUFEaGu6#vnwHA<* ztGeV-fhf4@T-}~EaTepWILEGka>8QlYFCraJIuo|I8W=B)@oqNHEM({sHo<=yY7bX z((n>GpDD=U9{8zzC&6Fv$Hel5ql3fWehrL3QiHiQ6PqWdVLbUXl$2Jw46`gM{4Yd} z;L-H9$FmCf0~DVE-r|Nz$dc7UfPNgMVsMEh{x>G3Tx~rQCI;%47zl9}7xnZiPxtQi2Pr6Y+xNY7Eq(q`G5O)+T5_+%+oPX* z{5BdJooY+1xc=#>c(M^~&_)pDj_GW`%t^P<(7>W{L`r{knYThmp86z(Ms`0oI9(c~xCG|s zTZO+kD5*i_S@_q(wlxd>^}3d$Jj(&yDZEu=E?TAF_d$bjU5f<;BmF>t%|$^9WIzi zp9f{%%tIH}lw`9L2YKfQW)wZ;I3HoS@3xO#)Xb#b#(s5i>W{_A&u65{HY^|XMguPKb(+kT~#w4Rj*p!?Y2;cqOHn-vEOVfT~N zD>lqGD}{3zK{hWhXeDfx4b~3)l2|PtqGp!QU8f2;?zi(dwqmtwb&__vDpRVs5pgDE zmkP)9lCl3RM_)t7=9;KvNql5&=wUu=a5J(X@L4^c&T(s=5@}8Mjrcu@vY|iEVE!C! zNN^U~UDxnrOP*=gJdU{>cW#a!b(hd^97!(9!hfiwb?*Gx#aSfK6+X-Gfym(H`;VJG z3GHR&Gh!~n8Eo!?3XjF`DH8*D8aG9UdB)c#ZY7bDqp~1(Wm4DE3W3)7vew6XS=Vos z0&@?p3dQZ^`jlK;Q%sT2qGfT#(Uj3y5M9u2Gw-&lVrkrC%*(f9(LX(b@3zcNmMkey z2=3yWt_y)^Ud(&VyZ8_DVKn)<>Uf$ZB6{4v;NO-y#9haV+GVWV7D5*t{O@X=(JRW~ zyV|@rZx`J2)#pQmR$f&WNUic-vMc>_xpk-3rsB(C{h>4D@h}(+Xm8+SeA1bP*+&Q( zu{n{=bb5(hC*|fDkrmf8NtJVaRa<}$`90;s8R|f>K;|dxXV8@yw{QME`dfyhSqof;={EYGmjt!bqgsxTYWvn z!%QBzsF6uwdFLXeich?LQoH}X!tylH^4BKW3m0hLL#G&to z5d4x1kv>fRUFlB*Y3+@(?aAW%)9hyoz5NaS<(YoPCS2=Te(>kJqJZ~9BCa>Y_UmQa z>r?3ZA=`<;I|ff%%XjQDVTLe1its#{w``SIi_O%oK=geZ$Op|e?d?IzM`YoF>{rH7 d#Z9OW{NVpAA@#Cn4*zo{$x13o)QFjc{2xcO)a3vG diff --git a/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml b/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml deleted file mode 100644 index 09e83c0b..00000000 --- a/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml +++ /dev/null @@ -1,4 +0,0 @@ - - GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]] - -1.8000000000000000e+02, 2.0000000000000000e+00, 0.0000000000000000e+00, 9.0000000000000000e+01, 0.0000000000000000e+00, -1.0000000000000000e+00 - diff --git a/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png b/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png deleted file mode 100644 index b3e809c1bf080d3334e12dd6b94a594d1f019761..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9527 zcmd^lRa2b{)9eEaS-88qySqCqBm~z0!QC|kcXtUM+}+(FxCeI+0fNJ~-#S0z%*E7P zj8u0`-*u!KL=FX!5D@?X6a{%{4FG_$`mfdlgZ|5f@51*000PvMHDy3|VE7Ys#3M9} zeI$f^EbJX5)Ex}i9YXvqM9eKTv@J~V79nVhjBFi_cpZUY9R+&>4RZq%YYhu^9Uozx z7^(ddpVKjfjX&1rj6~KHh zKvb!MldD2j{fQ|56WuxmWRnPE9}i<44|a@!v5f({MS#3QL2jY&j!|$<;b7w&7=s_M z2037pOgODvWQ$Z-iv&c26m-5W6z)cRvQ`F!4iVT6afEsqP`5a!LlV>~3u;gV)hUCj z)PQbDphE^|Q2^>8K-DMALM6~oEl`mz=!X#~(+nun27c-Sg@!=x7a;R1Zh|f--T@Tu z0VLZ2F|I(AI}qlb_VvF;P?{NRjwvX^6z;#dY$Le;7W~u(6=}m37=Q|N;BxihD>Xo6 znxHZb*kVoi|1SUUz9MY19H>DKu38mTtpryFLG%d%`3Hi00%5(r!v*?-zWc!V`N4;J zfx_KEq3*EXJ>jEVLD5byaSkwHF7S!ApcHFRtPMEP43XPc4G8iF0s{g6AiygeaES#R z5&_#3z&Z{1nhBWY048~WUJ0OG2543S8r6VGJs{t@Mh)7a`0uSihj^fKOwct7=oSI= z01F!82TgE-=9oasw4g->&#03bLFa8Cm4 zGXU!xz_bX^tOKOG0O1jUcLHFa1DKZp+8+SrFMxOtfFA(ZX8`mX3q%9J2LM$6r-J_% zj`U!r|HJ{fX+S=G{Qn)mrw4c%03Zhxq$M=HaxVQ5bg*$+NiCJ?uSn zu{o-OFeG?Eh512&v-I>0jQl&2_}L-Sf+*?rE<)|fZ+NokDa|i&-zYpa zx#edgc5lj$pO~0gnGwH3W~MIjG$wU>ZkMuad!B!le@exfB}+iidfh^Nb?hgTo&+~g zKFf$=5e9{2{g}yo!5yjQ9T&_S4`J>1eb~6}PEE9F=y#On=E5_&2o%l;N=)?7k|1pi zU8ThLcK+7ZjTF?e#R^oz7Sqz+{RUk4^3;C#%DlA$a=wCX-h`gH5)>D0-MGkD-;{90e&qo$Z>n zbc_Zg*S$SZKwHq(B~{K`q+58%2Di&>*M4K8^%99r;KLq@iRt#%Cx*r7{`X9pPPAe7 zQorj>(s93wPLr(G@*x+Vm^^)(N}vN*B&>=`ywDYDua~fAbEukO0vZl;aM-64FP?yp zPfuqy;A4A2TW#_Kt;zExN8@hEH3`>yJ(cdI+hr#sU(ewZ{iN#V%HC)5jd{%6%!8vy z5nqIklvG%i)ln8qj3Cy*?DXIH${J}y;oMldYG2ZDsn$xhGw zW^4YqMvQsX&dE8`x8-g5E+=D;S2)+=?Mj@CEviQRUCqEA(f|IQ_P!Y`;|?yUqOV23 zs;;Ba+G5w@>))2$G~fSf(cWx}G3Fj-!`mThIi^(yWYqPcE-ZTHs*Cn3o^@ z+Z^A@tZ`XZ9R+GKRAy@JaYPrFrS&gTdp^0NfPlT2nf;}hx7lWjNUOctPZgE4*?{2Vj=W|1pjaiLuUoQOU+V{88B}h^rat)Ny(inq3%}fOZL9ziw6~$m-Drom9;65 zel;K|uVSS$mDP1_{F|Q5qkM*DT_s|IZ~iq8ze)WDSy6X}yPthm-YLjWyh%keP>+6| zef3zr+7y83d%|j|Yp*TOxIJV^7r&+fPk|vJFnMnX%LSsia|OPbm{@*Io=j!7w)z8Z zK9dX#Oq9zt`t=b17dIXXEvUDQ)@ZbcGDf zbMTVxJt(NN>~rkDLqy4Ispr-(DeEVihD}gu!U3Lh3An5pmfjs>DV8tJ2x`CVzW-ZwcgrDp&iOUG zguDBedn%tpNVYLjWn3*X+Mu`2)!eErCgKK-WWc8s7dwA(EC0otk;;25xp=usDr~LF zjqI}Q3`VLRgWhqyHzKTJgeeQVi&!ax2&2LQQk1YW_?sx)7+{{Of-UV7V_WT|4SG6q#0x1Zt1#r@gmr(~bQwPHU^%%Z5zC&se}SRtnN zQOTvhTRyFlY-kHz_FV=3Qh7?wwfwDp-G!VxEvS?sv5;4vXM%szTsR;wP`J~PU)}RS-mWmOx4EAqmmSHIS<<`KBcZ`F!X_rG zCabIPuks3pc~LPiOGv+%cfRk?gj*#@N~%Fd{8_U6y}X7w3-0wHt&MzWge&5CHq7b+ z3#uPMj2KY`9IK=jy^yf5xF93WL@`84{iXW%gh2aR`#vvB?i>H5tu+h57rD`XQ|MEKF@8# ziL2ejVg-!WK0W2mZeE++@<`&9hH+<)k9*UpUZ7sA6KfLFtL~ z!Bf9zPgb6dYfemKdJ(~#ueu2;J0Xzmyrqs1(ZjBFX^`>*$j~L?nmVxAvON+t^5nrN z&j;7>w?t9+eeRrU&k>`gw@vV`3}snlco4^EQfSHsdJTHVjM6 zJfXuWWC4%6isHo25xWR)^p$O~r0^cuHspQ-)DR))IY0kdK~enylT%9fvPMtc+TzQ*R+4kK1&(GTx0Ym&g?fnmBhEbN{Z$osg?Qh z+Fj-MZZ@CjF{T=8>&R6LK{}1}Ch3e{7zFuQSC2=8Lai2vG-b@j+`?L1N9T?jm0b`6 z%LF`>zO>lOaDYXN0b$q#_}MiR1%J0ciGda81&c*1`>1_{L;k%&k3~@g>8%bQBhpeL z%e4J{elyA}xDmFqSNY)em$w0WFy~IgDVESq{|F_(Y#Z1OLv+*%J_rpy*yH!%t{oW5 zLWDl74{fp%6pMgEO-a`bOkvXe%cDn=sdb14c9LN2E9&1yA_3a{9&MpXot0L(33c=o zZOcGX-L_Ws;lcKjmC+wXcWNpLENefvf((oezU^=53Wg1b8@6{aIcYsJ7w(JtR}G_I z0@oPXze?t!PV52^olf02(S(2fG_H<<_ZAEG<5fe#<;6`~UL2u)ex$xgPpHUt5+>i( z8@ORZ`(|(7Ahv2@3;mesaWqup1$HzxdlZ!*X*~n13$e21(ST-{m#B+m)57Irz*#(tldY z3wauT;XH8NF%@#~D<&Jk2-Y$v3Dia4bQoHy{w}UV<8gYQO)gOJN3EM0`tjd-q}6)j z`LJA(WIWsu9Wh&&N~n!q`K+2R9Wr)c`VmL!b^u4_T?&66BGkNRAbo(&JmL2UsL6E%=clQKW=#E!@e zQwu#i9iI{-#@qUL-o4-3L+F-oK?2SzDb8v}9VCmjgb>{qEWni7{I(l=0iX~T@ zfuLs{c8KahHebIWA`0VMIsO4I9*#zNOp=@v|1!DEWVokyG8G}YKTk1ilF(xrvZFNA zn5>CKn{#)LL})D|%K)(kmm23RpKO+|W4*+B5yuY&1p@qZbJbK$I~#~B1oeADiYEo1 z2;Uk54+{g!%K&UEvbH!v6Mh##w?ur6(P!v1qd|vY%HYaEC_KPK7?I>pjay3c%L^l+ znzET%d}#I1oRxW9h9Vm*CBB+MpCop=OI0w6_-=S~j%w0sjldqO5q56aT?&KYBUfd%k<`xf>YVG`L7! z)82}L3hbHs9uVl_XA$X#3=w^d3#BzIZB4Ds-85NhK;S6ogjVh)Z5&2mp$p1)bR|AP zd`3bq%vYLHE@EkEwAHh7j_DYA^2*25T*3|+XP1l=yDu3+D#w7KQ45`NHWj571&gqQ9+Bqt&gO*jALXdrI^g^Wtc!_)R&qc6HsL>G;J2c; z1(8d+d{070d{Gp{3#|RX?CZw|7Ff$g>#aiCG@9&cDjb`bI2$m?`Ea3SWb7OioMP+i zcR9!#aAzZH)no)K&IO8g$42KDaf2096lTaVGH!mTunAr_wKR(m;exLk!7y5Ka#;06 zW@VR<=ubUZviJD6esXf*iZ(Cyrn0VBTtbFhTr5;{Z;~F6n#$55eOb;CGPHSrWCQh4 z5P{pE|7UVEbFL^;f3N?5*c6(YO}A-zHRYf_cV@0pT&J|R^Op(=)EF0;J2eIr5XWXw&a$$93|v&vsTPzZ&&tuK2ectVuhvh3}Nv zU(c2%j|wac)%(lnKUyA_PR8og$O%bp#PX7IkD}&8yIA+Nk_IAofYocaCzKfg7|3-=Ub zM5MITD}-nVaf_3A$Y2?-z%1)u@&;SauXwccmIo(zln*4f&E#07%?^5WEX+^gi8w={6gTFDLRJgs@Ui$ti7QW2c#=XlIk;ik2@2nw z%Ck923-(<8WMww0BcmY_*DYv))omngl+yxOFuO3KOs|G=(a)oc?QoDq?WI`l&JAnj zeniO{M;paP7@n~T0ypB|prJ|0n3H`1eBN>VGeLNX z^Ak(S$K=&k+QlX6VD?Sfi>H$_hT1P0R8 z5pGlCx_rEG3?IkB23f1vb)QFEgmFvMa1rI)$%#^;I5T?WRgaw<82dM=C!&6~Bdw$_Ay!Csx#eB6Lu zHnNGR`Mj;6pAkl^M2|fX_Czs4OYjXOah*g?RdnN=(RdSC$lf~5CM%a$_e0fqJ8YY^~o1w9{#Q?%|b)0-k`3FZ+Y}nEd z9h$rEhV!<)cc*aB^m&IEXCH^-`sqJtsXOv#N5+E5j*2MLehXKq9+jkL= z5s`jm_~H*bwE?AZv9Yo73T>gbVvgUQ-89*ezp1lPGYqVJab6h9&~dZu6xL&%e~Q>S zJaIKWKHPO#CoC$B6?|w{FYH$`vqV znRb~DJW0|rFmW+OCzKbK?1wftZyU(H^vsMFL`*Y0Ewf&DRZz)li@YBBprq>bUJnv1 z?J5ZbT4Z7_!+ndXxX1ywAJ3TKaWj}CJ_c@%FLmA{t8yCJ1)Uv@{%#wrRE?>d< zuDIN@l+H-I=VEe;wX_C;gKR@r?Ue@2j2p`J7_NWYEh(iJS`BRgmqfF?Fl|`wI8j@| zPB4Mp!;eN`h~c~vth{7W+h$kj#UYQ8YEnt`XV}#0sm!AnU5tcgvu-lY3Fg#@?tzB z_(|N(1@8`pfrf8szc5)&%cUXtMY`g%_qvR&lA`xF=UCI2i4hd6{wGco-O@|K)0gMx z^-|S!H_wNuRf5o~^E^XqDE#ckEq zhb7!YfE6W^M6ODW`%B`+R~xz$Sz!r*>k1oj6m~!Wrqix_kQ$Sic$SPd?fiJyC^@~+ zO}ukKtR>Ynm)O1Y~~z)Nwn&z*~jkV9=*5|nBOONdz{1Wu^5YS zV9{LjlB$WGyA7VGqODkj#QK_;At}9_x5J6v0|{{7qB*M23zLrM6Y?E zprEK!*ZyU-Fq*oXm|C28#k{VIJz<9)C2+b-iy3u&idDFn9Fv`aY;x8wK0yLT&^vhC z3@Shg{ezOYAE0}wN^&b42Nc!|OJgbvm7-{CLL`z~pYZO(9^a~egbuRV*5l*nXZddG zSym~*g?qX5JvXW;=UF zMo1$OS8iHj){diZQIJMQNB8~wrBc-BRX!p4`NwBAm#?v)QP#J+pKMv06TSR`WIO6e ztwR~>ioE}e&-)f5B|uv(_F8wmLWw%+;?EL3czS+mto-nLOq7IxxG*C?EK@#~NDz!6 zGWYoB&qUeZJ|lMKp;mg@5xjN?8{gnPb-!q)m+@=Uq4OlA`{lG`pBCAzwH?_3}WE*w&#(7=^0C3HB={nFAKPDRlk#5Hz$etw$F&ZPze+fb4+$ae5uQU`&Vh@ajxR%2*~?d&ZrttA@i_u|TtNT}h{Fsto3!y}BwfsBTKf}>*E8=#DJewmcw_{tig2k-0^{O#@aO>j+`g!1C!JYDCOEhnJPGT#l| zqrCU-UkYA7hr4QPb4BiwC{YcWO)|kz&cM&%i4iBJ3g_V)9MfHl<t=YM~&{CPn!WyM=zLm|7gl6FGbsOd%^sLJl&O;XODE3747cs zZn-W<#1PnZeUcNRh%DB|17BgT@AIx6k_EU_1|E+27>CIkhMS!)J_0-j4vvoAoB5yR zgPhyz<9gcLle>*g#(M~@S&dcaww`G_?d+J8XTOeca8PvcgOF|~3`Y`AoHpymDRFU; za$1_n?FPT*g%fl9gPf+*VG)xY`R*M{jop0~D^iVYe_}Yeg5rIE;;YZTj!$o04V~wc zDL;RH9u9p{l)Bq=)vM{SdQs`HY7%vcje^2#gcvgUeY&T*Isi&L3D!MsIj56@$Sub< zMc9H9VijEf$a_rp^B+sF3X~fi<<&Cmdom&jk#Z?~4Mv9RJ9K$)=)dMfTjX23m)XN% zp8Uh*c+Lu@GF%HWz$UCV)w;^Ji+z!`;6f}4MXt>H#kMRX$ZDUmC-r+`k_z2b9v0SH zniA^#5Ymxzl;v>5f2+_hKyF(|CPp%ja?DxAz<&x5n(`;eV92t6&0et#yn$Np{V^s*hL{z?HRjC zKN6EJJ4iZ3rSZ(dHd}oK`R=O*rvjA`>pJvwSPZLL(pundf$*PfFGx=BP0y2Z`D1&y!b|G~Yv9 z^_95Jw{m@1s##d3a~~YU@Z#`d;`umcFgPKlw5Pd4kKZJNlp_O{1^!wd8COI`E*}-V zFAm7Q@oWL;q}^`)-VdZmS*+%k<|cP{C`xIfV;n?6u8}bF)xIn$%vzs_B8R^;w}3}S zr6uR~>4MU7ZV5unQ5r?^I0-T_FPG+@&NSy09KFpG_$+4o9!v$=ijfw5e+#hrxje8k z(2f7gKFrV%Ac~68EzY?krYwleOwJ7u=*qWDR^#|nRYUcVAceC(c zVVCUNg(l(oP2tzqe$_4KG=@1|7_Hm9#^hOBNkXbG#qBCLzF%riT!Z&BySP>5=sa;^ zIiI{Qyek^1l?o@}v*mP)uQe|1_y|a&`}ij6PPb7jvg0z!_S*Y#hHq0J5!iGml1>Qo zpRAG2Udxk9TPNz4wn@;ufX_dr7l1$-ul2R@kax;XSql&;_q7!w|Dl_?^;}o5RBr3F zDAp@3TjN_U$M(f&Tj1%ak-5ShPA9L{Xilx`>w`2(+5WAAT$r?{q5bIzdedZ0|2M|5 zn3HP6z1*q|k`G}ADTS8l@n417N4bhBh9jQXAJGzNqJ*eterea!E0a(Jq}?&$3Ln?j z+uK#iuQRb?0i=^Fky%!{nKa*8&(Xbw5m@D3eh|YJt7*vFb%5A<3Oh!IwCeh51=?1%D z{v>(%tJXlQA-Xx>%gi?CSkTVqe7$h2x!3sR!{X*0A6M$`*u2N09(Qc~gEkCL`fkIz zw&PCrcxh)ldWT_3v{+ZV*KjsedR@!XhFz|}!tT*?=r}Sh(c&#z?=SU{V3R8Pn{VX{ xTYP-vyY$v*@rT4~Qr+II%zK~9&i{aXplN-;=Qb=E{raB|L_r23T_b4{^nY@?dKLfx diff --git a/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml b/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml deleted file mode 100644 index d7fb56b8..00000000 --- a/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml +++ /dev/null @@ -1,4 +0,0 @@ - - GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]] - -1.8000000000000000e+02, 1.0000000000000000e+00, 0.0000000000000000e+00, 9.0000000000000000e+01, 0.0000000000000000e+00, -2.0000000000000000e+00 - diff --git a/test/sambah/utilities.py b/test/sambah/utilities.py new file mode 100644 index 00000000..c5ab337a --- /dev/null +++ b/test/sambah/utilities.py @@ -0,0 +1,65 @@ +""" A module containing utility functionality used by the Harmony OPeNDAP + SubSetter regression tests. These functions are kept out of the Jupyter + notebook to increase the readability of the regression test suite. + +""" + +from os import listdir, remove, replace + +from harmony import Client, Request +from harmony.harmony import ProcessingFailedException +import xarray as xr + + +def compare_results_to_reference_file(results_file_name: str) -> None: + """Use native `xarray` functionality to compare data values and metadata + attributes of downloaded results to a reference file. + """ + results_data = xr.open_datatree(output_filename) + reference_data = xr.open_datatree(f"reference_files/{output_filename}") + + assert results_data.equals(reference_data), ( + 'Output and reference files ' 'do not match.' + ) + + reference_data.close() + results_data.close() + + +def submit_and_download( + harmony_client: Client, request: Request, output_file_name: str +): + """Submit a Harmony request via a `harmony-py` client. + + Wait for the Harmony job to finish, then download the results to the specified file path. + """ + downloaded_filename = None + + try: + assert request.is_valid() + job_id = harmony_client.submit(request) + + for filename in [ + file_future.result() + for file_future in harmony_client.download_all(job_id, overwrite=True) + ]: + + print(f'Downloaded: {filename}') + downloaded_filename = filename + + if downloaded_filename is not None: + replace(downloaded_filename, output_file_name) + print(f'Saved output to: {output_file_name}') + + except ProcessingFailedException as exception: + print('Harmony request failed to complete successfully.') + raise exception + + +def remove_results_files() -> None: + """Remove all NetCDF-4 files downloaded during the regression tests.""" + directory_files = listdir() + + for directory_file in directory_files: + if directory_file.endswith('.nc4'): + remove(directory_file) From e608ec71d863cec918837774cafeacc968d9d6fd Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 15:53:52 -0500 Subject: [PATCH 06/27] update environment.yaml --- test/sambah/environment.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test/sambah/environment.yaml diff --git a/test/sambah/environment.yaml b/test/sambah/environment.yaml new file mode 100644 index 00000000..6487be3e --- /dev/null +++ b/test/sambah/environment.yaml @@ -0,0 +1,13 @@ +name: papermill-sambah +channels: + - conda-forge + - defaults +dependencies: + - python=3.12 + - notebook=6.5.4 + - papermill=2.3.4 + - python-dateutil=2.8.2 + - numpy=2.1.3 + - netCDF4=1.7.2 + - xarray=2024.10.0 + - harmony-py=0.4.15 From 20e75a88a605a1473e994e772cde9884ff6a6dcc Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 16:52:30 -0500 Subject: [PATCH 07/27] fix variable name mismatch --- test/sambah/utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sambah/utilities.py b/test/sambah/utilities.py index c5ab337a..c89a8646 100644 --- a/test/sambah/utilities.py +++ b/test/sambah/utilities.py @@ -15,8 +15,8 @@ def compare_results_to_reference_file(results_file_name: str) -> None: """Use native `xarray` functionality to compare data values and metadata attributes of downloaded results to a reference file. """ - results_data = xr.open_datatree(output_filename) - reference_data = xr.open_datatree(f"reference_files/{output_filename}") + results_data = xr.open_datatree(results_file_name) + reference_data = xr.open_datatree(f"reference_files/{results_file_name}") assert results_data.equals(reference_data), ( 'Output and reference files ' 'do not match.' From 49ae4e0899ef8e49282424fd66cef8eb9fb4c72d Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 13 Nov 2024 16:53:02 -0500 Subject: [PATCH 08/27] merge cherry pick commit --- test/sambah/SAMBAH_Regression.ipynb | 434 ++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 test/sambah/SAMBAH_Regression.ipynb diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb new file mode 100644 index 00000000..3386b1f8 --- /dev/null +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -0,0 +1,434 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "raw", + "source": [ + "{\n", + " \"cells\": [\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"f7f9ded8\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"# Regression test suite for the SAMBAH:\\n\",\n", + " \"\\n\",\n", + " \"This notebook provides condensed examples of using Harmony to make requests against the SAMBAH service developed to process L2 TEMPO data. The features of SAMBAH include:\\n\",\n", + " \"\\n\",\n", + " \"* Variable subsetting, including required variables.\\n\",\n", + " \"* Temporal subsetting.\\n\",\n", + " \"* Bounding box spatial subsetting.\\n\",\n", + " \"\\n\",\n", + " \"## Prerequisites\\n\",\n", + " \"\\n\",\n", + " \"The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\\n\",\n", + " \"\\n\",\n", + " \"`conda env create -f ./environment.yaml && conda activate papermill-sambah`\\n\",\n", + " \"\\n\",\n", + " \"A `.netrc` file must also be located in the `test` directory of this repository.\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"009f1df8\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"## Import requirements:\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 1,\n", + " \"id\": \"7ce7f640\",\n", + " \"metadata\": {\n", + " \"ExecuteTime\": {\n", + " \"end_time\": \"2024-11-13T19:16:56.881691Z\",\n", + " \"start_time\": \"2024-11-13T19:16:55.207082Z\"\n", + " },\n", + " \"tags\": []\n", + " },\n", + " \"outputs\": [],\n", + " \"source\": [\n", + " \"from datetime import datetime\\n\",\n", + " \"from os.path import exists\\n\",\n", + " \"\\n\",\n", + " \"from harmony import BBox, Client, Collection, Dimension, Environment, Request\\n\",\n", + " \"\\n\",\n", + " \"from utilities import (\\n\",\n", + " \" compare_results_to_reference_file,\\n\",\n", + " \" remove_results_files,\\n\",\n", + " \" submit_and_download,\\n\",\n", + " \")\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"00758d6f\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"## Set default parameters:\\n\",\n", + " \"\\n\",\n", + " \"`papermill` requires default values for parameters used on the workflow. In this case, `harmony_host_url`.\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 2,\n", + " \"id\": \"45f380c6\",\n", + " \"metadata\": {\n", + " \"ExecuteTime\": {\n", + " \"end_time\": \"2024-11-13T19:17:01.668202Z\",\n", + " \"start_time\": \"2024-11-13T19:17:01.663791Z\"\n", + " },\n", + " \"tags\": [\n", + " \"parameters\"\n", + " ]\n", + " },\n", + " \"outputs\": [],\n", + " \"source\": [\n", + " \"harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"9237f449\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"### Identify Harmony environment (for easier reference):\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 3,\n", + " \"id\": \"4420bb6e\",\n", + " \"metadata\": {\n", + " \"ExecuteTime\": {\n", + " \"end_time\": \"2024-11-13T19:17:08.213477Z\",\n", + " \"start_time\": \"2024-11-13T19:17:05.311566Z\"\n", + " }\n", + " },\n", + " \"outputs\": [],\n", + " \"source\": [\n", + " \"host_environment = {\\n\",\n", + " \" 'http://localhost:3000': Environment.LOCAL,\\n\",\n", + " \" 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\\n\",\n", + " \" 'https://harmony.earthdata.nasa.gov': Environment.PROD,\\n\",\n", + " \"}\\n\",\n", + " \"\\n\",\n", + " \"data_environment = 'PROD' if harmony_host_url == '' else 'UAT'\\n\",\n", + " \"req_infos = {\\n\",\n", + " \" 'UAT': {\\n\",\n", + " \" # TEMPO NO2 tropospheric, stratospheric, and total columns V03\\n\",\n", + " \" 'collection': Collection(id='C1262899916-LARC_CLOUD'),\\n\",\n", + " \" 'temporal': {\\n\",\n", + " \" 'start': datetime(2024, 3, 28, 15, 34, 0),\\n\",\n", + " \" 'stop': datetime(2024, 3, 28, 16, 0, 0),\\n\",\n", + " \" },\\n\",\n", + " \" 'spatial': BBox(-170, 33, -10, 38),\\n\",\n", + " \" 'granule_id': [\\n\",\n", + " \" 'G1263137623-LARC_CLOUD',\\n\",\n", + " \" 'G1263137394-LARC_CLOUD',\\n\",\n", + " \" 'G1263137387-LARC_CLOUD',\\n\",\n", + " \" 'G1263137388-LARC_CLOUD',\\n\",\n", + " \" 'G1263137378-LARC_CLOUD',\\n\",\n", + " \" ],\\n\",\n", + " \" # chosen variables include one variable from each group\\n\",\n", + " \" # support/scattering_weights is 3D variable\\n\",\n", + " \" 'variables': [\\n\",\n", + " \" 'product/vertical_column_stratosphere',\\n\",\n", + " \" 'qa_statistics/fit_rms_residual',\\n\",\n", + " \" 'support_data/scattering_weights',\\n\",\n", + " \" ],\\n\",\n", + " \" },\\n\",\n", + " \"}\\n\",\n", + " \"\\n\",\n", + " \"req_info = req_infos.get(data_environment)\\n\",\n", + " \"harmony_environment = host_environment.get(harmony_host_url)\\n\",\n", + " \"\\n\",\n", + " \"if harmony_environment is not None:\\n\",\n", + " \" harmony_client = Client(env=harmony_environment)\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"fc7aef85\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"# Begin regression tests:\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"9dd58c71-0232-4a20-9dc4-ef6b3544e797\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"## SAMBAH\\n\",\n", + " \"\\n\",\n", + " \"SAMBAH is currently deployed to Sandbox, SIT, UAT and production.\\n\",\n", + " \"Requests will be made against the TEMPO NO2 L2 V03 collection.\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"ef9ec4d2-2228-47d0-a990-d5a4d77b7bc7\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"### SAMBAH: temporal, variable and bounding box subset request\\n\",\n", + " \"\\n\",\n", + " \"This is a request that exercises the full range of SAMBAH options: temporal, bounding box and variable subsetting.\\n\",\n", + " \"\\n\",\n", + " \"Requested variable names:\\n\",\n", + " \"\\n\",\n", + " \"* `2024-3-28 15:34:00 < time < 2024-3-28 16:00:00`\\n\",\n", + " \"* `product/vertical_column_total`\\n\",\n", + " \"* `BBox(-170,30,-10,40)`\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 7,\n", + " \"id\": \"d6aaf063-1f3c-4d84-b414-ca80e65ef78b\",\n", + " \"metadata\": {\n", + " \"ExecuteTime\": {\n", + " \"end_time\": \"2024-11-13T19:18:18.977791Z\",\n", + " \"start_time\": \"2024-11-13T19:17:24.118362Z\"\n", + " }\n", + " },\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"stdout\",\n", + " \"output_type\": \"stream\",\n", + " \"text\": [\n", + " \"Downloaded: 4901404_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", + " \"Saved output to: temp_var_bbox.nc4\\n\",\n", + " \"SAMBAH temporal, variable, bounding box request: success\\n\"\n", + " ]\n", + " }\n", + " ],\n", + " \"source\": [\n", + " \"temp_var_bbox_request = Request(\\n\",\n", + " \" collection=req_info['collection'],\\n\",\n", + " \" extend=['mirror_step'],\\n\",\n", + " \" concatenate=True,\\n\",\n", + " \" granule_id=req_info['granule_id'],\\n\",\n", + " \" temporal=req_info['temporal'],\\n\",\n", + " \" variables=req_info['variables'],\\n\",\n", + " \" spatial=req_info['spatial'],\\n\",\n", + " \")\\n\",\n", + " \"\\n\",\n", + " \"\\n\",\n", + " \"output_filename = 'temp_var_bbox.nc4'\\n\",\n", + " \"submit_and_download(harmony_client, temp_var_bbox_request, output_filename)\\n\",\n", + " \"assert exists(\\n\",\n", + " \" output_filename\\n\",\n", + " \"), 'Unsuccessful SAMBAH temporal, variable, bounding box request.'\\n\",\n", + " \"\\n\",\n", + " \"compare_results_to_reference_file(output_filename)\\n\",\n", + " \"\\n\",\n", + " \"\\n\",\n", + " \"print(\\\"SAMBAH temporal, variable, bounding box request: success\\\")\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"6603a5c2-9ef2-4620-93a5-029963eca195\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"### SAMBAH: variable subset request\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 11,\n", + " \"id\": \"90bb3455-f82c-4cd8-93f2-feba79fbed7b\",\n", + " \"metadata\": {},\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"stdout\",\n", + " \"output_type\": \"stream\",\n", + " \"text\": [\n", + " \"Downloaded: 4901438_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", + " \"Saved output to: var_only.nc4\\n\",\n", + " \"SAMBAH variable request: success\\n\"\n", + " ]\n", + " }\n", + " ],\n", + " \"source\": [\n", + " \"var_only_request = Request(\\n\",\n", + " \" collection=req_info['collection'],\\n\",\n", + " \" extend=['mirror_step'],\\n\",\n", + " \" concatenate=True,\\n\",\n", + " \" granule_id=req_info['granule_id'],\\n\",\n", + " \" variables=req_info['variables'],\\n\",\n", + " \")\\n\",\n", + " \"\\n\",\n", + " \"output_filename = 'var_only.nc4'\\n\",\n", + " \"submit_and_download(harmony_client, var_only_request, output_filename)\\n\",\n", + " \"assert exists(output_filename), 'Unsuccessful SAMBAH variable request.'\\n\",\n", + " \"\\n\",\n", + " \"compare_results_to_reference_file(output_filename)\\n\",\n", + " \"print(\\\"SAMBAH variable request: success\\\")\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"c8322c1b-7fad-4b87-a46d-abf5f0ee56df\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"### SAMBAH: spatial request\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 13,\n", + " \"id\": \"4330105a-f9f0-429e-9762-3049f408d635\",\n", + " \"metadata\": {},\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"stdout\",\n", + " \"output_type\": \"stream\",\n", + " \"text\": [\n", + " \"Downloaded: 4901458_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", + " \"Saved output to: spatial_only.nc4\\n\",\n", + " \"SAMBAH spatial request: success\\n\"\n", + " ]\n", + " }\n", + " ],\n", + " \"source\": [\n", + " \"spatial_only_request = Request(\\n\",\n", + " \" collection=req_info['collection'],\\n\",\n", + " \" extend=['mirror_step'],\\n\",\n", + " \" concatenate=True,\\n\",\n", + " \" granule_id=req_info['granule_id'],\\n\",\n", + " \" spatial=req_info['spatial'],\\n\",\n", + " \")\\n\",\n", + " \"\\n\",\n", + " \"output_filename = 'spatial_only.nc4'\\n\",\n", + " \"submit_and_download(harmony_client, spatial_only_request, output_filename)\\n\",\n", + " \"assert exists(output_filename), 'Unsuccessful SAMBAH temporal request.'\\n\",\n", + " \"\\n\",\n", + " \"compare_results_to_reference_file(output_filename)\\n\",\n", + " \"print(\\\"SAMBAH spatial request: success\\\")\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"d0629b4f-1990-42f7-b19d-025e46a96629\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"### SAMBAH: no subsetting required\\n\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 14,\n", + " \"id\": \"c747cfcc-e63a-4858-b406-a4ec87b6acb5\",\n", + " \"metadata\": {},\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"stdout\",\n", + " \"output_type\": \"stream\",\n", + " \"text\": [\n", + " \"Downloaded: 4901463_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", + " \"Saved output to: all_data.nc4\\n\",\n", + " \"SAMBAH temporal, variable, bounding box request: success\\n\"\n", + " ]\n", + " }\n", + " ],\n", + " \"source\": [\n", + " \"all_data_request = Request(\\n\",\n", + " \" collection=req_info['collection'],\\n\",\n", + " \" extend='mirror_step',\\n\",\n", + " \" concatenate=True,\\n\",\n", + " \" granule_id=req_info['granule_id'],\\n\",\n", + " \")\\n\",\n", + " \"\\n\",\n", + " \"output_filename = 'all_data.nc4'\\n\",\n", + " \"submit_and_download(harmony_client, all_data_request, output_filename)\\n\",\n", + " \"assert exists(\\n\",\n", + " \" output_filename\\n\",\n", + " \"), 'Unsuccessful SAMBAH temporal, variable, bounding box request.'\\n\",\n", + " \"\\n\",\n", + " \"compare_results_to_reference_file(output_filename)\\n\",\n", + " \"print(\\\"SAMBAH temporal, variable, bounding box request: success\\\")\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"markdown\",\n", + " \"id\": \"ca88eb71-2526-4b34-8ff7-441f869244e0\",\n", + " \"metadata\": {},\n", + " \"source\": [\n", + " \"# Clean up test outputs:\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": 15,\n", + " \"id\": \"73fdd147-74a0-48d3-957f-388e2abe356a\",\n", + " \"metadata\": {},\n", + " \"outputs\": [],\n", + " \"source\": [\n", + " \"remove_results_files()\"\n", + " ]\n", + " },\n", + " {\n", + " \"cell_type\": \"code\",\n", + " \"execution_count\": null,\n", + " \"id\": \"5a868b30-52d9-45aa-9359-a14d79548ce4\",\n", + " \"metadata\": {},\n", + " \"outputs\": [],\n", + " \"source\": []\n", + " }\n", + " ],\n", + " \"metadata\": {\n", + " \"celltoolbar\": \"Tags\",\n", + " \"kernelspec\": {\n", + " \"display_name\": \"papermill-sambah\",\n", + " \"language\": \"python\",\n", + " \"name\": \"papermill-sambah\"\n", + " },\n", + " \"language_info\": {\n", + " \"codemirror_mode\": {\n", + " \"name\": \"ipython\",\n", + " \"version\": 3\n", + " },\n", + " \"file_extension\": \".py\",\n", + " \"mimetype\": \"text/x-python\",\n", + " \"name\": \"python\",\n", + " \"nbconvert_exporter\": \"python\",\n", + " \"pygments_lexer\": \"ipython3\",\n", + " \"version\": \"3.11.10\"\n", + " },\n", + " \"name\": \"SDS_Regression.ipynb\"\n", + " },\n", + " \"nbformat\": 4,\n", + " \"nbformat_minor\": 5\n", + "}\n" + ], + "id": "93da5ee1a11dd456" + } + ], + "metadata": { + "celltoolbar": "Tags", + "kernelspec": { + "display_name": "papermill-sambah", + "language": "python", + "name": "papermill-sambah" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + }, + "name": "SDS_Regression.ipynb" + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 81f524704a43a3d6a1996c2238594f60c9f1f958 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:25:24 -0500 Subject: [PATCH 09/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/sambah/SAMBAH_Regression.ipynb | 131 ++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index 3386b1f8..ceca7f52 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -4,6 +4,7 @@ "metadata": {}, "cell_type": "raw", "source": [ +<<<<<<< HEAD "{\n", " \"cells\": [\n", " {\n", @@ -44,6 +45,136 @@ " \"ExecuteTime\": {\n", " \"end_time\": \"2024-11-13T19:16:56.881691Z\",\n", " \"start_time\": \"2024-11-13T19:16:55.207082Z\"\n", +======= + "# Regression test suite for the SAMBAH:\n", + "\n", + "This notebook provides condensed examples of using Harmony to make requests against the SAMBAH service developed to process L2 TEMPO data. The features of SAMBAH include:\n", + "\n", + "* Variable subsetting, including required variables.\n", + "* Temporal subsetting.\n", + "* Bounding box spatial subsetting.\n", + "\n", + "## Prerequisites\n", + "\n", + "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", + "\n", + "`conda env create -f ./environment.yaml && conda activate papermill-sambah`\n", + "\n", + "A `.netrc` file must also be located in the `test` directory of this repository." + ] + }, + { + "cell_type": "markdown", + "id": "009f1df8", + "metadata": {}, + "source": [ + "## Import requirements:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7ce7f640", + "metadata": { + "ExecuteTime": { + "end_time": "2024-11-13T19:16:56.881691Z", + "start_time": "2024-11-13T19:16:55.207082Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from os.path import exists\n", + "\n", + "from harmony import BBox, Client, Collection, Environment, Request\n", + "\n", + "from utilities import (\n", + " compare_results_to_reference_file,\n", + " remove_results_files,\n", + " submit_and_download,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "00758d6f", + "metadata": {}, + "source": [ + "## Set default parameters:\n", + "\n", + "`papermill` requires default values for parameters used on the workflow. In this case, `harmony_host_url`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "45f380c6", + "metadata": { + "ExecuteTime": { + "end_time": "2024-11-13T19:17:01.668202Z", + "start_time": "2024-11-13T19:17:01.663791Z" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" + ] + }, + { + "cell_type": "markdown", + "id": "9237f449", + "metadata": {}, + "source": [ + "### Identify Harmony environment (for easier reference):" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4420bb6e", + "metadata": { + "ExecuteTime": { + "end_time": "2024-11-13T19:17:08.213477Z", + "start_time": "2024-11-13T19:17:05.311566Z" + } + }, + "outputs": [], + "source": [ + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", + "\n", + "data_environment = 'PROD' if harmony_host_url == '' else 'UAT'\n", + "req_infos = {\n", + " 'UAT': {\n", + " # TEMPO NO2 tropospheric, stratospheric, and total columns V03\n", + " 'collection': Collection(id='C1262899916-LARC_CLOUD'),\n", + " 'temporal': {\n", + " 'start': datetime(2024, 3, 28, 15, 34, 0),\n", + " 'stop': datetime(2024, 3, 28, 16, 0, 0),\n", + " },\n", + " 'spatial': BBox(-170, 33, -10, 38),\n", + " 'granule_id': [\n", + " 'G1263137623-LARC_CLOUD',\n", + " 'G1263137394-LARC_CLOUD',\n", + " 'G1263137387-LARC_CLOUD',\n", + " 'G1263137388-LARC_CLOUD',\n", + " 'G1263137378-LARC_CLOUD',\n", + " ],\n", + " # chosen variables include one variable from each group\n", + " # support/scattering_weights is 3D variable\n", + " 'variables': [\n", + " 'product/vertical_column_stratosphere',\n", + " 'qa_statistics/fit_rms_residual',\n", + " 'support_data/scattering_weights',\n", + " ],\n", +>>>>>>> eccece1 ([pre-commit.ci] auto fixes from pre-commit.com hooks) " },\n", " \"tags\": []\n", " },\n", From 0cbd7633715c02143126d79d04910ead8e5cabd6 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 15 Nov 2024 17:17:06 -0500 Subject: [PATCH 10/27] update to use conditionals for test suites and fix environment mutability --- test/sambah/SAMBAH_Regression.ipynb | 402 +++++++++++++++++++++++++--- test/sambah/utilities.py | 23 +- 2 files changed, 391 insertions(+), 34 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index ceca7f52..926c3d18 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -48,13 +48,17 @@ ======= "# Regression test suite for the SAMBAH:\n", "\n", - "This notebook provides condensed examples of using Harmony to make requests against the SAMBAH service developed to process L2 TEMPO data. The features of SAMBAH include:\n", + "This notebook provides condensed examples of using Harmony to make requests against the SAMBAH service developed to process L2 TEMPO data. \n", + "\n", + "### Features of SAMBAH include:\n", "\n", "* Variable subsetting, including required variables.\n", "* Temporal subsetting.\n", "* Bounding box spatial subsetting.\n", + "* Concatenation within TEMPO east-west scans\n", + "* Concatenation across scans\n", "\n", - "## Prerequisites\n", + "### Prerequisites\n", "\n", "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", "\n", @@ -68,7 +72,7 @@ "id": "009f1df8", "metadata": {}, "source": [ - "## Import requirements:" + "# Import required packages:" ] }, { @@ -93,6 +97,7 @@ " compare_results_to_reference_file,\n", " remove_results_files,\n", " submit_and_download,\n", + " print_success\n", ")" ] }, @@ -134,7 +139,35 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, + "id": "97dc22bb-e04e-442c-ab12-160ff4842325", + "metadata": {}, + "outputs": [], + "source": [ + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", + "\n", + "harmony_environment = host_environment.get(harmony_host_url)\n", + "\n", + "if harmony_environment is not None:\n", + " harmony_client = Client(env=harmony_environment)" + ] + }, + { + "cell_type": "markdown", + "id": "36975d17-5f5c-4a57-8bc5-db9585546b96", + "metadata": {}, + "source": [ + "The request collection and granules are different for UAT and PROD:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "4420bb6e", "metadata": { "ExecuteTime": { @@ -144,36 +177,31 @@ }, "outputs": [], "source": [ - "host_environment = {\n", - " 'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "sambah_non_prod_information = {\n", + " # TEMPO NO2 tropospheric, stratospheric, and total columns V03\n", + " # https://cmr.uat.earthdata.nasa.gov/search/concepts/C1262899916-LARC_CLOUD.html\n", + " 'collection': Collection(id='C1262899916-LARC_CLOUD'),\n", + " 'granule_id': [\n", + " 'G1263137623-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T153020Z_S007G08.nc\n", + " 'G1263137394-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T153657Z_S007G09.nc\n", + " 'G1263137387-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T154353Z_S008G01.nc\n", + " 'G1263137388-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T155033Z_S008G02.nc\n", + " 'G1263137378-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T155713Z_S008G03.nc\n", + " ]\n", "}\n", "\n", - "data_environment = 'PROD' if harmony_host_url == '' else 'UAT'\n", - "req_infos = {\n", - " 'UAT': {\n", - " # TEMPO NO2 tropospheric, stratospheric, and total columns V03\n", - " 'collection': Collection(id='C1262899916-LARC_CLOUD'),\n", - " 'temporal': {\n", - " 'start': datetime(2024, 3, 28, 15, 34, 0),\n", - " 'stop': datetime(2024, 3, 28, 16, 0, 0),\n", - " },\n", - " 'spatial': BBox(-170, 33, -10, 38),\n", - " 'granule_id': [\n", - " 'G1263137623-LARC_CLOUD',\n", - " 'G1263137394-LARC_CLOUD',\n", - " 'G1263137387-LARC_CLOUD',\n", - " 'G1263137388-LARC_CLOUD',\n", - " 'G1263137378-LARC_CLOUD',\n", - " ],\n", - " # chosen variables include one variable from each group\n", - " # support/scattering_weights is 3D variable\n", - " 'variables': [\n", - " 'product/vertical_column_stratosphere',\n", - " 'qa_statistics/fit_rms_residual',\n", - " 'support_data/scattering_weights',\n", + "sambah_prod_information = {\n", + " # TEMPO NO2 tropospheric and stratospheric columns V03 (BETA)\n", + " # https://cmr.earthdata.nasa.gov/search/concepts/C2930725014-LARC_CLOUD.html\n", + " 'collection': Collection(id='C2930725014-LARC_CLOUD'),\n", + " 'granule_id': [\n", + " 'G3205017917-LARC_CLOUD',\n", + " 'G3205017777-LARC_CLOUD',\n", + " 'G3205052670-LARC_CLOUD',\n", + " 'G3205053541-LARC_CLOUD',\n", + " 'G3205052319-LARC_CLOUD',\n", " ],\n", +<<<<<<< HEAD >>>>>>> eccece1 ([pre-commit.ci] auto fixes from pre-commit.com hooks) " },\n", " \"tags\": []\n", @@ -537,6 +565,318 @@ "}\n" ], "id": "93da5ee1a11dd456" +======= + "}\n", + "\n", + "sambah_request_env = {\n", + " Environment.LOCAL: sambah_non_prod_information,\n", + " Environment.SIT: sambah_non_prod_information,\n", + " Environment.UAT: sambah_non_prod_information,\n", + " Environment.PROD: sambah_prod_information\n", + "}\n", + "\n", + "if harmony_environment in sambah_request_env:\n", + " sambah_info = sambah_request_env[harmony_environment]\n", + "else:\n", + " sambah_info = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0d7e83fa-20f1-4382-b07f-9f59ca1acb11", + "metadata": {}, + "outputs": [], + "source": [ + "request_info = {\n", + " # TEMPO NO2 tropospheric, stratospheric, and total columns V03\n", + " # https://cmr.uat.earthdata.nasa.gov/search/concepts/C1262899916-LARC_CLOUD.html\n", + " 'collection': sambah_info[\"collection\"],\n", + " 'temporal': {\n", + " 'start': datetime(2024, 3, 28, 15, 34, 0),\n", + " 'stop': datetime(2024, 3, 28, 16, 0, 0),\n", + " },\n", + " 'spatial': BBox(-170, 33, -10, 38),\n", + " 'granule_id': sambah_info[\"granule_id\"],\n", + " # chosen variables include one variable from each group\n", + " # support/scattering_weights is 3D variable\n", + " 'variables': [\n", + " 'product/vertical_column_stratosphere',\n", + " 'qa_statistics/fit_rms_residual',\n", + " 'support_data/scattering_weights',\n", + " ]\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "fc7aef85", + "metadata": {}, + "source": [ + "# Begin regression tests:" + ] + }, + { + "cell_type": "markdown", + "id": "9dd58c71-0232-4a20-9dc4-ef6b3544e797", + "metadata": {}, + "source": [ + "## SAMBAH\n", + "\n", + "SAMBAH is currently deployed to Sandbox, SIT, UAT and production.\n", + "Requests will be made against the TEMPO NO2 L2 V03 collection." + ] + }, + { + "cell_type": "markdown", + "id": "ef9ec4d2-2228-47d0-a990-d5a4d77b7bc7", + "metadata": {}, + "source": [ + "### SAMBAH: temporal, variable and bounding box subset request\n", + "\n", + "This is a request that exercises the full range of SAMBAH options: temporal, bounding box and variable subsetting.\n", + "\n", + "Requested variable names:\n", + "\n", + "* `2024-3-28 15:34:00 < time < 2024-3-28 16:00:00`\n", + "* `product/vertical_column_total`\n", + "* `BBox(-170,30,-10,40)`" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d6aaf063-1f3c-4d84-b414-ca80e65ef78b", + "metadata": { + "ExecuteTime": { + "end_time": "2024-11-13T19:18:18.977791Z", + "start_time": "2024-11-13T19:17:24.118362Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "81461799_C2930725014-LARC_CLOUD_merged.nc4\n", + "Downloaded: 81461799_C2930725014-LARC_CLOUD_merged.nc4\n", + "Saved output to: temp_var_bbox.nc4\n", + "\u001b[92mSuccess: SAMBAH temporal, variable, bounding box request\u001b[0m\n", + "81461808_C2930725014-LARC_CLOUD_merged.nc4\n" + ] + } + ], + "source": [ + "if request_info is not None:\n", + " temp_var_bbox_request = Request(\n", + " collection=request_info['collection'],\n", + " extend=['mirror_step'],\n", + " concatenate=True,\n", + " granule_id=request_info['granule_id'],\n", + " temporal=request_info['temporal'],\n", + " variables=request_info['variables'],\n", + " spatial=request_info['spatial'],\n", + " )\n", + " \n", + " request_name = 'SAMBAH temporal, variable, bounding box request'\n", + " output_filename = 'temp_var_bbox.nc4'\n", + " \n", + " submit_and_download(harmony_client, temp_var_bbox_request, output_filename)\n", + " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", + " \n", + " compare_results_to_reference_file(output_filename)\n", + " print_success(request_name)\n", + "else:\n", + " print(\n", + " f'SAMBAH is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "55aca1b5-0dec-4866-952d-b7c016bcf86e", + "metadata": {}, + "source": [ + "### SAMBAH: variable subset request, two files" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "90bb3455-f82c-4cd8-93f2-feba79fbed7b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloaded: 81461808_C2930725014-LARC_CLOUD_merged.nc4\n", + "Saved output to: var_only.nc4\n" + ] + }, + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '/Users/dkaufma3/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/reference_files/var_only.nc4'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/file_manager.py:211\u001b[0m, in \u001b[0;36mCachingFileManager._acquire_with_cache_info\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_key\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56\u001b[0m, in \u001b[0;36mLRUCache.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m---> 56\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache\u001b[38;5;241m.\u001b[39mmove_to_end(key)\n", + "\u001b[0;31mKeyError\u001b[0m: [, ('/Users/dkaufma3/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/reference_files/var_only.nc4',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '77e19ba6-0294-46f0-a392-e1fe3493a35a']", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[18], line 16\u001b[0m\n\u001b[1;32m 13\u001b[0m submit_and_download(harmony_client, var_only_request, output_filename)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m exists(output_filename), \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnsuccessful \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrequest_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m---> 16\u001b[0m \u001b[43mcompare_results_to_reference_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43moutput_filename\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 17\u001b[0m print_success(request_name)\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/utilities.py:19\u001b[0m, in \u001b[0;36mcompare_results_to_reference_file\u001b[0;34m(results_file_name)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Use native `xarray` functionality to compare data values and metadata\u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124;03mattributes of downloaded results to a reference file.\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 18\u001b[0m results_data \u001b[38;5;241m=\u001b[39m xr\u001b[38;5;241m.\u001b[39mopen_datatree(results_file_name)\n\u001b[0;32m---> 19\u001b[0m reference_data \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_datatree\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mreference_files/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mresults_file_name\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# We have to drop 'subset_files', \u001b[39;00m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;66;03m# because the values contain collection_ids, \u001b[39;00m\n\u001b[1;32m 23\u001b[0m \u001b[38;5;66;03m# which will be different for UAT and PROD\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;66;03m# results_data = results_data.map_over_datasets(drop_vars_partial, (\"subset_files\", ))\u001b[39;00m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;66;03m# reference_data = reference_data.map_over_datasets(drop_vars_partial, (\"subset_files\", ))\u001b[39;00m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m results_data\u001b[38;5;241m.\u001b[39mequals(reference_data), (\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOutput and reference files \u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdo not match.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 32\u001b[0m )\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/api.py:1089\u001b[0m, in \u001b[0;36mopen_datatree\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1077\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 1078\u001b[0m decode_cf,\n\u001b[1;32m 1079\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39m(),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1085\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 1086\u001b[0m )\n\u001b[1;32m 1087\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1089\u001b[0m backend_tree \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_datatree\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1090\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1091\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1094\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1096\u001b[0m tree \u001b[38;5;241m=\u001b[39m _datatree_from_backend_datatree(\n\u001b[1;32m 1097\u001b[0m backend_tree,\n\u001b[1;32m 1098\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1108\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 1109\u001b[0m )\n\u001b[1;32m 1111\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m tree\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:714\u001b[0m, in \u001b[0;36mNetCDF4BackendEntrypoint.open_datatree\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, format, clobber, diskless, persist, auto_complex, lock, autoclose, **kwargs)\u001b[0m\n\u001b[1;32m 693\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mopen_datatree\u001b[39m(\n\u001b[1;32m 694\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 695\u001b[0m filename_or_obj: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m os\u001b[38;5;241m.\u001b[39mPathLike[Any] \u001b[38;5;241m|\u001b[39m BufferedIOBase \u001b[38;5;241m|\u001b[39m AbstractDataStore,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 712\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 713\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataTree:\n\u001b[0;32m--> 714\u001b[0m groups_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_groups_as_dict\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 715\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 716\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 717\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 718\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 719\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 720\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 721\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 722\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 723\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 724\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 725\u001b[0m \u001b[43m \u001b[49m\u001b[43mclobber\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclobber\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 726\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiskless\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiskless\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 727\u001b[0m \u001b[43m \u001b[49m\u001b[43mpersist\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpersist\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 728\u001b[0m \u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 729\u001b[0m \u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 730\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 731\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 733\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m datatree_from_dict_with_io_cleanup(groups_dict)\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:760\u001b[0m, in \u001b[0;36mNetCDF4BackendEntrypoint.open_groups_as_dict\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, format, clobber, diskless, persist, auto_complex, lock, autoclose, **kwargs)\u001b[0m\n\u001b[1;32m 757\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mxarray\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtreenode\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m NodePath\n\u001b[1;32m 759\u001b[0m filename_or_obj \u001b[38;5;241m=\u001b[39m _normalize_path(filename_or_obj)\n\u001b[0;32m--> 760\u001b[0m store \u001b[38;5;241m=\u001b[39m \u001b[43mNetCDF4DataStore\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 761\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 762\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 763\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 764\u001b[0m \u001b[43m \u001b[49m\u001b[43mclobber\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclobber\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 765\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiskless\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiskless\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 766\u001b[0m \u001b[43m \u001b[49m\u001b[43mpersist\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpersist\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 767\u001b[0m \u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 768\u001b[0m \u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 769\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;66;03m# Check for a group and make it a parent if it exists\u001b[39;00m\n\u001b[1;32m 772\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m group:\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:453\u001b[0m, in \u001b[0;36mNetCDF4DataStore.open\u001b[0;34m(cls, filename, mode, format, group, clobber, diskless, persist, auto_complex, lock, lock_maker, autoclose)\u001b[0m\n\u001b[1;32m 449\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_complex\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m auto_complex\n\u001b[1;32m 450\u001b[0m manager \u001b[38;5;241m=\u001b[39m CachingFileManager(\n\u001b[1;32m 451\u001b[0m netCDF4\u001b[38;5;241m.\u001b[39mDataset, filename, mode\u001b[38;5;241m=\u001b[39mmode, kwargs\u001b[38;5;241m=\u001b[39mkwargs\n\u001b[1;32m 452\u001b[0m )\n\u001b[0;32m--> 453\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:394\u001b[0m, in \u001b[0;36mNetCDF4DataStore.__init__\u001b[0;34m(self, manager, group, mode, lock, autoclose)\u001b[0m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_group \u001b[38;5;241m=\u001b[39m group\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m mode\n\u001b[0;32m--> 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mds\u001b[49m\u001b[38;5;241m.\u001b[39mdata_model\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mds\u001b[38;5;241m.\u001b[39mfilepath()\n\u001b[1;32m 396\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_remote \u001b[38;5;241m=\u001b[39m is_remote_uri(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename)\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:462\u001b[0m, in \u001b[0;36mNetCDF4DataStore.ds\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 460\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mds\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:456\u001b[0m, in \u001b[0;36mNetCDF4DataStore._acquire\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 455\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_acquire\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m--> 456\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mwith\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire_context\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneeds_lock\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mas\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mroot\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mds\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m_nc4_require_group\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_group\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/contextlib.py:137\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkwds, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunc\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgen)\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 139\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgenerator didn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt yield\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/file_manager.py:199\u001b[0m, in \u001b[0;36mCachingFileManager.acquire_context\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;129m@contextlib\u001b[39m\u001b[38;5;241m.\u001b[39mcontextmanager\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21macquire_context\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m 198\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Context manager for acquiring a file.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 199\u001b[0m file, cached \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire_with_cache_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneeds_lock\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m file\n", + "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/file_manager.py:217\u001b[0m, in \u001b[0;36mCachingFileManager._acquire_with_cache_info\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 215\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m 216\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmode\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode\n\u001b[0;32m--> 217\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_opener\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# ensure file doesn't get overridden when opened again\u001b[39;00m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32msrc/netCDF4/_netCDF4.pyx:2521\u001b[0m, in \u001b[0;36mnetCDF4._netCDF4.Dataset.__init__\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32msrc/netCDF4/_netCDF4.pyx:2158\u001b[0m, in \u001b[0;36mnetCDF4._netCDF4._ensure_nc_success\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/Users/dkaufma3/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/reference_files/var_only.nc4'" + ] + } + ], + "source": [ + "if request_info is not None:\n", + " var_only_request = Request(\n", + " collection=request_info['collection'],\n", + " extend=['mirror_step'],\n", + " concatenate=True,\n", + " granule_id=request_info['granule_id'][:2],\n", + " variables=request_info['variables'],\n", + " )\n", + " \n", + " request_name = 'SAMBAH variable request'\n", + " output_filename = 'var_only.nc4'\n", + " \n", + " submit_and_download(harmony_client, var_only_request, output_filename)\n", + " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", + " \n", + " compare_results_to_reference_file(output_filename)\n", + " print_success(request_name)\n", + "else:\n", + " print(\n", + " f'SAMBAH is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "c8322c1b-7fad-4b87-a46d-abf5f0ee56df", + "metadata": {}, + "source": [ + "### SAMBAH: spatial request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4330105a-f9f0-429e-9762-3049f408d635", + "metadata": {}, + "outputs": [], + "source": [ + "if request_info is not None:\n", + " spatial_only_request = Request(\n", + " collection=request_info['collection'],\n", + " extend=['mirror_step'],\n", + " concatenate=True,\n", + " granule_id=request_info['granule_id'],\n", + " spatial=request_info['spatial'],\n", + " )\n", + " \n", + " request_name = 'SAMBAH spatial request'\n", + " output_filename = 'spatial_only.nc4'\n", + " \n", + " submit_and_download(harmony_client, spatial_only_request, output_filename)\n", + " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", + " \n", + " compare_results_to_reference_file(output_filename)\n", + " print_success(request_name)\n", + "else:\n", + " print(\n", + " f'SAMBAH is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "d0629b4f-1990-42f7-b19d-025e46a96629", + "metadata": {}, + "source": [ + "### SAMBAH: no subsetting required, single file\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c747cfcc-e63a-4858-b406-a4ec87b6acb5", + "metadata": {}, + "outputs": [], + "source": [ + "if request_info is not None:\n", + " all_data_request = Request(\n", + " collection=request_info['collection'],\n", + " extend='mirror_step',\n", + " concatenate=True,\n", + " granule_id=request_info['granule_id'][0],\n", + " )\n", + " \n", + " request_name = 'SAMBAH no subset single file request'\n", + " output_filename = 'all_data.nc4'\n", + " \n", + " submit_and_download(harmony_client, all_data_request, output_filename)\n", + " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", + " \n", + " compare_results_to_reference_file(output_filename)\n", + " print_success(request_name)\n", + "else:\n", + " print(\n", + " f'SAMBAH is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "ca88eb71-2526-4b34-8ff7-441f869244e0", + "metadata": {}, + "source": [ + "# Clean up test outputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73fdd147-74a0-48d3-957f-388e2abe356a", + "metadata": {}, + "outputs": [], + "source": [ + "remove_results_files()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a868b30-52d9-45aa-9359-a14d79548ce4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0332c0fd-1332-470f-8312-34eecd94280b", + "metadata": {}, + "outputs": [], + "source": [] +>>>>>>> f7db24f (update to use conditionals for test suites and fix environment mutability) } ], "metadata": { diff --git a/test/sambah/utilities.py b/test/sambah/utilities.py index c89a8646..2fe157d3 100644 --- a/test/sambah/utilities.py +++ b/test/sambah/utilities.py @@ -3,7 +3,7 @@ notebook to increase the readability of the regression test suite. """ - +from functools import partial from os import listdir, remove, replace from harmony import Client, Request @@ -18,6 +18,15 @@ def compare_results_to_reference_file(results_file_name: str) -> None: results_data = xr.open_datatree(results_file_name) reference_data = xr.open_datatree(f"reference_files/{results_file_name}") + # We have to drop 'subset_files', + # because the values contain collection_ids, + # which will be different for UAT and PROD + # def my_drop_vars(*arg, f): + # return lambda var_names: f("subset_files", ) + # drop_vars_partial = partial(xr.Dataset.drop_vars, errors="ignore") + # results_data = results_data.map_over_datasets(drop_vars_partial, ("subset_files", )) + # reference_data = reference_data.map_over_datasets(drop_vars_partial, ("subset_files", )) + assert results_data.equals(reference_data), ( 'Output and reference files ' 'do not match.' ) @@ -55,11 +64,19 @@ def submit_and_download( print('Harmony request failed to complete successfully.') raise exception - def remove_results_files() -> None: """Remove all NetCDF-4 files downloaded during the regression tests.""" directory_files = listdir() for directory_file in directory_files: - if directory_file.endswith('.nc4'): + if directory_file.endswith('.nc4') or directory_file.endswith('.nc'): remove(directory_file) + +def print_error(error_string: str) -> str: + """Print an error, with formatting for red text.""" + print(f'\033[91m{error_string}\033[0m') + +def print_success(success_string: str) -> str: + """Print a success message, with formatting for green text.""" + print(f'\033[92mSuccess: {success_string}\033[0m') + From 9da8a0b842f71fad6b69c4815131c68b8c938b23 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Mon, 18 Nov 2024 15:36:57 -0500 Subject: [PATCH 11/27] track and add reference data files --- test/sambah/reference_files/all_data.nc4 | 3 +++ test/sambah/reference_files/spatial_only.nc4 | 3 +++ test/sambah/reference_files/temp_var_bbox.nc4 | 3 +++ test/sambah/reference_files/var_only.nc4 | 3 +++ 4 files changed, 12 insertions(+) create mode 100644 test/sambah/reference_files/all_data.nc4 create mode 100644 test/sambah/reference_files/spatial_only.nc4 create mode 100644 test/sambah/reference_files/temp_var_bbox.nc4 create mode 100644 test/sambah/reference_files/var_only.nc4 diff --git a/test/sambah/reference_files/all_data.nc4 b/test/sambah/reference_files/all_data.nc4 new file mode 100644 index 00000000..edadd625 --- /dev/null +++ b/test/sambah/reference_files/all_data.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b063d2d045619a4fa596f9084204e705682958faa8a0606eef9c6a37aaa91847 +size 92642167 diff --git a/test/sambah/reference_files/spatial_only.nc4 b/test/sambah/reference_files/spatial_only.nc4 new file mode 100644 index 00000000..b152f6fb --- /dev/null +++ b/test/sambah/reference_files/spatial_only.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba92cce2fbb598fbd75e3240793201d65da7412efcadbe1823f7ec49c8e5159 +size 60496865 diff --git a/test/sambah/reference_files/temp_var_bbox.nc4 b/test/sambah/reference_files/temp_var_bbox.nc4 new file mode 100644 index 00000000..d323eca4 --- /dev/null +++ b/test/sambah/reference_files/temp_var_bbox.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b8de487dfe91f6e4de7d008fca65b48b86a3f1c7f8aedea9a3dd49737552e44 +size 25164633 diff --git a/test/sambah/reference_files/var_only.nc4 b/test/sambah/reference_files/var_only.nc4 new file mode 100644 index 00000000..e73cdf9d --- /dev/null +++ b/test/sambah/reference_files/var_only.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a781564ba24a3248b39972c3421cca4455a6235a42597f894ec28e32c1bb0ec7 +size 90702545 From dda0f1b4eda4baf08ebe65e8c47e0aa38cb7f4f6 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 15 Nov 2024 17:29:41 -0500 Subject: [PATCH 12/27] rename local utilities module and remove shared utils --- test/sambah/local_utilities.py | 43 ++++++++++++++++++ test/sambah/utilities.py | 82 ---------------------------------- 2 files changed, 43 insertions(+), 82 deletions(-) create mode 100644 test/sambah/local_utilities.py delete mode 100644 test/sambah/utilities.py diff --git a/test/sambah/local_utilities.py b/test/sambah/local_utilities.py new file mode 100644 index 00000000..9292f743 --- /dev/null +++ b/test/sambah/local_utilities.py @@ -0,0 +1,43 @@ +""" A module containing utility functionality used by the Harmony OPeNDAP + SubSetter regression tests. These functions are kept out of the Jupyter + notebook to increase the readability of the regression test suite. + +""" +from functools import partial +from os import listdir, remove, replace + +from harmony import Client, Request +from harmony.harmony import ProcessingFailedException +import xarray as xr + + +def compare_results_to_reference_file(results_file_name: str) -> None: + """Use native `xarray` functionality to compare data values and metadata + attributes of downloaded results to a reference file. + """ + results_data = xr.open_datatree(results_file_name) + reference_data = xr.open_datatree(f"reference_files/{results_file_name}") + + # We have to drop 'subset_files', + # because the values contain collection_ids, + # which will be different for UAT and PROD + drop_vars_partial = partial(xr.Dataset.drop_vars, errors="ignore") + results_data = results_data.map_over_datasets(drop_vars_partial, ("subset_files", )) + reference_data = reference_data.map_over_datasets(drop_vars_partial, ("subset_files", )) + + assert results_data.equals(reference_data), ( + 'Output and reference files ' 'do not match.' + ) + + reference_data.close() + results_data.close() + + +def remove_results_files() -> None: + """Remove all NetCDF-4 files downloaded during the regression tests.""" + directory_files = listdir() + + for directory_file in directory_files: + if directory_file.endswith('.nc4') or directory_file.endswith('.nc'): + remove(directory_file) + diff --git a/test/sambah/utilities.py b/test/sambah/utilities.py deleted file mode 100644 index 2fe157d3..00000000 --- a/test/sambah/utilities.py +++ /dev/null @@ -1,82 +0,0 @@ -""" A module containing utility functionality used by the Harmony OPeNDAP - SubSetter regression tests. These functions are kept out of the Jupyter - notebook to increase the readability of the regression test suite. - -""" -from functools import partial -from os import listdir, remove, replace - -from harmony import Client, Request -from harmony.harmony import ProcessingFailedException -import xarray as xr - - -def compare_results_to_reference_file(results_file_name: str) -> None: - """Use native `xarray` functionality to compare data values and metadata - attributes of downloaded results to a reference file. - """ - results_data = xr.open_datatree(results_file_name) - reference_data = xr.open_datatree(f"reference_files/{results_file_name}") - - # We have to drop 'subset_files', - # because the values contain collection_ids, - # which will be different for UAT and PROD - # def my_drop_vars(*arg, f): - # return lambda var_names: f("subset_files", ) - # drop_vars_partial = partial(xr.Dataset.drop_vars, errors="ignore") - # results_data = results_data.map_over_datasets(drop_vars_partial, ("subset_files", )) - # reference_data = reference_data.map_over_datasets(drop_vars_partial, ("subset_files", )) - - assert results_data.equals(reference_data), ( - 'Output and reference files ' 'do not match.' - ) - - reference_data.close() - results_data.close() - - -def submit_and_download( - harmony_client: Client, request: Request, output_file_name: str -): - """Submit a Harmony request via a `harmony-py` client. - - Wait for the Harmony job to finish, then download the results to the specified file path. - """ - downloaded_filename = None - - try: - assert request.is_valid() - job_id = harmony_client.submit(request) - - for filename in [ - file_future.result() - for file_future in harmony_client.download_all(job_id, overwrite=True) - ]: - - print(f'Downloaded: {filename}') - downloaded_filename = filename - - if downloaded_filename is not None: - replace(downloaded_filename, output_file_name) - print(f'Saved output to: {output_file_name}') - - except ProcessingFailedException as exception: - print('Harmony request failed to complete successfully.') - raise exception - -def remove_results_files() -> None: - """Remove all NetCDF-4 files downloaded during the regression tests.""" - directory_files = listdir() - - for directory_file in directory_files: - if directory_file.endswith('.nc4') or directory_file.endswith('.nc'): - remove(directory_file) - -def print_error(error_string: str) -> str: - """Print an error, with formatting for red text.""" - print(f'\033[91m{error_string}\033[0m') - -def print_success(success_string: str) -> str: - """Print a success message, with formatting for green text.""" - print(f'\033[92mSuccess: {success_string}\033[0m') - From 702ecf946163746d06811b87185ccbac681a2082 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 15 Nov 2024 17:29:57 -0500 Subject: [PATCH 13/27] update notebook to use shared_utils --- test/sambah/SAMBAH_Regression.ipynb | 78 +++++++---------------------- 1 file changed, 17 insertions(+), 61 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index 926c3d18..d598ee8e 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "7ce7f640", "metadata": { "ExecuteTime": { @@ -88,6 +88,14 @@ }, "outputs": [], "source": [ + "import sys\n", + "\n", + "sys.path.append('../shared_utils')\n", + "from utilities import (\n", + " print_success,\n", + " submit_and_download,\n", + ")\n", + "\n", "from datetime import datetime\n", "from os.path import exists\n", "\n", @@ -96,8 +104,6 @@ "from utilities import (\n", " compare_results_to_reference_file,\n", " remove_results_files,\n", - " submit_and_download,\n", - " print_success\n", ")" ] }, @@ -113,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "45f380c6", "metadata": { "ExecuteTime": { @@ -139,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "97dc22bb-e04e-442c-ab12-160ff4842325", "metadata": {}, "outputs": [], @@ -167,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "4420bb6e", "metadata": { "ExecuteTime": { @@ -583,7 +589,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "0d7e83fa-20f1-4382-b07f-9f59ca1acb11", "metadata": {}, "outputs": [], @@ -645,7 +651,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "d6aaf063-1f3c-4d84-b414-ca80e65ef78b", "metadata": { "ExecuteTime": { @@ -653,19 +659,7 @@ "start_time": "2024-11-13T19:17:24.118362Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "81461799_C2930725014-LARC_CLOUD_merged.nc4\n", - "Downloaded: 81461799_C2930725014-LARC_CLOUD_merged.nc4\n", - "Saved output to: temp_var_bbox.nc4\n", - "\u001b[92mSuccess: SAMBAH temporal, variable, bounding box request\u001b[0m\n", - "81461808_C2930725014-LARC_CLOUD_merged.nc4\n" - ] - } - ], + "outputs": [], "source": [ "if request_info is not None:\n", " temp_var_bbox_request = Request(\n", @@ -702,48 +696,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "90bb3455-f82c-4cd8-93f2-feba79fbed7b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloaded: 81461808_C2930725014-LARC_CLOUD_merged.nc4\n", - "Saved output to: var_only.nc4\n" - ] - }, - { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] No such file or directory: '/Users/dkaufma3/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/reference_files/var_only.nc4'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/file_manager.py:211\u001b[0m, in \u001b[0;36mCachingFileManager._acquire_with_cache_info\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_key\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56\u001b[0m, in \u001b[0;36mLRUCache.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m---> 56\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache\u001b[38;5;241m.\u001b[39mmove_to_end(key)\n", - "\u001b[0;31mKeyError\u001b[0m: [, ('/Users/dkaufma3/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/reference_files/var_only.nc4',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '77e19ba6-0294-46f0-a392-e1fe3493a35a']", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[18], line 16\u001b[0m\n\u001b[1;32m 13\u001b[0m submit_and_download(harmony_client, var_only_request, output_filename)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m exists(output_filename), \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnsuccessful \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrequest_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m---> 16\u001b[0m \u001b[43mcompare_results_to_reference_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43moutput_filename\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 17\u001b[0m print_success(request_name)\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/utilities.py:19\u001b[0m, in \u001b[0;36mcompare_results_to_reference_file\u001b[0;34m(results_file_name)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Use native `xarray` functionality to compare data values and metadata\u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124;03mattributes of downloaded results to a reference file.\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 18\u001b[0m results_data \u001b[38;5;241m=\u001b[39m xr\u001b[38;5;241m.\u001b[39mopen_datatree(results_file_name)\n\u001b[0;32m---> 19\u001b[0m reference_data \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_datatree\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mreference_files/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mresults_file_name\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# We have to drop 'subset_files', \u001b[39;00m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;66;03m# because the values contain collection_ids, \u001b[39;00m\n\u001b[1;32m 23\u001b[0m \u001b[38;5;66;03m# which will be different for UAT and PROD\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;66;03m# results_data = results_data.map_over_datasets(drop_vars_partial, (\"subset_files\", ))\u001b[39;00m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;66;03m# reference_data = reference_data.map_over_datasets(drop_vars_partial, (\"subset_files\", ))\u001b[39;00m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m results_data\u001b[38;5;241m.\u001b[39mequals(reference_data), (\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOutput and reference files \u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdo not match.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 32\u001b[0m )\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/api.py:1089\u001b[0m, in \u001b[0;36mopen_datatree\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1077\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 1078\u001b[0m decode_cf,\n\u001b[1;32m 1079\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39m(),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1085\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 1086\u001b[0m )\n\u001b[1;32m 1087\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1089\u001b[0m backend_tree \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_datatree\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1090\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1091\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1094\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1096\u001b[0m tree \u001b[38;5;241m=\u001b[39m _datatree_from_backend_datatree(\n\u001b[1;32m 1097\u001b[0m backend_tree,\n\u001b[1;32m 1098\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1108\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 1109\u001b[0m )\n\u001b[1;32m 1111\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m tree\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:714\u001b[0m, in \u001b[0;36mNetCDF4BackendEntrypoint.open_datatree\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, format, clobber, diskless, persist, auto_complex, lock, autoclose, **kwargs)\u001b[0m\n\u001b[1;32m 693\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mopen_datatree\u001b[39m(\n\u001b[1;32m 694\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 695\u001b[0m filename_or_obj: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m os\u001b[38;5;241m.\u001b[39mPathLike[Any] \u001b[38;5;241m|\u001b[39m BufferedIOBase \u001b[38;5;241m|\u001b[39m AbstractDataStore,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 712\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 713\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataTree:\n\u001b[0;32m--> 714\u001b[0m groups_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_groups_as_dict\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 715\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 716\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 717\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 718\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 719\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 720\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 721\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 722\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 723\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 724\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 725\u001b[0m \u001b[43m \u001b[49m\u001b[43mclobber\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclobber\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 726\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiskless\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiskless\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 727\u001b[0m \u001b[43m \u001b[49m\u001b[43mpersist\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpersist\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 728\u001b[0m \u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 729\u001b[0m \u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 730\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 731\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 733\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m datatree_from_dict_with_io_cleanup(groups_dict)\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:760\u001b[0m, in \u001b[0;36mNetCDF4BackendEntrypoint.open_groups_as_dict\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, format, clobber, diskless, persist, auto_complex, lock, autoclose, **kwargs)\u001b[0m\n\u001b[1;32m 757\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mxarray\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtreenode\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m NodePath\n\u001b[1;32m 759\u001b[0m filename_or_obj \u001b[38;5;241m=\u001b[39m _normalize_path(filename_or_obj)\n\u001b[0;32m--> 760\u001b[0m store \u001b[38;5;241m=\u001b[39m \u001b[43mNetCDF4DataStore\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 761\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 762\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 763\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 764\u001b[0m \u001b[43m \u001b[49m\u001b[43mclobber\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclobber\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 765\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiskless\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiskless\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 766\u001b[0m \u001b[43m \u001b[49m\u001b[43mpersist\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpersist\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 767\u001b[0m \u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 768\u001b[0m \u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 769\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;66;03m# Check for a group and make it a parent if it exists\u001b[39;00m\n\u001b[1;32m 772\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m group:\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:453\u001b[0m, in \u001b[0;36mNetCDF4DataStore.open\u001b[0;34m(cls, filename, mode, format, group, clobber, diskless, persist, auto_complex, lock, lock_maker, autoclose)\u001b[0m\n\u001b[1;32m 449\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_complex\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m auto_complex\n\u001b[1;32m 450\u001b[0m manager \u001b[38;5;241m=\u001b[39m CachingFileManager(\n\u001b[1;32m 451\u001b[0m netCDF4\u001b[38;5;241m.\u001b[39mDataset, filename, mode\u001b[38;5;241m=\u001b[39mmode, kwargs\u001b[38;5;241m=\u001b[39mkwargs\n\u001b[1;32m 452\u001b[0m )\n\u001b[0;32m--> 453\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:394\u001b[0m, in \u001b[0;36mNetCDF4DataStore.__init__\u001b[0;34m(self, manager, group, mode, lock, autoclose)\u001b[0m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_group \u001b[38;5;241m=\u001b[39m group\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m mode\n\u001b[0;32m--> 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mds\u001b[49m\u001b[38;5;241m.\u001b[39mdata_model\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mds\u001b[38;5;241m.\u001b[39mfilepath()\n\u001b[1;32m 396\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_remote \u001b[38;5;241m=\u001b[39m is_remote_uri(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename)\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:462\u001b[0m, in \u001b[0;36mNetCDF4DataStore.ds\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 460\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mds\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:456\u001b[0m, in \u001b[0;36mNetCDF4DataStore._acquire\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 455\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_acquire\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m--> 456\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mwith\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire_context\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneeds_lock\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mas\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mroot\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mds\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m_nc4_require_group\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_group\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/contextlib.py:137\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkwds, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunc\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgen)\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 139\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgenerator didn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt yield\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/file_manager.py:199\u001b[0m, in \u001b[0;36mCachingFileManager.acquire_context\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;129m@contextlib\u001b[39m\u001b[38;5;241m.\u001b[39mcontextmanager\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21macquire_context\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m 198\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Context manager for acquiring a file.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 199\u001b[0m file, cached \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire_with_cache_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneeds_lock\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m file\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/papermill-sambah/lib/python3.11/site-packages/xarray/backends/file_manager.py:217\u001b[0m, in \u001b[0;36mCachingFileManager._acquire_with_cache_info\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 215\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m 216\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmode\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode\n\u001b[0;32m--> 217\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_opener\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# ensure file doesn't get overridden when opened again\u001b[39;00m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", - "File \u001b[0;32msrc/netCDF4/_netCDF4.pyx:2521\u001b[0m, in \u001b[0;36mnetCDF4._netCDF4.Dataset.__init__\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32msrc/netCDF4/_netCDF4.pyx:2158\u001b[0m, in \u001b[0;36mnetCDF4._netCDF4._ensure_nc_success\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/Users/dkaufma3/projects/harmony-regression-tests_project/2_ExperimentFolder/harmony-regression-tests/test/sambah/reference_files/var_only.nc4'" - ] - } - ], + "outputs": [], "source": [ "if request_info is not None:\n", " var_only_request = Request(\n", From 93373b4754a4e2c771a0a985bbf6dcba0db90e96 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Mon, 18 Nov 2024 10:12:50 -0500 Subject: [PATCH 14/27] clean up imports --- test/sambah/SAMBAH_Regression.ipynb | 2 +- test/sambah/local_utilities.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index d598ee8e..2ae88860 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -101,7 +101,7 @@ "\n", "from harmony import BBox, Client, Collection, Environment, Request\n", "\n", - "from utilities import (\n", + "from local_utilities import (\n", " compare_results_to_reference_file,\n", " remove_results_files,\n", ")" diff --git a/test/sambah/local_utilities.py b/test/sambah/local_utilities.py index 9292f743..b0f1176c 100644 --- a/test/sambah/local_utilities.py +++ b/test/sambah/local_utilities.py @@ -4,10 +4,8 @@ """ from functools import partial -from os import listdir, remove, replace +from os import listdir, remove -from harmony import Client, Request -from harmony.harmony import ProcessingFailedException import xarray as xr From fe400a722cf197cc4e846b572c761f16f3acb7b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:16:15 -0500 Subject: [PATCH 15/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitattributes | 4 ++++ test/sambah/SAMBAH_Regression.ipynb | 34 ++++++++++++++++------------- test/sambah/local_utilities.py | 14 +++++++----- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.gitattributes b/.gitattributes index e64c0f27..6a05f673 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,6 @@ test/nsidc-icesat2/reference_files/*.h5 filter=lfs diff=lfs merge=lfs -text test/subset-band-name/reference_data/*.hdf filter=lfs diff=lfs merge=lfs -text +test/sambah/reference_files/all_data.nc4 filter=lfs diff=lfs merge=lfs -text +test/sambah/reference_files/spatial_only.nc4 filter=lfs diff=lfs merge=lfs -text +test/sambah/reference_files/temp_var_bbox.nc4 filter=lfs diff=lfs merge=lfs -text +test/sambah/reference_files/var_only.nc4 filter=lfs diff=lfs merge=lfs -text diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index 2ae88860..ae4bb2bd 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -193,7 +193,7 @@ " 'G1263137387-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T154353Z_S008G01.nc\n", " 'G1263137388-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T155033Z_S008G02.nc\n", " 'G1263137378-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T155713Z_S008G03.nc\n", - " ]\n", + " ],\n", "}\n", "\n", "sambah_prod_information = {\n", @@ -206,6 +206,7 @@ " 'G3205052670-LARC_CLOUD',\n", " 'G3205053541-LARC_CLOUD',\n", " 'G3205052319-LARC_CLOUD',\n", +<<<<<<< HEAD " ],\n", <<<<<<< HEAD >>>>>>> eccece1 ([pre-commit.ci] auto fixes from pre-commit.com hooks) @@ -572,13 +573,16 @@ ], "id": "93da5ee1a11dd456" ======= +======= + " ],\n", +>>>>>>> 66c779b ([pre-commit.ci] auto fixes from pre-commit.com hooks) "}\n", "\n", "sambah_request_env = {\n", " Environment.LOCAL: sambah_non_prod_information,\n", " Environment.SIT: sambah_non_prod_information,\n", " Environment.UAT: sambah_non_prod_information,\n", - " Environment.PROD: sambah_prod_information\n", + " Environment.PROD: sambah_prod_information,\n", "}\n", "\n", "if harmony_environment in sambah_request_env:\n", @@ -610,7 +614,7 @@ " 'product/vertical_column_stratosphere',\n", " 'qa_statistics/fit_rms_residual',\n", " 'support_data/scattering_weights',\n", - " ]\n", + " ],\n", "}" ] }, @@ -671,13 +675,13 @@ " variables=request_info['variables'],\n", " spatial=request_info['spatial'],\n", " )\n", - " \n", + "\n", " request_name = 'SAMBAH temporal, variable, bounding box request'\n", " output_filename = 'temp_var_bbox.nc4'\n", - " \n", + "\n", " submit_and_download(harmony_client, temp_var_bbox_request, output_filename)\n", " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", - " \n", + "\n", " compare_results_to_reference_file(output_filename)\n", " print_success(request_name)\n", "else:\n", @@ -709,13 +713,13 @@ " granule_id=request_info['granule_id'][:2],\n", " variables=request_info['variables'],\n", " )\n", - " \n", + "\n", " request_name = 'SAMBAH variable request'\n", " output_filename = 'var_only.nc4'\n", - " \n", + "\n", " submit_and_download(harmony_client, var_only_request, output_filename)\n", " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", - " \n", + "\n", " compare_results_to_reference_file(output_filename)\n", " print_success(request_name)\n", "else:\n", @@ -747,13 +751,13 @@ " granule_id=request_info['granule_id'],\n", " spatial=request_info['spatial'],\n", " )\n", - " \n", + "\n", " request_name = 'SAMBAH spatial request'\n", " output_filename = 'spatial_only.nc4'\n", - " \n", + "\n", " submit_and_download(harmony_client, spatial_only_request, output_filename)\n", " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", - " \n", + "\n", " compare_results_to_reference_file(output_filename)\n", " print_success(request_name)\n", "else:\n", @@ -784,13 +788,13 @@ " concatenate=True,\n", " granule_id=request_info['granule_id'][0],\n", " )\n", - " \n", + "\n", " request_name = 'SAMBAH no subset single file request'\n", " output_filename = 'all_data.nc4'\n", - " \n", + "\n", " submit_and_download(harmony_client, all_data_request, output_filename)\n", " assert exists(output_filename), f'Unsuccessful {request_name}.'\n", - " \n", + "\n", " compare_results_to_reference_file(output_filename)\n", " print_success(request_name)\n", "else:\n", diff --git a/test/sambah/local_utilities.py b/test/sambah/local_utilities.py index b0f1176c..258f8373 100644 --- a/test/sambah/local_utilities.py +++ b/test/sambah/local_utilities.py @@ -3,6 +3,7 @@ notebook to increase the readability of the regression test suite. """ + from functools import partial from os import listdir, remove @@ -16,13 +17,15 @@ def compare_results_to_reference_file(results_file_name: str) -> None: results_data = xr.open_datatree(results_file_name) reference_data = xr.open_datatree(f"reference_files/{results_file_name}") - # We have to drop 'subset_files', - # because the values contain collection_ids, + # We have to drop 'subset_files', + # because the values contain collection_ids, # which will be different for UAT and PROD drop_vars_partial = partial(xr.Dataset.drop_vars, errors="ignore") - results_data = results_data.map_over_datasets(drop_vars_partial, ("subset_files", )) - reference_data = reference_data.map_over_datasets(drop_vars_partial, ("subset_files", )) - + results_data = results_data.map_over_datasets(drop_vars_partial, ("subset_files",)) + reference_data = reference_data.map_over_datasets( + drop_vars_partial, ("subset_files",) + ) + assert results_data.equals(reference_data), ( 'Output and reference files ' 'do not match.' ) @@ -38,4 +41,3 @@ def remove_results_files() -> None: for directory_file in directory_files: if directory_file.endswith('.nc4') or directory_file.endswith('.nc'): remove(directory_file) - From 7537034487d8612310c5be2fecd0dd78d6cb02d7 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Mon, 18 Nov 2024 10:51:33 -0500 Subject: [PATCH 16/27] update comments and markdown text --- test/sambah/SAMBAH_Regression.ipynb | 36 +++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index ae4bb2bd..e9da2ad1 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -48,7 +48,7 @@ ======= "# Regression test suite for the SAMBAH:\n", "\n", - "This notebook provides condensed examples of using Harmony to make requests against the SAMBAH service developed to process L2 TEMPO data. \n", + "This notebook provides condensed examples of using Harmony to make requests against the [Subsetter And Multi-dimensional Batched Aggregation in Harmony (SAMBAH)](https://stitchee.readthedocs.io/en/latest/sambah_readme/) service developed to process Level 2 data from the [Tropospheric Emissions: Monitoring of Pollution (TEMPO)](https://asdc.larc.nasa.gov/project/TEMPO) instrument. \n", "\n", "### Features of SAMBAH include:\n", "\n", @@ -599,8 +599,6 @@ "outputs": [], "source": [ "request_info = {\n", - " # TEMPO NO2 tropospheric, stratospheric, and total columns V03\n", - " # https://cmr.uat.earthdata.nasa.gov/search/concepts/C1262899916-LARC_CLOUD.html\n", " 'collection': sambah_info[\"collection\"],\n", " 'temporal': {\n", " 'start': datetime(2024, 3, 28, 15, 34, 0),\n", @@ -631,8 +629,6 @@ "id": "9dd58c71-0232-4a20-9dc4-ef6b3544e797", "metadata": {}, "source": [ - "## SAMBAH\n", - "\n", "SAMBAH is currently deployed to Sandbox, SIT, UAT and production.\n", "Requests will be made against the TEMPO NO2 L2 V03 collection." ] @@ -644,13 +640,11 @@ "source": [ "### SAMBAH: temporal, variable and bounding box subset request\n", "\n", - "This is a request that exercises the full range of SAMBAH options: temporal, bounding box and variable subsetting.\n", - "\n", - "Requested variable names:\n", - "\n", - "* `2024-3-28 15:34:00 < time < 2024-3-28 16:00:00`\n", - "* `product/vertical_column_total`\n", - "* `BBox(-170,30,-10,40)`" + "This is a request that exercises the full range of SAMBAH options: \n", + "- time range\n", + "- spatial bounding box\n", + "- variable subsetting\n", + "- concatenating within (i.e., `extend`) and across scans" ] }, { @@ -695,7 +689,11 @@ "id": "55aca1b5-0dec-4866-952d-b7c016bcf86e", "metadata": {}, "source": [ - "### SAMBAH: variable subset request, two files" + "### SAMBAH: variable subset request, two files\n", + "\n", + "This is a request that includes:\n", + "- variable subsetting\n", + "- concatenating two granules within (i.e., `extend`) a scan" ] }, { @@ -733,7 +731,11 @@ "id": "c8322c1b-7fad-4b87-a46d-abf5f0ee56df", "metadata": {}, "source": [ - "### SAMBAH: spatial request" + "### SAMBAH: spatial request\n", + "\n", + "This is a request that includes:\n", + "- spatial bounding box subsetting\n", + "- concatenating granules within (i.e., `extend`) and across scans" ] }, { @@ -771,7 +773,11 @@ "id": "d0629b4f-1990-42f7-b19d-025e46a96629", "metadata": {}, "source": [ - "### SAMBAH: no subsetting required, single file\n" + "### SAMBAH: no subsetting required, single file\n", + "\n", + "This is a request that includes:\n", + "- a single granule\n", + "- no subsetting" ] }, { From f741fbe62819aff6a6896753b7d94b58bd37f043 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Mon, 18 Nov 2024 15:49:59 -0500 Subject: [PATCH 17/27] fix notebook source formatting --- test/sambah/SAMBAH_Regression.ipynb | 415 +--------------------------- 1 file changed, 2 insertions(+), 413 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index e9da2ad1..1e0d79e3 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -1,51 +1,10 @@ { "cells": [ { + "cell_type": "markdown", + "id": "5a9896b1", "metadata": {}, - "cell_type": "raw", "source": [ -<<<<<<< HEAD - "{\n", - " \"cells\": [\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"f7f9ded8\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"# Regression test suite for the SAMBAH:\\n\",\n", - " \"\\n\",\n", - " \"This notebook provides condensed examples of using Harmony to make requests against the SAMBAH service developed to process L2 TEMPO data. The features of SAMBAH include:\\n\",\n", - " \"\\n\",\n", - " \"* Variable subsetting, including required variables.\\n\",\n", - " \"* Temporal subsetting.\\n\",\n", - " \"* Bounding box spatial subsetting.\\n\",\n", - " \"\\n\",\n", - " \"## Prerequisites\\n\",\n", - " \"\\n\",\n", - " \"The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\\n\",\n", - " \"\\n\",\n", - " \"`conda env create -f ./environment.yaml && conda activate papermill-sambah`\\n\",\n", - " \"\\n\",\n", - " \"A `.netrc` file must also be located in the `test` directory of this repository.\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"009f1df8\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"## Import requirements:\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 1,\n", - " \"id\": \"7ce7f640\",\n", - " \"metadata\": {\n", - " \"ExecuteTime\": {\n", - " \"end_time\": \"2024-11-13T19:16:56.881691Z\",\n", - " \"start_time\": \"2024-11-13T19:16:55.207082Z\"\n", -======= "# Regression test suite for the SAMBAH:\n", "\n", "This notebook provides condensed examples of using Harmony to make requests against the [Subsetter And Multi-dimensional Batched Aggregation in Harmony (SAMBAH)](https://stitchee.readthedocs.io/en/latest/sambah_readme/) service developed to process Level 2 data from the [Tropospheric Emissions: Monitoring of Pollution (TEMPO)](https://asdc.larc.nasa.gov/project/TEMPO) instrument. \n", @@ -206,376 +165,7 @@ " 'G3205052670-LARC_CLOUD',\n", " 'G3205053541-LARC_CLOUD',\n", " 'G3205052319-LARC_CLOUD',\n", -<<<<<<< HEAD - " ],\n", -<<<<<<< HEAD ->>>>>>> eccece1 ([pre-commit.ci] auto fixes from pre-commit.com hooks) - " },\n", - " \"tags\": []\n", - " },\n", - " \"outputs\": [],\n", - " \"source\": [\n", - " \"from datetime import datetime\\n\",\n", - " \"from os.path import exists\\n\",\n", - " \"\\n\",\n", - " \"from harmony import BBox, Client, Collection, Dimension, Environment, Request\\n\",\n", - " \"\\n\",\n", - " \"from utilities import (\\n\",\n", - " \" compare_results_to_reference_file,\\n\",\n", - " \" remove_results_files,\\n\",\n", - " \" submit_and_download,\\n\",\n", - " \")\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"00758d6f\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"## Set default parameters:\\n\",\n", - " \"\\n\",\n", - " \"`papermill` requires default values for parameters used on the workflow. In this case, `harmony_host_url`.\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 2,\n", - " \"id\": \"45f380c6\",\n", - " \"metadata\": {\n", - " \"ExecuteTime\": {\n", - " \"end_time\": \"2024-11-13T19:17:01.668202Z\",\n", - " \"start_time\": \"2024-11-13T19:17:01.663791Z\"\n", - " },\n", - " \"tags\": [\n", - " \"parameters\"\n", - " ]\n", - " },\n", - " \"outputs\": [],\n", - " \"source\": [\n", - " \"harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"9237f449\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"### Identify Harmony environment (for easier reference):\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 3,\n", - " \"id\": \"4420bb6e\",\n", - " \"metadata\": {\n", - " \"ExecuteTime\": {\n", - " \"end_time\": \"2024-11-13T19:17:08.213477Z\",\n", - " \"start_time\": \"2024-11-13T19:17:05.311566Z\"\n", - " }\n", - " },\n", - " \"outputs\": [],\n", - " \"source\": [\n", - " \"host_environment = {\\n\",\n", - " \" 'http://localhost:3000': Environment.LOCAL,\\n\",\n", - " \" 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\\n\",\n", - " \" 'https://harmony.earthdata.nasa.gov': Environment.PROD,\\n\",\n", - " \"}\\n\",\n", - " \"\\n\",\n", - " \"data_environment = 'PROD' if harmony_host_url == '' else 'UAT'\\n\",\n", - " \"req_infos = {\\n\",\n", - " \" 'UAT': {\\n\",\n", - " \" # TEMPO NO2 tropospheric, stratospheric, and total columns V03\\n\",\n", - " \" 'collection': Collection(id='C1262899916-LARC_CLOUD'),\\n\",\n", - " \" 'temporal': {\\n\",\n", - " \" 'start': datetime(2024, 3, 28, 15, 34, 0),\\n\",\n", - " \" 'stop': datetime(2024, 3, 28, 16, 0, 0),\\n\",\n", - " \" },\\n\",\n", - " \" 'spatial': BBox(-170, 33, -10, 38),\\n\",\n", - " \" 'granule_id': [\\n\",\n", - " \" 'G1263137623-LARC_CLOUD',\\n\",\n", - " \" 'G1263137394-LARC_CLOUD',\\n\",\n", - " \" 'G1263137387-LARC_CLOUD',\\n\",\n", - " \" 'G1263137388-LARC_CLOUD',\\n\",\n", - " \" 'G1263137378-LARC_CLOUD',\\n\",\n", - " \" ],\\n\",\n", - " \" # chosen variables include one variable from each group\\n\",\n", - " \" # support/scattering_weights is 3D variable\\n\",\n", - " \" 'variables': [\\n\",\n", - " \" 'product/vertical_column_stratosphere',\\n\",\n", - " \" 'qa_statistics/fit_rms_residual',\\n\",\n", - " \" 'support_data/scattering_weights',\\n\",\n", - " \" ],\\n\",\n", - " \" },\\n\",\n", - " \"}\\n\",\n", - " \"\\n\",\n", - " \"req_info = req_infos.get(data_environment)\\n\",\n", - " \"harmony_environment = host_environment.get(harmony_host_url)\\n\",\n", - " \"\\n\",\n", - " \"if harmony_environment is not None:\\n\",\n", - " \" harmony_client = Client(env=harmony_environment)\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"fc7aef85\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"# Begin regression tests:\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"9dd58c71-0232-4a20-9dc4-ef6b3544e797\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"## SAMBAH\\n\",\n", - " \"\\n\",\n", - " \"SAMBAH is currently deployed to Sandbox, SIT, UAT and production.\\n\",\n", - " \"Requests will be made against the TEMPO NO2 L2 V03 collection.\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"ef9ec4d2-2228-47d0-a990-d5a4d77b7bc7\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"### SAMBAH: temporal, variable and bounding box subset request\\n\",\n", - " \"\\n\",\n", - " \"This is a request that exercises the full range of SAMBAH options: temporal, bounding box and variable subsetting.\\n\",\n", - " \"\\n\",\n", - " \"Requested variable names:\\n\",\n", - " \"\\n\",\n", - " \"* `2024-3-28 15:34:00 < time < 2024-3-28 16:00:00`\\n\",\n", - " \"* `product/vertical_column_total`\\n\",\n", - " \"* `BBox(-170,30,-10,40)`\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 7,\n", - " \"id\": \"d6aaf063-1f3c-4d84-b414-ca80e65ef78b\",\n", - " \"metadata\": {\n", - " \"ExecuteTime\": {\n", - " \"end_time\": \"2024-11-13T19:18:18.977791Z\",\n", - " \"start_time\": \"2024-11-13T19:17:24.118362Z\"\n", - " }\n", - " },\n", - " \"outputs\": [\n", - " {\n", - " \"name\": \"stdout\",\n", - " \"output_type\": \"stream\",\n", - " \"text\": [\n", - " \"Downloaded: 4901404_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", - " \"Saved output to: temp_var_bbox.nc4\\n\",\n", - " \"SAMBAH temporal, variable, bounding box request: success\\n\"\n", - " ]\n", - " }\n", - " ],\n", - " \"source\": [\n", - " \"temp_var_bbox_request = Request(\\n\",\n", - " \" collection=req_info['collection'],\\n\",\n", - " \" extend=['mirror_step'],\\n\",\n", - " \" concatenate=True,\\n\",\n", - " \" granule_id=req_info['granule_id'],\\n\",\n", - " \" temporal=req_info['temporal'],\\n\",\n", - " \" variables=req_info['variables'],\\n\",\n", - " \" spatial=req_info['spatial'],\\n\",\n", - " \")\\n\",\n", - " \"\\n\",\n", - " \"\\n\",\n", - " \"output_filename = 'temp_var_bbox.nc4'\\n\",\n", - " \"submit_and_download(harmony_client, temp_var_bbox_request, output_filename)\\n\",\n", - " \"assert exists(\\n\",\n", - " \" output_filename\\n\",\n", - " \"), 'Unsuccessful SAMBAH temporal, variable, bounding box request.'\\n\",\n", - " \"\\n\",\n", - " \"compare_results_to_reference_file(output_filename)\\n\",\n", - " \"\\n\",\n", - " \"\\n\",\n", - " \"print(\\\"SAMBAH temporal, variable, bounding box request: success\\\")\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"6603a5c2-9ef2-4620-93a5-029963eca195\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"### SAMBAH: variable subset request\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 11,\n", - " \"id\": \"90bb3455-f82c-4cd8-93f2-feba79fbed7b\",\n", - " \"metadata\": {},\n", - " \"outputs\": [\n", - " {\n", - " \"name\": \"stdout\",\n", - " \"output_type\": \"stream\",\n", - " \"text\": [\n", - " \"Downloaded: 4901438_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", - " \"Saved output to: var_only.nc4\\n\",\n", - " \"SAMBAH variable request: success\\n\"\n", - " ]\n", - " }\n", - " ],\n", - " \"source\": [\n", - " \"var_only_request = Request(\\n\",\n", - " \" collection=req_info['collection'],\\n\",\n", - " \" extend=['mirror_step'],\\n\",\n", - " \" concatenate=True,\\n\",\n", - " \" granule_id=req_info['granule_id'],\\n\",\n", - " \" variables=req_info['variables'],\\n\",\n", - " \")\\n\",\n", - " \"\\n\",\n", - " \"output_filename = 'var_only.nc4'\\n\",\n", - " \"submit_and_download(harmony_client, var_only_request, output_filename)\\n\",\n", - " \"assert exists(output_filename), 'Unsuccessful SAMBAH variable request.'\\n\",\n", - " \"\\n\",\n", - " \"compare_results_to_reference_file(output_filename)\\n\",\n", - " \"print(\\\"SAMBAH variable request: success\\\")\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"c8322c1b-7fad-4b87-a46d-abf5f0ee56df\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"### SAMBAH: spatial request\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 13,\n", - " \"id\": \"4330105a-f9f0-429e-9762-3049f408d635\",\n", - " \"metadata\": {},\n", - " \"outputs\": [\n", - " {\n", - " \"name\": \"stdout\",\n", - " \"output_type\": \"stream\",\n", - " \"text\": [\n", - " \"Downloaded: 4901458_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", - " \"Saved output to: spatial_only.nc4\\n\",\n", - " \"SAMBAH spatial request: success\\n\"\n", - " ]\n", - " }\n", - " ],\n", - " \"source\": [\n", - " \"spatial_only_request = Request(\\n\",\n", - " \" collection=req_info['collection'],\\n\",\n", - " \" extend=['mirror_step'],\\n\",\n", - " \" concatenate=True,\\n\",\n", - " \" granule_id=req_info['granule_id'],\\n\",\n", - " \" spatial=req_info['spatial'],\\n\",\n", - " \")\\n\",\n", - " \"\\n\",\n", - " \"output_filename = 'spatial_only.nc4'\\n\",\n", - " \"submit_and_download(harmony_client, spatial_only_request, output_filename)\\n\",\n", - " \"assert exists(output_filename), 'Unsuccessful SAMBAH temporal request.'\\n\",\n", - " \"\\n\",\n", - " \"compare_results_to_reference_file(output_filename)\\n\",\n", - " \"print(\\\"SAMBAH spatial request: success\\\")\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"d0629b4f-1990-42f7-b19d-025e46a96629\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"### SAMBAH: no subsetting required\\n\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 14,\n", - " \"id\": \"c747cfcc-e63a-4858-b406-a4ec87b6acb5\",\n", - " \"metadata\": {},\n", - " \"outputs\": [\n", - " {\n", - " \"name\": \"stdout\",\n", - " \"output_type\": \"stream\",\n", - " \"text\": [\n", - " \"Downloaded: 4901463_C1262899916-LARC_CLOUD_merged.nc4\\n\",\n", - " \"Saved output to: all_data.nc4\\n\",\n", - " \"SAMBAH temporal, variable, bounding box request: success\\n\"\n", - " ]\n", - " }\n", - " ],\n", - " \"source\": [\n", - " \"all_data_request = Request(\\n\",\n", - " \" collection=req_info['collection'],\\n\",\n", - " \" extend='mirror_step',\\n\",\n", - " \" concatenate=True,\\n\",\n", - " \" granule_id=req_info['granule_id'],\\n\",\n", - " \")\\n\",\n", - " \"\\n\",\n", - " \"output_filename = 'all_data.nc4'\\n\",\n", - " \"submit_and_download(harmony_client, all_data_request, output_filename)\\n\",\n", - " \"assert exists(\\n\",\n", - " \" output_filename\\n\",\n", - " \"), 'Unsuccessful SAMBAH temporal, variable, bounding box request.'\\n\",\n", - " \"\\n\",\n", - " \"compare_results_to_reference_file(output_filename)\\n\",\n", - " \"print(\\\"SAMBAH temporal, variable, bounding box request: success\\\")\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"markdown\",\n", - " \"id\": \"ca88eb71-2526-4b34-8ff7-441f869244e0\",\n", - " \"metadata\": {},\n", - " \"source\": [\n", - " \"# Clean up test outputs:\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": 15,\n", - " \"id\": \"73fdd147-74a0-48d3-957f-388e2abe356a\",\n", - " \"metadata\": {},\n", - " \"outputs\": [],\n", - " \"source\": [\n", - " \"remove_results_files()\"\n", - " ]\n", - " },\n", - " {\n", - " \"cell_type\": \"code\",\n", - " \"execution_count\": null,\n", - " \"id\": \"5a868b30-52d9-45aa-9359-a14d79548ce4\",\n", - " \"metadata\": {},\n", - " \"outputs\": [],\n", - " \"source\": []\n", - " }\n", - " ],\n", - " \"metadata\": {\n", - " \"celltoolbar\": \"Tags\",\n", - " \"kernelspec\": {\n", - " \"display_name\": \"papermill-sambah\",\n", - " \"language\": \"python\",\n", - " \"name\": \"papermill-sambah\"\n", - " },\n", - " \"language_info\": {\n", - " \"codemirror_mode\": {\n", - " \"name\": \"ipython\",\n", - " \"version\": 3\n", - " },\n", - " \"file_extension\": \".py\",\n", - " \"mimetype\": \"text/x-python\",\n", - " \"name\": \"python\",\n", - " \"nbconvert_exporter\": \"python\",\n", - " \"pygments_lexer\": \"ipython3\",\n", - " \"version\": \"3.11.10\"\n", - " },\n", - " \"name\": \"SDS_Regression.ipynb\"\n", - " },\n", - " \"nbformat\": 4,\n", - " \"nbformat_minor\": 5\n", - "}\n" - ], - "id": "93da5ee1a11dd456" -======= -======= " ],\n", ->>>>>>> 66c779b ([pre-commit.ci] auto fixes from pre-commit.com hooks) "}\n", "\n", "sambah_request_env = {\n", @@ -842,7 +432,6 @@ "metadata": {}, "outputs": [], "source": [] ->>>>>>> f7db24f (update to use conditionals for test suites and fix environment mutability) } ], "metadata": { From e73815fb9c37e0b1827f84712e29b78cbe73fa34 Mon Sep 17 00:00:00 2001 From: Daniel Kaufman <114174502+danielfromearth@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:45:14 -0500 Subject: [PATCH 18/27] add "sambah" to services_tests_config_prod.json --- config/services_tests_config_prod.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/services_tests_config_prod.json b/config/services_tests_config_prod.json index 7772476f..9b39ac40 100644 --- a/config/services_tests_config_prod.json +++ b/config/services_tests_config_prod.json @@ -1,5 +1,5 @@ { - "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,opera-rtc-s1-browse", + "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,sambah,opera-rtc-s1-browse", "batchee": "TBD", "geoloco": "geoloco", "giovanni-adapter": "TBD", @@ -18,5 +18,6 @@ "subset-band-name": "subset-band-name", "swath-projector": "swath-projector", "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter", + "sambah": "sambah", "opera-rtc-s1-browse": "opera-rtc-s1-browse" } From 7b18d7748f2669a598f7f1b48782a2c779e5701f Mon Sep 17 00:00:00 2001 From: Daniel Kaufman <114174502+danielfromearth@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:45:58 -0500 Subject: [PATCH 19/27] add "sambah" to services_tests_config_uat.json --- config/services_tests_config_uat.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/services_tests_config_uat.json b/config/services_tests_config_uat.json index 9cac553e..fa0596cf 100644 --- a/config/services_tests_config_uat.json +++ b/config/services_tests_config_uat.json @@ -1,5 +1,5 @@ { - "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,opera-rtc-s1-browse", + "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,sambah,opera-rtc-s1-browse", "batchee": "TBD", "geoloco": "geoloco", "giovanni-adapter": "TBD", @@ -18,5 +18,6 @@ "subset-band-name": "subset-band-name", "swath-projector": "swath-projector", "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter", + "sambah": "sambah", "opera-rtc-s1-browse": "opera-rtc-s1-browse" } From 456073d4099930b897e7a384bcd30c2cf7701cfc Mon Sep 17 00:00:00 2001 From: Daniel Kaufman <114174502+danielfromearth@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:09:19 -0500 Subject: [PATCH 20/27] Update services_tests_config_prod.json --- config/services_tests_config_prod.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/services_tests_config_prod.json b/config/services_tests_config_prod.json index 9b39ac40..84680198 100644 --- a/config/services_tests_config_prod.json +++ b/config/services_tests_config_prod.json @@ -1,6 +1,6 @@ { "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,sambah,opera-rtc-s1-browse", - "batchee": "TBD", + "batchee": "sambah", "geoloco": "geoloco", "giovanni-adapter": "TBD", "harmony-gdal-adapter": "hga", @@ -14,10 +14,9 @@ "podaac-l2-subsetter": "TBD", "query-cmr": "geoloco", "sds-maskfill": "TBD", - "stitchee": "TBD", + "stitchee": "sambah", "subset-band-name": "subset-band-name", "swath-projector": "swath-projector", "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter", - "sambah": "sambah", "opera-rtc-s1-browse": "opera-rtc-s1-browse" } From a88cb28499336554f3f31ba3630786353e1a4225 Mon Sep 17 00:00:00 2001 From: Daniel Kaufman <114174502+danielfromearth@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:09:37 -0500 Subject: [PATCH 21/27] Update services_tests_config_uat.json --- config/services_tests_config_uat.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/services_tests_config_uat.json b/config/services_tests_config_uat.json index fa0596cf..33d6841a 100644 --- a/config/services_tests_config_uat.json +++ b/config/services_tests_config_uat.json @@ -1,6 +1,6 @@ { "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,sambah,opera-rtc-s1-browse", - "batchee": "TBD", + "batchee": "sambah", "geoloco": "geoloco", "giovanni-adapter": "TBD", "harmony-gdal-adapter": "hga", @@ -14,10 +14,9 @@ "podaac-l2-subsetter": "TBD", "query-cmr": "variable-subsetter", "sds-maskfill": "TBD", - "stitchee": "TBD", + "stitchee": "sambah", "subset-band-name": "subset-band-name", "swath-projector": "swath-projector", "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter", - "sambah": "sambah", "opera-rtc-s1-browse": "opera-rtc-s1-browse" } From 79691251716f1ad7be8c7aca18593252fd8b2051 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 9 Dec 2024 15:15:58 -0700 Subject: [PATCH 22/27] trade-187: include share-util in docker images Update kernelspec metadata to use python3 instead of papermill-sambah. Removes defaults channel. Updates python packages. --- .github/workflows/build-all-images.yml | 1 + test/Makefile | 3 ++- test/sambah/SAMBAH_Regression.ipynb | 2 +- test/sambah/environment.yaml | 7 +++---- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-all-images.yml b/.github/workflows/build-all-images.yml index 704f96db..9abc2a16 100644 --- a/.github/workflows/build-all-images.yml +++ b/.github/workflows/build-all-images.yml @@ -61,6 +61,7 @@ jobs: - image: "sambah" notebook: "SAMBAH_Regression.ipynb" + shared-utils: "true" lfs: "true" - image: "opera-rtc-s1-browse" diff --git a/test/Makefile b/test/Makefile index 43074666..1e75823a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -59,7 +59,8 @@ net2cog-image: Dockerfile net2cog/environment.yaml sambah-image: Dockerfile sambah/environment.yaml docker build -t ghcr.io/nasa/regression-tests-sambah:latest -f ./Dockerfile \ - --build-arg notebook=SAMBAH_Regression.ipynb --build-arg sub_dir=sambah . + --build-arg notebook=SAMBAH_Regression.ipynb --build-arg sub_dir=sambah \ + --build-arg shared_utils=true . opera-rtc-s1-browse-image: Dockerfile opera-rtc-s1-browse/environment.yaml docker build -t ghcr.io/nasa/regression-tests-opera-rtc-s1-browse:latest -f ./Dockerfile \ diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index 1e0d79e3..62b17da5 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -439,7 +439,7 @@ "kernelspec": { "display_name": "papermill-sambah", "language": "python", - "name": "papermill-sambah" + "name": "python3" }, "language_info": { "codemirror_mode": { diff --git a/test/sambah/environment.yaml b/test/sambah/environment.yaml index 6487be3e..c1ec0c7a 100644 --- a/test/sambah/environment.yaml +++ b/test/sambah/environment.yaml @@ -1,12 +1,11 @@ name: papermill-sambah channels: - conda-forge - - defaults + - nodefaults dependencies: - python=3.12 - - notebook=6.5.4 - - papermill=2.3.4 - - python-dateutil=2.8.2 + - notebook=7.2.2 + - papermill==2.6.0 - numpy=2.1.3 - netCDF4=1.7.2 - xarray=2024.10.0 From 43d66c355da67655ee9807754d86c571d009b9b6 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 9 Dec 2024 15:40:42 -0700 Subject: [PATCH 23/27] trade-187: recover deleted files. --- .../VCF5KYR_1991001_001_2018224205008.png | Bin 0 -> 10181 bytes ...CF5KYR_1991001_001_2018224205008.png.aux.xml | 4 ++++ .../VCF5KYR_1991001_001_2018224205008.png | Bin 0 -> 9527 bytes ...CF5KYR_1991001_001_2018224205008.png.aux.xml | 4 ++++ 4 files changed, 8 insertions(+) create mode 100644 test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png create mode 100644 test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml create mode 100644 test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png create mode 100644 test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml diff --git a/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png b/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png new file mode 100644 index 0000000000000000000000000000000000000000..406a8aebbf1ee1f99257bb88c17a8338777be596 GIT binary patch literal 10181 zcmd^l1y7w_)Aa=hcb5ajwK&Dy-QBf#ad&s8KyfWr9E!U;94PKB?(X*9Prje=tz^&6 zPG)8$la=gDvZ7U#q*0N;NB{t!%F0Nn{dfQV-$4NV7m={<|BE0%ML|Onbc+Cgh=H_^ zj=6)3xPy(ejf}R93A+s@*hIqGL`UDmLf8a@Hc3fW;fPle30F~Z*3hxmu&`IK(N+l% zSBYWQNMTpVL2DG03kal(NTl5 zj&Ty6W*V7w43=>Wk$w!70)k2k!61U*kc{JlC&3&;a9sWHtV2lLJ&24WC`^NBjD1*S z!(hrjVq6Fj`V2YNBst0~CBh^F%;6<8{MC(!2D_}HA;k1ik^$KA$ z3y~D7;H0ZiluMChN-=CwLAJ>-_K7eyi3pByFm`bWZc!kwaFAO#ykiWUQzU|M4vaw_ ztU(TfNhX|DE{a7OtVI%%K`I7!Cn`rH0ci_8V!I%0y9i>vB&bUS)Gh{UkpeZyf$9`M zRVqN27|<>WG|K|@N6ZT`g3`@lb4)=Qrf~n&WgEf$*Pv7zRICkKXaFkIfy>p0uT%q- zYk0|2W3)Bpd2 zBOL+*06+n5YD(%K|9=Bee}Jb305U*ULR7;m=PVad1AAq$ToP5!>Vu~`)l%9nG_ zBxIn2{6&0VkKcSA9V}EgG^{oBFVwa#VS=#$K5|C3SzA(kz>)0g(|zQT%adHA%P#rY z$<*&+$B2jzgu_bhRoH{Rw%fnHb8m#Bq+cKJvbj%9S*fxT=*0CJHA}Ui_~3&g4_x~! zfb-w>R>s~!ui&8DOs(`WlwQim#9E5Uo4=8+uu~C`xm5gC*R%Cpw$RZCA?}^)SyLxj zPCNjw@c`gEW0!$9LD`kV_6yryoxr@B*5pm^CR;0k|4Qup^zs|@><}E*+d7qxg!*?C z{3q+rQ1tK00;sy$u0X+~Q-wp1y_VF`6C0zNE5)#LKmQARqu4n^TGA!QKcg9(92nWp z#q0SVE_^pP9E7~1t&!HYB1@3WA8tyY@yAK#Yq_eZir{#OX?{_!D2*7QXDD?1)l0 z$cNb4pyblb;-S2wx<4fQ~$X+46b4jPRU45jjp7ul0{6c@Gpe$+Yoc}DLVt|0(Z)V(qjGFwCMUbiH* zt_~ze5j5#pQtkBt%_G0ee_$zaj=Q^SSFK`OVP$Mvr2~_b-O!fE_M9~9;_81OI%|*- zhwvZyc(d#{sqil=wN4XbS9tJ=_3Wk%UPPl;5z)8yHU}dJ@5cowgzXj>Ow8$6nl0G~ zo*_eoP+PEe{%4;!x*?F29|cWj7PnWZ(_2->+je&O+xTDc&v#QX?9b#_DbDMKD|QYV zC#H|QdQHt_zZDF-pvaDlTIZFPavZd#u{DqR_M5BN_Wpg%xB0*|P=Xn`>zQTK{RVqU zv*qR8=54>(iHfHbjh-{{BB*WK3`0FgHkhZUrGkg6Emn)ng-u#ssgG&th^hClg{1wa zpW1!I-cdtevaQDW`iDqSW4QmN_Q@#-v=jz=!pC2=Vu3?Mpp)&izT_#bJ9d0WlKT6lwuYT!}2E+VN{=@R$D6ojdu~0txLg=w$~{WMWWK{e#HSvA=mrg^KKX zJ14-mlE>XNoBa^m`ZoD_wMbtBPJ|AC+-uEaP%_wwiQ5fmgJ=cqeHnU=j|0Q_8I4Qy z&06LTE2(KGqU(i+(|J2g45#8dzgKcaV!lOH68mdw+7JbrHp@xBK;PQzghM!+Kw<+F za*hiWRmFGKBBH-Ntqo;0+%T_#wmb={L&48siZK1W{X8<)`>y6`FEnozgLd(V>e-|< ze>n#!fpwG~Z=Cbl+3`Bc@Izq=x{TK_gd+Rj#!i~cOmd2f2G;gsV%6BQc-Ur#8Y!gR z&CMS538qmVMU_iFRvVhlrg}n_$eh@V%sNFjwO@=d{Db`o`o$nVIPp<2l&fHW^HV)B zssP`**Je^Fb#ZATYYe=Blvc!u8T)L+m9@?h<{UVCm!~Q2@0rQ&cbvjEz$cv9VyN8c zIco))L|sD_eNt%)xJYYQ7;N+81_Dc_u2QYa>&xn$wAtC0<4Q)e39|aioe$pNOgOeUemx!>%rmmpImWgB?^U z+H#hA$MjKEwGa4vcNx?AtGywp{JB9t97j)AQdi?!UGr2(e5~JtE{&gQBbjZ!R+8@I zY;;nKJhmyW#`Kzv5&ns~hPKL~tAb}90~6VEPz@jsBY?G`=Km=0_R(qi6ce%)>N)psXT5Z`b-tpRDg*n|Q`>7a zQebDd-|q)5z#_)d3*qr3Z62}qMgsv$$qxiksHOwHUASrz0LMZ_lpI|YgCvIH_Pq>2 z&EVm8QPuuI$|m1G_un77fq}x`-zWM^UQQxE(v6dhT`BX`^zU}=Tof) z?bgERr)<+qTIqaZ_DIf0CHvaEzZ|A?m>`K6K07OU z{uCT6j}p(illA=UkWJeThh%}B8$q_fiiy;JeH0F9#DqCJD0~~b6Yyaxj_4CFzjdN% z!L#Vck}JDu4G*7Jip7S#BX2QBu2CPixpU`i32=eNWkt7r^lJXTO0p$<4QZY}eG} z9Om~K)?#r!pQoO@@4_kM@1rg@KJ?}>(a>=7k4{QfX6b$=*y;j&auhwfxfwc(@S@~5 zl(67S3_Z-;7Zm!a!b=QG;p(2Ax}J!MmuT=SvYv)WiQ2CnRpEaU>7@Li@7blZCJ$Z> z@5g*L?iR)1cs4Ttw-jV%qZBZ*#5<{DLKsY=%ha@zi|$Ll+ug4i}+nd^ij@ z)P6SsiDHeSTZ&Yp^~3@M{4@bzmryP4y6A)Cla5g+posD*O8E3n5(wlusa5|DWdbak-?p<#lt|?uI4w3tT)vC(&hZF>Jj>B z0qd{{I0cC=BKT)=hFWZ2gg!6xtPZ#B7o*kGSQ!W|{&mVIQIf%Ez!1&9BbCme69L^+ z{6!GV*OI`Dd~{xMelzz$;e(pQ0gnrMB&jxx+jlpSU?H32{F+EaV}>93b$mR&!}yvL zV|%>EI&uLsw*OD!>nHoW1pg?)AN>UfEXdbeJG(MpI9v{h=Ita#>u!ZjU)5Eel-Ko*tZe|Z@sPMxXa)R_Ch#>`d&W0@zk0WoEYdW-eX1?Jd zTqnWKd)j?v>OzHEvZ|Cl1ARiokdUh2yR%OVrX50?1XDsZlm7Km-mY<8*TO;O3hhpaZ3MKU{OK4;2u=v9HMwdg zoHt2^#~glE#l`M6{yh7cdJ1bwca85``BMy~{`)%sQoIY+Gm6`lsgR z&5)d?@o$`3i9ZkirKhvZs7!AFGhz1)su=&N`68E-0Z$Ih%}iHuR5ED28F- zuAzS=kebDSj_)fjo|N?J#79fa7VgOyJN=5|+D4Z+y#w1z2D9dbR0GGR%73hSaLGza zqIY54F%Dn6*Ep%k0`k0fYw*P5<$tjKoVhDHPGvQOvy7SbWT&8Z`5`Oqiy1e9r){T4 zM~!28U&PsfQtTCWbU}v|t!(f$?qe0{B1SF0za4rJHuvxgjFf~INyQIe8x<;QfhhUd z=%NcL`y`e0JLsw_7kT2%Gb~;P+8c_tHr_93If&SVR%7RIV>6^z-#M;~w=vz`Wc~i( z)9p=MmdPB!l}@S^W3-<*Rs{Z~7|z4LQqo$kJr<7{%0U1%dknf~=zz|C_}2F|d}5m~ z=x7@cn-`FgyTDWRUFT5rzRty! znws$b9oH^lFuOQ6PFMVouAUEqzpU9XMQ3xUE@4%O=?*)=yA@6^HyMkE%#aVf9j%yz z7m8~nNQMd-8LOUbI3XzmRGFRJ*QX`aQ(W=}OW@PCkKDIG?h>=@-Y9M9YCeE2JLWZ2 z6i>J+0C#;_F7CdTWV@=Kr|0hrtM=YN>JRd5AAyF7(kBUJ(xo)uzn#B=b))6};M#Yz_ z#`ub=Z@3gT5F(1OdVB0(=UZI7@m4t#DHRY%iG>(QN{EM;SLUPR>(vpq|AmT~E>xe~ z?O-VGG^n%vC6h3Zh%JKpuP8s1JipQD(s)#6U7>YQX|L3gEyhEtrY)KMuMmCtAEL*} z+Wm`|?-;^m_0hXR3f$njp4$dPghJ^}B+1uB4|}!o6#^W%B{?rd$Ai;?m+_Udv5eL# z8V2G`CsElkRXe9OY%&}eM0MzRFXolXA=qB6R}qJi+87itzbf$_S@1MYJc1F8?{*Dv zVHsky6D3236GPS_LuReGFfH&xoyHtrRvwq~pO&7#(yV}fax@~IpSN2sTnTl6P|~GD z_!*c(RB{Bqk23#b>Rkp@e&=)pL4p|gkF8O@>#XFn1%`peX{BF zLiR`ehZj9UoM>|O*N3DPFaU2ScH%`zQ1bfFJ5($7zg$R3_P2e>|jsM`O_m$WTbL?ZVu&5%? zd!gugf+uggW28sy!Mo=@PNHzgxHh9pR&=lzdtF!`i(EBHw)a6 zvPQ__i8U@qw8TRxF)@ihdHc&i?3StZOYYX?;pRZ3(k8lbV(;hIsY+3?eTyl>D;+`N z^NIdqwKs8gb}cAy4zafvO))1o3!@^d8PTPA*)Po3^}ZnG6DvqX~dyx23MHvM#* zuXQ_{BVO^D)p5z2=b1QyozC+wvu;cRcJG^Akv7XECV0fT-F4 zw4lAdmaI-m{T;M9sJ?q0y4EdA*Nuf!Jjs_D(Q*1jwBzM|<{RQDhDTl$i$w%QJuA>w zAlu0uLgWoFF;&zV@0FgTw0*cl3G*y-2oIIaqs-n2y6{st6qdyE{8+FR_Y8{7LwCVe z&w2BnPttKJS6G2H8^eei&MZs>WcxkbQdu@*jJOb_;3NhpW-GVG)==X@FSyXrEq$)C zjaNqUITBu!7ol^SQ_PBS+7Z9<<=T&Tx8iZl9|!#&`K19v;Vk%IG${bb0(K|FMT_>7 ztV1Tl@g1czPE9{5Y1%q?uZ+}Y+2~mfcv+qayZA-voQ9&ImBx;24dkGIn>v?q;U-u+ z(+U`RQAPYUSch!{jmjJXp@0I;YRSJSaK$JKb)1d&(36Va*V1O?D?)j6&I zD@t658x$WT6=7szUSVFTA!_Ns?)@%L&M3E4eGDr^K>bvSWca)PE|Y4Oo@6Wm)(VHM zC)@PjRrO%SR_tJ(o?d;v>&&n{U!r0cl1QKtUFEljqK%-R#L}FHe3Xi^q9|#T&y4D? zm&0VV%aTP`XTDC#SOr^Mq>j>UDSzg2T4I-WzM1M5ym9{GTqqNnsOF}4WvGoVqvRVw zKTRq_99+z?-A=qTD0T8xgm;`4{-0f%N>whTmoEues&fgx^D!xamriwrG&l`vO(T@g?ZbC;BBT`+tDcB?M*wj z)+c?j)kB1Z1ancgZ~&PfaFl)uksW9D{#pNd@zbqlhrY}NzZaJ@gmw9jJtRQ0#^d0r zc>jg67KUPZh{P4k8TmL@rZTVbU@zGmL5%3U2efHb2`v^tGO60`YYuR zr-gkkxIR&p@lEjxE%{sLxjEnS{`ftJs(bkONK3?%5x^2I6?GAijuDeg7) zb83 z9nIFo-rlq?9ujZ2+8}|La~(9)g0&^)Ssvw+0yvzx^TB9lSh<3}JkG>zs8#>iV`H*P zRI}t0tDHAGT258E5W9VFJ>esrtO-emmwV~7;Y1CMhe6fpUg)>n8h zR9ZXS+C4fkF`ZUR1RsFo=+mC2xpyq`;iLv9_qG;`HW`V5Z$@Ihq zwtbx_W$NP;5B=k#TtpT1@bn#;4VJd1BXZ#B0B&fUTp2lEO#Aq!kkDLpnllTfZu0d< z6nv2GB~dNB#OFyjEy$Mqrg#VWnIkb;WA;tPKn16k*BFcc2C*vzvS`zkxeQotr$(5wZ+Rd1Q$x6nxMGY;LdyIQcyD8YVLMP`t{2+N1wNX;7%_xq7c5Wul;t<0%6?qK}z{ zvjSuc-%{LMi(c|xIq)GNTRgeM&VP?X&u8BrwHFH^HEGu)&!4A_`?3ZqCwplYCCIcMoc;Xrg;Ub%k!=#OLrtl)GlpFKrk2gzYh9d{$T2sAGX`YHwff$t$q!P&mhwGlQj8&2DNo@pJ60 zMmZR+dYd3}ZR)s2$J}JQZb?kEq+au4ZBNIMXcEqfR3RVqTfj&cUFEaGu6#vnwHA<* ztGeV-fhf4@T-}~EaTepWILEGka>8QlYFCraJIuo|I8W=B)@oqNHEM({sHo<=yY7bX z((n>GpDD=U9{8zzC&6Fv$Hel5ql3fWehrL3QiHiQ6PqWdVLbUXl$2Jw46`gM{4Yd} z;L-H9$FmCf0~DVE-r|Nz$dc7UfPNgMVsMEh{x>G3Tx~rQCI;%47zl9}7xnZiPxtQi2Pr6Y+xNY7Eq(q`G5O)+T5_+%+oPX* z{5BdJooY+1xc=#>c(M^~&_)pDj_GW`%t^P<(7>W{L`r{knYThmp86z(Ms`0oI9(c~xCG|s zTZO+kD5*i_S@_q(wlxd>^}3d$Jj(&yDZEu=E?TAF_d$bjU5f<;BmF>t%|$^9WIzi zp9f{%%tIH}lw`9L2YKfQW)wZ;I3HoS@3xO#)Xb#b#(s5i>W{_A&u65{HY^|XMguPKb(+kT~#w4Rj*p!?Y2;cqOHn-vEOVfT~N zD>lqGD}{3zK{hWhXeDfx4b~3)l2|PtqGp!QU8f2;?zi(dwqmtwb&__vDpRVs5pgDE zmkP)9lCl3RM_)t7=9;KvNql5&=wUu=a5J(X@L4^c&T(s=5@}8Mjrcu@vY|iEVE!C! zNN^U~UDxnrOP*=gJdU{>cW#a!b(hd^97!(9!hfiwb?*Gx#aSfK6+X-Gfym(H`;VJG z3GHR&Gh!~n8Eo!?3XjF`DH8*D8aG9UdB)c#ZY7bDqp~1(Wm4DE3W3)7vew6XS=Vos z0&@?p3dQZ^`jlK;Q%sT2qGfT#(Uj3y5M9u2Gw-&lVrkrC%*(f9(LX(b@3zcNmMkey z2=3yWt_y)^Ud(&VyZ8_DVKn)<>Uf$ZB6{4v;NO-y#9haV+GVWV7D5*t{O@X=(JRW~ zyV|@rZx`J2)#pQmR$f&WNUic-vMc>_xpk-3rsB(C{h>4D@h}(+Xm8+SeA1bP*+&Q( zu{n{=bb5(hC*|fDkrmf8NtJVaRa<}$`90;s8R|f>K;|dxXV8@yw{QME`dfyhSqof;={EYGmjt!bqgsxTYWvn z!%QBzsF6uwdFLXeich?LQoH}X!tylH^4BKW3m0hLL#G&to z5d4x1kv>fRUFlB*Y3+@(?aAW%)9hyoz5NaS<(YoPCS2=Te(>kJqJZ~9BCa>Y_UmQa z>r?3ZA=`<;I|ff%%XjQDVTLe1its#{w``SIi_O%oK=geZ$Op|e?d?IzM`YoF>{rH7 d#Z9OW{NVpAA@#Cn4*zo{$x13o)QFjc{2xcO)a3vG literal 0 HcmV?d00001 diff --git a/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml b/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml new file mode 100644 index 00000000..09e83c0b --- /dev/null +++ b/test/hybig/reference_data/dimensions/VCF5KYR_1991001_001_2018224205008.png.aux.xml @@ -0,0 +1,4 @@ + + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]] + -1.8000000000000000e+02, 2.0000000000000000e+00, 0.0000000000000000e+00, 9.0000000000000000e+01, 0.0000000000000000e+00, -1.0000000000000000e+00 + diff --git a/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png b/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png new file mode 100644 index 0000000000000000000000000000000000000000..b3e809c1bf080d3334e12dd6b94a594d1f019761 GIT binary patch literal 9527 zcmd^lRa2b{)9eEaS-88qySqCqBm~z0!QC|kcXtUM+}+(FxCeI+0fNJ~-#S0z%*E7P zj8u0`-*u!KL=FX!5D@?X6a{%{4FG_$`mfdlgZ|5f@51*000PvMHDy3|VE7Ys#3M9} zeI$f^EbJX5)Ex}i9YXvqM9eKTv@J~V79nVhjBFi_cpZUY9R+&>4RZq%YYhu^9Uozx z7^(ddpVKjfjX&1rj6~KHh zKvb!MldD2j{fQ|56WuxmWRnPE9}i<44|a@!v5f({MS#3QL2jY&j!|$<;b7w&7=s_M z2037pOgODvWQ$Z-iv&c26m-5W6z)cRvQ`F!4iVT6afEsqP`5a!LlV>~3u;gV)hUCj z)PQbDphE^|Q2^>8K-DMALM6~oEl`mz=!X#~(+nun27c-Sg@!=x7a;R1Zh|f--T@Tu z0VLZ2F|I(AI}qlb_VvF;P?{NRjwvX^6z;#dY$Le;7W~u(6=}m37=Q|N;BxihD>Xo6 znxHZb*kVoi|1SUUz9MY19H>DKu38mTtpryFLG%d%`3Hi00%5(r!v*?-zWc!V`N4;J zfx_KEq3*EXJ>jEVLD5byaSkwHF7S!ApcHFRtPMEP43XPc4G8iF0s{g6AiygeaES#R z5&_#3z&Z{1nhBWY048~WUJ0OG2543S8r6VGJs{t@Mh)7a`0uSihj^fKOwct7=oSI= z01F!82TgE-=9oasw4g->&#03bLFa8Cm4 zGXU!xz_bX^tOKOG0O1jUcLHFa1DKZp+8+SrFMxOtfFA(ZX8`mX3q%9J2LM$6r-J_% zj`U!r|HJ{fX+S=G{Qn)mrw4c%03Zhxq$M=HaxVQ5bg*$+NiCJ?uSn zu{o-OFeG?Eh512&v-I>0jQl&2_}L-Sf+*?rE<)|fZ+NokDa|i&-zYpa zx#edgc5lj$pO~0gnGwH3W~MIjG$wU>ZkMuad!B!le@exfB}+iidfh^Nb?hgTo&+~g zKFf$=5e9{2{g}yo!5yjQ9T&_S4`J>1eb~6}PEE9F=y#On=E5_&2o%l;N=)?7k|1pi zU8ThLcK+7ZjTF?e#R^oz7Sqz+{RUk4^3;C#%DlA$a=wCX-h`gH5)>D0-MGkD-;{90e&qo$Z>n zbc_Zg*S$SZKwHq(B~{K`q+58%2Di&>*M4K8^%99r;KLq@iRt#%Cx*r7{`X9pPPAe7 zQorj>(s93wPLr(G@*x+Vm^^)(N}vN*B&>=`ywDYDua~fAbEukO0vZl;aM-64FP?yp zPfuqy;A4A2TW#_Kt;zExN8@hEH3`>yJ(cdI+hr#sU(ewZ{iN#V%HC)5jd{%6%!8vy z5nqIklvG%i)ln8qj3Cy*?DXIH${J}y;oMldYG2ZDsn$xhGw zW^4YqMvQsX&dE8`x8-g5E+=D;S2)+=?Mj@CEviQRUCqEA(f|IQ_P!Y`;|?yUqOV23 zs;;Ba+G5w@>))2$G~fSf(cWx}G3Fj-!`mThIi^(yWYqPcE-ZTHs*Cn3o^@ z+Z^A@tZ`XZ9R+GKRAy@JaYPrFrS&gTdp^0NfPlT2nf;}hx7lWjNUOctPZgE4*?{2Vj=W|1pjaiLuUoQOU+V{88B}h^rat)Ny(inq3%}fOZL9ziw6~$m-Drom9;65 zel;K|uVSS$mDP1_{F|Q5qkM*DT_s|IZ~iq8ze)WDSy6X}yPthm-YLjWyh%keP>+6| zef3zr+7y83d%|j|Yp*TOxIJV^7r&+fPk|vJFnMnX%LSsia|OPbm{@*Io=j!7w)z8Z zK9dX#Oq9zt`t=b17dIXXEvUDQ)@ZbcGDf zbMTVxJt(NN>~rkDLqy4Ispr-(DeEVihD}gu!U3Lh3An5pmfjs>DV8tJ2x`CVzW-ZwcgrDp&iOUG zguDBedn%tpNVYLjWn3*X+Mu`2)!eErCgKK-WWc8s7dwA(EC0otk;;25xp=usDr~LF zjqI}Q3`VLRgWhqyHzKTJgeeQVi&!ax2&2LQQk1YW_?sx)7+{{Of-UV7V_WT|4SG6q#0x1Zt1#r@gmr(~bQwPHU^%%Z5zC&se}SRtnN zQOTvhTRyFlY-kHz_FV=3Qh7?wwfwDp-G!VxEvS?sv5;4vXM%szTsR;wP`J~PU)}RS-mWmOx4EAqmmSHIS<<`KBcZ`F!X_rG zCabIPuks3pc~LPiOGv+%cfRk?gj*#@N~%Fd{8_U6y}X7w3-0wHt&MzWge&5CHq7b+ z3#uPMj2KY`9IK=jy^yf5xF93WL@`84{iXW%gh2aR`#vvB?i>H5tu+h57rD`XQ|MEKF@8# ziL2ejVg-!WK0W2mZeE++@<`&9hH+<)k9*UpUZ7sA6KfLFtL~ z!Bf9zPgb6dYfemKdJ(~#ueu2;J0Xzmyrqs1(ZjBFX^`>*$j~L?nmVxAvON+t^5nrN z&j;7>w?t9+eeRrU&k>`gw@vV`3}snlco4^EQfSHsdJTHVjM6 zJfXuWWC4%6isHo25xWR)^p$O~r0^cuHspQ-)DR))IY0kdK~enylT%9fvPMtc+TzQ*R+4kK1&(GTx0Ym&g?fnmBhEbN{Z$osg?Qh z+Fj-MZZ@CjF{T=8>&R6LK{}1}Ch3e{7zFuQSC2=8Lai2vG-b@j+`?L1N9T?jm0b`6 z%LF`>zO>lOaDYXN0b$q#_}MiR1%J0ciGda81&c*1`>1_{L;k%&k3~@g>8%bQBhpeL z%e4J{elyA}xDmFqSNY)em$w0WFy~IgDVESq{|F_(Y#Z1OLv+*%J_rpy*yH!%t{oW5 zLWDl74{fp%6pMgEO-a`bOkvXe%cDn=sdb14c9LN2E9&1yA_3a{9&MpXot0L(33c=o zZOcGX-L_Ws;lcKjmC+wXcWNpLENefvf((oezU^=53Wg1b8@6{aIcYsJ7w(JtR}G_I z0@oPXze?t!PV52^olf02(S(2fG_H<<_ZAEG<5fe#<;6`~UL2u)ex$xgPpHUt5+>i( z8@ORZ`(|(7Ahv2@3;mesaWqup1$HzxdlZ!*X*~n13$e21(ST-{m#B+m)57Irz*#(tldY z3wauT;XH8NF%@#~D<&Jk2-Y$v3Dia4bQoHy{w}UV<8gYQO)gOJN3EM0`tjd-q}6)j z`LJA(WIWsu9Wh&&N~n!q`K+2R9Wr)c`VmL!b^u4_T?&66BGkNRAbo(&JmL2UsL6E%=clQKW=#E!@e zQwu#i9iI{-#@qUL-o4-3L+F-oK?2SzDb8v}9VCmjgb>{qEWni7{I(l=0iX~T@ zfuLs{c8KahHebIWA`0VMIsO4I9*#zNOp=@v|1!DEWVokyG8G}YKTk1ilF(xrvZFNA zn5>CKn{#)LL})D|%K)(kmm23RpKO+|W4*+B5yuY&1p@qZbJbK$I~#~B1oeADiYEo1 z2;Uk54+{g!%K&UEvbH!v6Mh##w?ur6(P!v1qd|vY%HYaEC_KPK7?I>pjay3c%L^l+ znzET%d}#I1oRxW9h9Vm*CBB+MpCop=OI0w6_-=S~j%w0sjldqO5q56aT?&KYBUfd%k<`xf>YVG`L7! z)82}L3hbHs9uVl_XA$X#3=w^d3#BzIZB4Ds-85NhK;S6ogjVh)Z5&2mp$p1)bR|AP zd`3bq%vYLHE@EkEwAHh7j_DYA^2*25T*3|+XP1l=yDu3+D#w7KQ45`NHWj571&gqQ9+Bqt&gO*jALXdrI^g^Wtc!_)R&qc6HsL>G;J2c; z1(8d+d{070d{Gp{3#|RX?CZw|7Ff$g>#aiCG@9&cDjb`bI2$m?`Ea3SWb7OioMP+i zcR9!#aAzZH)no)K&IO8g$42KDaf2096lTaVGH!mTunAr_wKR(m;exLk!7y5Ka#;06 zW@VR<=ubUZviJD6esXf*iZ(Cyrn0VBTtbFhTr5;{Z;~F6n#$55eOb;CGPHSrWCQh4 z5P{pE|7UVEbFL^;f3N?5*c6(YO}A-zHRYf_cV@0pT&J|R^Op(=)EF0;J2eIr5XWXw&a$$93|v&vsTPzZ&&tuK2ectVuhvh3}Nv zU(c2%j|wac)%(lnKUyA_PR8og$O%bp#PX7IkD}&8yIA+Nk_IAofYocaCzKfg7|3-=Ub zM5MITD}-nVaf_3A$Y2?-z%1)u@&;SauXwccmIo(zln*4f&E#07%?^5WEX+^gi8w={6gTFDLRJgs@Ui$ti7QW2c#=XlIk;ik2@2nw z%Ck923-(<8WMww0BcmY_*DYv))omngl+yxOFuO3KOs|G=(a)oc?QoDq?WI`l&JAnj zeniO{M;paP7@n~T0ypB|prJ|0n3H`1eBN>VGeLNX z^Ak(S$K=&k+QlX6VD?Sfi>H$_hT1P0R8 z5pGlCx_rEG3?IkB23f1vb)QFEgmFvMa1rI)$%#^;I5T?WRgaw<82dM=C!&6~Bdw$_Ay!Csx#eB6Lu zHnNGR`Mj;6pAkl^M2|fX_Czs4OYjXOah*g?RdnN=(RdSC$lf~5CM%a$_e0fqJ8YY^~o1w9{#Q?%|b)0-k`3FZ+Y}nEd z9h$rEhV!<)cc*aB^m&IEXCH^-`sqJtsXOv#N5+E5j*2MLehXKq9+jkL= z5s`jm_~H*bwE?AZv9Yo73T>gbVvgUQ-89*ezp1lPGYqVJab6h9&~dZu6xL&%e~Q>S zJaIKWKHPO#CoC$B6?|w{FYH$`vqV znRb~DJW0|rFmW+OCzKbK?1wftZyU(H^vsMFL`*Y0Ewf&DRZz)li@YBBprq>bUJnv1 z?J5ZbT4Z7_!+ndXxX1ywAJ3TKaWj}CJ_c@%FLmA{t8yCJ1)Uv@{%#wrRE?>d< zuDIN@l+H-I=VEe;wX_C;gKR@r?Ue@2j2p`J7_NWYEh(iJS`BRgmqfF?Fl|`wI8j@| zPB4Mp!;eN`h~c~vth{7W+h$kj#UYQ8YEnt`XV}#0sm!AnU5tcgvu-lY3Fg#@?tzB z_(|N(1@8`pfrf8szc5)&%cUXtMY`g%_qvR&lA`xF=UCI2i4hd6{wGco-O@|K)0gMx z^-|S!H_wNuRf5o~^E^XqDE#ckEq zhb7!YfE6W^M6ODW`%B`+R~xz$Sz!r*>k1oj6m~!Wrqix_kQ$Sic$SPd?fiJyC^@~+ zO}ukKtR>Ynm)O1Y~~z)Nwn&z*~jkV9=*5|nBOONdz{1Wu^5YS zV9{LjlB$WGyA7VGqODkj#QK_;At}9_x5J6v0|{{7qB*M23zLrM6Y?E zprEK!*ZyU-Fq*oXm|C28#k{VIJz<9)C2+b-iy3u&idDFn9Fv`aY;x8wK0yLT&^vhC z3@Shg{ezOYAE0}wN^&b42Nc!|OJgbvm7-{CLL`z~pYZO(9^a~egbuRV*5l*nXZddG zSym~*g?qX5JvXW;=UF zMo1$OS8iHj){diZQIJMQNB8~wrBc-BRX!p4`NwBAm#?v)QP#J+pKMv06TSR`WIO6e ztwR~>ioE}e&-)f5B|uv(_F8wmLWw%+;?EL3czS+mto-nLOq7IxxG*C?EK@#~NDz!6 zGWYoB&qUeZJ|lMKp;mg@5xjN?8{gnPb-!q)m+@=Uq4OlA`{lG`pBCAzwH?_3}WE*w&#(7=^0C3HB={nFAKPDRlk#5Hz$etw$F&ZPze+fb4+$ae5uQU`&Vh@ajxR%2*~?d&ZrttA@i_u|TtNT}h{Fsto3!y}BwfsBTKf}>*E8=#DJewmcw_{tig2k-0^{O#@aO>j+`g!1C!JYDCOEhnJPGT#l| zqrCU-UkYA7hr4QPb4BiwC{YcWO)|kz&cM&%i4iBJ3g_V)9MfHl<t=YM~&{CPn!WyM=zLm|7gl6FGbsOd%^sLJl&O;XODE3747cs zZn-W<#1PnZeUcNRh%DB|17BgT@AIx6k_EU_1|E+27>CIkhMS!)J_0-j4vvoAoB5yR zgPhyz<9gcLle>*g#(M~@S&dcaww`G_?d+J8XTOeca8PvcgOF|~3`Y`AoHpymDRFU; za$1_n?FPT*g%fl9gPf+*VG)xY`R*M{jop0~D^iVYe_}Yeg5rIE;;YZTj!$o04V~wc zDL;RH9u9p{l)Bq=)vM{SdQs`HY7%vcje^2#gcvgUeY&T*Isi&L3D!MsIj56@$Sub< zMc9H9VijEf$a_rp^B+sF3X~fi<<&Cmdom&jk#Z?~4Mv9RJ9K$)=)dMfTjX23m)XN% zp8Uh*c+Lu@GF%HWz$UCV)w;^Ji+z!`;6f}4MXt>H#kMRX$ZDUmC-r+`k_z2b9v0SH zniA^#5Ymxzl;v>5f2+_hKyF(|CPp%ja?DxAz<&x5n(`;eV92t6&0et#yn$Np{V^s*hL{z?HRjC zKN6EJJ4iZ3rSZ(dHd}oK`R=O*rvjA`>pJvwSPZLL(pundf$*PfFGx=BP0y2Z`D1&y!b|G~Yv9 z^_95Jw{m@1s##d3a~~YU@Z#`d;`umcFgPKlw5Pd4kKZJNlp_O{1^!wd8COI`E*}-V zFAm7Q@oWL;q}^`)-VdZmS*+%k<|cP{C`xIfV;n?6u8}bF)xIn$%vzs_B8R^;w}3}S zr6uR~>4MU7ZV5unQ5r?^I0-T_FPG+@&NSy09KFpG_$+4o9!v$=ijfw5e+#hrxje8k z(2f7gKFrV%Ac~68EzY?krYwleOwJ7u=*qWDR^#|nRYUcVAceC(c zVVCUNg(l(oP2tzqe$_4KG=@1|7_Hm9#^hOBNkXbG#qBCLzF%riT!Z&BySP>5=sa;^ zIiI{Qyek^1l?o@}v*mP)uQe|1_y|a&`}ij6PPb7jvg0z!_S*Y#hHq0J5!iGml1>Qo zpRAG2Udxk9TPNz4wn@;ufX_dr7l1$-ul2R@kax;XSql&;_q7!w|Dl_?^;}o5RBr3F zDAp@3TjN_U$M(f&Tj1%ak-5ShPA9L{Xilx`>w`2(+5WAAT$r?{q5bIzdedZ0|2M|5 zn3HP6z1*q|k`G}ADTS8l@n417N4bhBh9jQXAJGzNqJ*eterea!E0a(Jq}?&$3Ln?j z+uK#iuQRb?0i=^Fky%!{nKa*8&(Xbw5m@D3eh|YJt7*vFb%5A<3Oh!IwCeh51=?1%D z{v>(%tJXlQA-Xx>%gi?CSkTVqe7$h2x!3sR!{X*0A6M$`*u2N09(Qc~gEkCL`fkIz zw&PCrcxh)ldWT_3v{+ZV*KjsedR@!XhFz|}!tT*?=r}Sh(c&#z?=SU{V3R8Pn{VX{ xTYP-vyY$v*@rT4~Qr+II%zK~9&i{aXplN-;=Qb=E{raB|L_r23T_b4{^nY@?dKLfx literal 0 HcmV?d00001 diff --git a/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml b/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml new file mode 100644 index 00000000..d7fb56b8 --- /dev/null +++ b/test/hybig/reference_data/scale_size/VCF5KYR_1991001_001_2018224205008.png.aux.xml @@ -0,0 +1,4 @@ + + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]] + -1.8000000000000000e+02, 1.0000000000000000e+00, 0.0000000000000000e+00, 9.0000000000000000e+01, 0.0000000000000000e+00, -2.0000000000000000e+00 + From 254c70d85755f86677d7ec499a1df7a94e8a9802 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 10 Dec 2024 08:43:49 -0700 Subject: [PATCH 24/27] trade-187: reword module comment to specify sambah --- test/sambah/local_utilities.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/sambah/local_utilities.py b/test/sambah/local_utilities.py index 258f8373..9591738e 100644 --- a/test/sambah/local_utilities.py +++ b/test/sambah/local_utilities.py @@ -1,8 +1,4 @@ -""" A module containing utility functionality used by the Harmony OPeNDAP - SubSetter regression tests. These functions are kept out of the Jupyter - notebook to increase the readability of the regression test suite. - -""" +"""Utility functions for the SAMBAH regression tests.""" from functools import partial from os import listdir, remove From 729ba352aa4c289fb8fcb93bb247e56ba560102b Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 7 Feb 2025 16:16:44 -0500 Subject: [PATCH 25/27] use updated granules and new time bounds --- test/sambah/SAMBAH_Regression.ipynb | 104 +++++++++++++++++++--------- 1 file changed, 73 insertions(+), 31 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index 62b17da5..b4de4cfc 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "7ce7f640", "metadata": { "ExecuteTime": { @@ -78,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "45f380c6", "metadata": { "ExecuteTime": { @@ -90,9 +90,7 @@ ] }, "outputs": [], - "source": [ - "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" - ] + "source": "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" }, { "cell_type": "markdown", @@ -104,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "97dc22bb-e04e-442c-ab12-160ff4842325", "metadata": {}, "outputs": [], @@ -132,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "4420bb6e", "metadata": { "ExecuteTime": { @@ -147,11 +145,11 @@ " # https://cmr.uat.earthdata.nasa.gov/search/concepts/C1262899916-LARC_CLOUD.html\n", " 'collection': Collection(id='C1262899916-LARC_CLOUD'),\n", " 'granule_id': [\n", - " 'G1263137623-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T153020Z_S007G08.nc\n", - " 'G1263137394-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T153657Z_S007G09.nc\n", - " 'G1263137387-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T154353Z_S008G01.nc\n", - " 'G1263137388-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T155033Z_S008G02.nc\n", - " 'G1263137378-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240328T155713Z_S008G03.nc\n", + " 'G1269044486-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T153258Z_S007G07.nc\n", + " 'G1269044632-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T153935Z_S007G08.nc\n", + " 'G1269044623-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T154612Z_S007G09.nc\n", + " 'G1269044612-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T155308Z_S008G01.nc\n", + " 'G1269044756-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T155948Z_S008G02.nc\n", " ],\n", "}\n", "\n", @@ -160,11 +158,11 @@ " # https://cmr.earthdata.nasa.gov/search/concepts/C2930725014-LARC_CLOUD.html\n", " 'collection': Collection(id='C2930725014-LARC_CLOUD'),\n", " 'granule_id': [\n", - " 'G3205017917-LARC_CLOUD',\n", - " 'G3205017777-LARC_CLOUD',\n", - " 'G3205052670-LARC_CLOUD',\n", - " 'G3205053541-LARC_CLOUD',\n", - " 'G3205052319-LARC_CLOUD',\n", + " 'G3181300053-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T153258Z_S007G07.nc\n", + " 'G3181300108-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T153935Z_S007G08.nc\n", + " 'G3181299889-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T154612Z_S007G09.nc\n", + " 'G3181345515-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T155308Z_S008G01.nc\n", + " 'G3181345531-LARC_CLOUD', # TEMPO_NO2_L2_V03_20240801T155948Z_S008G02.nc\n", " ],\n", "}\n", "\n", @@ -183,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "0d7e83fa-20f1-4382-b07f-9f59ca1acb11", "metadata": {}, "outputs": [], @@ -191,8 +189,8 @@ "request_info = {\n", " 'collection': sambah_info[\"collection\"],\n", " 'temporal': {\n", - " 'start': datetime(2024, 3, 28, 15, 34, 0),\n", - " 'stop': datetime(2024, 3, 28, 16, 0, 0),\n", + " 'start': datetime(2024, 8, 1, 15, 34, 0),\n", + " 'stop': datetime(2024, 8, 1, 16, 0, 0),\n", " },\n", " 'spatial': BBox(-170, 33, -10, 38),\n", " 'granule_id': sambah_info[\"granule_id\"],\n", @@ -239,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "d6aaf063-1f3c-4d84-b414-ca80e65ef78b", "metadata": { "ExecuteTime": { @@ -247,7 +245,21 @@ "start_time": "2024-11-13T19:17:24.118362Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "89362838_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "Downloaded: 89362838_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "Saved output to: temp_var_bbox.nc4\n", + "\u001B[92mSuccess: SAMBAH temporal, variable, bounding box request\u001B[0m\n", + "89362844_C2930725014-LARC_CLOUD_batch_of_2_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T153935Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "89362853_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "89362857_C2930725014-LARC_CLOUD_batch_of_1_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_stitched_20240801T153258Z_C2930725014-LARC_CLOUD_merged.nc4\n" + ] + } + ], "source": [ "if request_info is not None:\n", " temp_var_bbox_request = Request(\n", @@ -288,10 +300,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "90bb3455-f82c-4cd8-93f2-feba79fbed7b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloaded: 89362844_C2930725014-LARC_CLOUD_batch_of_2_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T153935Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "Saved output to: var_only.nc4\n", + "\u001B[92mSuccess: SAMBAH variable request\u001B[0m\n" + ] + } + ], "source": [ "if request_info is not None:\n", " var_only_request = Request(\n", @@ -330,10 +352,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "4330105a-f9f0-429e-9762-3049f408d635", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloaded: 89362853_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "Saved output to: spatial_only.nc4\n", + "\u001B[92mSuccess: SAMBAH spatial request\u001B[0m\n" + ] + } + ], "source": [ "if request_info is not None:\n", " spatial_only_request = Request(\n", @@ -372,10 +404,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "c747cfcc-e63a-4858-b406-a4ec87b6acb5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloaded: 89362857_C2930725014-LARC_CLOUD_batch_of_1_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_stitched_20240801T153258Z_C2930725014-LARC_CLOUD_merged.nc4\n", + "Saved output to: all_data.nc4\n", + "\u001B[92mSuccess: SAMBAH no subset single file request\u001B[0m\n" + ] + } + ], "source": [ "if request_info is not None:\n", " all_data_request = Request(\n", @@ -409,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "73fdd147-74a0-48d3-957f-388e2abe356a", "metadata": {}, "outputs": [], @@ -437,7 +479,7 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "papermill-sambah", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -451,7 +493,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.12.8" }, "name": "SDS_Regression.ipynb" }, From ef8a88092a2452cccf0c073106abe8c6210c94d9 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 7 Feb 2025 16:17:04 -0500 Subject: [PATCH 26/27] update sambah test reference files --- test/sambah/reference_files/all_data.nc4 | 4 ++-- test/sambah/reference_files/spatial_only.nc4 | 4 ++-- test/sambah/reference_files/temp_var_bbox.nc4 | 4 ++-- test/sambah/reference_files/var_only.nc4 | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/sambah/reference_files/all_data.nc4 b/test/sambah/reference_files/all_data.nc4 index edadd625..e496b460 100644 --- a/test/sambah/reference_files/all_data.nc4 +++ b/test/sambah/reference_files/all_data.nc4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b063d2d045619a4fa596f9084204e705682958faa8a0606eef9c6a37aaa91847 -size 92642167 +oid sha256:09fe85f29b4b1d8dae33fbbc0157ed93430e50277b6df314ac3084eba4a4594d +size 95736520 diff --git a/test/sambah/reference_files/spatial_only.nc4 b/test/sambah/reference_files/spatial_only.nc4 index b152f6fb..2fa3eb14 100644 --- a/test/sambah/reference_files/spatial_only.nc4 +++ b/test/sambah/reference_files/spatial_only.nc4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bba92cce2fbb598fbd75e3240793201d65da7412efcadbe1823f7ec49c8e5159 -size 60496865 +oid sha256:4ed22fdebe325bc61df85d3e90d7080c476d92741eb2bca9cc0050ba7048b44d +size 61491536 diff --git a/test/sambah/reference_files/temp_var_bbox.nc4 b/test/sambah/reference_files/temp_var_bbox.nc4 index d323eca4..18deee1a 100644 --- a/test/sambah/reference_files/temp_var_bbox.nc4 +++ b/test/sambah/reference_files/temp_var_bbox.nc4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b8de487dfe91f6e4de7d008fca65b48b86a3f1c7f8aedea9a3dd49737552e44 -size 25164633 +oid sha256:5570b7ce72638bb250fdb4a1f2d2c0257061cc361bea5a753e7cc2cd3913a0e7 +size 25118557 diff --git a/test/sambah/reference_files/var_only.nc4 b/test/sambah/reference_files/var_only.nc4 index e73cdf9d..d51be3fc 100644 --- a/test/sambah/reference_files/var_only.nc4 +++ b/test/sambah/reference_files/var_only.nc4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a781564ba24a3248b39972c3421cca4455a6235a42597f894ec28e32c1bb0ec7 -size 90702545 +oid sha256:27e3c1d9b13fcd77e8aee0d2d8839828a5bbefd91504a9aff78a7d99ab36a90e +size 94790167 From 2e80db02c394d616f899c4b837ad7e5367ef733c Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 14 Feb 2025 10:42:48 -0500 Subject: [PATCH 27/27] clean up notebook outputs and remove empty cells at end --- test/sambah/SAMBAH_Regression.ipynb | 92 ++++++----------------------- 1 file changed, 17 insertions(+), 75 deletions(-) diff --git a/test/sambah/SAMBAH_Regression.ipynb b/test/sambah/SAMBAH_Regression.ipynb index b4de4cfc..376df135 100644 --- a/test/sambah/SAMBAH_Regression.ipynb +++ b/test/sambah/SAMBAH_Regression.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "7ce7f640", "metadata": { "ExecuteTime": { @@ -78,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "45f380c6", "metadata": { "ExecuteTime": { @@ -90,7 +90,9 @@ ] }, "outputs": [], - "source": "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" + "source": [ + "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" + ] }, { "cell_type": "markdown", @@ -102,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "97dc22bb-e04e-442c-ab12-160ff4842325", "metadata": {}, "outputs": [], @@ -130,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "4420bb6e", "metadata": { "ExecuteTime": { @@ -181,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "0d7e83fa-20f1-4382-b07f-9f59ca1acb11", "metadata": {}, "outputs": [], @@ -237,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "d6aaf063-1f3c-4d84-b414-ca80e65ef78b", "metadata": { "ExecuteTime": { @@ -245,21 +247,7 @@ "start_time": "2024-11-13T19:17:24.118362Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "89362838_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "Downloaded: 89362838_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "Saved output to: temp_var_bbox.nc4\n", - "\u001B[92mSuccess: SAMBAH temporal, variable, bounding box request\u001B[0m\n", - "89362844_C2930725014-LARC_CLOUD_batch_of_2_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T153935Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "89362853_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "89362857_C2930725014-LARC_CLOUD_batch_of_1_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_stitched_20240801T153258Z_C2930725014-LARC_CLOUD_merged.nc4\n" - ] - } - ], + "outputs": [], "source": [ "if request_info is not None:\n", " temp_var_bbox_request = Request(\n", @@ -300,20 +288,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "90bb3455-f82c-4cd8-93f2-feba79fbed7b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloaded: 89362844_C2930725014-LARC_CLOUD_batch_of_2_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T153935Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "Saved output to: var_only.nc4\n", - "\u001B[92mSuccess: SAMBAH variable request\u001B[0m\n" - ] - } - ], + "outputs": [], "source": [ "if request_info is not None:\n", " var_only_request = Request(\n", @@ -352,20 +330,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "4330105a-f9f0-429e-9762-3049f408d635", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloaded: 89362853_C2930725014-LARC_CLOUD_batch_of_5_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_subsetted_stitched_20240801T155948Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "Saved output to: spatial_only.nc4\n", - "\u001B[92mSuccess: SAMBAH spatial request\u001B[0m\n" - ] - } - ], + "outputs": [], "source": [ "if request_info is not None:\n", " spatial_only_request = Request(\n", @@ -404,20 +372,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "c747cfcc-e63a-4858-b406-a4ec87b6acb5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloaded: 89362857_C2930725014-LARC_CLOUD_batch_of_1_starting_from_TEMPO_NO2_L2_V03_20240801T153258Z_S007G07_stitched_20240801T153258Z_C2930725014-LARC_CLOUD_merged.nc4\n", - "Saved output to: all_data.nc4\n", - "\u001B[92mSuccess: SAMBAH no subset single file request\u001B[0m\n" - ] - } - ], + "outputs": [], "source": [ "if request_info is not None:\n", " all_data_request = Request(\n", @@ -451,29 +409,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "73fdd147-74a0-48d3-957f-388e2abe356a", "metadata": {}, "outputs": [], "source": [ "remove_results_files()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a868b30-52d9-45aa-9359-a14d79548ce4", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0332c0fd-1332-470f-8312-34eecd94280b", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {