From afcfb4b7b6cf40d4a4daab5c4e0420cedf8c1f6b Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Thu, 19 Mar 2020 20:08:22 -1000 Subject: [PATCH] build: implement `diff_packages` When activated default packages are not installed but only the packages actually requested. This is different from web interfaces where default plus additional packages are installed. This function is usfull when upgrading a running device which sends all installed packages to the server. Signed-off-by: Paul Spooren --- asu/build.py | 58 ++++++++--- tests/test_build.py | 91 +++++++++++++++++- ...sttarget-testsubtarget.Linux-x86_64.tar.xz | Bin 2001 -> 2307 bytes .../Makefile | 3 + ...-testtarget-testsubtarget-testprofile.info | 19 ++++ .../testtarget/testsubtarget/sha256sums | 2 +- .../testtarget/testsubtarget/sha256sums.sig | 2 +- 7 files changed, 153 insertions(+), 22 deletions(-) create mode 100644 tests/upstream/snapshots/targets/testtarget/testsubtarget/openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64/openwrt-testtarget-testsubtarget-testprofile.info diff --git a/asu/build.py b/asu/build.py index 4b84ddc8..340294fe 100644 --- a/asu/build.py +++ b/asu/build.py @@ -25,6 +25,16 @@ def build(request: dict): Args: request (dict): Contains all properties of requested image """ + + if "packages" in request: + assert isinstance( + request["packages"], set + ), "packages must be type set not list" + else: + request["packages"] = set() + + assert (request["store_path"]).is_dir(), "store_path must be existing directory" + job = get_current_job() log.debug(f"Building {request}") @@ -104,23 +114,19 @@ def download_file(filename: str, dest: str = None): cache.mkdir(parents=True, exist_ok=True) - if not (request["store_path"]).is_dir(): - (request["store_path"]).mkdir(parents=True, exist_ok=True) - if sig_file.is_file(): + sig_file_headers = urllib.request.urlopen( + request["upstream_url"] + + "/" + + request["version_data"]["path"] + + "/targets/" + + request["target"] + + "/sha256sums.sig" + ).info() + log.debug("sig_file_headers:", sig_file_headers) last_modified = time.mktime( time.strptime( - urllib.request.urlopen( - request["upstream_url"] - + "/" - + request["version_data"]["path"] - + "/targets/" - + request["target"] - + "/sha256sums.sig" - ) - .info() - .get("Last-Modified"), - "%a, %d %b %Y %H:%M:%S %Z", + sig_file_headers.get("Last-Modified"), "%a, %d %b %Y %H:%M:%S %Z" ) ) log.debug( @@ -134,6 +140,27 @@ def download_file(filename: str, dest: str = None): else: setup_ib() + if request.get("diff_packages", False) and request.get("packages"): + info_run = subprocess.run( + ["make", "info"], text=True, capture_output=True, cwd=cache / subtarget + ) + default_packages = set( + re.search(r"Default Packages: (.*)\n", info_run.stdout).group(1).split() + ) + profile_packages = set( + re.search( + r"{}:\n .+\n Packages: (.+?)\n".format(request["profile"]), + info_run.stdout, + re.MULTILINE, + ) + .group(1) + .split() + ) + remove_packages = (default_packages | profile_packages) - request["packages"] + request["packages"] = request["packages"] | set( + map(lambda p: f"-{p}", remove_packages) + ) + manifest_run = subprocess.run( [ "make", @@ -166,8 +193,7 @@ def download_file(filename: str, dest: str = None): / packages_hash ) - if not (request["store_path"] / bin_dir).is_dir(): - (request["store_path"] / bin_dir).mkdir(parents=True, exist_ok=True) + (request["store_path"] / bin_dir).mkdir(parents=True, exist_ok=True) image_build = subprocess.run( [ diff --git a/tests/test_build.py b/tests/test_build.py index 07aca43b..a88769b9 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -6,7 +6,8 @@ from pytest_httpserver import HTTPServer -def test_build_fake(app, httpserver: HTTPServer): +@pytest.fixture +def upstream(httpserver): base_url = "/snapshots/targets/testtarget/testsubtarget" upstream_path = Path("./tests/upstream/snapshots/targets/testtarget/testsubtarget/") expected_file_requests = [ @@ -17,9 +18,51 @@ def test_build_fake(app, httpserver: HTTPServer): for f in expected_file_requests: httpserver.expect_request(f"{base_url}/{f}").respond_with_data( - (upstream_path / f).read_bytes() + (upstream_path / f).read_bytes(), + headers={"Last-Modified": "Thu, 19 Mar 2020 20:27:41 GMT"}, ) + +def test_build_fake(app, upstream): + request_data = dict( + version_data={ + "branch": "master", + "path": "snapshots", + "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89", + }, + target="testtarget/testsubtarget", + store_path=app.config["STORE_PATH"], + cache_path=app.config["CACHE_PATH"], + upstream_url="http://localhost:8001", + version="SNAPSHOT", + profile="testprofile", + packages={"test1", "test2"}, + ) + result = build(request_data) + assert result["id"] == "testprofile" + + +def test_build_fake_diff_packages(app, upstream): + request_data = dict( + version_data={ + "branch": "master", + "path": "snapshots", + "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89", + }, + target="testtarget/testsubtarget", + store_path=app.config["STORE_PATH"], + cache_path=app.config["CACHE_PATH"], + upstream_url="http://localhost:8001", + version="SNAPSHOT", + profile="testprofile", + packages={"test1", "test2"}, + diff_packages=True, + ) + result = build(request_data) + assert result["id"] == "testprofile" + + +def test_build_fake_no_packages(app, upstream): request_data = dict( version_data={ "branch": "master", @@ -32,12 +75,52 @@ def test_build_fake(app, httpserver: HTTPServer): upstream_url="http://localhost:8001", version="SNAPSHOT", profile="testprofile", - packages=["test1", "test2"], ) result = build(request_data) assert result["id"] == "testprofile" +def test_build_fake_list_packages(app, upstream): + request_data = dict( + version_data={ + "branch": "master", + "path": "snapshots", + "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89", + }, + target="testtarget/testsubtarget", + store_path=app.config["STORE_PATH"], + cache_path=app.config["CACHE_PATH"], + upstream_url="http://localhost:8001", + version="SNAPSHOT", + profile="testprofile", + packages=["test1"], + ) + with pytest.raises(Exception) as execinfo: + result = build(request_data) + assert str(execinfo.value) == "packages must be type set not list" + + +def test_build_fake_store_path_not_exists(app, upstream): + app.config["STORE_PATH"].rmdir() + + request_data = dict( + version_data={ + "branch": "master", + "path": "snapshots", + "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89", + }, + target="testtarget/testsubtarget", + store_path=app.config["STORE_PATH"], + cache_path=app.config["CACHE_PATH"], + upstream_url="http://localhost:8001", + version="SNAPSHOT", + profile="testprofile", + ) + with pytest.raises(Exception) as execinfo: + result = build(request_data) + assert str(execinfo.value) == "store_path must be existing directory" + + @pytest.mark.slow def test_build_real(app, httpserver: HTTPServer): request_data = dict( @@ -52,7 +135,7 @@ def test_build_real(app, httpserver: HTTPServer): upstream_url="https://cdn.openwrt.org", version="SNAPSHOT", profile="tplink_tl-wdr4300-v1", - packages=["tmux", "vim"], + packages={"tmux", "vim"}, ) result = build(request_data) assert result["id"] == "tplink_tl-wdr4300-v1" diff --git a/tests/upstream/snapshots/targets/testtarget/testsubtarget/openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64.tar.xz b/tests/upstream/snapshots/targets/testtarget/testsubtarget/openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64.tar.xz index d85bbb9198b3bde1663437f09d751e8f66fed239..9a7ab6a9602f2d1a8a7023e42318967b8207d17b 100644 GIT binary patch literal 2307 zcmV+e3H%qxi^uZtQB(8ZrvZu*B}ZWZ)J=d-Kx9QUUDR`e1-5t8)@l_nYd37pbpd^ayCrKha{v7BPVKEDnh(s}zQxZk& zl^>nl!dFn&K;L}-l{kVs#{Y{_@FsZkLy<(IGjq3R*Dd#dWvWJ?`I>-2r*8e{L zPcWg@|5L!kgaM9GLcssya~R_b?TYQ3xTui)xi&V{E*$F* z`1@a{U9MZ9L@TatblqfS0cheFP@yX8R;w$eZ!$UR!IqKMf_h~K8}?6cwmQ|60N|t$^IW_8;B{Uo8VhdMvd8(b* zkH7x(;p4yl{prH{!+Y+TIMH*^SNQ*5#I?5T;Ak61{6CJOaI^o9DD3|krFioH=OE7w z{H6CdS^2?}wsbrG*#G-?|9toB)z9yK`~J-rNOK#+ua%LDA~V)Yj^%hmDRlBjDcUa; z>UmjV$}bdRtG-AN`y%qcIC+@=B~$))4mwl*x8p3HyVny9^aAofcw$rj_e^A96X(0Z zf05y7qHjzuBmR?z@t;ib{~UC#^WR|?{oMVXXq#R@{7(o?9>jk(<^N})R5Tjd725d7 z2eYG^P|7(XBw-1eagumx*W9P&9q>uONI($rGG)-HUOQ!~=7-8TOq;#5uk>QqqYx$$< z1ge|F=oyWG4HK>wg(~wkf+9u`)+`i;grLEqgrTk!vd5%lF~Bglzfy2XQ$avWG>t$aJndk$e8!EQ+!27==s_%!~o$?BNC&xz#J1K${fd5XnT(7*{bs zu612#D=rKO{Kn7$gwv2qjtNdglo%OnqX;ums3T*HAfb1im(U+G=NNX(qq;vtL5OMz zX0wtlXkKL{O1ri#3$!TPYT3=lZLuCT?)yfMf!BiRiUQ0muMhHTeSlSn_5i0eW?2fg zhr86OahmVrWhs)iL{et$KFPU|$E}jLNxZA{5B(5QJA_8@8BHY~ZWluj z(q)vKFU<_;?j+;!7=8Cg~L16A-%S;z7Kq1?sY+ItyCRR+b zUgBLJmB`jCDCg>6tnYiW9*xe;<$2aZ#{(QD0pl@A5H(`Pxic{!kf2sc7+R-XT`9$Ii*6O+uO3fJ`oLju(oSX1>ZNV z&QAbCn$ebEyPNoAa34I5VcUhyHA*4*ZyqhZHKAghriwQHnQ z3t7c-z2J%6&T2B7aU7?Nu}T@CB8F)`q+r+jzRrsI$b-P1m}8E$ zVWDlXpQh`j7$J#gMpJCvUvCH+gaPlL&^U^&%j;5ZdPJ?plHrsxZP#dBNwvLzwb2nV zAvETgPNvN>j-z_7eH^!1)y^Xk*bJ{3bqzU5&q*M=#9nhN?f%=EfKacqqlt8Jv&lCg zNwGGwth<9g#k#+PZqG8+VeU%PeV%aT2EpUCa&gcJb~Z$8*&nuoS~JSrKCtU^AJk~w zM}WStn2&XWd`c!JN#oQe99@&mH(Aika$&Lv#+e50E%-t3xD9N~<|ob8d6O_pOhKNi4pvX?e1K8zfhO>BJO{t_y3=b&UOD|w{*|l*NGm| z3%LIw5cOB@e}*AVsJs86@f82hL2tXdhU~-twggP+P?gM>5s0M+!)b9g`9 zh-_)a`5$`c*j##p*)H9y{c9WceE&NuU)7m?f&Y7l*jTsEmR;dOH1F&({$=&OhD#rN zT`2|5^=^)l7g?1ly@Eb$i@5TS!&fJM?wh!DjlJGz z;BWr*MggK%?9-;;2?}kBV7AnN_t7xmPt?^a)PL#X^+-2=^#2Rp#!WQQL=#Ok(L@tX dG|@y8O*GL&6HPSHL=!!rKLIC$3;h64005X*n0WvI literal 2001 zcmV;?2QK&@iwFSgMOa<{1MOQ`b0fDA)@%4HSgJfEaWIQ(1{gm%$3tATaa^|FQYjS% z2RSRwkvT-FT>kHA%!t%bB1LMqmRdXbc1zqre`uf^KsS2Ct!i7Yn-1lLTG*`5SH?D| zvu)R@W?{SGpKYIwPtp6l>^Ep5lbeLUyE+#bAmZ465)tF+=N4ClB{adD3dXK5B{3IQ z^Z4u$J^_6Tw$0~P>ICnU{8x(TUG(mo_f@%orM>-3WB3J5INz@K(@#vsy|LR1m=M?s zSM&FuX+C?{u76Wi-AlYv@?R;ck3aw0uE*HJQ{7Z9U)ktGTa|BZYru0{V*MxQ|4{rV zn9w2q$B7U85DOS3gk8a79{+!y&F9y-x&C^7Jvpw|e;%6sp17$S>++2!yyn19 zG_=t7-|#e_|2+JJ_EHu0wHi@--}d#QQN~7DUJe5c%G#xZt>e|fW_oPy?(1QU$8Ohy z=ok3;pJ8@|?UYfSy1BDWn^z^U$sm9Zb=7p%+!%Y8>xl(>PTDCNC%g6G`1EeikkK7o z-mMO~qo|Bs`R+eKAM$9zoh?n(41%Sdd*S?VRZ%~+<6-(Yx+;o%m`0O&pmdAtJzm)t znH|*r_}yQ=|M4IH`ssT1%j~UpV!&JA6XO3r)vfjC;N%!5;y>mwdvgEx2_VG(OJFuE z{MWPJ>3SZ$>Pq({9?yUOhrfRN-Ob;>{o#-AK0@1YLHtx5g(`FBqvTZ2Y}*P(exp_Q zu}XiN0rHEe{}S=M{u818y96#&|2+v7Z*6tJ7GFU92Tx=EkK+W&KRVWbBIN%|z`-SM zJ}mxkGrtAAj4vbqljr%Lg#3RAT_kP*0=XrwE;Pa7>NiXGrh=BUilU=Gi6(-_P@S+ zBrzg#W}2$bEO-@~Fi9gJ5u3TZv1_$jJv71qh*AXlxg=bHPSdd(21Se@Y#CRMaG1fef~Bqv@~=s&a!bR~|J6!r zni-0cSzafd%0`EfNJb1%XnWqYMy?;siHGM8?XsNC9V5;Z!OS_BoPW6iPVVf3Phd=n zFh^S0lu45S<<>T(_3C&;;BNv3Ae?cjB_>#?SU8JV4iI!DYulmJwEH<*Hwm;5eE9<~@WeIN9w@2lTJt8Vq zcSKS-x2lvmAYED04l_R_%QK`KjkM0YLz)YvPkW{4MDldfUyVyh{SrDOQ<`Z!eXaC9HY!pP4ISJBg=~+hQ=>3{mhZ(K*14VLyp_Fwz?S)!BMkd7BN^A54iw7 zkr-QbzU;bsH%XstgdbW#N+QC<+ukD&zeFlmsLRXyU2X$+0g;jvW2r|<-7OnCgwd&s zd&u0ri_hvW+0a9lf9_7hV&kMTkDLcx;zgd$fyTh+d^)5+RvO6!gTy@2R=F)ZphC4r z*}g_+U1FGFqsGmcmB`*Ms21jEt{*MgtwulW<@K3Fzd7}p+%n$`@#7TNpKxW0yxiVx&goFvZe2CEXR_fA z*7fbV#TjHMGr9_VcbA+^9+Sr@Znw0BMHv+TEhNwrJ&JDFk?{;%k~=fm(QxA< zP}OLX`7=_PrLGgbJMhF`XDvxniFt-dG7`*SPgSn&r;0dAOvVUR32gIi33hAmo4j01 zA_&}xC6?Ga#{GcfHr*Y?7-^h3OR*1sqa$dJ%m8C!xtPug>M1Rp$db$#9DSQF9;=|#OJTDpCYc548sZ>C+!iio3zO}f zlth@#)-@OfAuWI0!B2~*w=Qb~&cr9Y{~5n`)%HbuVvJM2|B2(>_x~uP!hipRlO(+V zxdfy$DiK%-X5!QtoG_XS?06PKu9E%{GnHa#34=e1IH4Wq#t5R~jA1qZ)2qMbF%!~J zPMlSs3ESAj4hDh?W0uA-hlvOUX)sGOD~Og#a+yiZwG_l+24g0#9CP;t4wofN6YErr zDNh{eHL2D%NnD(z%1I&|F>$PgWJ*iQsrCO4A!ANU;*@;lxKDoN9YSJ2KtMo1KtMo1 jKtMo1KtMo1KtMo1KtMo1KtMpiXT<*i