build: implement diff_packages

When activated default packages are not installed but only the packages
actually requested. This is different from web interfaces where default
plus additional packages are installed. This function is usfull when
upgrading a running device which sends all installed packages to the
server.

Signed-off-by: Paul Spooren <mail@aparcar.org>

asu/build.py

@@ -25,6 +25,16 @@ def build(request: dict):
     Args:
         request (dict): Contains all properties of requested image
     """
+
+    if "packages" in request:
+        assert isinstance(
+            request["packages"], set
+        ), "packages must be type set not list"
+    else:
+        request["packages"] = set()
+
+    assert (request["store_path"]).is_dir(), "store_path must be existing directory"
+
     job = get_current_job()
 
     log.debug(f"Building {request}")
@@ -104,23 +114,19 @@ def download_file(filename: str, dest: str = None):
 
     cache.mkdir(parents=True, exist_ok=True)
 
-    if not (request["store_path"]).is_dir():
-        (request["store_path"]).mkdir(parents=True, exist_ok=True)
-
     if sig_file.is_file():
+        sig_file_headers = urllib.request.urlopen(
+            request["upstream_url"]
+            + "/"
+            + request["version_data"]["path"]
+            + "/targets/"
+            + request["target"]
+            + "/sha256sums.sig"
+        ).info()
+        log.debug("sig_file_headers:", sig_file_headers)
         last_modified = time.mktime(
             time.strptime(
-                urllib.request.urlopen(
-                    request["upstream_url"]
-                    + "/"
-                    + request["version_data"]["path"]
-                    + "/targets/"
-                    + request["target"]
-                    + "/sha256sums.sig"
-                )
-                .info()
-                .get("Last-Modified"),
-                "%a, %d %b %Y %H:%M:%S %Z",
+                sig_file_headers.get("Last-Modified"), "%a, %d %b %Y %H:%M:%S %Z"
             )
         )
         log.debug(
@@ -134,6 +140,27 @@ def download_file(filename: str, dest: str = None):
     else:
         setup_ib()
 
+    if request.get("diff_packages", False) and request.get("packages"):
+        info_run = subprocess.run(
+            ["make", "info"], text=True, capture_output=True, cwd=cache / subtarget
+        )
+        default_packages = set(
+            re.search(r"Default Packages: (.*)\n", info_run.stdout).group(1).split()
+        )
+        profile_packages = set(
+            re.search(
+                r"{}:\n    .+\n    Packages: (.+?)\n".format(request["profile"]),
+                info_run.stdout,
+                re.MULTILINE,
+            )
+            .group(1)
+            .split()
+        )
+        remove_packages = (default_packages | profile_packages) - request["packages"]
+        request["packages"] = request["packages"] | set(
+            map(lambda p: f"-{p}", remove_packages)
+        )
+
     manifest_run = subprocess.run(
         [
             "make",
@@ -166,8 +193,7 @@ def download_file(filename: str, dest: str = None):
         / packages_hash
     )
 
-    if not (request["store_path"] / bin_dir).is_dir():
-        (request["store_path"] / bin_dir).mkdir(parents=True, exist_ok=True)
+    (request["store_path"] / bin_dir).mkdir(parents=True, exist_ok=True)
 
     image_build = subprocess.run(
         [

tests/test_build.py

@@ -6,7 +6,8 @@
 from pytest_httpserver import HTTPServer
 
 
-def test_build_fake(app, httpserver: HTTPServer):
+@pytest.fixture
+def upstream(httpserver):
     base_url = "/snapshots/targets/testtarget/testsubtarget"
     upstream_path = Path("./tests/upstream/snapshots/targets/testtarget/testsubtarget/")
     expected_file_requests = [
@@ -17,9 +18,51 @@ def test_build_fake(app, httpserver: HTTPServer):
 
     for f in expected_file_requests:
         httpserver.expect_request(f"{base_url}/{f}").respond_with_data(
-            (upstream_path / f).read_bytes()
+            (upstream_path / f).read_bytes(),
+            headers={"Last-Modified": "Thu, 19 Mar 2020 20:27:41 GMT"},
         )
 
+
+def test_build_fake(app, upstream):
+    request_data = dict(
+        version_data={
+            "branch": "master",
+            "path": "snapshots",
+            "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89",
+        },
+        target="testtarget/testsubtarget",
+        store_path=app.config["STORE_PATH"],
+        cache_path=app.config["CACHE_PATH"],
+        upstream_url="http://localhost:8001",
+        version="SNAPSHOT",
+        profile="testprofile",
+        packages={"test1", "test2"},
+    )
+    result = build(request_data)
+    assert result["id"] == "testprofile"
+
+
+def test_build_fake_diff_packages(app, upstream):
+    request_data = dict(
+        version_data={
+            "branch": "master",
+            "path": "snapshots",
+            "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89",
+        },
+        target="testtarget/testsubtarget",
+        store_path=app.config["STORE_PATH"],
+        cache_path=app.config["CACHE_PATH"],
+        upstream_url="http://localhost:8001",
+        version="SNAPSHOT",
+        profile="testprofile",
+        packages={"test1", "test2"},
+        diff_packages=True,
+    )
+    result = build(request_data)
+    assert result["id"] == "testprofile"
+
+
+def test_build_fake_no_packages(app, upstream):
     request_data = dict(
         version_data={
             "branch": "master",
@@ -32,12 +75,52 @@ def test_build_fake(app, httpserver: HTTPServer):
         upstream_url="http://localhost:8001",
         version="SNAPSHOT",
         profile="testprofile",
-        packages=["test1", "test2"],
     )
     result = build(request_data)
     assert result["id"] == "testprofile"
 
 
+def test_build_fake_list_packages(app, upstream):
+    request_data = dict(
+        version_data={
+            "branch": "master",
+            "path": "snapshots",
+            "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89",
+        },
+        target="testtarget/testsubtarget",
+        store_path=app.config["STORE_PATH"],
+        cache_path=app.config["CACHE_PATH"],
+        upstream_url="http://localhost:8001",
+        version="SNAPSHOT",
+        profile="testprofile",
+        packages=["test1"],
+    )
+    with pytest.raises(Exception) as execinfo:
+        result = build(request_data)
+    assert str(execinfo.value) == "packages must be type set not list"
+
+
+def test_build_fake_store_path_not_exists(app, upstream):
+    app.config["STORE_PATH"].rmdir()
+
+    request_data = dict(
+        version_data={
+            "branch": "master",
+            "path": "snapshots",
+            "pubkey": "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89",
+        },
+        target="testtarget/testsubtarget",
+        store_path=app.config["STORE_PATH"],
+        cache_path=app.config["CACHE_PATH"],
+        upstream_url="http://localhost:8001",
+        version="SNAPSHOT",
+        profile="testprofile",
+    )
+    with pytest.raises(Exception) as execinfo:
+        result = build(request_data)
+    assert str(execinfo.value) == "store_path must be existing directory"
+
+
 @pytest.mark.slow
 def test_build_real(app, httpserver: HTTPServer):
     request_data = dict(
@@ -52,7 +135,7 @@ def test_build_real(app, httpserver: HTTPServer):
         upstream_url="https://cdn.openwrt.org",
         version="SNAPSHOT",
         profile="tplink_tl-wdr4300-v1",
-        packages=["tmux", "vim"],
+        packages={"tmux", "vim"},
     )
     result = build(request_data)
     assert result["id"] == "tplink_tl-wdr4300-v1"

tests/upstream/snapshots/targets/testtarget/testsubtarget/openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64/Makefile

@@ -6,3 +6,6 @@ image:
 
 manifest:
 	@cat ./openwrt-testtarget-testsubtarget-testprofile.manifest
+
+info:
+	@cat ./openwrt-testtarget-testsubtarget-testprofile.info

tests/upstream/snapshots/targets/testtarget/testsubtarget/openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64/openwrt-testtarget-testsubtarget-testprofile.info

@@ -0,0 +1,19 @@
+Current Target: "testtarget/testsubtarget"
+Current Revision: "r12647-cb44ab4f5d"
+Default Packages: base-files libc libgcc busybox dropbear mtd uci opkg netifd fstools uclient-fetch logd urandom-seed urngd kmod-gpio-button-hotplug swconfig kmod-ath9k uboot-envtools wpad-basic dnsmasq iptables ip6tables ppp ppp-mod-pppoe firewall odhcpd-ipv6only odhcp6c kmod-ipt-offload
+Available Profiles:
+
+Default:
+    Default Profile (all drivers)
+    Packages: iwinfo
+    hasImageMetadata: 0
+8dev_carambola2:
+    8devices Carambola2
+    Packages: kmod-usb2 kmod-usb-chipidea2
+    hasImageMetadata: 1
+    SupportedDevices: 8dev,carambola2 carambola2
+testprofile:
+    Testprofile
+    Packages: kmod-usb2 kmod-usb-chipidea2 kmod-usb-storage -swconfig
+    hasImageMetadata: 1
+    SupportedDevices: testvendor,testprofile testprofile

tests/upstream/snapshots/targets/testtarget/testsubtarget/sha256sums

@@ -1 +1 @@
-7924ada680e61be0fc6f053f1d944b15a758d82bf66b1f0e43810d159e20b0c7 *openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64.tar.xz
+46a66cc7aee80f55eb2956e8fefc6b1b547826d05e2bf50abb1e881dc6d7ea46 *openwrt-imagebuilder-testtarget-testsubtarget.Linux-x86_64.tar.xz

tests/upstream/snapshots/targets/testtarget/testsubtarget/sha256sums.sig

@@ -1,2 +1,2 @@
 untrusted comment: signed by key ab1df166947b2551
-RWSrHfFmlHslUeSxO2Ancb2Lx0hn6XSzfRBevWFNlWNCfcuDLTVbWVeDxbvak0Oxlj9kSSkKhkCrF0h3w6jqpFD2R9Xz9RggWQM=
+RWSrHfFmlHslUSMOpii5K4+83aPGncrIBKWInwUdXBGCSttWIY38I9cCXyLlo5wEMXwKTR3LG2h9a9Sma/y3Dx223n1VR21Q4AI=