tests: add janitor tests

Signed-off-by: Paul Spooren <mail@aparcar.org>

tests/test_janitor.py

@@ -0,0 +1,112 @@
+from asu.build import build
+from pathlib import Path
+import redis
+
+import pytest
+
+from pytest_httpserver import HTTPServer
+
+from asu.janitor import *
+
+
+def test_get_packages_arch(app, httpserver: HTTPServer, redis):
+    base_url = "/snapshots/packages/x86_64/base"
+    upstream_path = Path("./tests/upstream/snapshots/packages/x86_64/base")
+    expected_file_requests = ["Packages"]
+
+    for f in expected_file_requests:
+        httpserver.expect_request(f"{base_url}/{f}").respond_with_data(
+            (upstream_path / f).read_bytes()
+        )
+
+    version = app.config["VERSIONS"]["branches"][0]
+    with app.app_context():
+        get_packages_arch(version, sources=["base"])
+    assert b"base-files" in redis.smembers("packages-snapshot")
+
+
+def test_get_packages_target(app, httpserver: HTTPServer, redis):
+    base_url = "/snapshots/targets/testtarget/testsubtarget/packages"
+    upstream_path = Path(
+        "./tests/upstream/snapshots/targets/testtarget/testsubtarget/packages"
+    )
+    expected_file_requests = ["Packages"]
+
+    for f in expected_file_requests:
+        httpserver.expect_request(f"{base_url}/{f}").respond_with_data(
+            (upstream_path / f).read_bytes()
+        )
+
+    version = app.config["VERSIONS"]["branches"][0]
+    with app.app_context():
+        assert get_packages_target((version, "testtarget/testsubtarget")) == (
+            "testtarget/testsubtarget",
+            ["base-files", "block-mount", "blockd"],
+        )
+
+
+def test_get_packages_targets(app, httpserver: HTTPServer, redis):
+    base_url = "/snapshots/targets/testtarget/testsubtarget/packages"
+    upstream_path = Path(
+        "./tests/upstream/snapshots/targets/testtarget/testsubtarget/packages"
+    )
+    expected_file_requests = ["Packages"]
+
+    for f in expected_file_requests:
+        httpserver.expect_request(f"{base_url}/{f}").respond_with_data(
+            (upstream_path / f).read_bytes()
+        )
+
+    version = app.config["VERSIONS"]["branches"][0]
+    with app.app_context():
+        get_packages_targets(version)
+    assert redis.smembers("packages-snapshot-testtarget/testsubtarget") == {
+        b"base-files",
+        b"block-mount",
+        b"blockd",
+    }
+
+
+def test_get_json_files(app, httpserver: HTTPServer, redis):
+    base_url = "/snapshots/targets"
+    upstream_path = Path("./tests/upstream/snapshots/targets")
+    expected_file_requests = [""]
+
+    httpserver.expect_request(f"{base_url}/", query_string="json").respond_with_data(
+        (upstream_path / "?json").read_bytes()
+    )
+    httpserver.expect_request(
+        f"{base_url}/testtarget/testsubtarget/openwrt-testtarget-testsubtarget-testprofile.json"
+    ).respond_with_data(
+        (
+            upstream_path
+            / "testtarget/testsubtarget/openwrt-testtarget-testsubtarget-testprofile.json"
+        ).read_bytes()
+    )
+
+    version = app.config["VERSIONS"]["branches"][0]
+    with app.app_context():
+        get_json_files(version)
+    assert redis.hgetall("profiles-snapshot") == {
+        b"8devices_carambola": b"ramips/rt305x",
+        b"testprofile": b"testtarget/testsubtarget",
+    }
+    assert len(redis.hgetall("profiles-snapshot")) == 2
+
+
+def test_get_packages_arch_real(app, httpserver: HTTPServer, redis):
+    app.config["UPSTREAM_URL"] = "https://cdn.openwrt.org"
+    version = app.config["VERSIONS"]["branches"][0]
+    with app.app_context():
+        get_packages_arch(version, sources=["base", "luci"])
+    assert len(redis.smembers("packages-snapshot")) > 2000
+
+
+@pytest.mark.slow
+@pytest.mark.skip
+def test_get_json_files_real(app, httpserver: HTTPServer, redis):
+    app.config["UPSTREAM_URL"] = "https://cdn.openwrt.org"
+    version = app.config["VERSIONS"]["branches"][0]
+    with app.app_context():
+        get_json_files(version)
+    assert len(redis.hgetall("profiles-snapshot")) > 900

tests/upstream/snapshots/packages/x86_64/base/Packages

@@ -0,0 +1,35 @@
+Package: base-files
+Version: 214-r12467-5ea1b1ecd1
+Depends: libc, netifd, procd, jsonfilter, usign, openwrt-keyring, fstools, fwtool
+License: GPL-2.0
+Section: base
+Architecture: mips_mips32
+Installed-Size: 45996
+Filename: base-files_214-r12467-5ea1b1ecd1_mips_mips32.ipk
+Size: 46979
+SHA256sum:81184dbc1753154f6420eb7f9f20b5cc50aab071d9ee65b51d91a14a6817b1e6
+Description:  This package contains a base filesystem and system scripts for OpenWrt.
+
+Package: block-mount
+Version: 2020-01-21-deb745f8-1
+Depends: libc, ubox, libubox20191228, libuci20130104, libblobmsg-json, libjson-c4
+License: GPL-2.0
+Section: base
+Architecture: mips_mips32
+Installed-Size: 22892
+Filename: block-mount_2020-01-21-deb745f8-1_mips_mips32.ipk
+Size: 23512
+SHA256sum:5eaa1bb7e3df6c354fa8dec3c30e77d0e8dc0b601923eba22b5ae3319b868a8a
+Description:  Block device mounting and checking
+
+Package: blockd
+Version: 2020-01-21-deb745f8-1
+Depends: libc, block-mount, fstools, libubus20191227, kmod-fs-autofs4, libblobmsg-json, libjson-c4
+License: GPL-2.0
+Section: base
+Architecture: mips_mips32
+Installed-Size: 5153
+Filename: blockd_2020-01-21-deb745f8-1_mips_mips32.ipk
+Size: 5919
+SHA256sum:4186637e16373c4485bcb5a8d70193dd398d5f74810e30c5268e75af3c6e0535
+Description:  Block device automounting

tests/upstream/snapshots/targets/?json

@@ -0,0 +1 @@
+["testtarget/testsubtarget/openwrt-testtarget-testsubtarget-testprofile.json"]

tests/upstream/snapshots/targets/testtarget/testsubtarget/openwrt-testtarget-testsubtarget-testprofile.json

@@ -0,0 +1,24 @@
+{
+  "id": "testprofile",
+  "image_prefix": "openwrt-testtarget-testsubtarget-testprofile",
+  "images": [
+    {
+      "name": "openwrt-testtarget-testsubtarget-testprofile-sysupgrade.bin",
+      "sha256": "000",
+      "type": "sysupgrade"
+    }
+  ],
+  "metadata_version": 1,
+  "supported_devices": [
+    "testprofile"
+  ],
+  "target": "testtarget/testsubtarget",
+  "titles": [
+    {
+      "model": "Test1",
+      "vendor": "The Test Comp"
+    }
+  ],
+  "version_commit": "r99999-999999999",
+  "version_number": "SNAPSHOT"
+}

tests/upstream/snapshots/targets/testtarget/testsubtarget/packages/Packages

@@ -0,0 +1,35 @@
+Package: base-files
+Version: 214-r12467-5ea1b1ecd1
+Depends: libc, netifd, procd, jsonfilter, usign, openwrt-keyring, fstools, fwtool
+License: GPL-2.0
+Section: base
+Architecture: mips_mips32
+Installed-Size: 45996
+Filename: base-files_214-r12467-5ea1b1ecd1_mips_mips32.ipk
+Size: 46979
+SHA256sum:81184dbc1753154f6420eb7f9f20b5cc50aab071d9ee65b51d91a14a6817b1e6
+Description:  This package contains a base filesystem and system scripts for OpenWrt.
+
+Package: block-mount
+Version: 2020-01-21-deb745f8-1
+Depends: libc, ubox, libubox20191228, libuci20130104, libblobmsg-json, libjson-c4
+License: GPL-2.0
+Section: base
+Architecture: mips_mips32
+Installed-Size: 22892
+Filename: block-mount_2020-01-21-deb745f8-1_mips_mips32.ipk
+Size: 23512
+SHA256sum:5eaa1bb7e3df6c354fa8dec3c30e77d0e8dc0b601923eba22b5ae3319b868a8a
+Description:  Block device mounting and checking
+
+Package: blockd
+Version: 2020-01-21-deb745f8-1
+Depends: libc, block-mount, fstools, libubus20191227, kmod-fs-autofs4, libblobmsg-json, libjson-c4
+License: GPL-2.0
+Section: base
+Architecture: mips_mips32
+Installed-Size: 5153
+Filename: blockd_2020-01-21-deb745f8-1_mips_mips32.ipk
+Size: 5919
+SHA256sum:4186637e16373c4485bcb5a8d70193dd398d5f74810e30c5268e75af3c6e0535
+Description:  Block device automounting