config: branches as dict

Represent branches as dict rather than a list to avoid mapping each
time.

Also forward calls to /api/latest and /api/branches to
/json/{latest,branches}.json to allow static hosting, future clients
should request these files directly.

Also add some more testcases

Signed-off-by: Paul Spooren <mail@aparcar.org>

asu/api.py

@@ -1,6 +1,6 @@
 from uuid import uuid4
 
-from flask import Blueprint, current_app, g, request
+from flask import Blueprint, current_app, g, redirect, request
 from rq import Connection, Queue
 
 from .build import build
@@ -18,17 +18,6 @@ def get_distros() -> list:
     return ["openwrt"]
 
 
-@bp.route("/branches")
-def get_branches() -> dict:
-    if "branches" not in g:
-        g.branches = {}
-        for branch in current_app.config["BRANCHES"]:
-            if branch["enabled"]:
-                g.branches[branch["name"]] = branch
-        current_app.logger.info(f"Loaded {len(g.branches)} branches")
-    return g.branches
-
-
 def get_redis():
     """Return Redis connectio
 
@@ -40,24 +29,14 @@ def get_redis():
     return g.redis
 
 
-def get_latest() -> dict:
-    if "latest" not in g:
-        g.latest = list(
-            map(
-                lambda b: b["versions"][0],
-                filter(
-                    lambda b: b.get("enabled"),
-                    current_app.config["BRANCHES"],
-                ),
-            )
-        )
-
-    return g.latest
-
-
 @bp.route("/latest")
 def api_latest():
-    return {"latest": get_latest()}
+    return redirect("/json/latest.json")
+
+
+@bp.route("/branches")
+def api_branches():
+    return redirect("/json/branches.json")
 
 
 def get_queue() -> Queue:
@@ -162,7 +141,7 @@ def validate_request(req):
         # e.g. snapshot, 21.02.0-rc1 or 19.07.7
         req["branch"] = req["version"].rsplit(".", maxsplit=1)[0]
 
-    if req["branch"] not in get_branches().keys():
+    if req["branch"] not in current_app.config["BRANCHES"].keys():
         return (
             {
                 "status": "bad_branch",
@@ -171,7 +150,7 @@ def validate_request(req):
             400,
         )
 
-    if req["version"] not in get_branches()[req["branch"]]["versions"]:
+    if req["version"] not in current_app.config["BRANCHES"][req["branch"]]["versions"]:
         return (
             {
                 "status": "bad_version",
@@ -196,7 +175,9 @@ def validate_request(req):
             400,
         )
 
-    req["arch"] = get_branches()[req["branch"]]["targets"][req["target"]]
+    req["arch"] = current_app.config["BRANCHES"][req["branch"]]["targets"][
+        req["target"]
+    ]
 
     if req["target"] in ["x86/64", "x86/generic", "x86/geode", "x86/legacy"]:
         current_app.logger.debug("Use generic profile for {req['target']}")
@@ -331,7 +312,7 @@ def api_build():
 
         req["store_path"] = current_app.config["STORE_PATH"]
         req["upstream_url"] = current_app.config["UPSTREAM_URL"]
-        req["branch_data"] = get_branches()[req["branch"]]
+        req["branch_data"] = current_app.config["BRANCHES"][req["branch"]]
 
         if req["branch_data"]["snapshot"]:
             result_ttl = "15m"

asu/asu.py

@@ -66,7 +66,30 @@ def store_path(path="index.html"):
     app.register_blueprint(api.bp)
 
     (app.config["JSON_PATH"] / "branches.json").write_text(
-        json.dumps(app.config["BRANCHES"])
+        json.dumps(
+            dict(
+                map(
+                    lambda b: (b["name"], b),
+                    filter(lambda b: b.get("enabled"), app.config["BRANCHES"].values()),
+                )
+            )
+        )
+    )
+
+    (app.config["JSON_PATH"] / "latest.json").write_text(
+        json.dumps(
+            {
+                "latest": list(
+                    map(
+                        lambda b: b["versions"][0],
+                        filter(
+                            lambda b: b.get("enabled"),
+                            app.config["BRANCHES"].values(),
+                        ),
+                    )
+                )
+            }
+        )
     )
 
     return app

asu/common.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+import json
 import struct
 from pathlib import Path
 
@@ -55,7 +56,7 @@ def get_manifest_hash(manifest: dict) -> str:
     Returns:
         str: hash of `req`
     """
-    return str(hash(frozenset(sorted(manifest.items()))))
+    return get_str_hash(json.dumps(manifest, sort_keys=True))
 
 
 def get_request_hash(req: dict) -> str:

asu/janitor.py

@@ -336,7 +336,7 @@ def update(interval):
     """
     current_app.logger.info("Init ASU janitor")
     while True:
-        for branch in current_app.config["BRANCHES"]:
+        for branch in current_app.config["BRANCHES"].values():
             if not branch.get("enabled"):
                 current_app.logger.info(f"Skip disabled version {branch['name']}")
                 continue

tests/conftest.py

@@ -39,6 +39,7 @@ def redis():
     )
     r.sadd("targets-SNAPSHOT", "testtarget/testsubtarget", "x86/64")
     r.sadd("targets-21.02", "testtarget/testsubtarget")
+    r.hset("mapping-abi", mapping={"test1-1": "test1"})
     yield r
 
 
@@ -52,8 +53,8 @@ def app(redis):
             "STORE_PATH": test_path + "/store",
             "TESTING": True,
             "UPSTREAM_URL": "http://localhost:8001",
-            "BRANCHES": [
-                {
+            "BRANCHES": {
+                "SNAPSHOT": {
                     "name": "SNAPSHOT",
                     "enabled": True,
                     "snapshot": True,
@@ -69,7 +70,7 @@ def app(redis):
                         "x86/64": "x86_64",
                     },
                 },
-                {
+                "21.02": {
                     "name": "21.02",
                     "enabled": True,
                     "snapshot": True,
@@ -82,7 +83,7 @@ def app(redis):
                     "updates": "rc",
                     "targets": {"testtarget/testsubtarget": "testarch"},
                 },
-                {
+                "19.07": {
                     "name": "19.07",
                     "enabled": True,
                     "versions": ["19.07.7", "19.07.6"],
@@ -94,7 +95,7 @@ def app(redis):
                     "updates": "stable",
                     "targets": {"testtarget/testsubtarget": "testarch"},
                 },
-            ],
+            },
         }
     )
 

tests/test_api.py

@@ -1,10 +1,6 @@
 def test_api_version(client, app):
     response = client.get("/api/branches")
-    branches = {}
-    for branch in app.config["BRANCHES"]:
-        if branch["enabled"]:
-            branches[branch["name"]] = branch
-    assert response.json == branches
+    assert response.status == "302 FOUND"
 
 
 def test_api_build(client):
@@ -19,15 +15,12 @@ def test_api_build(client):
     )
     assert response.status == "202 ACCEPTED"
     assert response.json.get("status") == "queued"
-    assert response.json.get("request_hash") == "3128aff5c6db"
+    assert response.json.get("request_hash") == "e360f833a191"
 
 
 def test_api_latest_default(client):
     response = client.get("/api/latest")
-    assert "19.07.7" in response.json["latest"]
-    assert "21.02.0-rc1" in response.json["latest"]
-    assert "SNAPSHOT" in response.json["latest"]
-    assert response.status == "200 OK"
+    assert response.status == "302 FOUND"
 
 
 def test_api_build_mapping(client):
@@ -42,11 +35,44 @@ def test_api_build_mapping(client):
     )
     assert response.status == "202 ACCEPTED"
     assert response.json.get("status") == "queued"
-    assert response.json.get("request_hash") == "d0318d0bba8d"
+    assert response.json.get("request_hash") == "19bb42f198c9"
+
+
+def test_api_build_mapping_abi(client):
+    response = client.post(
+        "/api/build",
+        json=dict(
+            version="SNAPSHOT",
+            target="testtarget/testsubtarget",
+            profile="testvendor,testprofile",
+            packages=["test1-1", "test2"],
+        ),
+    )
+    assert response.status == "202 ACCEPTED"
+    assert response.json.get("status") == "queued"
+    assert response.json.get("request_hash") == "7d099fb07fb3"
+
+
+def test_api_build_bad_target(client):
+    response = client.post(
+        "/api/build",
+        json=dict(
+            version="SNAPSHOT",
+            target="testtarget/testsubtargetbad",
+            profile="testvendor,testprofile",
+            packages=["test1", "test2"],
+        ),
+    )
+    assert response.status == "400 BAD REQUEST"
+    assert (
+        response.json.get("message")
+        == "Unsupported target: testtarget/testsubtargetbad"
+    )
+    assert response.json.get("status") == "bad_target"
 
 
 def test_api_build_get(client):
-    client.post(
+    response = client.post(
         "/api/build",
         json=dict(
             version="SNAPSHOT",
@@ -55,10 +81,42 @@ def test_api_build_get(client):
             packages=["test1", "test2"],
         ),
     )
-    response = client.get("/api/build/3128aff5c6db")
+    assert response.json["request_hash"] == "e360f833a191"
+    response = client.get("/api/build/e360f833a191")
     assert response.status == "202 ACCEPTED"
     assert response.json.get("status") == "queued"
-    assert response.json.get("request_hash") == "3128aff5c6db"
+    assert response.json.get("request_hash") == "e360f833a191"
+
+
+def test_api_build_packages_versions(client):
+    response = client.post(
+        "/api/build",
+        json=dict(
+            version="SNAPSHOT",
+            target="testtarget/testsubtarget",
+            profile="testprofile",
+            packages_versions={"test1": "1.0", "test2": "2.0"},
+        ),
+    )
+    assert response.json["request_hash"] == "552b9e328888"
+    response = client.get("/api/build/552b9e328888")
+    assert response.status == "202 ACCEPTED"
+    assert response.json.get("status") == "queued"
+    assert response.json.get("request_hash") == "552b9e328888"
+
+
+def test_api_build_packages_duplicate(client):
+    response = client.post(
+        "/api/build",
+        json=dict(
+            version="SNAPSHOT",
+            target="testtarget/testsubtarget",
+            profile="testprofile",
+            packages=["test1", "test2"],
+            packages_versions={"test1": "1.0", "test2": "2.0"},
+        ),
+    )
+    assert response.status == "400 BAD REQUEST"
 
 
 def test_api_build_get_not_found(client):
@@ -83,7 +141,7 @@ def test_api_build_empty_packages_list(client):
     )
     assert response.status == "202 ACCEPTED"
     assert response.json.get("status") == "queued"
-    assert response.json.get("request_hash") == "c6022275d623"
+    assert response.json.get("request_hash") == "66cb932c37a4"
 
 
 def test_api_build_withouth_packages_list(client):
@@ -97,7 +155,7 @@ def test_api_build_withouth_packages_list(client):
     )
     assert response.status == "202 ACCEPTED"
     assert response.json.get("status") == "queued"
-    assert response.json.get("request_hash") == "c6022275d623"
+    assert response.json.get("request_hash") == "66cb932c37a4"
 
 
 def test_api_build_prerelease_snapshot(client):
@@ -162,7 +220,7 @@ def test_api_build_x86(client):
 
     assert response.status == "202 ACCEPTED"
     assert response.json.get("status") == "queued"
-    assert response.json.get("request_hash") == "a30a7dc8e19b"
+    assert response.json.get("request_hash") == "1fda145d439f"
 
 
 def test_api_build_needed(client):

tests/test_common.py

@@ -32,7 +32,7 @@ def test_get_request_hash():
         "package_hash": get_packages_hash(["test"]),
     }
 
-    assert get_request_hash(request) == "e8dc1720525b"
+    assert get_request_hash(request) == "c753cbe6356a"
 
 
 def test_get_request_hash_diff_packages():
@@ -44,7 +44,12 @@ def test_get_request_hash_diff_packages():
         "diff_packages": True,
     }
 
-    assert get_request_hash(request) == "75967363f440"
+    assert get_request_hash(request) == "a7ff5158d58b"
+
+
+def test_fingerprint_pubkey_usign():
+    pub_key = "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89"
+    assert fingerprint_pubkey_usign(pub_key) == "ab1df166947b2551"
 
 
 def test_verify_usign():

tests/test_factory.py

@@ -11,3 +11,23 @@ def test_pathlib(app):
 
 def test_other(app):
     assert app.config["UPSTREAM_URL"] == "http://localhost:8001"
+
+
+def test_json_path_latest(client):
+    response = client.get("/json/latest.json")
+    assert "19.07.7" in response.json["latest"]
+    assert "21.02.0-rc1" in response.json["latest"]
+    assert "SNAPSHOT" in response.json["latest"]
+    assert response.status == "200 OK"
+
+
+def test_json_path_branches(client):
+    response = client.get("/json/branches.json")
+    assert "19.07" in response.json.keys()
+    assert "SNAPSHOT" in response.json.keys()
+    assert response.status == "200 OK"
+
+
+def test_json_store(client):
+    response = client.get("/store/")
+    assert response.status == "404 NOT FOUND"