From bac6de18348b87ab7680e89847fd88224b02efbc Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Sat, 14 Aug 2021 13:15:59 -1000 Subject: [PATCH] config: branches as dict Represent branches as dict rather than a list to avoid mapping each time. Also forward calls to /api/latest and /api/branches to /json/{latest,branches}.json to allow static hosting, future clients should request these files directly. Also add some more testcases Signed-off-by: Paul Spooren --- asu/api.py | 45 ++++++--------------- asu/asu.py | 25 +++++++++++- asu/common.py | 3 +- asu/janitor.py | 2 +- tests/conftest.py | 11 +++--- tests/test_api.py | 92 +++++++++++++++++++++++++++++++++++-------- tests/test_common.py | 9 ++++- tests/test_factory.py | 20 ++++++++++ tests/test_janitor.py | 8 ++-- 9 files changed, 152 insertions(+), 63 deletions(-) diff --git a/asu/api.py b/asu/api.py index 2aa03a96..54be9d9f 100644 --- a/asu/api.py +++ b/asu/api.py @@ -1,6 +1,6 @@ from uuid import uuid4 -from flask import Blueprint, current_app, g, request +from flask import Blueprint, current_app, g, redirect, request from rq import Connection, Queue from .build import build @@ -18,17 +18,6 @@ def get_distros() -> list: return ["openwrt"] -@bp.route("/branches") -def get_branches() -> dict: - if "branches" not in g: - g.branches = {} - for branch in current_app.config["BRANCHES"]: - if branch["enabled"]: - g.branches[branch["name"]] = branch - current_app.logger.info(f"Loaded {len(g.branches)} branches") - return g.branches - - def get_redis(): """Return Redis connectio @@ -40,24 +29,14 @@ def get_redis(): return g.redis -def get_latest() -> dict: - if "latest" not in g: - g.latest = list( - map( - lambda b: b["versions"][0], - filter( - lambda b: b.get("enabled"), - current_app.config["BRANCHES"], - ), - ) - ) - - return g.latest - - @bp.route("/latest") def api_latest(): - return {"latest": get_latest()} + return redirect("/json/latest.json") + + +@bp.route("/branches") +def api_branches(): + return redirect("/json/branches.json") def get_queue() -> Queue: @@ -162,7 +141,7 @@ def validate_request(req): # e.g. snapshot, 21.02.0-rc1 or 19.07.7 req["branch"] = req["version"].rsplit(".", maxsplit=1)[0] - if req["branch"] not in get_branches().keys(): + if req["branch"] not in current_app.config["BRANCHES"].keys(): return ( { "status": "bad_branch", @@ -171,7 +150,7 @@ def validate_request(req): 400, ) - if req["version"] not in get_branches()[req["branch"]]["versions"]: + if req["version"] not in current_app.config["BRANCHES"][req["branch"]]["versions"]: return ( { "status": "bad_version", @@ -196,7 +175,9 @@ def validate_request(req): 400, ) - req["arch"] = get_branches()[req["branch"]]["targets"][req["target"]] + req["arch"] = current_app.config["BRANCHES"][req["branch"]]["targets"][ + req["target"] + ] if req["target"] in ["x86/64", "x86/generic", "x86/geode", "x86/legacy"]: current_app.logger.debug("Use generic profile for {req['target']}") @@ -331,7 +312,7 @@ def api_build(): req["store_path"] = current_app.config["STORE_PATH"] req["upstream_url"] = current_app.config["UPSTREAM_URL"] - req["branch_data"] = get_branches()[req["branch"]] + req["branch_data"] = current_app.config["BRANCHES"][req["branch"]] if req["branch_data"]["snapshot"]: result_ttl = "15m" diff --git a/asu/asu.py b/asu/asu.py index 57a15169..8934d9a5 100644 --- a/asu/asu.py +++ b/asu/asu.py @@ -66,7 +66,30 @@ def store_path(path="index.html"): app.register_blueprint(api.bp) (app.config["JSON_PATH"] / "branches.json").write_text( - json.dumps(app.config["BRANCHES"]) + json.dumps( + dict( + map( + lambda b: (b["name"], b), + filter(lambda b: b.get("enabled"), app.config["BRANCHES"].values()), + ) + ) + ) + ) + + (app.config["JSON_PATH"] / "latest.json").write_text( + json.dumps( + { + "latest": list( + map( + lambda b: b["versions"][0], + filter( + lambda b: b.get("enabled"), + app.config["BRANCHES"].values(), + ), + ) + ) + } + ) ) return app diff --git a/asu/common.py b/asu/common.py index cbdfcf59..eaa0e13f 100644 --- a/asu/common.py +++ b/asu/common.py @@ -1,5 +1,6 @@ import base64 import hashlib +import json import struct from pathlib import Path @@ -55,7 +56,7 @@ def get_manifest_hash(manifest: dict) -> str: Returns: str: hash of `req` """ - return str(hash(frozenset(sorted(manifest.items())))) + return get_str_hash(json.dumps(manifest, sort_keys=True)) def get_request_hash(req: dict) -> str: diff --git a/asu/janitor.py b/asu/janitor.py index 8e3752e9..ccd014bc 100644 --- a/asu/janitor.py +++ b/asu/janitor.py @@ -336,7 +336,7 @@ def update(interval): """ current_app.logger.info("Init ASU janitor") while True: - for branch in current_app.config["BRANCHES"]: + for branch in current_app.config["BRANCHES"].values(): if not branch.get("enabled"): current_app.logger.info(f"Skip disabled version {branch['name']}") continue diff --git a/tests/conftest.py b/tests/conftest.py index 452a9537..7c178258 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,7 @@ def redis(): ) r.sadd("targets-SNAPSHOT", "testtarget/testsubtarget", "x86/64") r.sadd("targets-21.02", "testtarget/testsubtarget") + r.hset("mapping-abi", mapping={"test1-1": "test1"}) yield r @@ -52,8 +53,8 @@ def app(redis): "STORE_PATH": test_path + "/store", "TESTING": True, "UPSTREAM_URL": "http://localhost:8001", - "BRANCHES": [ - { + "BRANCHES": { + "SNAPSHOT": { "name": "SNAPSHOT", "enabled": True, "snapshot": True, @@ -69,7 +70,7 @@ def app(redis): "x86/64": "x86_64", }, }, - { + "21.02": { "name": "21.02", "enabled": True, "snapshot": True, @@ -82,7 +83,7 @@ def app(redis): "updates": "rc", "targets": {"testtarget/testsubtarget": "testarch"}, }, - { + "19.07": { "name": "19.07", "enabled": True, "versions": ["19.07.7", "19.07.6"], @@ -94,7 +95,7 @@ def app(redis): "updates": "stable", "targets": {"testtarget/testsubtarget": "testarch"}, }, - ], + }, } ) diff --git a/tests/test_api.py b/tests/test_api.py index 605e36fd..b1c3f8a3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,10 +1,6 @@ def test_api_version(client, app): response = client.get("/api/branches") - branches = {} - for branch in app.config["BRANCHES"]: - if branch["enabled"]: - branches[branch["name"]] = branch - assert response.json == branches + assert response.status == "302 FOUND" def test_api_build(client): @@ -19,15 +15,12 @@ def test_api_build(client): ) assert response.status == "202 ACCEPTED" assert response.json.get("status") == "queued" - assert response.json.get("request_hash") == "3128aff5c6db" + assert response.json.get("request_hash") == "e360f833a191" def test_api_latest_default(client): response = client.get("/api/latest") - assert "19.07.7" in response.json["latest"] - assert "21.02.0-rc1" in response.json["latest"] - assert "SNAPSHOT" in response.json["latest"] - assert response.status == "200 OK" + assert response.status == "302 FOUND" def test_api_build_mapping(client): @@ -42,11 +35,44 @@ def test_api_build_mapping(client): ) assert response.status == "202 ACCEPTED" assert response.json.get("status") == "queued" - assert response.json.get("request_hash") == "d0318d0bba8d" + assert response.json.get("request_hash") == "19bb42f198c9" + + +def test_api_build_mapping_abi(client): + response = client.post( + "/api/build", + json=dict( + version="SNAPSHOT", + target="testtarget/testsubtarget", + profile="testvendor,testprofile", + packages=["test1-1", "test2"], + ), + ) + assert response.status == "202 ACCEPTED" + assert response.json.get("status") == "queued" + assert response.json.get("request_hash") == "7d099fb07fb3" + + +def test_api_build_bad_target(client): + response = client.post( + "/api/build", + json=dict( + version="SNAPSHOT", + target="testtarget/testsubtargetbad", + profile="testvendor,testprofile", + packages=["test1", "test2"], + ), + ) + assert response.status == "400 BAD REQUEST" + assert ( + response.json.get("message") + == "Unsupported target: testtarget/testsubtargetbad" + ) + assert response.json.get("status") == "bad_target" def test_api_build_get(client): - client.post( + response = client.post( "/api/build", json=dict( version="SNAPSHOT", @@ -55,10 +81,42 @@ def test_api_build_get(client): packages=["test1", "test2"], ), ) - response = client.get("/api/build/3128aff5c6db") + assert response.json["request_hash"] == "e360f833a191" + response = client.get("/api/build/e360f833a191") assert response.status == "202 ACCEPTED" assert response.json.get("status") == "queued" - assert response.json.get("request_hash") == "3128aff5c6db" + assert response.json.get("request_hash") == "e360f833a191" + + +def test_api_build_packages_versions(client): + response = client.post( + "/api/build", + json=dict( + version="SNAPSHOT", + target="testtarget/testsubtarget", + profile="testprofile", + packages_versions={"test1": "1.0", "test2": "2.0"}, + ), + ) + assert response.json["request_hash"] == "552b9e328888" + response = client.get("/api/build/552b9e328888") + assert response.status == "202 ACCEPTED" + assert response.json.get("status") == "queued" + assert response.json.get("request_hash") == "552b9e328888" + + +def test_api_build_packages_duplicate(client): + response = client.post( + "/api/build", + json=dict( + version="SNAPSHOT", + target="testtarget/testsubtarget", + profile="testprofile", + packages=["test1", "test2"], + packages_versions={"test1": "1.0", "test2": "2.0"}, + ), + ) + assert response.status == "400 BAD REQUEST" def test_api_build_get_not_found(client): @@ -83,7 +141,7 @@ def test_api_build_empty_packages_list(client): ) assert response.status == "202 ACCEPTED" assert response.json.get("status") == "queued" - assert response.json.get("request_hash") == "c6022275d623" + assert response.json.get("request_hash") == "66cb932c37a4" def test_api_build_withouth_packages_list(client): @@ -97,7 +155,7 @@ def test_api_build_withouth_packages_list(client): ) assert response.status == "202 ACCEPTED" assert response.json.get("status") == "queued" - assert response.json.get("request_hash") == "c6022275d623" + assert response.json.get("request_hash") == "66cb932c37a4" def test_api_build_prerelease_snapshot(client): @@ -162,7 +220,7 @@ def test_api_build_x86(client): assert response.status == "202 ACCEPTED" assert response.json.get("status") == "queued" - assert response.json.get("request_hash") == "a30a7dc8e19b" + assert response.json.get("request_hash") == "1fda145d439f" def test_api_build_needed(client): diff --git a/tests/test_common.py b/tests/test_common.py index ef7e9af9..19a20f4b 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -32,7 +32,7 @@ def test_get_request_hash(): "package_hash": get_packages_hash(["test"]), } - assert get_request_hash(request) == "e8dc1720525b" + assert get_request_hash(request) == "c753cbe6356a" def test_get_request_hash_diff_packages(): @@ -44,7 +44,12 @@ def test_get_request_hash_diff_packages(): "diff_packages": True, } - assert get_request_hash(request) == "75967363f440" + assert get_request_hash(request) == "a7ff5158d58b" + + +def test_fingerprint_pubkey_usign(): + pub_key = "RWSrHfFmlHslUcLbXFIRp+eEikWF9z1N77IJiX5Bt/nJd1a/x+L+SU89" + assert fingerprint_pubkey_usign(pub_key) == "ab1df166947b2551" def test_verify_usign(): diff --git a/tests/test_factory.py b/tests/test_factory.py index 8ae67a65..9e206169 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -11,3 +11,23 @@ def test_pathlib(app): def test_other(app): assert app.config["UPSTREAM_URL"] == "http://localhost:8001" + + +def test_json_path_latest(client): + response = client.get("/json/latest.json") + assert "19.07.7" in response.json["latest"] + assert "21.02.0-rc1" in response.json["latest"] + assert "SNAPSHOT" in response.json["latest"] + assert response.status == "200 OK" + + +def test_json_path_branches(client): + response = client.get("/json/branches.json") + assert "19.07" in response.json.keys() + assert "SNAPSHOT" in response.json.keys() + assert response.status == "200 OK" + + +def test_json_store(client): + response = client.get("/store/") + assert response.status == "404 NOT FOUND" diff --git a/tests/test_janitor.py b/tests/test_janitor.py index ee0a679e..4fca5fa7 100644 --- a/tests/test_janitor.py +++ b/tests/test_janitor.py @@ -26,7 +26,7 @@ def upstream(httpserver): def test_update_branch(app, upstream): with app.app_context(): - update_branch(app.config["BRANCHES"][0]) + update_branch(app.config["BRANCHES"]["SNAPSHOT"]) assert (app.config["JSON_PATH"] / "snapshots/overview.json").is_file() @@ -47,7 +47,7 @@ def test_parse_packages_file_bad(app, upstream): def test_get_packages_target_base(app, upstream): - branch = app.config["BRANCHES"][0] + branch = app.config["BRANCHES"]["SNAPSHOT"] version = "snapshots" target = "testtarget/testsubtarget" with app.app_context(): @@ -56,7 +56,7 @@ def test_get_packages_target_base(app, upstream): def test_update_target_packages(app, upstream): - branch = app.config["BRANCHES"][0] + branch = app.config["BRANCHES"]["SNAPSHOT"] version = "snapshots" target = "testtarget/testsubtarget" with app.app_context(): @@ -68,7 +68,7 @@ def test_update_target_packages(app, upstream): def test_update_arch_packages(app, upstream): - branch = app.config["BRANCHES"][0] + branch = app.config["BRANCHES"]["SNAPSHOT"] arch = "testarch" with app.app_context(): packages = update_arch_packages(branch, arch)