api: add /update/<version>/<target> call

this call can be called by a remote process to trigger an update of a
specific version/target combination, instead of requestion *everything*
all the time. This should be a massive performance improvement and keep
things better in sync.

If you enable UPDATE_TOKEN, the janitor process won't run every x
minutes anymore.

For openwrt.org, this can be called via buildbots, for other projects
this could be called via webhooks from just about any CI.

Signed-off-by: Paul Spooren <[email protected]>

asu/api.py

@@ -2,7 +2,14 @@
 from rq import Connection, Queue
 
 from asu.build import build
-from asu.common import get_redis_client, get_request_hash, remove_prefix
+from asu.common import (
+    get_branch,
+    get_redis_client,
+    get_request_hash,
+    remove_prefix,
+    update_profiles,
+    update_targets,
+)
 
 bp = Blueprint("api", __name__, url_prefix="/api")
 
@@ -89,20 +96,17 @@ def validate_request(req):
             400,
         )
 
-    if req["version"].endswith("-SNAPSHOT"):
-        # e.g. 21.02-snapshot
-        req["branch"] = req["version"].rsplit("-", maxsplit=1)[0]
-    else:
-        # e.g. snapshot, 21.02.0-rc1 or 19.07.7
-        req["branch"] = req["version"].rsplit(".", maxsplit=1)[0]
+    req["branch"] = get_branch(req["version"])
+
+    r = redis_client()
 
-    if req["branch"] not in current_app.config["BRANCHES"].keys():
+    if not r.sismember("branches", req["branch"]):
         return (
             {"detail": f"Unsupported branch: {req['version']}", "status": 400},
             400,
         )
 
-    if req["version"] not in current_app.config["BRANCHES"][req["branch"]]["versions"]:
+    if not r.sismember(f"versions:{req['branch']}", req["version"]):
         return (
             {"detail": f"Unsupported version: {req['version']}", "status": 400},
             400,
@@ -115,22 +119,10 @@ def validate_request(req):
         )
     )
 
-    r = redis_client()
-
     current_app.logger.debug("Profile before mapping " + req["profile"])
 
-    if not r.sismember(
-        f"targets:{req['branch']}",
-        req["target"],
-    ):
-        return (
-            {"detail": f"Unsupported target: {req['target']}", "status": 400},
-            400,
-        )
-
-    req["arch"] = (
-        r.hget(f"architecture:{req['branch']}", req["target"]) or b""
-    ).decode()
+    if not r.hexists(f"targets:{req['branch']}", req["target"]):
+        return ({"detail": f"Unsupported target: {req['target']}", "status": 400}, 400)
 
     if req["target"] in [
         "x86/64",
@@ -203,6 +195,38 @@ def return_job_v1(job):
     return response, response["status"], headers
 
 
+def api_v1_update(version, target, subtarget):
+    print(f"update_token {current_app.config['update_token']}")
+    print(f"X-Update-Token {request.headers.get('X-Update-Token')}")
+    if current_app.config["UPDATE_TOKEN"] != "" and current_app.config[
+        "UPDATE_TOKEN"
+    ] == request.headers.get("X-Update-Token"):
+        config = {
+            "JSON_PATH": current_app.config["PUBLIC_PATH"] / "json/v1",
+            "BRANCHES": current_app.config["BRANCHES"],
+            "UPSTREAM_URL": current_app.config["UPSTREAM_URL"],
+            "ALLOW_DEFAULTS": current_app.config["ALLOW_DEFAULTS"],
+            "REPOSITORY_ALLOW_LIST": current_app.config["REPOSITORY_ALLOW_LIST"],
+            "REDIS_URL": current_app.config["REDIS_URL"],
+        }
+        get_queue().enqueue(
+            update_targets,
+            config=config,
+            branch=current_app.config["BRANCHES"][get_branch(version)],
+            job_timeout="10m",
+        )
+        get_queue().enqueue(
+            update_profiles,
+            config=config,
+            version=version,
+            target=f"{target}/{subtarget}",
+            job_timeout="10m",
+        )
+        return None, 204
+    else:
+        return {"status": 403, "detail": "Forbidden"}, 403
+
+
 # legacy offering /api/overview
 def api_v1_build_get(request_hash):
     job = get_queue().fetch_job(request_hash)

asu/asu.py

@@ -123,7 +123,7 @@ def overview():
         validate_responses=app.config["TESTING"],
     )
 
-    if not app.config["TESTING"]:
+    if not app.config["TESTING"] and app.config["UPDATE_TOKEN"] == "":
         queue = Queue(
             connection=redis_client,
             is_async=app.config["ASYNC_QUEUE"],

asu/branches.yml

@@ -7,17 +7,7 @@ branches:
     branch_off_rev: 23069
     name: "23.05"
     path: releases/{version}
-    path_packages: releases/packages-{branch}
-    pubkey: RWS1BD5w+adc3j2Hqg9+b66CvLR7NlHbsj7wjNVj0XGt/othDgIAOJS+
-    release_date: "2023-10-14"
-    repos:
-      - base
-      - packages
-      - luci
-      - routing
-      - telephony
     snapshot: false
-    updates: features
     versions:
       - 23.05.2
       - 23.05.0
@@ -50,20 +40,14 @@ branches:
     branch_off_rev: 19160
     name: "22.03"
     path: releases/{version}
-    path_packages: releases/packages-{branch}
-    pubkey: RWRNAX5vHtXWFmt+n5di7XX8rTu0w+c8X7Ihv4oCyD6tzsUwmH0A6kO0
-    release_date: "2022-10-17"
-    repos:
-      - base
-      - packages
-      - luci
-      - routing
-      - telephony
     snapshot: false
-    updates: features
     versions:
       - 22.03.5
       - 22.03.4
+      - 22.03.3
+      - 22.03.2
+      - 22.03.1
+      - 22.03.0
       - 22.03-SNAPSHOT
     package_changes:
       - source: kmod-nft-nat6
@@ -81,20 +65,16 @@ branches:
     branch_off_rev: 15812
     name: "21.02"
     path: releases/{version}
-    path_packages: releases/packages-{branch}
-    pubkey: RWQviwuY4IMGvwLfs6842A0m4EZU1IjczTxKMSk3BQP8DAQLHBwdQiaU
-    release_date: "2022-10-17"
-    repos:
-      - base
-      - packages
-      - luci
-      - routing
-      - telephony
     snapshot: false
-    updates: features
     versions:
       - 21.02.7
       - 21.02.6
+      - 21.02.5
+      - 21.02.4
+      - 21.02.3
+      - 21.02.2
+      - 21.02.1
+      - 21.02.0
       - 21.02-SNAPSHOT
 
   SNAPSHOT:
@@ -104,16 +84,7 @@ branches:
     git_branch: master
     name: SNAPSHOT
     path: snapshots
-    path_packages: snapshots/packages
-    pubkey: RWS1BD5w+adc3j2Hqg9+b66CvLR7NlHbsj7wjNVj0XGt/othDgIAOJS+
-    repos:
-      - base
-      - packages
-      - luci
-      - routing
-      - telephony
     snapshot: true
-    updates: dev
     versions:
       - SNAPSHOT
     package_changes:

asu/common.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import struct
+from datetime import datetime
 from os import getenv
 from pathlib import Path
 from re import match
@@ -16,6 +17,15 @@
 import redis
 
 
+def get_branch(version):
+    if version.endswith("-SNAPSHOT"):
+        # e.g. 21.02-snapshot
+        return version.rsplit("-", maxsplit=1)[0]
+    else:
+        # e.g. snapshot, 21.02.0-rc1 or 19.07.7
+        return version.rsplit(".", maxsplit=1)[0]
+
+
 def get_redis_client(config):
     return redis.from_url(getenv("REDIS_URL") or config["REDIS_URL"])
 
@@ -351,3 +361,119 @@ def check_manifest(manifest, packages_versions):
                 f"Impossible package selection: {package} version not as requested: "
                 f"{version} vs. {manifest[package]}"
             )
+
+
+def get_targets_upstream(config: dict, version: str) -> list:
+    """Return list of targets for a specific version
+
+    Args:
+        config (dict): Configuration
+        version (str): Version within branch
+
+    Returns:
+        list: List of targets
+    """
+    branch = config["BRANCHES"][get_branch(version)]
+    version_path = branch["path"].format(version=version)
+
+    req = requests.get(config["UPSTREAM_URL"] + f"/{version_path}/.targets.json")
+
+    if req.status_code != 200:
+        logging.warning("Couldn't download %s", targets_url)
+        return []
+
+    return list(req.json().keys())
+
+
+def update_targets(config: dict, branch: dict) -> list:
+    version_path = branch["path"].format(version=branch["versions"][0])
+
+    targets = requests.get(
+        config["UPSTREAM_URL"] + f"/{version_path}/.targets.json"
+    ).json()
+
+    logging.warning(f"{branch['name']}: Found {len(targets)} targets")
+    pipeline = get_redis_client(config).pipeline(True)
+    pipeline.delete(f"targets:{branch['name']}")
+    pipeline.hset(f"targets:{branch['name']}", mapping=targets)
+    pipeline.execute()
+
+    return targets
+
+
+def update_profiles(config, version: str, target: str) -> str:
+    """Update available profiles of a specific version
+
+    Args:
+        config (dict): Configuration
+        version(str): Version within branch
+        target(str): Target within version
+    """
+    branch = config["BRANCHES"][get_branch(version)]
+    version_path = branch["path"].format(version=version)
+    logging.info(f"{version}/{target}: Update profiles")
+
+    r = get_redis_client(config)
+
+    r.sadd(f"branches", branch["name"])
+    r.sadd(f"versions:{branch['name']}", version)
+
+    profiles_url = (
+        config["UPSTREAM_URL"] + f"/{version_path}/targets/{target}/profiles.json"
+    )
+
+    req = requests.get(profiles_url)
+
+    if req.status_code != 200:
+        logging.warning("Couldn't download %s", profiles_url)
+        return False
+
+    metadata = req.json()
+    profiles = metadata.pop("profiles", {})
+
+    r.set(
+        f"revision:{version}:{target}",
+        metadata["version_code"],
+    )
+    logging.info(f"{version}/{target}: Found revision {metadata['version_code']}")
+    logging.info(f"{version}/{target}: Found {len(profiles)} profiles")
+
+    pipeline = r.pipeline(True)
+    pipeline.delete(f"profiles:{branch['name']}:{version}:{target}")
+
+    for profile, data in profiles.items():
+        for supported in data.get("supported_devices", []):
+            if not r.hexists(f"mapping:{branch['name']}:{version}:{target}", supported):
+                logging.info(
+                    f"{version}/{target}: Add profile mapping {supported} -> {profile}"
+                )
+                r.hset(
+                    f"mapping:{branch['name']}:{version}:{target}", supported, profile
+                )
+
+        pipeline.sadd(f"profiles:{branch['name']}:{version}:{target}", profile)
+
+        profile_path = (
+            config["JSON_PATH"] / version_path / "targets" / target / profile
+        ).with_suffix(".json")
+
+        profile_path.parent.mkdir(exist_ok=True, parents=True)
+        profile_path.write_text(
+            json.dumps(
+                {
+                    **metadata,
+                    **data,
+                    "id": profile,
+                    "build_at": datetime.utcfromtimestamp(
+                        int(metadata.get("source_date_epoch", 0))
+                    ).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                },
+                sort_keys=True,
+                separators=(",", ":"),
+            )
+        )
+
+        data["target"] = target
+
+    pipeline.execute()
+    return profiles