diff --git a/asu/api.py b/asu/api.py index a761b692..503730f9 100644 --- a/asu/api.py +++ b/asu/api.py @@ -7,8 +7,7 @@ get_redis_client, get_request_hash, remove_prefix, - update_profiles, - update_targets, + update, ) bp = Blueprint("api", __name__, url_prefix="/api") @@ -196,11 +195,7 @@ def return_job_v1(job): def api_v1_update(version, target, subtarget): - print(f"update_token {current_app.config['update_token']}") - print(f"X-Update-Token {request.headers.get('X-Update-Token')}") - if current_app.config["UPDATE_TOKEN"] != "" and current_app.config[ - "UPDATE_TOKEN" - ] == request.headers.get("X-Update-Token"): + if current_app.config.get("UPDATE_TOKEN") == request.headers.get("X-Update-Token"): config = { "JSON_PATH": current_app.config["PUBLIC_PATH"] / "json/v1", "BRANCHES": current_app.config["BRANCHES"], @@ -210,18 +205,13 @@ def api_v1_update(version, target, subtarget): "REDIS_URL": current_app.config["REDIS_URL"], } get_queue().enqueue( - update_targets, - config=config, - branch=current_app.config["BRANCHES"][get_branch(version)], - job_timeout="10m", - ) - get_queue().enqueue( - update_profiles, + update, config=config, version=version, target=f"{target}/{subtarget}", job_timeout="10m", ) + return None, 204 else: return {"status": 403, "detail": "Forbidden"}, 403 diff --git a/asu/asu.py b/asu/asu.py index b213ce9f..a656e75d 100644 --- a/asu/asu.py +++ b/asu/asu.py @@ -11,7 +11,7 @@ from asu import __version__ from asu.common import get_redis_client -from asu.janitor import update +from asu.janitor import update_branches def create_app(test_config: dict = None) -> Flask: @@ -123,13 +123,13 @@ def overview(): validate_responses=app.config["TESTING"], ) - if not app.config["TESTING"] and app.config["UPDATE_TOKEN"] == "": + if not app.config.get("UPDATE_TOKEN"): queue = Queue( connection=redis_client, is_async=app.config["ASYNC_QUEUE"], ) queue.enqueue( - update, + update_branches, { "JSON_PATH": app.config["PUBLIC_PATH"] / "json/v1", "BRANCHES": app.config["BRANCHES"], @@ -138,7 +138,7 @@ def overview(): "REPOSITORY_ALLOW_LIST": app.config["REPOSITORY_ALLOW_LIST"], "REDIS_URL": app.config["REDIS_URL"], }, - job_timeout="10m", + job_timeout="15m", ) return app diff --git a/asu/common.py b/asu/common.py index 3a71dd53..c24485d3 100644 --- a/asu/common.py +++ b/asu/common.py @@ -16,6 +16,8 @@ import redis +from . import __version__ + def get_branch(version): if version.endswith("-SNAPSHOT"): @@ -381,14 +383,15 @@ def get_targets_upstream(config: dict, version: str) -> list: return list(req.json().keys()) -def update_targets(config: dict, branch: dict) -> list: +def update_targets(config: dict, version) -> list: + branch = config["BRANCHES"][get_branch(version)] version_path = branch["path"].format(version=branch["versions"][0]) targets = requests.get( config["UPSTREAM_URL"] + f"/{version_path}/.targets.json" ).json() - logging.warning(f"{branch['name']}: Found {len(targets)} targets") + logging.info(f"{branch['name']}: Found {len(targets)} targets") pipeline = get_redis_client(config).pipeline(True) pipeline.delete(f"targets:{branch['name']}") pipeline.hset(f"targets:{branch['name']}", mapping=targets) @@ -407,7 +410,7 @@ def update_profiles(config, version: str, target: str) -> str: """ branch = config["BRANCHES"][get_branch(version)] version_path = branch["path"].format(version=version) - logging.info(f"{version}/{target}: Update profiles") + logging.debug(f"{version}/{target}: Update profiles") r = get_redis_client(config) @@ -426,13 +429,13 @@ def update_profiles(config, version: str, target: str) -> str: metadata = req.json() profiles = metadata.pop("profiles", {}) + logging.info(f"{version}/{target}: Found {len(profiles)} profiles") r.set( f"revision:{version}:{target}", metadata["version_code"], ) logging.info(f"{version}/{target}: Found revision {metadata['version_code']}") - logging.info(f"{version}/{target}: Found {len(profiles)} profiles") pipeline = r.pipeline(True) pipeline.delete(f"profiles:{branch['name']}:{version}:{target}") @@ -472,4 +475,67 @@ def update_profiles(config, version: str, target: str) -> str: data["target"] = target pipeline.execute() - return profiles + + +def update_meta_json(config): + latest = list( + map( + lambda b: b["versions"][0], + filter( + lambda b: b.get("enabled"), + config["BRANCHES"].values(), + ), + ) + ) + + branches = dict( + map( + lambda b: ( + b["name"], + { + **b, + "targets": dict( + map( + lambda a: (a[0].decode(), a[1].decode()), + get_redis_client(config) + .hgetall(f"architecture:{b['name']}") + .items(), + ) + ), + }, + ), + filter( + lambda b: b.get("enabled"), + config["BRANCHES"].values(), + ), + ) + ) + + config["OVERVIEW"] = { + "latest": latest, + "branches": branches, + "server": { + "version": __version__, + "contact": "mail@aparcar.org", + "allow_defaults": config["ALLOW_DEFAULTS"], + "repository_allow_list": config["REPOSITORY_ALLOW_LIST"], + }, + } + + config["JSON_PATH"].mkdir(exist_ok=True, parents=True) + + (config["JSON_PATH"] / "overview.json").write_text( + json.dumps(config["OVERVIEW"], indent=2, sort_keys=False, default=str) + ) + + (config["JSON_PATH"] / "branches.json").write_text( + json.dumps(list(branches.values()), indent=2, sort_keys=False, default=str) + ) + + (config["JSON_PATH"] / "latest.json").write_text(json.dumps({"latest": latest})) + + +def update(config, version: str, target: str): + update_targets(config, version) + update_profiles(config, version, target) + update_meta_json(config) diff --git a/asu/janitor.py b/asu/janitor.py index 5be84da7..58f028d2 100644 --- a/asu/janitor.py +++ b/asu/janitor.py @@ -1,4 +1,3 @@ -import json import logging from datetime import timedelta @@ -6,8 +5,12 @@ from flask import Blueprint from rq import Queue -from asu import __version__ -from asu.common import get_redis_client, update_profiles, update_targets +from asu.common import ( + get_redis_client, + update_meta_json, + update_profiles, + update_targets, +) bp = Blueprint("janitor", __name__) @@ -15,94 +18,36 @@ def update_branch(config, branch): - logging.warning(f"Update {branch['name']}") + logging.debug(f"Update {branch['name']}") - targets = list(update_targets(config, branch).keys()) + targets = list(update_targets(config, branch["versions"][0]).keys()) - logging.warning(f"Targets: {targets}") + logging.debug(f"Targets: {targets}") if not targets: logging.warning(f"No targets found for {branch['name']}") return for version in branch["versions"]: - logging.warning(f"Update {branch['name']}/{version}") + logging.info(f"Update {branch['name']}/{version}") for target in targets: - logging.warning(f"Update {branch['name']}/{version}/{target}") logging.info(f"Update {branch['name']}/{version}/{target}") update_profiles(config, version, target) -def update_meta_json(config): - latest = list( - map( - lambda b: b["versions"][0], - filter( - lambda b: b.get("enabled"), - config["BRANCHES"].values(), - ), - ) - ) - - branches = dict( - map( - lambda b: ( - b["name"], - { - **b, - "targets": dict( - map( - lambda a: (a[0].decode(), a[1].decode()), - get_redis_client(config) - .hgetall(f"architecture:{b['name']}") - .items(), - ) - ), - }, - ), - filter( - lambda b: b.get("enabled"), - config["BRANCHES"].values(), - ), - ) - ) - - config["OVERVIEW"] = { - "latest": latest, - "branches": branches, - "server": { - "version": __version__, - "contact": "mail@aparcar.org", - "allow_defaults": config["ALLOW_DEFAULTS"], - "repository_allow_list": config["REPOSITORY_ALLOW_LIST"], - }, - } - - config["JSON_PATH"].mkdir(exist_ok=True, parents=True) - - (config["JSON_PATH"] / "overview.json").write_text( - json.dumps(config["OVERVIEW"], indent=2, sort_keys=False, default=str) - ) - - (config["JSON_PATH"] / "branches.json").write_text( - json.dumps(list(branches.values()), indent=2, sort_keys=False, default=str) - ) - - (config["JSON_PATH"] / "latest.json").write_text(json.dumps({"latest": latest})) - - -def update(config): +def update_branches(config): """Update the data required to run the server For this all available profiles for all enabled versions is downloaded and stored in the Redis database. """ + if not config["BRANCHES"]: + logging.error("No BRANCHES defined in config, nothing to do, exiting") + return + try: - if not config["BRANCHES"]: - logging.error("No BRANCHES defined in config, nothing to do, exiting") - return for branch in config["BRANCHES"].values(): if not branch.get("enabled"): logging.info(f"{branch['name']}: Skip disabled branch") @@ -112,12 +57,12 @@ def update(config): update_branch(config, branch) update_meta_json(config) - except Exception: - logging.exception("Failed to update") + except Exception as e: + logging.exception(e) Queue(connection=get_redis_client(config)).enqueue_in( - timedelta(minutes=10), - update, + timedelta(hours=1), + update_branches, config, - job_timeout="10m", + job_timeout="15m", ) diff --git a/misc/config.py b/misc/config.py index 7a861aac..cf66a207 100644 --- a/misc/config.py +++ b/misc/config.py @@ -44,3 +44,7 @@ # where to downlaod the images from # UPSTREAM_PATH = "https://downloads.openwrt.org" + +# token used to trigger an update of targets +# if an empty string is set, the update happens periodically +UPDATE_TOKEN=""