Skip to content

Commit

Permalink
working version
Browse files Browse the repository at this point in the history 
Signed-off-by: Paul Spooren <[email protected]>
  • Loading branch information
@aparcar
aparcar committed Feb 16, 2024
1 parent 9220421 commit cfe187b
Show file tree
Hide file tree
Showing 5 changed files with 103 additions and 98 deletions.
18 changes: 4 additions & 14 deletions asu/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
get_redis_client,
get_request_hash,
remove_prefix,
update_profiles,
update_targets,
update,
)

bp = Blueprint("api", __name__, url_prefix="/api")
Expand Down Expand Up @@ -196,11 +195,7 @@ def return_job_v1(job):


def api_v1_update(version, target, subtarget):
print(f"update_token {current_app.config['update_token']}")
print(f"X-Update-Token {request.headers.get('X-Update-Token')}")
if current_app.config["UPDATE_TOKEN"] != "" and current_app.config[
"UPDATE_TOKEN"
] == request.headers.get("X-Update-Token"):
if current_app.config.get("UPDATE_TOKEN") == request.headers.get("X-Update-Token"):
config = {
"JSON_PATH": current_app.config["PUBLIC_PATH"] / "json/v1",
"BRANCHES": current_app.config["BRANCHES"],
Expand All @@ -210,18 +205,13 @@ def api_v1_update(version, target, subtarget):
"REDIS_URL": current_app.config["REDIS_URL"],
}
get_queue().enqueue(
update_targets,
config=config,
branch=current_app.config["BRANCHES"][get_branch(version)],
job_timeout="10m",
)
get_queue().enqueue(
update_profiles,
update,
config=config,
version=version,
target=f"{target}/{subtarget}",
job_timeout="10m",
)

return None, 204
else:
return {"status": 403, "detail": "Forbidden"}, 403
Expand Down
8 changes: 4 additions & 4 deletions asu/asu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from asu import __version__
from asu.common import get_redis_client
from asu.janitor import update
from asu.janitor import update_branches


def create_app(test_config: dict = None) -> Flask:
Expand Down Expand Up @@ -123,13 +123,13 @@ def overview():
validate_responses=app.config["TESTING"],
)

if not app.config["TESTING"] and app.config["UPDATE_TOKEN"] == "":
if not app.config.get("UPDATE_TOKEN"):
queue = Queue(
connection=redis_client,
is_async=app.config["ASYNC_QUEUE"],
)
queue.enqueue(
update,
update_branches,
{
"JSON_PATH": app.config["PUBLIC_PATH"] / "json/v1",
"BRANCHES": app.config["BRANCHES"],
Expand All @@ -138,7 +138,7 @@ def overview():
"REPOSITORY_ALLOW_LIST": app.config["REPOSITORY_ALLOW_LIST"],
"REDIS_URL": app.config["REDIS_URL"],
},
job_timeout="10m",
job_timeout="15m",
)

return app
76 changes: 71 additions & 5 deletions asu/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

import redis

from . import __version__


def get_branch(version):
if version.endswith("-SNAPSHOT"):
Expand Down Expand Up @@ -381,14 +383,15 @@ def get_targets_upstream(config: dict, version: str) -> list:
return list(req.json().keys())


def update_targets(config: dict, branch: dict) -> list:
def update_targets(config: dict, version) -> list:
branch = config["BRANCHES"][get_branch(version)]
version_path = branch["path"].format(version=branch["versions"][0])

targets = requests.get(
config["UPSTREAM_URL"] + f"/{version_path}/.targets.json"
).json()

logging.warning(f"{branch['name']}: Found {len(targets)} targets")
logging.info(f"{branch['name']}: Found {len(targets)} targets")
pipeline = get_redis_client(config).pipeline(True)
pipeline.delete(f"targets:{branch['name']}")
pipeline.hset(f"targets:{branch['name']}", mapping=targets)
Expand All @@ -407,7 +410,7 @@ def update_profiles(config, version: str, target: str) -> str:
"""
branch = config["BRANCHES"][get_branch(version)]
version_path = branch["path"].format(version=version)
logging.info(f"{version}/{target}: Update profiles")
logging.debug(f"{version}/{target}: Update profiles")

r = get_redis_client(config)

Expand All @@ -426,13 +429,13 @@ def update_profiles(config, version: str, target: str) -> str:

metadata = req.json()
profiles = metadata.pop("profiles", {})
logging.info(f"{version}/{target}: Found {len(profiles)} profiles")

r.set(
f"revision:{version}:{target}",
metadata["version_code"],
)
logging.info(f"{version}/{target}: Found revision {metadata['version_code']}")
logging.info(f"{version}/{target}: Found {len(profiles)} profiles")

pipeline = r.pipeline(True)
pipeline.delete(f"profiles:{branch['name']}:{version}:{target}")
Expand Down Expand Up @@ -472,4 +475,67 @@ def update_profiles(config, version: str, target: str) -> str:
data["target"] = target

pipeline.execute()
return profiles


def update_meta_json(config):
latest = list(
map(
lambda b: b["versions"][0],
filter(
lambda b: b.get("enabled"),
config["BRANCHES"].values(),
),
)
)

branches = dict(
map(
lambda b: (
b["name"],
{
**b,
"targets": dict(
map(
lambda a: (a[0].decode(), a[1].decode()),
get_redis_client(config)
.hgetall(f"architecture:{b['name']}")
.items(),
)
),
},
),
filter(
lambda b: b.get("enabled"),
config["BRANCHES"].values(),
),
)
)

config["OVERVIEW"] = {
"latest": latest,
"branches": branches,
"server": {
"version": __version__,
"contact": "[email protected]",
"allow_defaults": config["ALLOW_DEFAULTS"],
"repository_allow_list": config["REPOSITORY_ALLOW_LIST"],
},
}

config["JSON_PATH"].mkdir(exist_ok=True, parents=True)

(config["JSON_PATH"] / "overview.json").write_text(
json.dumps(config["OVERVIEW"], indent=2, sort_keys=False, default=str)
)

(config["JSON_PATH"] / "branches.json").write_text(
json.dumps(list(branches.values()), indent=2, sort_keys=False, default=str)
)

(config["JSON_PATH"] / "latest.json").write_text(json.dumps({"latest": latest}))


def update(config, version: str, target: str):
update_targets(config, version)
update_profiles(config, version, target)
update_meta_json(config)
95 changes: 20 additions & 75 deletions asu/janitor.py
View file
Original file line number Diff line number Diff line change
@@ -1,108 +1,53 @@
import json
import logging
from datetime import timedelta

import requests
from flask import Blueprint
from rq import Queue

from asu import __version__
from asu.common import get_redis_client, update_profiles, update_targets
from asu.common import (
get_redis_client,
update_meta_json,
update_profiles,
update_targets,
)

bp = Blueprint("janitor", __name__)

session = requests.Session()


def update_branch(config, branch):
logging.warning(f"Update {branch['name']}")
logging.debug(f"Update {branch['name']}")

targets = list(update_targets(config, branch).keys())
targets = list(update_targets(config, branch["versions"][0]).keys())

logging.warning(f"Targets: {targets}")
logging.debug(f"Targets: {targets}")

if not targets:
logging.warning(f"No targets found for {branch['name']}")
return

for version in branch["versions"]:
logging.warning(f"Update {branch['name']}/{version}")
logging.info(f"Update {branch['name']}/{version}")

for target in targets:
logging.warning(f"Update {branch['name']}/{version}/{target}")
logging.info(f"Update {branch['name']}/{version}/{target}")
update_profiles(config, version, target)


def update_meta_json(config):
latest = list(
map(
lambda b: b["versions"][0],
filter(
lambda b: b.get("enabled"),
config["BRANCHES"].values(),
),
)
)

branches = dict(
map(
lambda b: (
b["name"],
{
**b,
"targets": dict(
map(
lambda a: (a[0].decode(), a[1].decode()),
get_redis_client(config)
.hgetall(f"architecture:{b['name']}")
.items(),
)
),
},
),
filter(
lambda b: b.get("enabled"),
config["BRANCHES"].values(),
),
)
)

config["OVERVIEW"] = {
"latest": latest,
"branches": branches,
"server": {
"version": __version__,
"contact": "[email protected]",
"allow_defaults": config["ALLOW_DEFAULTS"],
"repository_allow_list": config["REPOSITORY_ALLOW_LIST"],
},
}

config["JSON_PATH"].mkdir(exist_ok=True, parents=True)

(config["JSON_PATH"] / "overview.json").write_text(
json.dumps(config["OVERVIEW"], indent=2, sort_keys=False, default=str)
)

(config["JSON_PATH"] / "branches.json").write_text(
json.dumps(list(branches.values()), indent=2, sort_keys=False, default=str)
)

(config["JSON_PATH"] / "latest.json").write_text(json.dumps({"latest": latest}))


def update(config):
def update_branches(config):
"""Update the data required to run the server
For this all available profiles for all enabled versions is
downloaded and stored in the Redis database.
"""

if not config["BRANCHES"]:
logging.error("No BRANCHES defined in config, nothing to do, exiting")
return

try:
if not config["BRANCHES"]:
logging.error("No BRANCHES defined in config, nothing to do, exiting")
return
for branch in config["BRANCHES"].values():
if not branch.get("enabled"):
logging.info(f"{branch['name']}: Skip disabled branch")
Expand All @@ -112,12 +57,12 @@ def update(config):
update_branch(config, branch)

update_meta_json(config)
except Exception:
logging.exception("Failed to update")
except Exception as e:
logging.exception(e)

Queue(connection=get_redis_client(config)).enqueue_in(
timedelta(minutes=10),
update,
timedelta(hours=1),
update_branches,
config,
job_timeout="10m",
job_timeout="15m",
)
4 changes: 4 additions & 0 deletions misc/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,7 @@

# where to downlaod the images from
# UPSTREAM_PATH = "https://downloads.openwrt.org"

# token used to trigger an update of targets
# if an empty string is set, the update happens periodically
UPDATE_TOKEN=""

0 comments on commit cfe187b

Please sign in to comment.