From 5f28e25ba093abcad0d6cd41998592a5040fb64a Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Tue, 2 Apr 2024 01:53:41 +0200 Subject: [PATCH] build: allow uploading to s3 This allows remote workers and unified storage. Signed-off-by: Paul Spooren --- asu/api.py | 4 +++ asu/asu.py | 13 +++++-- asu/build.py | 30 +++++++++++++++- poetry.lock | 93 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 5 files changed, 137 insertions(+), 4 deletions(-) diff --git a/asu/api.py b/asu/api.py index fac98e71..ff90da23 100644 --- a/asu/api.py +++ b/asu/api.py @@ -258,6 +258,10 @@ def api_v1_build_post(): req["repository_allow_list"] = current_app.config["REPOSITORY_ALLOW_LIST"] req["request_hash"] = request_hash req["base_container"] = current_app.config["BASE_CONTAINER"] + req["s3_bucket"] = current_app.config["S3_BUCKET"] + req["s3_access_key"] = current_app.config["S3_ACCESS_KEY"] + req["s3_secret_key"] = current_app.config["S3_SECRET_KEY"] + req["s3_server"] = current_app.config["S3_SERVER"] job = get_queue().enqueue( build, diff --git a/asu/asu.py b/asu/asu.py index 569e82f8..cfd05116 100644 --- a/asu/asu.py +++ b/asu/asu.py @@ -2,7 +2,7 @@ from pathlib import Path import connexion -from flask import Flask, render_template, send_from_directory +from flask import Flask, redirect, render_template, send_from_directory from pkg_resources import resource_filename from prometheus_client import CollectorRegistry, make_wsgi_app from rq import Queue @@ -38,6 +38,10 @@ def create_app(test_config: dict = None) -> Flask: MAX_CUSTOM_ROOTFS_SIZE_MB=100, REPOSITORY_ALLOW_LIST=[], BASE_CONTAINER="ghcr.io/openwrt/imagebuilder", + S3_BUCKET=None, + S3_ACCESS_KEY=None, + S3_SECRET_KEY=None, + S3_SERVER=None, ) if not test_config: @@ -80,7 +84,12 @@ def json_path(path="index.html"): @app.route("/store/") @app.route("/store/") def store_path(path="index.html"): - return send_from_directory(app.config["PUBLIC_PATH"] / "public", path) + if app.config.get("S3_SERVER"): + return redirect( + f"{app.config['S3_SERVER']}/{app.config['S3_BUCKET']}/{path}" + ) + else: + return send_from_directory(app.config["PUBLIC_PATH"] / "public", path) from . import janitor diff --git a/asu/build.py b/asu/build.py index 5ae2a0bd..9569e398 100644 --- a/asu/build.py +++ b/asu/build.py @@ -1,10 +1,13 @@ import json import logging import re +import tempfile from datetime import datetime from os import getenv from pathlib import Path +from shutil import rmtree +import boto3 from podman import PodmanClient from rq import get_current_job @@ -31,7 +34,13 @@ def build(req: dict, job=None): Args: request (dict): Contains all properties of requested image """ - store_path = Path(req["public_path"]) / "store" + if req["s3_server"]: + temp_path = tempfile.TemporaryDirectory() + store_path = Path(temp_path.name) + else: + temp_path = None + store_path = Path(req["public_path"]) / "store" + store_path.mkdir(parents=True, exist_ok=True) log.debug(f"Store path: {store_path}") @@ -320,6 +329,25 @@ def build(req: dict, job=None): log.debug("JSON content %s", json_content) + # Upload to S3 + s3 = boto3.client( + "s3", + endpoint_url=req["s3_server"], + aws_access_key_id=req["s3_access_key"], + aws_secret_access_key=req["s3_secret_key"], + ) + for image in json_content["images"]: + print(f"Uploading {image['name']} to S3") + s3.upload_file( + str(store_path / bin_dir / image["name"]), + req["s3_bucket"], + f"{req['request_hash']}/{image['name']}", + ) + + if temp_path: + temp_path.cleanup() + rmtree(store_path, ignore_errors=True) + # Increment stats job.connection.hincrby( "stats:builds", diff --git a/poetry.lock b/poetry.lock index 7eaf9cbe..65b2d57a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -30,6 +30,44 @@ tests = ["attrs[tests-no-zope]", "zope-interface"] tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +[[package]] +name = "boto3" +version = "1.34.75" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.34.75-py3-none-any.whl", hash = "sha256:ba5d2104bba4370766036d64ad9021eb6289d154265852a2a821ec6a5e816faa"}, + {file = "boto3-1.34.75.tar.gz", hash = "sha256:eaec72fda124084105a31bcd67eafa1355b34df6da70cadae0c0f262d8a4294f"}, +] + +[package.dependencies] +botocore = ">=1.34.75,<1.35.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.34.75" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.34.75-py3-none-any.whl", hash = "sha256:1d7f683d99eba65076dfb9af3b42fa967c64f11111d9699b65757420902aa002"}, + {file = "botocore-1.34.75.tar.gz", hash = "sha256:06113ee2587e6160211a6bd797e135efa6aa21b5bde97bf455c02f7dff40203c"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.19.19)"] + [[package]] name = "certifi" version = "2024.2.2" @@ -476,6 +514,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jsonschema" version = "4.21.1" @@ -710,6 +759,20 @@ files = [ [package.dependencies] Werkzeug = ">=2.0.0" +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "pyxdg" version = "0.28" @@ -983,6 +1046,23 @@ files = [ {file = "ruff-0.3.5.tar.gz", hash = "sha256:a067daaeb1dc2baf9b82a32dae67d154d95212080c80435eb052d95da647763d"}, ] +[[package]] +name = "s3transfer" +version = "0.10.1" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"}, + {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "setuptools" version = "69.2.0" @@ -999,6 +1079,17 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1061,4 +1152,4 @@ watchdog = ["watchdog"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "d28acb3fb5775938750dd6a0e392ae6504db27d10bf8551f67969d73b69454ab" +content-hash = "64f59a9367cfc88c9bf050aac7fe032ba24dcf7019d5c1a7258cb4f232caf8f9" diff --git a/pyproject.toml b/pyproject.toml index 10ab5f8c..66296814 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ prometheus-client = ">=0.13.1,<0.21.0" gunicorn = ">=20.1,<22.0" podman = ">=4.4.1,<6.0.0" setuptools = "^69.0.3" +boto3 = "^1.34.75" [tool.poetry.group.dev.dependencies] ruff = ">=0.2.2,<0.4.0"