From 963687d05d5c7f4725de40a98f47b1b24d9d1e55 Mon Sep 17 00:00:00 2001 From: Tianyu Chen Date: Thu, 1 Feb 2024 17:38:43 +0800 Subject: [PATCH] init --- .github/workflows/deploy.yaml | 48 +++++++++++++++++++++++++++++++ .gitignore | 2 ++ packages/__init__.py | 0 packages/__main__.py | 35 +++++++++++++++++++++++ packages/download.py | 53 +++++++++++++++++++++++++++++++++++ packages/templates/index.html | 34 ++++++++++++++++++++++ 6 files changed, 172 insertions(+) create mode 100644 .github/workflows/deploy.yaml create mode 100644 .gitignore create mode 100644 packages/__init__.py create mode 100644 packages/__main__.py create mode 100644 packages/download.py create mode 100644 packages/templates/index.html diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..02ca4bf --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,48 @@ +name: Deploy packages to Pages +on: + # Runs on pushes targeting the default branch + push: + branches: ["master"] + # Every 6 hours + schedule: + - cron: '0 0/6 * * *' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Install Dependencies + run: | + sudo apt update && sudo apt -y install python3-jinja2 python3-debian python3-requests + - name: Build Pages + run: | + python -m packages + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + # Upload ./output + path: './output' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..575c2cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +/data/ \ No newline at end of file diff --git a/packages/__init__.py b/packages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/__main__.py b/packages/__main__.py new file mode 100644 index 0000000..6a92930 --- /dev/null +++ b/packages/__main__.py @@ -0,0 +1,35 @@ + +from os import mkdir +from jinja2 import Environment, PackageLoader +from jinja2 import select_autoescape + +from debian.debian_support import version_compare + +from .download import prepare + +if __name__ == '__main__': + + env = Environment( + loader=PackageLoader('packages'), + autoescape=select_autoescape(), + ) + env.globals.update(version_compare=version_compare) + tmpl = env.get_template('index.html') + + raw_data = prepare() + + mkdir('output') + + with open('output/index.html', 'w', encoding='utf-8') as f: + f.write(tmpl.render({ + 'packages': raw_data, + 'distros': ['beige', 'sid'], + })) + + ESSENTIAL = 'acl apt attr audit base-files base-passwd bash binutils build-essential bzip2 cdebconf coreutils dash db5.3 debianutils diffutils dpkg dwz e2fsprogs elfutils elogind file findutils gawk gcc-13 gcc-defaults gdbm gettext glibc gmp gnupg2 gnutls28 grep groff guile-3.0 gzip hostname icu isl jansson keyutils krb5 libcap-ng libcap2 libffi libgc libgcrypt20 libgpg-error libidn2 libmd libnsl libpipeline libseccomp libselinux libsigsegv libtasn1-6 libtirpc libunistring libxcrypt libxml2 libzstd lz4 m4 make-dfsg man-db mpclib3 mpfr4 ncurses nettle openssl p11-kit pam patch pcre2 perl readline rpcsvc-proto sed shadow systemd sysvinit tar uchardet util-linux xxhash xz-utils zlib'.split() + + with open('output/index-essential.html', 'w', encoding='utf-8') as f: + f.write(tmpl.render({ + 'packages': {k: v for k, v in raw_data.items() if k in ESSENTIAL}, + 'distros': ['beige', 'sid'], + })) diff --git a/packages/download.py b/packages/download.py new file mode 100644 index 0000000..680784c --- /dev/null +++ b/packages/download.py @@ -0,0 +1,53 @@ +import gzip +from pathlib import Path +import requests + +from debian.deb822 import Sources +from debian.debian_support import version_compare + + +data_source = { + 'beige': { + 'sources': 'https://ci.deepin.com/repo/deepin/deepin-community/stable/dists/beige/main/source/Sources.gz', + }, + 'sid': { + 'sources': 'https://ftp.debian.org/debian/dists/sid/main/source/Sources.gz', + }, +} + +TARGET_DIR = 'data' + +def sync_data(): + + for repo, data in data_source.items(): + + Path(TARGET_DIR + '/' + repo).mkdir(parents=True, exist_ok=True) + + for file, url in data.items(): + r = requests.get(url, timeout=60) + assert r.url.endswith('.gz') + decompressed = gzip.decompress(r.content) + match file.split('-'): + case ['sources']: + with open(TARGET_DIR + '/' + repo + '/' + 'Sources', 'wb') as f: + f.write(decompressed) + case ['packages', arch]: + with open(TARGET_DIR + '/' + repo + '/' + 'Packages-' + arch, 'wb') as f: + f.write(decompressed) + +def prepare(): + sync_data() + # Stable + Testing + packages = {} + for repo in ['beige', 'sid']: + with open(TARGET_DIR + '/' + repo + '/' + 'Sources') as f: + for item in Sources.iter_paragraphs(f): + package: str = item['package'] + version: str = item['version'] + if package not in packages: + packages[package] = {} + if repo not in packages[package]: + packages[package][repo] = version + elif version_compare(version, packages[package][repo]) >= 0: + packages[package][repo] = version + return packages diff --git a/packages/templates/index.html b/packages/templates/index.html new file mode 100644 index 0000000..23e2e7e --- /dev/null +++ b/packages/templates/index.html @@ -0,0 +1,34 @@ + + + + + Index Page + + + + + + + + + + {% for package, versions in packages.items() %} + + + + + + {% endfor %} +
+ Package + beigeunstable/sid
0 %} + style="background-color: green;" + {% endif %} + {% endif %} + >{{ package }}{{ versions.beige }}{{ versions.sid }}
+ +