refactor: update toolchain usage, improve CI/CD

.dockerignore

@@ -4,6 +4,7 @@
 
 # git subrepos (==submodules)
 android_*
+*_kernel_*
 clang*
 rtl8812au
 AnyKernel3

.github/workflows/cd.yml → .github/workflows/github-ci.yml

@@ -1,4 +1,4 @@
-name: Slim Release
+name: Release
 
 on:
   push:
@@ -30,7 +30,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Build and assemble
         run: |
           python3 -m pip install poetry
@@ -41,7 +41,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: build-artifacts
-          path: "multi-slim"
+          path: "multi-build"
           retention-days: 1
   release:
     runs-on: ubuntu-latest
@@ -53,18 +53,19 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: build-artifacts
-          path: "multi-slim"
+          path: "multi-build"
       - name: Get current version
         id: version
         run: echo "version=$(python3 scripts/get_version.py)" >> $GITHUB_OUTPUT
       - name: Form a tag name
         id: tagname
-        run: echo "tagname=v${{ steps.version.outputs.version }}.${{ github.run_number }}" >> $GITHUB_OUTPUT
+        run: echo "tagname=v${{ steps.version.outputs.version }}" >> $GITHUB_OUTPUT
       - name: Release
         uses: ncipollo/release-action@v1
         with:
           draft: true
+          skipIfReleaseExists: true
           tag: ${{ steps.tagname.outputs.tagname }}
           prerelease: ${{ env.IS_PRERELEASE }}
           token: ${{ secrets.GITHUB_TOKEN }}
-          artifacts: "multi-slim/*.zip"
+          artifacts: "multi-build/*.zip"

.gitignore

@@ -5,11 +5,11 @@
 
 # git subrepos (==submodules)
 /android_*
+/*_kernel_*
 /clang*
 /rtl8812au
 /AnyKernel3
 /KernelSU
-/*_kernel_*
 
 # misc local artifacts
 /*.log
@@ -18,4 +18,4 @@
 /source
 /bundle
 /localversion
-/multi-slim
+/multi-build

.gitlab-ci.yml

@@ -0,0 +1,46 @@
+stages:
+- build
+- tag
+
+variables:
+  GIT_STRATEGY: "clone"
+
+job-build:
+  stage: build
+  tags:
+    - docker
+    - linux
+  image: docker:24.0-cli
+  services:
+    - docker:dind
+  script:
+    - export DOCKER_HOST=tcp://docker:2375/
+    - apk update && apk add python3 py3-pip
+    - python3 -m pip install poetry
+    - python3 -m poetry config virtualenvs.create false
+    - python3 -m poetry install --no-root
+    - python3 scripts/multi_build.py
+  artifacts:
+    paths:
+      - "multi-build/"
+    when: on_success
+    expire_in: never
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - when: manual
+
+job-tag:
+  stage: tag
+  needs:
+    - job-build
+  script:
+    - USERNAME=$CUSTOM_CI_USERNAME
+    - PASSWD=$CUSTOM_CI_PASSWD
+    - TAGNAME=$(python3 scripts/get_version.py)
+    - git config --global user.name "${GITLAB_USER_NAME}"
+    - git config --global user.email "${GITLAB_USER_EMAIL}"
+    - git remote remove origin
+    - git remote add origin https://$USERNAME:[email protected]/$CI_PROJECT_PATH
+    - if [ $(git tag | grep "$TAGNAME") ]; then echo "[ * ] Tag already exists, skipping.."; else git tag $TAGNAME && git push origin $TAGNAME; fi
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

Dockerfile

@@ -4,6 +4,10 @@ FROM python:3.12-slim-bookworm as base
 ARG WDIR="/zero_build"
 ENV CONAN_UPLOAD_CUSTOM 0
 
+# place sources from host to container
+COPY . $WDIR
+WORKDIR $WDIR
+
 # install basic packages
 RUN \
     apt-get update \
@@ -23,17 +27,20 @@ RUN \
         bison \
         flex
 
-# place sources from host to container
-COPY . $WDIR
-WORKDIR $WDIR
-
 # configure Python environment
 RUN python3 -m pip install pip --upgrade && \
     python3 -m pip install poetry && \
     python3 -m poetry config virtualenvs.create false && \
     python3 -m poetry install --no-root
 
-# install shared tools from tools.json
+# install shared tools from tools.json;
+#
+# The idea here is that we pre-pack all the tools into the Docker/Podman image that can be used for any device:
+# (which are toolchains, binutils -- everything except device-specific kernel source);
+# 
+# This significantly reduces the total build time, as each time we make a build call for a device,
+# only device-specific kernel source is being downloaded into the container.
+#
 RUN python3 $WDIR/wrapper/bridge.py --tools
 
 # launch app

README.md

@@ -34,7 +34,7 @@ The kernel has the following features:
 - Kali NetHunter support;
 - RTL8812/21AU + RTL8814AU + RTL8187 Wi-Fi drivers;
 - packet injection support for internal Wi-Fi chipset;
-- KernelSU support.
+- optional KernelSU support.
 
 ## Supported ROMs and devices
 
@@ -91,7 +91,7 @@ optional arguments:
 To run this tool in a `local` environment, you will need:
 
 - a Debian-based Linux distribution (other types of distros are untested);
-- packages installed via apt: `libssl-dev`, `wget`, `git`, `make`, `gcc`, `zip`.
+- a few [packages](Dockerfile#L15) installed in your system.
 
 You will also need a few Python packages. To install them, use:
 

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "zero-kernel-builder"
 version = "0.3.3"
-description = ""
+description = "An Android kernel with Kali NetHunter functionality."
 authors = ["seppzer0"]
 packages = [{include = "wrapper"}]
 

scripts/multi_build.py

@@ -1,10 +1,22 @@
 import os
 import shutil
+import argparse
 import subprocess
 from pathlib import Path
 
 
-def ucopy(src: Path, dst: Path, exceptions: list[str] = []) -> None:
+def parse_args() -> None:
+    """Parse arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--env",
+        choices=("docker", "podman", "local"),
+        default="docker"
+    )
+    return parser.parse_args()
+
+
+def ucopy(src: Path, dst: Path) -> None:
     """A universal method to copy files into desired destinations.
 
     :param src: Source path.
@@ -18,7 +30,7 @@ def ucopy(src: Path, dst: Path, exceptions: list[str] = []) -> None:
         contents = os.listdir(src)
         for e in contents:
             # do not copy restricted files
-            if e not in exceptions and e != src:
+            if e != src:
                 src_e = Path(src, e)
                 dst_e = Path(dst, e)
                 if src_e.is_dir():
@@ -30,88 +42,67 @@ def ucopy(src: Path, dst: Path, exceptions: list[str] = []) -> None:
         shutil.copy(src, dst)
 
 
-# launch the builds
-apath = Path(Path(__file__).absolute().parents[1])
-# those with "bundle" module usage are primarily tested builds
-argsets = (
-    #{
-    #    "module": "bundle",
-    #    "rom": "los",
-    #    "codename": "dumpling",
-    #    "ksu": "false",
-    #    "size": "slim"
-    #},
-    #{
-    #    "module": "bundle",
-    #    "rom": "pa",
-    #    "codename": "dumpling",
-    #    "ksu": "false",
-    #    "size": "slim"
-    #},
-    #{
-    #    "module": "kernel",
-    #    "rom": "los",
-    #    "codename": "dumpling",
-    #    "ksu": "true"
-    #},
-    #{
-    #    "module": "kernel",
-    #    "rom": "pa",
-    #    "codename": "dumpling",
-    #    "ksu": "true"
-    #},
-    {
-        "module": "kernel",
-        "rom": "x",
-        "codename": "dumpling",
-        "ksu": "false"
-    },
-    {
-        "module": "kernel",
-        "rom": "x",
-        "codename": "dumpling",
-        "ksu": "true"
-    },
-    {
-        "module": "assets",
-        "rom": "los",
-        "codename": "cheeseburger",
-        "ksu": "true"
-    },
-    {
-        "module": "assets",
-        "rom": "pa",
-        "codename": "cheeseburger",
-        "ksu": "true"
-    },
-)
-os.chdir(apath)
-dir_shared = "multi-slim"
-shutil.rmtree(dir_shared, ignore_errors=True)
-for count, argset in enumerate(argsets, 1):
-    # create artifact holder directory
-    if dir_shared not in os.listdir():
-        os.mkdir(dir_shared)
-    # extract individual values
-    module = argset["module"]
-    rom = argset["rom"]
-    codename = argset["codename"]
-    ksu = "--ksu" if argset["ksu"] == "true" else ""
-    size = argset["size"] if argset["module"] == "bundle" else ""
-    extra = "minimal --rom-only --clean" if argset["module"] == "assets" else ""
-    # if the build is last, make it automatically remove the Docker image from runner
-    clean = "--clean-image" if count == len(argsets) else ""
-    # form and launch the command
-    cmd = f"python3 wrapper {module} docker {rom} {codename} {size} {ksu} {clean} {extra}"
-    print(f"[CMD]: {cmd}")
-    subprocess.run(cmd.strip(), shell=True, check=True)
-    # copy artifacts into the shared directory
-    out = ""
-    match argset["module"]:
-        case "bundle":
-            out = "bundle"
-        case "kernel":
-            out = "kernel"
-        case "assets":
-            out = "assets"
-    ucopy(Path(out), Path(dir_shared))
+def main(args: argparse.Namespace) -> None:
+    """Run multi build."""
+    apath = Path(Path(__file__).absolute().parents[1])
+    argsets = (
+        {
+            "module": "kernel",
+            "rom": "x",
+            "codename": "dumpling",
+            "ksu": "false"
+        },
+        {
+            "module": "kernel",
+            "rom": "x",
+            "codename": "dumpling",
+            "ksu": "true"
+        },
+        {
+            "module": "assets",
+            "rom": "los",
+            "codename": "cheeseburger",
+            "ksu": "true"
+        },
+        {
+            "module": "assets",
+            "rom": "pa",
+            "codename": "cheeseburger",
+            "ksu": "true"
+        },
+    )
+    os.chdir(apath)
+    dir_shared = "multi-build"
+    shutil.rmtree(dir_shared, ignore_errors=True)
+    for count, argset in enumerate(argsets, 1):
+        # create artifact holder directory
+        if dir_shared not in os.listdir():
+            os.mkdir(dir_shared)
+        # extract individual values
+        module = argset["module"]
+        rom = argset["rom"]
+        codename = argset["codename"]
+        ksu = "--ksu" if argset["ksu"] == "true" else ""
+        size = argset["size"] if argset["module"] == "bundle" else ""
+        extra = "minimal --rom-only --clean" if argset["module"] == "assets" else ""
+        # if the build is last, make it automatically remove the Docker/Podman image from runner
+        clean = "--clean-image" if count == len(argsets) and args.env in ("docker", "podman") else ""
+        # form and launch the command
+        cmd = f"python3 wrapper {module} {args.env} {rom} {codename} {size} {ksu} {clean} {extra}"
+        print(f"[CMD]: {cmd}")
+        subprocess.run(cmd.strip(), shell=True, check=True)
+        # copy artifacts into the shared directory
+        out = ""
+        match argset["module"]:
+            case "bundle":
+                out = "bundle"
+            case "kernel":
+                out = "kernel"
+            case "assets":
+                out = "assets"
+        ucopy(Path(out), Path(dir_shared))
+
+
+if __name__ == "__main__":
+    # launch the builds
+    main(parse_args())

wrapper/bridge.py

@@ -105,7 +105,7 @@ def main(args: argparse.Namespace) -> None:
                 ksu = args.ksu,
             ).run()
         case _:
-            # if no module was selected, then the shared tools are installed
+            # if no module was selected, then shared tools are installed
             tconf = Resources()
             tconf.path_gen()
             tconf.download()