From a3f8c5dc133e3be3a2e0f16cd344e47328786948 Mon Sep 17 00:00:00 2001 From: Joseph P Date: Fri, 10 Nov 2023 11:39:39 +0000 Subject: [PATCH] refactor: update toolchain usage, improve CI/CD --- .dockerignore | 1 + .github/workflows/{cd.yml => github-ci.yml} | 13 +- .gitignore | 4 +- .gitlab-ci.yml | 46 ++++++ Dockerfile | 17 +- README.md | 4 +- pyproject.toml | 2 +- scripts/multi_build.py | 165 +++++++++----------- wrapper/bridge.py | 2 +- wrapper/engines/container.py | 6 +- wrapper/manifests/tools.json | 7 - wrapper/models/kernel.py | 11 +- wrapper/tools/cleaning.py | 3 +- wrapper/utils/resources.py | 7 +- 14 files changed, 162 insertions(+), 126 deletions(-) rename .github/workflows/{cd.yml => github-ci.yml} (89%) create mode 100644 .gitlab-ci.yml diff --git a/.dockerignore b/.dockerignore index faecc19..e996593 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ # git subrepos (==submodules) android_* +*_kernel_* clang* rtl8812au AnyKernel3 diff --git a/.github/workflows/cd.yml b/.github/workflows/github-ci.yml similarity index 89% rename from .github/workflows/cd.yml rename to .github/workflows/github-ci.yml index 4c0ed23..769acef 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/github-ci.yml @@ -1,4 +1,4 @@ -name: Slim Release +name: Release on: push: @@ -30,7 +30,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" - name: Build and assemble run: | python3 -m pip install poetry @@ -41,7 +41,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: build-artifacts - path: "multi-slim" + path: "multi-build" retention-days: 1 release: runs-on: ubuntu-latest @@ -53,18 +53,19 @@ jobs: uses: actions/download-artifact@v3 with: name: build-artifacts - path: "multi-slim" + path: "multi-build" - name: Get current version id: version run: echo "version=$(python3 scripts/get_version.py)" >> $GITHUB_OUTPUT - name: Form a tag name id: tagname - run: echo "tagname=v${{ steps.version.outputs.version }}.${{ github.run_number }}" >> $GITHUB_OUTPUT + run: echo "tagname=v${{ steps.version.outputs.version }}" >> $GITHUB_OUTPUT - name: Release uses: ncipollo/release-action@v1 with: draft: true + skipIfReleaseExists: true tag: ${{ steps.tagname.outputs.tagname }} prerelease: ${{ env.IS_PRERELEASE }} token: ${{ secrets.GITHUB_TOKEN }} - artifacts: "multi-slim/*.zip" + artifacts: "multi-build/*.zip" diff --git a/.gitignore b/.gitignore index d21d7f6..4de04fe 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,11 @@ # git subrepos (==submodules) /android_* +/*_kernel_* /clang* /rtl8812au /AnyKernel3 /KernelSU -/*_kernel_* # misc local artifacts /*.log @@ -18,4 +18,4 @@ /source /bundle /localversion -/multi-slim \ No newline at end of file +/multi-build \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..beb0380 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,46 @@ +stages: +- build +- tag + +variables: + GIT_STRATEGY: "clone" + +job-build: + stage: build + tags: + - docker + - linux + image: docker:24.0-cli + services: + - docker:dind + script: + - export DOCKER_HOST=tcp://docker:2375/ + - apk update && apk add python3 py3-pip + - python3 -m pip install poetry + - python3 -m poetry config virtualenvs.create false + - python3 -m poetry install --no-root + - python3 scripts/multi_build.py + artifacts: + paths: + - "multi-build/" + when: on_success + expire_in: never + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - when: manual + +job-tag: + stage: tag + needs: + - job-build + script: + - USERNAME=$CUSTOM_CI_USERNAME + - PASSWD=$CUSTOM_CI_PASSWD + - TAGNAME=$(python3 scripts/get_version.py) + - git config --global user.name "${GITLAB_USER_NAME}" + - git config --global user.email "${GITLAB_USER_EMAIL}" + - git remote remove origin + - git remote add origin https://$USERNAME:$PASSWD@gitlab.com/$CI_PROJECT_PATH + - if [ $(git tag | grep "$TAGNAME") ]; then echo "[ * ] Tag already exists, skipping.."; else git tag $TAGNAME && git push origin $TAGNAME; fi + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/Dockerfile b/Dockerfile index bed217e..26bd8f0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,10 @@ FROM python:3.12-slim-bookworm as base ARG WDIR="/zero_build" ENV CONAN_UPLOAD_CUSTOM 0 +# place sources from host to container +COPY . $WDIR +WORKDIR $WDIR + # install basic packages RUN \ apt-get update \ @@ -23,17 +27,20 @@ RUN \ bison \ flex -# place sources from host to container -COPY . $WDIR -WORKDIR $WDIR - # configure Python environment RUN python3 -m pip install pip --upgrade && \ python3 -m pip install poetry && \ python3 -m poetry config virtualenvs.create false && \ python3 -m poetry install --no-root -# install shared tools from tools.json +# install shared tools from tools.json; +# +# The idea here is that we pre-pack all the tools into the Docker/Podman image that can be used for any device: +# (which are toolchains, binutils -- everything except device-specific kernel source); +# +# This significantly reduces the total build time, as each time we make a build call for a device, +# only device-specific kernel source is being downloaded into the container. +# RUN python3 $WDIR/wrapper/bridge.py --tools # launch app diff --git a/README.md b/README.md index 2bc7f1a..d041095 100755 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The kernel has the following features: - Kali NetHunter support; - RTL8812/21AU + RTL8814AU + RTL8187 Wi-Fi drivers; - packet injection support for internal Wi-Fi chipset; -- KernelSU support. +- optional KernelSU support. ## Supported ROMs and devices @@ -91,7 +91,7 @@ optional arguments: To run this tool in a `local` environment, you will need: - a Debian-based Linux distribution (other types of distros are untested); -- packages installed via apt: `libssl-dev`, `wget`, `git`, `make`, `gcc`, `zip`. +- a few [packages](Dockerfile#L15) installed in your system. You will also need a few Python packages. To install them, use: diff --git a/pyproject.toml b/pyproject.toml index a334ce2..6260890 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "zero-kernel-builder" version = "0.3.3" -description = "" +description = "An Android kernel with Kali NetHunter functionality." authors = ["seppzer0"] packages = [{include = "wrapper"}] diff --git a/scripts/multi_build.py b/scripts/multi_build.py index 145782f..80d7dd6 100644 --- a/scripts/multi_build.py +++ b/scripts/multi_build.py @@ -1,10 +1,22 @@ import os import shutil +import argparse import subprocess from pathlib import Path -def ucopy(src: Path, dst: Path, exceptions: list[str] = []) -> None: +def parse_args() -> None: + """Parse arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--env", + choices=("docker", "podman", "local"), + default="docker" + ) + return parser.parse_args() + + +def ucopy(src: Path, dst: Path) -> None: """A universal method to copy files into desired destinations. :param src: Source path. @@ -18,7 +30,7 @@ def ucopy(src: Path, dst: Path, exceptions: list[str] = []) -> None: contents = os.listdir(src) for e in contents: # do not copy restricted files - if e not in exceptions and e != src: + if e != src: src_e = Path(src, e) dst_e = Path(dst, e) if src_e.is_dir(): @@ -30,88 +42,67 @@ def ucopy(src: Path, dst: Path, exceptions: list[str] = []) -> None: shutil.copy(src, dst) -# launch the builds -apath = Path(Path(__file__).absolute().parents[1]) -# those with "bundle" module usage are primarily tested builds -argsets = ( - #{ - # "module": "bundle", - # "rom": "los", - # "codename": "dumpling", - # "ksu": "false", - # "size": "slim" - #}, - #{ - # "module": "bundle", - # "rom": "pa", - # "codename": "dumpling", - # "ksu": "false", - # "size": "slim" - #}, - #{ - # "module": "kernel", - # "rom": "los", - # "codename": "dumpling", - # "ksu": "true" - #}, - #{ - # "module": "kernel", - # "rom": "pa", - # "codename": "dumpling", - # "ksu": "true" - #}, - { - "module": "kernel", - "rom": "x", - "codename": "dumpling", - "ksu": "false" - }, - { - "module": "kernel", - "rom": "x", - "codename": "dumpling", - "ksu": "true" - }, - { - "module": "assets", - "rom": "los", - "codename": "cheeseburger", - "ksu": "true" - }, - { - "module": "assets", - "rom": "pa", - "codename": "cheeseburger", - "ksu": "true" - }, -) -os.chdir(apath) -dir_shared = "multi-slim" -shutil.rmtree(dir_shared, ignore_errors=True) -for count, argset in enumerate(argsets, 1): - # create artifact holder directory - if dir_shared not in os.listdir(): - os.mkdir(dir_shared) - # extract individual values - module = argset["module"] - rom = argset["rom"] - codename = argset["codename"] - ksu = "--ksu" if argset["ksu"] == "true" else "" - size = argset["size"] if argset["module"] == "bundle" else "" - extra = "minimal --rom-only --clean" if argset["module"] == "assets" else "" - # if the build is last, make it automatically remove the Docker image from runner - clean = "--clean-image" if count == len(argsets) else "" - # form and launch the command - cmd = f"python3 wrapper {module} docker {rom} {codename} {size} {ksu} {clean} {extra}" - print(f"[CMD]: {cmd}") - subprocess.run(cmd.strip(), shell=True, check=True) - # copy artifacts into the shared directory - out = "" - match argset["module"]: - case "bundle": - out = "bundle" - case "kernel": - out = "kernel" - case "assets": - out = "assets" - ucopy(Path(out), Path(dir_shared)) +def main(args: argparse.Namespace) -> None: + """Run multi build.""" + apath = Path(Path(__file__).absolute().parents[1]) + argsets = ( + { + "module": "kernel", + "rom": "x", + "codename": "dumpling", + "ksu": "false" + }, + { + "module": "kernel", + "rom": "x", + "codename": "dumpling", + "ksu": "true" + }, + { + "module": "assets", + "rom": "los", + "codename": "cheeseburger", + "ksu": "true" + }, + { + "module": "assets", + "rom": "pa", + "codename": "cheeseburger", + "ksu": "true" + }, + ) + os.chdir(apath) + dir_shared = "multi-build" + shutil.rmtree(dir_shared, ignore_errors=True) + for count, argset in enumerate(argsets, 1): + # create artifact holder directory + if dir_shared not in os.listdir(): + os.mkdir(dir_shared) + # extract individual values + module = argset["module"] + rom = argset["rom"] + codename = argset["codename"] + ksu = "--ksu" if argset["ksu"] == "true" else "" + size = argset["size"] if argset["module"] == "bundle" else "" + extra = "minimal --rom-only --clean" if argset["module"] == "assets" else "" + # if the build is last, make it automatically remove the Docker/Podman image from runner + clean = "--clean-image" if count == len(argsets) and args.env in ("docker", "podman") else "" + # form and launch the command + cmd = f"python3 wrapper {module} {args.env} {rom} {codename} {size} {ksu} {clean} {extra}" + print(f"[CMD]: {cmd}") + subprocess.run(cmd.strip(), shell=True, check=True) + # copy artifacts into the shared directory + out = "" + match argset["module"]: + case "bundle": + out = "bundle" + case "kernel": + out = "kernel" + case "assets": + out = "assets" + ucopy(Path(out), Path(dir_shared)) + + +if __name__ == "__main__": + # launch the builds + main(parse_args()) diff --git a/wrapper/bridge.py b/wrapper/bridge.py index 2c9043a..7b02706 100644 --- a/wrapper/bridge.py +++ b/wrapper/bridge.py @@ -105,7 +105,7 @@ def main(args: argparse.Namespace) -> None: ksu = args.ksu, ).run() case _: - # if no module was selected, then the shared tools are installed + # if no module was selected, then shared tools are installed tconf = Resources() tconf.path_gen() tconf.download() diff --git a/wrapper/engines/container.py b/wrapper/engines/container.py index 6c2231a..ce7fa0d 100644 --- a/wrapper/engines/container.py +++ b/wrapper/engines/container.py @@ -37,6 +37,7 @@ def __init__(self, config: dict) -> None: @property def _dir_bundle_conan(self) -> Path: + """Determine path to Conan's local cache.""" res = "" if os.getenv("CONAN_USER_HOME"): res = Path(os.getenv("CONAN_USER_HOME")) @@ -152,7 +153,8 @@ def _build(self) -> None: msg.note(f"Building the {alias} image..") os.chdir(self._wdir_local) # build only if it is not present in local cache - if self._name_image not in ccmd.launch("docker images --format '{{.Repository}}'", get_output=True): + #if self._name_image not in ccmd.launch(f"{self._buildenv} image list --format '{{.Repository}}'", get_output=True): + if 1 == 1: # force enable Docker Buildkit if self._buildenv == "docker": os.environ["DOCKER_BUILDKIT"] = "1" @@ -169,7 +171,7 @@ def _build(self) -> None: def run(self) -> None: self._build() - # form the final "docker run" command + # form the final "docker/podman run" command cmd = '{} run {} {} /bin/bash -c "{}"'.format( self._buildenv, " ".join(self._container_options), diff --git a/wrapper/manifests/tools.json b/wrapper/manifests/tools.json index 320f333..b0cdfef 100644 --- a/wrapper/manifests/tools.json +++ b/wrapper/manifests/tools.json @@ -1,11 +1,4 @@ { - "android_prebuilts_gcc_linux-x86_aarch64_aarch64-linux-android-4.9": { - "type": "git", - "path": "android_prebuilts_gcc_linux-x86_aarch64_aarch64-linux-android-4.9", - "url": "https://android.googlesource.com/platform/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9", - "branch": "android-13.0.0_r0.52", - "commit": "" - }, "android_prebuilts_gcc_linux-x86_arm_arm-linux-androideabi-4.9": { "type": "git", "path": "android_prebuilts_gcc_linux-x86_arm_arm-linux-androideabi-4.9", diff --git a/wrapper/models/kernel.py b/wrapper/models/kernel.py index 57e9a8d..3a41454 100755 --- a/wrapper/models/kernel.py +++ b/wrapper/models/kernel.py @@ -536,20 +536,21 @@ def _build(self) -> None: print("\n", end="") msg.note("Launching the build..") os.chdir(self._rcs.paths[self._codename]["path"]) - # launch "make" with the number of all available processing units + # launch "make" punits = ccmd.launch("nproc --all", get_output=True) cmd1 = "make -j{} O=out {} "\ "ARCH=arm64 "\ "SUBARCH=arm64 "\ - "HOSTCC=clang "\ - "HOSTCXX=clang+"\ + "LLVM=1"\ .format(punits, self._defconfig) cmd2 = "make -j{} O=out "\ "ARCH=arm64 "\ "SUBARCH=arm64 "\ - "CROSS_COMPILE=aarch64-linux-android- "\ + "CROSS_COMPILE=llvm- "\ "CROSS_COMPILE_ARM32=arm-linux-androideabi- "\ "CLANG_TRIPLE=aarch64-linux-gnu- "\ + "LLVM=1 "\ + "LLVM_IAS=1 "\ "HOSTCC=clang "\ "HOSTCXX=clang++ "\ "CC=clang "\ @@ -557,8 +558,6 @@ def _build(self) -> None: "AR=llvm-ar "\ "NM=llvm-nm "\ "AS=llvm-as "\ - "LLVM=1 "\ - "LLVM_IAS=1 "\ "OBJCOPY=llvm-objcopy "\ "OBJDUMP=llvm-objdump "\ "STRIP=llvm-strip"\ diff --git a/wrapper/tools/cleaning.py b/wrapper/tools/cleaning.py index 10e64c1..ea93fb0 100755 --- a/wrapper/tools/cleaning.py +++ b/wrapper/tools/cleaning.py @@ -59,13 +59,14 @@ def root(extra: list[str] = []) -> None: cfg.DIR_ASSETS, cfg.DIR_BUNDLE, "android_*", + "*_kernel_*", "clang*", "AnyKernel3", "rtl8812au", "source", "localversion", "KernelSU", - "multi_slim", + "multi_build", ] # add extra elements to clean up from root directory if extra: diff --git a/wrapper/utils/resources.py b/wrapper/utils/resources.py index 076fc5c..f03ef7b 100644 --- a/wrapper/utils/resources.py +++ b/wrapper/utils/resources.py @@ -100,9 +100,4 @@ def export_path(self) -> None: for elem in self.paths: p = self.paths[elem]["path"] pathenv = str(f"{p}/bin/") - # special rules for "android_prebuilts_build-tools" - if p.name == "android_prebuilts_build-tools": - os.environ["PATH"] += os.pathsep + pathenv.replace("/bin/:", "/linux-86/bin/:") - os.environ["PATH"] += os.pathsep + pathenv.replace("/bin/:", "/path/linux-86/:") - else: - os.environ["PATH"] += os.pathsep + pathenv + os.environ["PATH"] = pathenv + os.pathsep + os.getenv("PATH")