Skip to content

πŸ›οΈ Build πŸ“€ (toolpack_aarch64-Linux) Packages πŸ“¦πŸ“€ #7

πŸ›οΈ Build πŸ“€ (toolpack_aarch64-Linux) Packages πŸ“¦πŸ“€

πŸ›οΈ Build πŸ“€ (toolpack_aarch64-Linux) Packages πŸ“¦πŸ“€ #7

name: πŸ›οΈ Build πŸ“€ (toolpack_aarch64-Linux) Packages πŸ“¦πŸ“€
on:
workflow_dispatch:
schedule:
- cron: "0 6 * * 4" # @ 06:00 AM UTC (11:45 AM NPT Morning Fri)
#Sane env
env:
GITHUB_TOKEN: "${{ secrets.TOOLPACKS_GHRO }}"
GITLAB_TOKEN: "${{ secrets.TOOLPACKS_GLRO }}"
HF_TOKEN: "${{ secrets.TOOLPACKS_HFRO }}"
RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}"
R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras"
jobs:
#------------------------------------------------------------------------------------#
presetup:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
path: main
filter: "blob:none"
#------------------------------------------------------------------------------------#
- name: Install Addons
run: |
#presets
set -x ; set +e
#-------------#
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh")
continue-on-error: true
#------------------------------------------------------------------------------------#
- name: Setup Env
run: |
##presets
set +x ; set +e
#-------------#
##CoreUtils
sudo apt update -y
sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y
sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null
##tmp
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
#GH ENV
echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}"
#GH Dirs
mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux"
#-------------#
mkdir -p "${HOME}/bin"
sudo apt update -y
sudo apt install dos2unix -y
##Setup rClone
mkdir -p "${HOME}/.config/rclone"
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf"
##User-Agent
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}"
continue-on-error: true
- name: Set TZ to (Asia/Kathmandu)
run: |
# Presets
set +x ; set +e
#--------------#
sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata
sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime"
sudo dpkg-reconfigure --frontend noninteractive tzdata
sudo apt-get install apt-utils software-properties-common -y
sudo apt-get update -y
continue-on-error: true
#------------------------------------------------------------------------------------#
- name: rClone BackUp Repo ("https://pub.ajam.dev/repos/${GITHUB_REPOSITORY}")
run: |
# Presets
set +x ; set +e
#--------------#
#copy & chdir to Repo
cp -r "${GITHUB_WORKSPACE}/main" "${SYSTMP}/REPO" && cd "${SYSTMP}/REPO"
#Git pull
git pull origin main --no-edit 2>/dev/null
#Del Bloat
rm -rf "$(pwd)/.git"
#Upload to Pub
echo -e "[+] Syncing ${GITHUB_REPOSITORY} to pub.ajam.dev/repos/${GITHUB_REPOSITORY} \n"
rclone sync "." "r2:/pub/repos/${GITHUB_REPOSITORY}/" --user-agent="${USER_AGENT}" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress
##Scripts_aarch64-Linux
#Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/
curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq"
rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks-Extras/.github/scripts/aarch64-Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/.github/scripts/aarch64-Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp"
#Clean Pretty Formatted
echo "[" $(cat metadata.json.tmp | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json"
#Update Metadata
rclone copyto --checksum "./metadata.json" "r2:/pub/repos/Azathothas/Toolpacks-Extras/.github/scripts/aarch64-Linux/bins/metadata.json"
continue-on-error: true
#------------------------------------------------------------------------------------#
#------------------------------------------------------------------------------------#
build-fetch-packages:
#runs-on: ubuntu-latest
runs-on: arm64-linux-toolpacker
needs: [presetup]
timeout-minutes: 2400
permissions:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
path: main
filter: "blob:none"
- name: Install Addons
run: |
#presets
set -x ; set +e
#-------------#
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh")
continue-on-error: true
#------------------------------------------------------------------------------------#
- name: Setup Env
run: |
##presets
set +x ; set +e
#-------------#
##CoreUtils
sudo apt update -y
sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y
sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null
##tmp
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
#GH ENV
echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}"
#GH Dirs
mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux"
#-------------#
mkdir -p "${HOME}/bin"
sudo apt update -y
sudo apt install dos2unix -y
##Setup rClone
mkdir -p "${HOME}/.config/rclone"
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf"
##User-Agent
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}"
continue-on-error: true
- name: Set TZ to (Asia/Kathmandu)
run: |
# Presets
set +x ; set +e
#--------------#
sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata
sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime"
sudo dpkg-reconfigure --frontend noninteractive tzdata
sudo apt-get install apt-utils software-properties-common -y
sudo apt-get update -y
continue-on-error: true
#------------------------------------------------------------------------------------#
#Main Build Script
- name: BUILD
run: |
#Presets
set +x ; set +e
#--------------#
##Main
curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/.github/scripts/aarch64-Linux/build_debian.sh" -o "${SYSTMP}/BUILD.sh"
dos2unix --quiet "${SYSTMP}/BUILD.sh" ; chmod +xwr "${SYSTMP}/BUILD.sh"
##Run with STDOUT [Slow, Not Recommended]
#bash "${SYSTMP}/BUILD.sh"
##Run with LOGS only
#bash "${SYSTMP}/BUILD.sh" > "${SYSTMP}/BUILD.log" 2>&1
##Run with STDOUT + LOGS
bash "${SYSTMP}/BUILD.sh" | tee "${SYSTMP}/BUILD.log" 2>&1
##Run without STDOUT/Logs
#bash "${SYSTMP}/BUILD.sh" >/dev/null 2>&1
if [[ -f "${SYSTMP}/BUILD.log" ]] && [[ $(stat -c%s "${SYSTMP}/BUILD.log") -gt 1024 ]]; then
##Purge Tokens (in case set -x & gh didn't redact)
cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.gh.log.txt"
sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
cat "${SYSTMP}/BUILD.gh.log.txt" > "${SYSTMP}/BUILD.log"
if grep -iq "completed" "${SYSTMP}/BUILD.gh.log.txt"; then
mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD.log"
elif grep -iq "initializing" "${SYSTMP}/BUILD.gh.log.txt"; then
mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD_FAILED.log"
fi
##rClone Upload logs
rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD_FAILED.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
fi
continue-on-error: true
#------------------------------------------------------------------------------------#
#Logs & Artifacts
- name: Upload (aarch64-Linux) Artifacts
uses: actions/upload-artifact@v4
with:
name: Toolpack_aarch64-Linux
path: |
/tmp/BUILD.log
compression-level: 0 # no compression, [Default: 6 (GNU Gzip)]
retention-days: 7
overwrite: true
continue-on-error: true
#------------------------------------------------------------------------------------#