Skip to content

🐧🧹 HealthChecks πŸ–³πŸ—‘οΈ #149

🐧🧹 HealthChecks πŸ–³πŸ—‘οΈ

🐧🧹 HealthChecks πŸ–³πŸ—‘οΈ #149

name: 🐧🧹 HealthChecks πŸ–³πŸ—‘οΈ
#MAX_RUNTIME: 02 Minutes */10 * * * *
on:
#push:
workflow_dispatch:
schedule:
- cron: "0 */1 * * *" # Every 01 Hrs
env:
GITHUB_TOKEN: "${{ secrets.TOOLPACKS_GHRO }}"
GITLAB_TOKEN: "${{ secrets.TOOLPACKS_GLRO }}"
HF_TOKEN: "${{ secrets.TOOLPACKS_HFRO }}"
RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}"
R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras"
jobs:
#------------------------------------------------------------------------------------#
check_build_status:
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.check.outputs.should_run }}
steps:
- name: Check Build Packages workflow status
id: check
run: |
#presets
set +x ; set +e
#-------------#
AARCH64_STATUS="$(curl -qfsSL -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/Azathothas/Toolpacks-Extras/actions/workflows/build_aarch64_Linux.yaml/runs?status=in_progress" | jq '.total_count')" && export AARCH64_STATUS="${AARCH64_STATUS}"
X86_64_STATUS="$(curl -qfsSL -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/Azathothas/Toolpacks-Extras/actions/workflows/build_x86_64_Linux.yaml/runs?status=in_progress" | jq '.total_count')" && export X86_64_STATUS="${X86_64_STATUS}"
if [ "${X86_64_STATUS}" -eq "0" ] && [ "${AARCH64_STATUS}" -eq "0" ]; then
echo "should_run=true" >> "${GITHUB_OUTPUT}"
else
echo "should_run=false" >> "${GITHUB_OUTPUT}"
fi
continue-on-error: true
#------------------------------------------------------------------------------------#
sync-metadata:
needs: check_build_status
if: needs.check_build_status.outputs.should_run == 'true'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
#- name: Debloat Runner
# run: |
# #Presets
# set +x ; set +e
# #--------------#
# bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/debloat.sh")
# continue-on-error: true
- name: Checkout repository
uses: actions/checkout@v4
with:
path: main
filter: "blob:none"
- name: Install Addons
run: |
#presets
set +x ; set +e
#-------------#
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh")
continue-on-error: true
- name: Setup Env
run: |
#presets
set +x ; set +e
#-------------#
#tmp
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
echo "SYSTMP=${SYSTMP}" >> "$GITHUB_ENV"
#GH Dirs
mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux"
mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux"
#Setup HF
git config --global "credential.helper" store
git config --global "user.email" "[email protected]"
git config --global "user.name" "Azathothas"
#huggingface-cli login --token "${HF_TOKEN}" --add-to-git-credential
huggingface-cli login --token "${HF_TOKEN}" --add-to-git-credential >/dev/null 2>&1
#-------------#
##Setup rClone
mkdir -p "$HOME/.config/rclone"
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf"
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "${GITHUB_ENV}"
#-------------#
##User-Agent
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}"
continue-on-error: true
- name: Dos2Unix Everything
run: |
#Presets
set +x ; set +e
#--------------#
cd "${GITHUB_WORKSPACE}/main"
find . -type f ! -path "./.git/*" -exec dos2unix {} \; 2>/dev/null
continue-on-error: true
- name: ActionLint
run: |
#Presets
set +x ; set +e
#--------------#
cd "${GITHUB_WORKSPACE}/main"
find ".github/workflows" -type f -name "*ml" -exec actionlint {} \;
continue-on-error: true
- name: Shellcheck
run: |
#Presets
set +x ; set +e
#--------------#
cd "${GITHUB_WORKSPACE}/main"
find ".github" -type f -name '*.sh' -exec shellcheck --exclude="SC2261" --severity=error "{}" \; 2>/dev/null | tee "${GITHUB_WORKSPACE}/main/.github/shellcheck.txt"
continue-on-error: true
- name: YAMLint
run: |
#Presets
set +x ; set +e
#--------------#
cd "${GITHUB_WORKSPACE}/main/.github/scripts/aarch64-Linux/pkgs"
find "." -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} +
cd "${GITHUB_WORKSPACE}/main/.github/scripts/x86_64-Linux/pkgs"
find "." -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} +
continue-on-error: true
- name: Sync Metadata (aarch64-Linux)
run: |
#Presets
set +x ; set +e
#--------------#
pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1
mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/BUILD.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.log.txt"
curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.gh.log.txt"
curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD_FAILED.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_FAILED.gh.log.txt"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/BLAKE3SUM.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BLAKE3SUM.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/FILE.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/FILE.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.min.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.min.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/SHA256SUM.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/SHA256SUM.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/SIZE.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.json"
popd >/dev/null 2>&1
continue-on-error: true
- name: Sync Metadata (x86_64-Linux)
run: |
#Presets
set +x ; set +e
#--------------#
pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1
mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/BUILD.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.log.txt"
curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.gh.log.txt"
curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD_FAILED.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_FAILED.gh.log.txt"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/BLAKE3SUM.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BLAKE3SUM.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/FILE.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/FILE.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.min.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.min.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/SHA256SUM.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/SHA256SUM.json"
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/SIZE.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.json"
popd >/dev/null 2>&1
continue-on-error: true
- name: Sync Metadata (ROOT)
run: |
#Presets
set +x ; set +e
#--------------#
#Generate
pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1
##Get JSON
#aarch64-Linux
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.json" -o "${SYSTMP}/aarch64-Linux-METADATA.json" || echo '{}' > "${SYSTMP}/aarch64-Linux-METADATA.json"
#x86_64-Linux
curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.json" -o "${SYSTMP}/x86_64-Linux-METADATA.json" || echo '{}' > "${SYSTMP}/x86_64-Linux-METADATA.json"
##Merge
jq -s '[{ "host": "aarch64-Linux", "main": .[0] },{ "host": "x86_64-Linux", "main": .[1] }]' "${SYSTMP}/aarch64-Linux-METADATA.json" "${SYSTMP}/x86_64-Linux-METADATA.json" | jq . > "${GITHUB_WORKSPACE}/main/METADATA.json"
sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux|https://pkg.ajam.dev/aarch64-Linux|g" "${GITHUB_WORKSPACE}/main/METADATA.json" | sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux|https://pkg.ajam.dev/x86_64-Linux|g" | jq -r tostring > "${GITHUB_WORKSPACE}/main/METADATA.min.json"
#Gen Yaml
cat "${GITHUB_WORKSPACE}/main/METADATA.json" | yj -jy | jq . > "${GITHUB_WORKSPACE}/main/METADATA.yaml"
cat "${GITHUB_WORKSPACE}/main/METADATA.min.json" | yj -jy | yq . > "${GITHUB_WORKSPACE}/main/METADATA.min.yaml"
popd >/dev/null 2>&1
##Sync
if jq --exit-status . "${GITHUB_WORKSPACE}/main/METADATA.json" >/dev/null 2>&1; then
pushd "$(mktemp -d)" >/dev/null 2>&1 && git clone --depth="1" --filter="blob:none" --no-checkout "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras" && cd "./Toolpacks-Extras"
git lfs install
huggingface-cli lfs-enable-largefiles "."
git fetch origin main ; git lfs track "./METADATA.json" "./METADATA.min.json" "./METADATA.yaml" "./METADATA.min.yaml"
git sparse-checkout disable
git sparse-checkout set --no-cone --sparse-index "/METADATA.json" "/METADATA.min.json" "/METADATA.yaml" "/METADATA.min.yaml"
git checkout ; ls -lah "."
cat "${GITHUB_WORKSPACE}/main/METADATA.json" | jq . > "./METADATA.json"
cat "${GITHUB_WORKSPACE}/main/METADATA.min.json" | jq . > "./METADATA.min.json"
cat "${GITHUB_WORKSPACE}/main/METADATA.yaml" | yq . > "./METADATA.yaml"
cat "${GITHUB_WORKSPACE}/main/METADATA.min.yaml" | yq . > "./METADATA.min.yaml"
find "." -type f -size -3c -delete
git add --all --verbose && git commit -m "[+] METADATA (ROOT) $(TZ='UTC' date +'%Y_%m_%d')]"
git pull origin main ; git push origin main
popd >/dev/null 2>&1
fi
continue-on-error: true
- name: Generate All-In-One (aarch64-Linux)
run: |
#Presets
set +x ; set +e
#--------------#
##Main (bin.ajam.dev)
curl -qfsSL "https://bin.ajam.dev/aarch64/METADATA.json" -o "${SYSTMP}/METADATA.json.src"
if [ "$(jq '. | length' "${SYSTMP}/METADATA.json.src")" -gt 1000 ]; then
cat "${SYSTMP}/METADATA.json.src" |\
jq -r '
.[] |
"[[bin]]\n" +
"name = \"\(.name)\"\n" +
"bin_name = \"\(.name | gsub("\\.no_strip"; ""))\"\n" +
"description = \"\(.description)\"\n" +
"note = \"\(.note)\"\n" +
"version = \"" + (if has("repo_version") then (if .repo_version == "" then "latest" else .repo_version end) else "latest" end) + "\"\n" +
"download_url = \"" + (.download_url | sub("https://bin.ajam.dev/aarch64_arm64_Linux/"; "https://bin.ajam.dev/aarch64/")) + "\"\n" +
"size = \"\(.size | ascii_upcase)\"\n" +
"bsum = \"\(.b3sum)\"\n" +
"shasum = \"\(.sha256)\"\n" +
"build_date = \"\(.build_date)\"\n" +
"src_url = \"\(.repo_url)\"\n" +
"web_url = \"\(.web_url)\"\n" +
"build_script = \"\(.build_script)\"\n" +
"build_log = \"https://bin.ajam.dev/aarch64/\(.name).log.txt\"\n" +
"category = \"" + (if .repo_topics == "" then "utility" else .repo_topics end) + "\"\n" +
"icon = \"https://bin.ajam.dev/aarch64/" + (.build_script | split("/")[-1] | sub("\\.sh$"; ".preview.png")) + "\"\n" +
"extra_bins = \"\(.extra_bins)\"\n"
' > "${SYSTMP}/aarch64-Linux-METADATA.toml"
validtoml "${SYSTMP}/aarch64-Linux-METADATA.toml"
##BaseUtils (bin.ajam.dev)
curl -qfsSL "https://bin.ajam.dev/aarch64/Baseutils/METADATA.json" |\
jq -r '
.[] |
"[[base]]\n" +
"name = \"\(.name)\"\n" +
"bin_name = \"\(.name | gsub("\\.no_strip"; ""))\"\n" +
"description = \"\(.description)\"\n" +
"note = \"\(.note)\"\n" +
"version = \"" + (if has("repo_version") then (if .repo_version == "" then "latest" else .repo_version end) else "latest" end) + "\"\n" +
"download_url = \"" + (.download_url | sub("https://bin.ajam.dev/aarch64_arm64_Linux/"; "https://bin.ajam.dev/aarch64/")) + "\"\n" +
"size = \"\(.size | ascii_upcase)\"\n" +
"bsum = \"\(.b3sum)\"\n" +
"shasum = \"\(.sha256)\"\n" +
"build_date = \"\(.build_date)\"\n" +
"src_url = \"\(.repo_url)\"\n" +
"web_url = \"\(.web_url)\"\n" +
"build_script = \"\(.build_script)\"\n" +
"build_log = \"https://bin.ajam.dev/aarch64/\(.name).log.txt\"\n" +
"category = \"" + (if .repo_topics == "" then "utility" else .repo_topics end) + "\"\n" +
"icon = \"https://bin.ajam.dev/aarch64/" + (.build_script | split("/")[-1] | sub("\\.sh$"; ".preview.png")) + "\"\n" +
"extra_bins = \"\(.extra_bins)\"\n"
' >> "${SYSTMP}/aarch64-Linux-METADATA.toml"
validtoml "${SYSTMP}/aarch64-Linux-METADATA.toml"
##Main (pkg.ajam.dev)
curl -qfsSL "https://pkg.ajam.dev/aarch64/METADATA.min.json" |\
jq -r '
.[] |
"[[pkg]]\n" +
"name = \"\(.name)\"\n" +
"bin_name = \"\(.bin_name)\"\n" +
"description = \"\(.description)\"\n" +
"note = \"\(.note)\"\n" +
"version = \"\(.version)\"\n" +
"download_url = \"" + (.download_url | sub("https://bin.ajam.dev/aarch64_arm64_Linux/"; "https://bin.ajam.dev/aarch64/")) + "\"\n" +
"size = \"\(.size | ascii_upcase)\"\n" +
"bsum = \"\(.bsum)\"\n" +
"shasum = \"\(.shasum)\"\n" +
"build_date = \"\(.build_date)\"\n" +
"src_url = \"\(.src_url)\"\n" +
"web_url = \"\(.web_url)\"\n" +
"build_script = \"\(.build_script)\"\n" +
"build_log = \"\(.build_log)\"\n" +
"category = \"\(.category)\"\n" +
"icon = \"https://pkg.ajam.dev/aarch64/\(.bin_name).icon.png\"\n" +
"extra_bins = \"\(.extra_bins)\"\n"
' >> "${SYSTMP}/aarch64-Linux-METADATA.toml"
validtoml "${SYSTMP}/aarch64-Linux-METADATA.toml"
taplo check "${SYSTMP}/aarch64-Linux-METADATA.toml" && cp "${SYSTMP}/aarch64-Linux-METADATA.toml" "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml"
cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml" | yj -tj | jq 'to_entries | sort_by(.key) | from_entries' > "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.json"
cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.json" | jq -r tostring > "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.min.json"
cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml" | yj -ty | yq . > "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.yaml"
fi
continue-on-error: true
- name: Sync Metadata (aarch64-Linux-TOML)
run: |
#Presets
set +x ; set +e
#--------------#
##Sync
pushd "$(mktemp -d)" >/dev/null 2>&1 && git clone --depth="1" --filter="blob:none" --no-checkout "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras" && cd "./Toolpacks-Extras"
git lfs install
huggingface-cli lfs-enable-largefiles "."
git sparse-checkout disable
git sparse-checkout set --no-cone --sparse-index "aarch64-Linux/*.json" "aarch64-Linux/*.log" "aarch64-Linux/*.temp" "aarch64-Linux/*.toml" "aarch64-Linux/*.tmp" "aarch64-Linux/*.txt" "aarch64-Linux/*.yaml" "aarch64-Linux/*.yml"
git checkout ; ls -lah "." "./aarch64-Linux"
git sparse-checkout list
#copy TOML
taplo check "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml" && cp "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml" "./aarch64-Linux/METADATA.AIO.toml"
#copy json
if jq --exit-status . "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.json" >/dev/null 2>&1; then
cp "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.json" "./aarch64-Linux/METADATA.AIO.json"
fi
#copy min.json
if jq --exit-status . "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.min.json" >/dev/null 2>&1; then
cp "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.min.json" "./aarch64-Linux/METADATA.AIO.min.json"
fi
#copy yaml
if yq --exit-status . "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.yaml" >/dev/null 2>&1; then
cp "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.yaml" "./aarch64-Linux/METADATA.AIO.yaml"
fi
find "." -type f -size -3c -delete
git add --all --verbose && git commit -m "[+] METADATA (AIO) $(TZ='UTC' date +'%Y_%m_%d')]"
git pull origin main ; git push origin main
popd >/dev/null 2>&1
continue-on-error: true
- name: Generate All-In-One (x86_64-Linux)
run: |
#Presets
set +x ; set +e
#--------------#
##Main (bin.ajam.dev)
curl -qfsSL "https://bin.ajam.dev/x86_64/METADATA.json" -o "${SYSTMP}/METADATA.json.src"
if [ "$(jq '. | length' "${SYSTMP}/METADATA.json.src")" -gt 1000 ]; then
cat "${SYSTMP}/METADATA.json.src" |\
jq -r '
.[] |
"[[bin]]\n" +
"name = \"\(.name)\"\n" +
"bin_name = \"\(.name | gsub("\\.no_strip"; ""))\"\n" +
"description = \"\(.description)\"\n" +
"note = \"\(.note)\"\n" +
"version = \"" + (if has("repo_version") then (if .repo_version == "" then "latest" else .repo_version end) else "latest" end) + "\"\n" +
"download_url = \"" + (.download_url | sub("https://bin.ajam.dev/x86_64_Linux/"; "https://bin.ajam.dev/x86_64/")) + "\"\n" +
"size = \"\(.size | ascii_upcase)\"\n" +
"bsum = \"\(.b3sum)\"\n" +
"shasum = \"\(.sha256)\"\n" +
"build_date = \"\(.build_date)\"\n" +
"src_url = \"\(.repo_url)\"\n" +
"web_url = \"\(.web_url)\"\n" +
"build_script = \"\(.build_script)\"\n" +
"build_log = \"https://bin.ajam.dev/x86_64/\(.name).log.txt\"\n" +
"category = \"" + (if .repo_topics == "" then "utility" else .repo_topics end) + "\"\n" +
"icon = \"https://bin.ajam.dev/x86_64/" + (.build_script | split("/")[-1] | sub("\\.sh$"; ".preview.png")) + "\"\n" +
"extra_bins = \"\(.extra_bins)\"\n"
' > "${SYSTMP}/x86_64-Linux-METADATA.toml"
validtoml "${SYSTMP}/x86_64-Linux-METADATA.toml"
##BaseUtils (bin.ajam.dev)
curl -qfsSL "https://bin.ajam.dev/x86_64/Baseutils/METADATA.json" |\
jq -r '
.[] |
"[[base]]\n" +
"name = \"\(.name)\"\n" +
"bin_name = \"\(.name | gsub("\\.no_strip"; ""))\"\n" +
"description = \"\(.description)\"\n" +
"note = \"\(.note)\"\n" +
"version = \"" + (if has("repo_version") then (if .repo_version == "" then "latest" else .repo_version end) else "latest" end) + "\"\n" +
"download_url = \"" + (.download_url | sub("https://bin.ajam.dev/x86_64_Linux/"; "https://bin.ajam.dev/x86_64/")) + "\"\n" +
"size = \"\(.size | ascii_upcase)\"\n" +
"bsum = \"\(.b3sum)\"\n" +
"shasum = \"\(.sha256)\"\n" +
"build_date = \"\(.build_date)\"\n" +
"src_url = \"\(.repo_url)\"\n" +
"web_url = \"\(.web_url)\"\n" +
"build_script = \"\(.build_script)\"\n" +
"build_log = \"https://bin.ajam.dev/x86_64/\(.name).log.txt\"\n" +
"category = \"" + (if .repo_topics == "" then "utility" else .repo_topics end) + "\"\n" +
"icon = \"https://bin.ajam.dev/x86_64/" + (.build_script | split("/")[-1] | sub("\\.sh$"; ".preview.png")) + "\"\n" +
"extra_bins = \"\(.extra_bins)\"\n"
' >> "${SYSTMP}/x86_64-Linux-METADATA.toml"
validtoml "${SYSTMP}/x86_64-Linux-METADATA.toml"
##Main (pkg.ajam.dev)
curl -qfsSL "https://pkg.ajam.dev/x86_64/METADATA.min.json" |\
jq -r '
.[] |
"[[pkg]]\n" +
"name = \"\(.name)\"\n" +
"bin_name = \"\(.bin_name)\"\n" +
"description = \"\(.description)\"\n" +
"note = \"\(.note)\"\n" +
"version = \"\(.version)\"\n" +
"download_url = \"\(.download_url)\"\n" +
"size = \"\(.size | ascii_upcase)\"\n" +
"bsum = \"\(.bsum)\"\n" +
"shasum = \"\(.shasum)\"\n" +
"build_date = \"\(.build_date)\"\n" +
"src_url = \"\(.src_url)\"\n" +
"web_url = \"\(.web_url)\"\n" +
"build_script = \"\(.build_script)\"\n" +
"build_log = \"\(.build_log)\"\n" +
"category = \"\(.category)\"\n" +
"icon = \"https://pkg.ajam.dev/x86_64/\(.bin_name).icon.png\"\n" +
"extra_bins = \"\(.extra_bins)\"\n"
' >> "${SYSTMP}/x86_64-Linux-METADATA.toml"
taplo check "${SYSTMP}/x86_64-Linux-METADATA.toml" && cp "${SYSTMP}/x86_64-Linux-METADATA.toml" "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml"
cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml" | yj -tj | jq 'to_entries | sort_by(.key) | from_entries' > "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.json"
cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.json" | jq -r tostring > "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.min.json"
cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml" | yj -ty | yq . > "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.yaml"
fi
continue-on-error: true
- name: Sync Metadata (x86_64-Linux-TOML)
run: |
#Presets
set +x ; set +e
#--------------#
##Sync
pushd "$(mktemp -d)" >/dev/null 2>&1 && git clone --depth="1" --filter="blob:none" --no-checkout "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras" && cd "./Toolpacks-Extras"
git lfs install
huggingface-cli lfs-enable-largefiles "."
git sparse-checkout disable
git sparse-checkout set --no-cone --sparse-index "x86_64-Linux/*.json" "x86_64-Linux/*.log" "x86_64-Linux/*.temp" "x86_64-Linux/*.toml" "x86_64-Linux/*.tmp" "x86_64-Linux/*.txt" "x86_64-Linux/*.yaml" "x86_64-Linux/*.yml"
git checkout ; ls -lah "." "./x86_64-Linux"
git sparse-checkout list
#copy TOML
taplo check "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml" && cp "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml" "./x86_64-Linux/METADATA.AIO.toml"
#copy json
if jq --exit-status . "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.json" >/dev/null 2>&1; then
cp "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.json" "./x86_64-Linux/METADATA.AIO.json"
fi
#copy min.json
if jq --exit-status . "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.min.json" >/dev/null 2>&1; then
cp "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.min.json" "./x86_64-Linux/METADATA.AIO.min.json"
fi
#copy yaml
if yq --exit-status . "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.yaml" >/dev/null 2>&1; then
cp "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.yaml" "./x86_64-Linux/METADATA.AIO.yaml"
fi
find "." -type f -size -3c -delete
git add --all --verbose && git commit -m "[+] METADATA (AIO) $(TZ='UTC' date +'%Y_%m_%d')]"
git pull origin main ; git push origin main
popd >/dev/null 2>&1
continue-on-error: true
- name: Update TOTAL_PACKAGES
run: |
#Presets
set +x ; set +e
#--------------#
#Presets
set +x ; set +e
#--------------#
##aarch64-Linux/
unset TOTAL_PACKAGES
TOTAL_PACKAGES="$(curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export TOTAL_PACKAGES="${TOTAL_PACKAGES}"
sed 's#\(tree/main/aarch64-Linux)| \)[0-9]*\( |.*\)#\1'"${TOTAL_PACKAGES}"'\2#' -i "${GITHUB_WORKSPACE}/main/README.md"
##x86_64-Linux/
unset TOTAL_PACKAGES
TOTAL_PACKAGES="$(curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export TOTAL_PACKAGES="${TOTAL_PACKAGES}"
sed 's#\(tree/main/x86_64-Linux)| \)[0-9]*\( |.*\)#\1'"${TOTAL_PACKAGES}"'\2#' -i "${GITHUB_WORKSPACE}/main/README.md"
continue-on-error: true
- name: Generate README (aarch64-Linux)
run: |
#presets
set +x ; set +e
#-------------#
if jq empty "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" 2>/dev/null; then
#Convert to YAML
cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | yj -jy | yq . > "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.yaml"
##Gen aarch64-Linux/DETAILED.md
cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | jq -r '.[] | "### [`\(.name // "" | if . == null or . == "" then "" else . end)` πŸ“€](\(.download_url // "" | if . == null or . == "" then "" else . end))\n- **PKG_Name πŸ“¦:** \(.bin_name // "" | if . == null or . == "" then "" else . end)\n- **Version 🧬:** `\(.version // "" | if . == null or . == "" then "" else . end)`\n- **Description ℹ️:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note ⚠️:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Homepage 🌐:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Source πŸ“‘:** [\(.src_url // "" | if . == null or . == "" then "" else . end)](\(.src_url // "" | if . == null or . == "" then "" else . end))\n- **Download ⬇️:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Size πŸ’Ύ:** `\(.size // "" | if . == null or . == "" then "" else . end)`\n- **SHA256SUM πŸ”:** `\(.shasum // "" | if . == null or . == "" then "" else . end)`\n- **B3SUM πŸ”:** `\(.bsum // "" | if . == null or . == "" then "" else . end)`\n- **Script βš™οΈ:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Log 🧾:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Icon πŸ–ΌοΈ:** \(.build_log | sub("\\.log$"; ".icon.png"))\n- **Desktop πŸ–₯️:** \(.build_log | sub("\\.log$"; ".desktop"))\n- **Zsync ♻️:** \(.build_log | sub("\\.log$"; ".zsync"))\n\n---\n"' > "${GITHUB_WORKSPACE}/main/aarch64-Linux/DETAILED.md"
sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux|https://pkg.ajam.dev/aarch64-Linux|g" "${GITHUB_WORKSPACE}/main/aarch64-Linux/DETAILED.md" > "${GITHUB_WORKSPACE}/main/aarch64-Linux/DETAILED.min.md"
##aarch64-Linux/README.md
{ echo '|ICON πŸ–ΌοΈ|PKG πŸ“€|PKG_Name πŸ“¦|Version 🧬| Description ℹ️|Note ⚠️|Homepage 🌐|Source πŸ“‘|Size πŸ’Ύ|SHA256SUM πŸ”|B3SUM πŸ”|Script βš™οΈ|Log 🧾|' ; echo '| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |' ; cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | sed 's/|/,/g' | jq -r '.[] | "| <img src=\"\(.build_log // "" | if . == null or . == "" then "" else (. | sub("\\.log$"; ".icon.png")) end)\" width=\"28\" height=\"28\"> | [`\(.name // "" | if . == null or . == "" then "" else . end)`](\(.download_url // "" | if . == null or . == "" then "" else . end)) | \(.bin_name // "" | if . == null or . == "" then "" else . end) | \(.version // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.note // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.src_url // "" | if . == null or . == "" then "" else . end)](\(.src_url // "" | if . == null or . == "" then "" else . end)) |\(.size // "" | if . == null or . == "" then "" else . end) | `\(.shasum // "" | if . == null or . == "" then "" else . end)` | `\(.bsum // "" | if . == null or . == "" then "" else . end)` | \(.build_script // "" | if . == null or . == "" then "" else . end) | \(.build_log // "" | if . == null or . == "" then "" else . end) | \(.category // "" | if . == null or . == "" then "" else . end) |"' ; } > "${GITHUB_WORKSPACE}/main/aarch64-Linux/README.md"
sed 's/||//g' -i "${GITHUB_WORKSPACE}/main/aarch64-Linux/README.md"
sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux|https://pkg.ajam.dev/aarch64-Linux|g" "${GITHUB_WORKSPACE}/main/aarch64-Linux/README.md" > "${GITHUB_WORKSPACE}/main/aarch64-Linux/README.min.md"
fi
continue-on-error: true
- name: Generate README (x86_64-Linux)
run: |
#presets
set +x ; set +e
#-------------#
if jq empty "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" 2>/dev/null; then
#Convert to YAML
cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | yj -jy | yq . > "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.yaml"
##Gen x86_64-Linux/DETAILED.md
cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | jq -r '.[] | "### [`\(.name // "" | if . == null or . == "" then "" else . end)` πŸ“€](\(.download_url // "" | if . == null or . == "" then "" else . end))\n- **PKG_Name πŸ“¦:** \(.bin_name // "" | if . == null or . == "" then "" else . end)\n- **Version 🧬:** `\(.version // "" | if . == null or . == "" then "" else . end)`\n- **Description ℹ️:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note ⚠️:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Homepage 🌐:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Source πŸ“‘:** [\(.src_url // "" | if . == null or . == "" then "" else . end)](\(.src_url // "" | if . == null or . == "" then "" else . end))\n- **Download ⬇️:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Size πŸ’Ύ:** `\(.size // "" | if . == null or . == "" then "" else . end)`\n- **SHA256SUM πŸ”:** `\(.shasum // "" | if . == null or . == "" then "" else . end)`\n- **B3SUM πŸ”:** `\(.bsum // "" | if . == null or . == "" then "" else . end)`\n- **Script βš™οΈ:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Log 🧾:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Icon πŸ–ΌοΈ:** \(.build_log | sub("\\.log$"; ".icon.png"))\n- **Desktop πŸ–₯️:** \(.build_log | sub("\\.log$"; ".desktop"))\n- **Zsync ♻️:** \(.build_log | sub("\\.log$"; ".zsync"))\n\n---\n"' > "${GITHUB_WORKSPACE}/main/x86_64-Linux/DETAILED.md"
sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux|https://pkg.ajam.dev/x86_64-Linux|g" "${GITHUB_WORKSPACE}/main/x86_64-Linux/DETAILED.md" > "${GITHUB_WORKSPACE}/main/x86_64-Linux/DETAILED.min.md"
##x86_64-Linux/README.md
{ echo '|ICON πŸ–ΌοΈ|PKG πŸ“€|PKG_Name πŸ“¦|Version 🧬| Description ℹ️|Note ⚠️|Homepage 🌐|Source πŸ“‘|Size πŸ’Ύ|SHA256SUM πŸ”|B3SUM πŸ”|Script βš™οΈ|Log 🧾|' ; echo '| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |' ; cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | sed 's/|/,/g' | jq -r '.[] | "| <img src=\"\(.build_log // "" | if . == null or . == "" then "" else (. | sub("\\.log$"; ".icon.png")) end)\" width=\"28\" height=\"28\"> | [`\(.name // "" | if . == null or . == "" then "" else . end)`](\(.download_url // "" | if . == null or . == "" then "" else . end)) | \(.bin_name // "" | if . == null or . == "" then "" else . end) | \(.version // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.note // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.src_url // "" | if . == null or . == "" then "" else . end)](\(.src_url // "" | if . == null or . == "" then "" else . end)) |\(.size // "" | if . == null or . == "" then "" else . end) | `\(.shasum // "" | if . == null or . == "" then "" else . end)` | `\(.bsum // "" | if . == null or . == "" then "" else . end)` | \(.build_script // "" | if . == null or . == "" then "" else . end) | \(.build_log // "" | if . == null or . == "" then "" else . end) | \(.category // "" | if . == null or . == "" then "" else . end) |"' ; } > "${GITHUB_WORKSPACE}/main/x86_64-Linux/README.md"
sed 's/||//g' -i "${GITHUB_WORKSPACE}/main/x86_64-Linux/README.md"
sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux|https://pkg.ajam.dev/x86_64-Linux|g" "${GITHUB_WORKSPACE}/main/x86_64-Linux/README.md" > "${GITHUB_WORKSPACE}/main/x86_64-Linux/README.min.md"
fi
continue-on-error: true
- name: Get DateTime
run: |
#Presets
set +x ; set +e
#--------------#
UTC_TIME="$(TZ='UTC' date +'%Y-%m-%d (%I:%M:%S %p)')"
echo "UTC_TIME=$UTC_TIME" >> $GITHUB_ENV
continue-on-error: true
- uses: stefanzweifel/git-auto-commit-action@v5
with:
repository: ./main
commit_user_name: Azathothas
commit_user_email: [email protected]
commit_message: "βœ… Updated (Metadata) πŸ›οΈ <-- ${{ env.UTC_TIME }} ⌚"
#push_options: '--force'
continue-on-error: true
#------------------------------------------------------------------------------------#
- name: rClone BackUp Repo ("https://pub.ajam.dev/repos/${GITHUB_REPOSITORY}")
run: |
# Presets
set +x ; set +e
#--------------#
#touch "$HOME/.rclone.conf"
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf"
#chdir to Repo
cd "${GITHUB_WORKSPACE}/main"
#Git pull
git pull origin main --no-edit 2>/dev/null
#Del Bloat
rm -rf "$(pwd)/.git"
#Upload to Pub
echo -e "[+] Syncing ${GITHUB_REPOSITORY} to pub.ajam.dev/repos/${GITHUB_REPOSITORY} \n"
rclone sync "." "r2:/pub/repos/${GITHUB_REPOSITORY}/" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress
#Upload AIO files (aarch64-Linux) ==> (https://bin.ajam.dev/aarch64_arm64_Linux/)
rclone copyto "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.json" "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.json" --checksum --check-first --user-agent="${USER_AGENT}"
rclone copyto "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.min.json" "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.min.json" --checksum --check-first --user-agent="${USER_AGENT}"
rclone copyto "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml" "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.toml" --checksum --check-first --user-agent="${USER_AGENT}"
rclone copyto "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.yaml" "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.yaml" --checksum --check-first --user-agent="${USER_AGENT}"
#Upload AIO files (x86_64-Linux) ==> (https://bin.ajam.dev/x86_64_Linux/)
rclone copyto "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.json" "r2:/bin/x86_64_Linux/METADATA.AIO.json" --checksum --check-first --user-agent="${USER_AGENT}"
rclone copyto "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.min.json" "r2:/bin/x86_64_Linux/METADATA.AIO.min.json" --checksum --check-first --user-agent="${USER_AGENT}"
rclone copyto "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml" "r2:/bin/x86_64_Linux/METADATA.AIO.toml" --checksum --check-first --user-agent="${USER_AGENT}"
rclone copyto "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.yaml" "r2:/bin/x86_64_Linux/METADATA.AIO.yaml" --checksum --check-first --user-agent="${USER_AGENT}"
continue-on-error: true
#------------------------------------------------------------------------------------#