Skip to content

MakeTrie

MakeTrie #213

name: MakeTrie
on:
# docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#schedule
schedule:
# at 21:53 on 1st, 8th, 16th, 24th of every month
- cron: '53 21 1,8,16,24 * *'
workflow_dispatch:
inputs:
publish:
description: 'publish blocklists?'
required: true
default: 'test'
type: choice
options:
# "yes" / "no" convert to bools, hence use "yeah" / "nope" instead.
# bools are bools when accessed with input-context, but are
# stringified when accessed with github.events context. And so:
# for the sake of everyone's sanity, stick to non-boolean inputs.
# ref: docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#providing-inputs
- yeah
- nope
- test
env:
INDIR: "blocklistfiles"
OUTDIR: "result"
S3DIR: "blocklists"
S3_TEST_DIR: "testblocklists"
BLCONFIG: "config.json"
PREFER_S3_OVER_R2: "false"
jobs:
version:
runs-on: ubuntu-latest
timeout-minutes: 1
# docs.github.com/en/actions/learn-github-actions/contexts#example-usage-of-the-needs-context
outputs:
epoch: ${{ steps.v.outputs.epoch }}
steps:
- name: Gen version
id: v
run: |
# stackoverflow.com/a/69400542
v=$(date --utc +"%s.%3N")
echo "epoch=${v}" >> $GITHUB_OUTPUT
echo "::notice::Version ${v} / ${SCHED}"
shell: bash
env:
SCHED: ${{ github.event.schedule }}
make:
needs: version
strategy:
matrix:
codec: ["u8", "u6"]
runs-on: ubuntu-latest
timeout-minutes: 100
steps:
- name: 📃 Checkout repo
uses: actions/[email protected]
- name: 🐉 Setup python
uses: actions/[email protected]
with:
python-version: '3.x'
- name: ⭕ Setup node
uses: actions/[email protected]
with:
node-version: '19'
- name: ⏳ Make swap
run: |
free -hm
df -h
# make swap: community.fly.io/t/6782/10
sudo fallocate -l 16G /tmp/swapfile
# unix.stackexchange.com/a/297155
sudo chown root:root /tmp/swapfile
sudo mkswap /tmp/swapfile
# stackoverflow.com/a/550808
sudo cat /proc/sys/vm/swappiness
echo "20" | sudo tee /proc/sys/vm/swappiness > /dev/null
# github.com/pierotofy/set-swap-space/blob/master/action.yml
sudo swapon /tmp/swapfile
free -hm
df -h
shell: bash
- name: 🎨 Setup
id: setup
run: |
# install deps
python -m pip install aiohttp
npm install
# determine config
# all output vars are strings
# docs.github.com/en/actions/learn-github-actions/contexts#steps-context
echo "upload=y" >> $GITHUB_OUTPUT
if [ "$PUBLISH_BLOCKLISTS" = "test" ]; then
echo "S3DIR=$S3_TEST_DIR" >> $GITHUB_ENV
elif [ "$PUBLISH_BLOCKLISTS" = "nope" ]; then
echo "S3DIR=" >> $GITHUB_ENV
# do not upload
echo "upload=n" >> $GITHUB_OUTPUT
fi
echo "UNIX_EPOCH_SEC=${V}" >> $GITHUB_ENV
echo "::notice::Building ${GIT_REF}/${COMMIT_SHA} @ ${V}"
shell: bash
env:
COMMIT_SHA: ${{ github.sha }}
GIT_REF: ${{ github.ref }}
V: ${{ needs.version.outputs.epoch }}
PUBLISH_BLOCKLISTS: ${{ inputs.publish }}
- name: 📥 Download blocklists
run: |
python download.py
env:
BLCONFIG: ${{ env.BLCONFIG }}
- name: 🌲 Make compact trie
run: |
mkdir -p "$OUTDIR"
# 32768/1024 => 32G RAM
node --max-old-space-size=32768 --expose-gc ./src/build.js
env:
BLCONFIG: ${{ env.BLCONFIG }}
OUTDIR: ${{ env.OUTDIR }}
INDIR: ${{ env.INDIR }}
UNIX_EPOCH_SEC: ${{ env.UNIX_EPOCH_SEC }}
CODEC: ${{ matrix.codec }}
- name: 🪄 Show artifacts
run: |
# note: split now happens in serverless-dns/trie:trie.js
# creates td00.txt, td01.txt, ... , td98.txt, td99.txt, td100.txt, ...
# cd "$OUTDIR" && split --bytes=30M -d --additional-suffix=.txt td.txt td
cd "$OUTDIR" && ls -lhtr
env:
OUTDIR: ${{ env.OUTDIR }}
- name: 🏆 Upload?
# stackoverflow.com/a/68317611
# github.com/actions/runner/issues/866
if: ${{ steps.setup.outputs.upload == 'y' }}
run: |
node ./src/upload.js
env:
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
PREFER_S3_OVER_R2: ${{ env.PREFER_S3_OVER_R2 }}
UNIX_EPOCH_SEC: ${{ env.UNIX_EPOCH_SEC }}
CODEC: ${{ matrix.codec }}
S3DIR: ${{ env.S3DIR }}
OUTDIR: ${{ env.OUTDIR }}