Feat/actions crawler

.ci/crawl.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+devp2p discv4 crawl -timeout "$ETH_DNS_DISCV4_CRAWLTIME" all.json

.ci/deploy.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+proto_groups=(all les)
+
+for network in "$@"; do
+
+    echo "Deploy: $network"
+
+    for p in "${proto_groups[@]}"; do
+        echo -n "Deploy: ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}"
+
+        # Ensure that we actually have a nodeset to deploy to DNS.
+        [[ ! -d ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN} ]] || [[ ! -f ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json ]] && { echo " | DNE, skipping"; continue; }
+
+        echo
+        devp2p dns to-cloudflare --zoneid "$ETH_DNS_CLOUDFLARE_ZONEID" "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}"
+    done
+done

.ci/deps.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+set -e
+
+# Check programs we depend on.
+command -v devp2p >/dev/null 2>&1 && echo "OK: devp2p command in PATH" || { echo "Please install devp2p"; exit 1; }
+command -v ethkey >/dev/null 2>&1 && echo "OK: ethkey command in PATH" || { echo "Please install ethkey"; exit 1; }
+
+# Check that we have key and keypass file.
+if [ ! -f $ETH_DNS_DISCV4_KEY_PATH ] || [ ! -f $ETH_DNS_DISCV4_KEYPASS_PATH ]; then
+    echo "
+No key found at key file path or no password file found at ${ETH_DNS_DISCV4_KEYPASS_PATH}. 
+Use 'ethkey generate ${ETH_DNS_DISCV4_KEY_PATH}'
+Save the password in plaintext in ${ETH_DNS_DISCV4_KEYPASS_PATH}
+"
+    exit 1
+else
+    echo "OK: Key and password file exist."
+fi
+
+# Check that we have deploy variables set.
+if [ -z $CLOUDFLARE_API_TOKEN ]; then
+    echo "Missing CLOUDFLARE_API_TOKEN env var"
+    exit 1
+else
+    echo "OK: environment variable CLOUDFLARE_API_TOKEN is not empty."
+fi
+
+# I'm not sure why I couldn't get devp2p to work without using the --zoneid flag; kept getting 403 bad perms.
+# Using the flag seems to fix it, and this var gets set as the value for that flag.
+# It's associated with the specific domain name that Cloudflare is managing and that we want to deploy to.
+if [ -z $ETH_DNS_CLOUDFLARE_ZONEID ]; then
+    echo "Missing ETH_DNS_CLOUDFLARE_ZONEID env var"
+    exit 1
+else
+    echo "OK: environment variable ETH_DNS_CLOUDFLARE_ZONEID is not empty."
+fi
+

.ci/filter_and_sign.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+proto_groups=(all les)
+
+for network in "$@"; do
+
+    echo "Filter: $network"
+
+    mkdir -p "all.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}"
+    devp2p nodeset filter all.json -eth-network "$network" >"all.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json"
+
+    mkdir -p "les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}"
+    devp2p nodeset filter all.json -les-server -eth-network "$network" >"les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json"
+
+
+    echo "Sign: $network"
+
+    for p in "${proto_groups[@]}"; do
+        echo -n "Sign: ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}"
+
+        # Ensure that we actually have a nodeset to sign.
+        [ ! -d ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN} ] || [ ! -f ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json ] && { echo " | DNE, skipping"; continue; }
+
+        echo
+        cat "${ETH_DNS_DISCV4_KEYPASS_PATH}" | devp2p dns sign "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" "${ETH_DNS_DISCV4_KEY_PATH}" && echo "OK"
+
+    done
+
+done

.github/workflows/crawl.yml

@@ -0,0 +1,71 @@
+name: Discv4 Crawl and DNS Update
+on:
+  schedule:
+    - cron: '0 */2 * * *'
+jobs:
+  build:
+    if: github.repository == 'ethereum/discv4-dns-lists'
+    name: Discv4-DNS-Crawler
+    runs-on: ubuntu-latest
+    env:
+      ETH_DNS_DISCV4_CRAWLTIME: 30m
+      ETH_DNS_DISCV4_PARENT_DOMAIN: ethdisco.net
+      ETH_DNS_DISCV4_KEY_PATH: ./secrets/dnskey.json
+      ETH_DNS_DISCV4_KEYPASS_PATH: ./secrets/dnskey_password.txt
+      CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      ETH_DNS_CLOUDFLARE_ZONEID: ${{ secrets.ETH_DNS_CLOUDFLARE_ZONEID }}
+      ETH_DNS_DISCV4_KEY: ${{ secrets.ETH_DNS_DISCV4_KEY }}
+      ETH_DNS_DISCV4_KEYPASS: ${{ secrets.ETH_DNS_DISCV4_KEYPASS }}
+
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v2-beta
+      with:
+        go-version: 1.13.8
+      id: go
+
+    - run: go version
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: |
+        go get -u github.com/ethereum/go-ethereum/cmd/devp2p
+        go get -u github.com/ethereum/go-ethereum/cmd/ethkey
+
+    - name: Setup secrets
+      run: |
+        mkdir secrets
+        echo "$ETH_DNS_DISCV4_KEY" > "$ETH_DNS_DISCV4_KEY_PATH"
+        echo "$ETH_DNS_DISCV4_KEYPASS" > "$ETH_DNS_DISCV4_KEYPASS_PATH"
+
+    - name: Check env and secrets
+      run: |
+        ./.ci/deps.sh
+
+    - name: Crawl
+      run: |
+        ./.ci/crawl.sh
+
+    - name: Filter and sign
+      run: |
+        ./.ci/filter_and_sign.sh mainnet ropsten rinkeby goerli
+
+    - name: Commit and Push
+      env:
+        GITHUB_USER: FIXME
+        GITHUB_PAT: FIXME
+      run: |
+        git config --local user.name 'crawler'
+        git config --local user.email '[email protected]'
+        git add all* les*
+        git commit --author 'crawler <>' -m "automatic update: crawl time ${ETH_DNS_DISCV4_CRAWLTIME} ci ${GITHUB_RUN_ID}:${GITHUB_RUN_NUMBER}"
+        git remote set-url origin https://${GITHUB_USER}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}.git
+        git push origin master
+
+    - name: Deploy to DNS
+      run: |
+        ./.ci/deploy.sh mainnet ropsten rinkeby goerli
+

.gitignore

@@ -0,0 +1 @@
+secrets/