Initial: cluster creation and proxy recreation

Proxy Style, i.e. with only one pub ip
Incl. DNS

Author: Gunther Klessinger

.gitignore

@@ -0,0 +1,12 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+README.md
+mysecrets.py

environ

@@ -0,0 +1,27 @@
+# vim: ft=sh
+# shellcheck disable=all
+
+NAME="company"
+
+DNS_API_TOKEN=""
+DNS_PROVIDER="digitalocean"
+DOMAIN="py:keyval"
+EMAIL="py:keyval"
+FN_SSH_KEY="$HOME/.ssh/hetzner-cluster-lois"
+
+HK_HOST_NETWORK=8
+HK_VER="2.2.3"
+HK_LOCATION="fsn1"
+
+# flux
+GITOPS_BRANCH="main"
+GITOPS_HOST="py:keyval"
+GITOPS_OWNER="company"
+GITOPS_PATH="clusters/staging"
+GITOPS_REPO="k8s"
+GITOPS_TOKEN=""
+
+DNS_API_TOKEN="pass:DO/pat"
+GITOPS_TOKEN="pass:AX/gitlabflux"
+HCLOUD_TOKEN="pass:HCloudLoisRO"
+HCLOUD_TOKEN_WRITE="pass:HCloudLois"

justfile

@@ -0,0 +1,53 @@
+# https://cheatography.com/linux-china/cheat-sheets/justfile/
+set dotenv-filename := 'environ'
+set dotenv-load := true
+set dotenv-required := true
+set export
+set unstable
+set script-interpreter := ['uv', 'run']
+alias c := create-cluster
+alias cfg := render-config
+alias t := test
+alias p := pyhk3
+alias pf := port-forward
+alias elk := download-kubectl
+
+default:
+  @just --list
+
+pyhk3 *ARGS:
+  uv run pyhk3 {{ARGS}}
+
+do *ARGS:
+  just p do {{ARGS}}
+
+ssh *ARGS:
+  just p do ssh {{ARGS}}
+
+render-config:
+  just p hk3s render_config
+  
+port-forward:
+  just p do port_forward
+
+[confirm('Sure to destroy all servers of the cluster?')]
+rm:
+  just p do remove all
+
+
+
+[confirm('Sure to destroy proxy (if existing) and recreate it?')]
+recover-proxy:
+  just p do delete proxy
+  just p create proxy proxylb dns
+  just p recover hk3sconfig
+  just p recover kubeconfig
+
+create-cluster:
+  just p create proxy k3s proxylb dns
+
+download-kubectl:
+  just p do download_kubectl
+
+test:
+   uv run pytest ./tests/test_setup.py

keyval.py

@@ -0,0 +1,16 @@
+"""
+Custom calc-ed values for <key>='py:<this module path>' constructs
+❗ module path w/o '.py'
+
+Evaluated only once, not per key.
+"""
+
+try:
+    from mysecrets import sec  # whatever
+except ImportError:
+    sec = {}
+
+
+DOMAIN = sec.get('DOMAIN', 'cluster.company.net')
+EMAIL = sec.get('EMAIL', '[email protected]')
+GITOPS_HOST = sec.get('GITOPS_HOST', 'gitlab.company.com')

pyproject.toml

@@ -0,0 +1,80 @@
+[project]
+name = "pyhk3"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Gunther Klessinger", email = "[email protected]" }]
+dependencies = [
+  "absl-py>=2.1.0",
+  "pyyaml>=6.0.2",
+  "requests>=2.32.3",
+  "rich>=13.9.4",
+  "sh>=2.2.1",
+  "structlog>=25.1.0",
+]
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+
+[tool.hatch.build.targets.sdist]
+include = ["src/pyhk3"]
+
+
+[dependency-groups]
+dev = ["pytest>=8.3.4"]
+
+
+[project.scripts]
+pyhk3 = "pyhk3.cli:main"
+
+
+[tool.pyright]
+# https://github.com/microsoft/pyright/blob/main/docs/configuration.md
+# ruff is enough
+reportSelfClsParameterName = false
+pythonPlatform = "Linux"
+typeCheckingMode = "off"
+reportMissingImports = true
+reportMissingTypeStubs = false
+reportUndefinedVariable = true
+reportGeneralTypeIssues = false
+reportUnusedExpression = false
+root = "src"
+include = ["src", "tests"]
+# pythonVersion = "3.8"
+executionEnvironments = [{ root = "src" }, { root = "tests" }]
+venvPath = "./"
+venv = ".venv"
+
+[tool.ruff]
+# https://docs.astral.sh/ruff/rules/
+line-length = 90
+extend-select = ["Q"]
+select = ["E", "F", "B"] # Enable flake8-bugbear (`B`) rules.
+ignore = [
+  "E501", # Never enforce `E501` (line length violations).
+  "E741", # short var names
+  "E731", # no lambda
+  "B006", # mutables in signature
+]
+
+[tool.ruff.lint]
+fixable = ["ALL"]
+unfixable = [
+  "B",    # Avoid trying to fix flake8-bugbear (`B`) violations.
+  "F401", # Unused Import
+  "F841", # variable assigned but not used
+]
+
+
+[tool.ruff.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "single"
+
+[tool.ruff.format]
+# Prefer single quotes over double quotes
+quote-style = "single"

src/pyhk3/__init__.py

@@ -0,0 +1,78 @@
+T_UNIT_FWD = (
+    lambda net: f"""
+[Unit]
+Description=Bastion IP Forwarder
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c 'echo 1 > /proc/sys/net/ipv4/ip_forward'
+ExecStart=/bin/bash -c 'iptables -t nat -A POSTROUTING -s "10.{net}.0.0/16" -o eth0 -j MASQUERADE'
+
+[Install]
+WantedBy=multi-user.target
+"""
+)
+
+
+T_SSHD = """
+grep -q "HCLOUD_TOKEN" /etc/sshd/sshd_config || {
+    echo 'AcceptEnv HCLOUD_TOKEN' >> /etc/ssh/sshd_config
+    echo 'PasswordAuthentication no' >> /etc/ssh/sshd_config
+    echo 'ClientAliveInterval 60' >> /etc/ssh/sshd_config
+    echo 'ClientAliveCountMax 4' >> /etc/ssh/sshd_config
+    systemctl daemon-reload
+    systemctl reload sshd || systemctl restart ssh
+}
+type binenv 2>/dev/null || sed -i '1iexport PATH="$HOME/.binenv:$PATH"' ~/.bashrc
+test -e "/root/.ssh/id_ed25519" || ssh-keygen -q -t ecdsa -N '' -f "/root/.ssh/id_ed25519" >/dev/null
+touch /etc/postinstalled
+"""
+
+
+T_INST_TOOLS_PROXY = """
+    function have_ { type "$1" 2>/dev/null ; }
+    function install_binenv {
+        echo "Installing $binenv"
+        wget -q "https://raw.githubusercontent.com/axgkl/binaries/refs/heads/master/binenv-install.sh" -O - | bash
+        have_ "binenv" || { echo "binenv install failed" && exit 1; }
+    }
+    export URL_BINENV_DISTRIS="%(URL_BINENV_DISTRIS|)s"
+    export BINENV_TOOLS="%(BINENV_TOOLS_PROXY|)s %(BINENV_ADD_TOOLS_PROXY|)s"
+    have_ "binenv" || install_binenv
+    eval "binenv install $BINENV_TOOLS"
+"""
+
+
+T_CADDY = (
+    lambda cfg: f"""
+useradd -m -d /home/caddy caddy || true
+mkdir -p /opt/caddy
+test -f /root/caddy && mv /root/caddy /opt/caddy/caddy
+chmod +x /opt/caddy/caddy
+echo -e '{cfg}' > /opt/caddy/config.json
+chown -R caddy:caddy /opt/caddy
+echo -e '
+[Unit]
+Description=Caddy Web Server
+Documentation=https://caddyserver.com/docs/
+After=network.target
+
+[Service]
+User=caddy
+ExecStartPre=/usr/bin/mkdir -p /home/caddy/.config
+ExecStart=/opt/caddy/caddy run --config /opt/caddy/config.json
+TimeoutStopSec=5s
+LimitNOFILE=1048576
+LimitNPROC=512
+PrivateTmp=true
+ProtectSystem=full
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+
+[Install]
+WantedBy=multi-user.target
+' > /etc/systemd/system/caddy.service
+systemctl daemon-reload
+for a in enable restart; do systemctl $a caddy; done
+systemctl status caddy
+"""
+)

src/pyhk3/add.py

@@ -0,0 +1,71 @@
+---
+cluster_name: "%(NAME)s"
+kubeconfig_path: "/root/.kube/config"
+k3s_version: "%(HK_VER_K3S)s"
+networking:
+  ssh:
+    port: 22
+    use_agent: false # set to true if your key has a passphrase
+    public_key_path: "~/.ssh/id_ed25519.pub"
+    private_key_path: "~/.ssh/id_ed25519"
+  allowed_networks:
+    ssh:
+      - 0.0.0.0/0
+    api:
+      - 0.0.0.0/0
+  public_network:
+    ipv4: false
+    ipv6: true
+  private_network:
+    enabled: true
+    subnet: 10.%(HK_HOST_NETWORK)s.0.0/16
+    existing_network_name: ten-%(HK_HOST_NETWORK)s
+  cni:
+    enabled: true
+    encryption: false
+    mode: T(HK_CNI)s
+  cluster_cidr: "%(HK_CIDR_CLUSTER)s"
+  service_cidr: "%(HK_CIDR_SERVICE)s"
+  cluster_dns: "%(HK_DNS_CLUSTER)s"
+datastore:
+  mode: etcd # etcd (default) or external
+  external_datastore_endpoint: postgres://....
+schedule_workloads_on_masters: T(HK_MASTERS_ARE_WORKERS)s
+create_load_balancer_for_the_kubernetes_api: false
+protect_against_deletion: false
+
+masters_pool:
+  instance_type: "%(HK_MASTERS_TYPE)s"
+  instance_count: T(HK_MASTERS_COUNT)s
+  location: "%(HK_LOCATION)s"
+  image: "%(HK_MASTERS_IMG)s"
+
+worker_node_pools:
+  - name: "%(NAME)s-small-static"
+    instance_type: "%(HK_WORKERS_TYPE)s"
+    instance_count: T(HK_WORKERS_COUNT)s
+    location: "%(HK_LOCATION)s"
+    image: "%(HK_WORKERS_IMG)s"
+    # labels:
+    #   - key: purpose
+    #     value: blah
+    # taints:
+    #   - key: something
+    #     value: value1:NoSchedule
+
+  - name: "%(NAME)s-medium-autoscaled"
+    instance_type: "%(HK_AUTOSCL_TYPE)s"
+    location: "%(HK_LOCATION)s"
+    image: "%(HK_AUTOSCL_IMG)s"
+    autoscaling:
+      enabled: true
+      min_instances: 0
+      max_instances: T(HK_AUTOSCL_COUNT)s
+
+embedded_registry_mirror:
+  enabled: T(HK_REGISTRY_MIRROR)s
+
+additional_packages:
+  - ifupdown
+
+api_server_hostname: "first_master"

src/pyhk3/assets/create_templ.py

@@ -0,0 +1,26 @@
+post_create_commands:
+  - echo "Started" > /.status
+  - timedatectl set-timezone Europe/Berlin
+  - echo '%(key_proxy)s' >> /root/.ssh/authorized_keys
+  - echo '%(key_local)s' >> /root/.ssh/authorized_keys
+  - echo "root:$(head -c 50 /dev/urandom | base64)" | chpasswd
+  - mkdir -p /etc/network/interfaces.d
+  - iface="$(ip -o -4 addr list | grep " 10.%(HK_HOST_NETWORK)s." | cut -d " " -f 2)"
+  - |
+    cat > /etc/network/interfaces.d/$iface <<EOF
+    auto $iface
+    iface $iface inet dhcp
+      post-up ip route add default via 10.%(HK_HOST_NETWORK)s.0.1
+      post-up ip route add 169.254.169.254 via 172.31.1.1
+    EOF
+  - rm -f /etc/resolv.conf
+  - |
+    cat > /etc/resolv.conf <<EOF
+    nameserver 185.12.64.1
+    nameserver 185.12.64.2
+    edns edns0 trust-ad
+    search .
+    EOF
+  - ip route add 169.254.0.0/16 via 172.31.1.1
+  - ip route add default via 10.%(HK_HOST_NETWORK)s.0.1
+  - echo "Done" > /.status

src/pyhk3/assets/hk3s.yml

@@ -0,0 +1,24 @@
+nil = b'\x00'
+
+
+class Cache:
+    active = True
+
+    def __init__(self):
+        self.cache = {}
+
+    def clear(self, path=None):
+        if path is None:
+            return self.cache.clear()
+        self.cache.pop(path, None)
+
+    def get(self, key):
+        if not self.active:
+            return nil
+        return self.cache.get(key, nil)
+
+    def set(self, key, value):
+        self.cache[key] = value
+
+
+cache = Cache()