feat flux install

with pass key insert when not existing, after create with age

Author: Gunther Klessinger

environ

@@ -27,3 +27,4 @@ GITOPS_OWNER="company"
 GITOPS_PATH="clusters/staging"
 GITOPS_REPO="k8s"
 GITOPS_TOKEN="py:keyval"
+GITOPS_FLUX_PRIV_SECRET="py:keyval"

justfile

@@ -53,8 +53,8 @@ port-forward:
   just p do port_forward
 
 
-install-gitops:
-  just p gitops install
+install-flux:
+  just p flux install
 
 
 test:

keyval.py

@@ -19,3 +19,6 @@
 GITOPS_TOKEN = sec.get('GITOPS_TOKEN', '...')
 HCLOUD_TOKEN = sec.get('HCLOUD_TOKEN', '...')
 HCLOUD_TOKEN_WRITE = sec.get('HCLOUD_TOKEN_WRITE', '...')
+
+# delivering a pass value is also possible:
+GITOPS_FLUX_PRIV_SECRET = sec.get('GITOPS_FLUX_PRIV_SECRET', 'pass:my/flux_priv_secret')

pyproject.toml

@@ -57,6 +57,7 @@ extend-select = ["Q"]
 select = ["E", "F", "B"] # Enable flake8-bugbear (`B`) rules.
 ignore = [
   "E501", # Never enforce `E501` (line length violations).
+  "E713", # not in condition
   "E741", # short var names
   "E731", # no lambda
   "B006", # mutables in signature

src/pyhk3/assets/kubectl.py

@@ -0,0 +1,19 @@
+T_NS = """
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "%(namespace)s"
+"""
+
+
+T_SECRET = """
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "%(name)s"
+  namespace: "%(namespace)s"
+data:
+%(data)s
+"""

src/pyhk3/cli.py

@@ -4,7 +4,7 @@
 from inspect import signature as sig
 from functools import partial
 from .create import create, hk3s
-from .gitops import gitops
+from .flux import flux
 from .do import do, recover
 from rich.console import Console
 from rich.tree import Tree
@@ -17,7 +17,7 @@ class pyhk3:
     do = do
     recover = recover
     hk3s = hk3s
-    gitops = gitops
+    flux = flux
 
 
 console = Console()

src/pyhk3/defaults.py

@@ -1,16 +1,19 @@
 binenv_custom_base = 'https://github.com/axgkl/binaries/raw/master'
 dflt_img = 'ubuntu-24.04'
 dflt_type = 'cx22'
+tools_proxy = 'helm 3.16.3 kubectl 1.31.3 hetzner-k3s 2.2.3 btop 1.4.0'
+tools_local = 'flux 2.4.0 yq 4.44.5 age-keygen 1.2.0 sops 3.9.1'
 
 
 class envdefaults:
-    BINENV_TOOLS_PROXY = 'helm 3.16.3 kubectl 1.31.3 hetzner-k3s 2.2.3 btop 1.4.0'
+    BINENV_TOOLS_PROXY = tools_proxy
     DNS_API_TOKEN = 'your-token-to-add-a-wildcard-dns-entry-at-provider'
     DNS_PROVIDER = 'digitalocean'  # see dns.py for others
     DNS_TTL = 60
     DOMAIN = 'k8s.mycompany.net'
     EMAIL = '[email protected]'
     FN_SSH_KEY = '$HOME/.ssh/hetzner-cluster'  # created if not exists, also on hcloud
+    GITOPS_FLUX_PRIV_SECRET = ''
     GITOPS_BRANCH = 'main'
     GITOPS_HOST = 'gitlab.mycompany.com'
     GITOPS_OWNER = 'company'

src/pyhk3/flux.py

@@ -0,0 +1,59 @@
+from .ssh import ensure_forward
+from .tools import run, log, die, need_env as E, read_file, env_key_on_missing
+from .tools import add_to_pass
+from .kubectl import ensure_namespace
+import os
+import sh
+# from kubernetes import client, config
+# # Load the kubeconfig file
+# config.load_kube_config()
+# # Create a V1Namespace object
+# namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name='my-namespace'))
+# # Create the namespace using the CoreV1Api
+# v1 = client.CoreV1Api()
+# breakpoint()  # FIXME BREAKPOINT
+# v1.create_namespace(namespace)
+
+
+def install():
+    """To uninstall use: flux uninstall."""
+    host = E('GITOPS_HOST')
+    os.environ['GITLAB_TOKEN'] = E('GITOPS_TOKEN')
+    if not 'gitlab' in host:
+        die('Only gitlab is supported at this time')
+    ensure_forward()
+    run('flux check --pre')
+    ensure_namespace('flux-system')
+    ns = ('--namespace', 'flux-system')
+    have = sh.kubectl.get('secrets', *ns)
+    if 'sops-age' in have:
+        die('Secret already exists', hint='run: flux check, possibly then flux uninstall')
+    S = 'AGE-SECRET-KEY'
+    s = priv = E('GITOPS_FLUX_PRIV_SECRET', env_key_on_missing)
+    if S in s:
+        log.info('Using existing secret')
+    else:
+        log.warn('No GITOPS_FLUX_PRIV_SECRET - gen.ing new one using age')
+        priv = sh.age_keygen().strip().split('\n')[-1]
+        assert priv.startswith(S)
+
+    _ = '--from-file=age.agekey=/dev/stdin'
+    sh.kubectl.create.secret.generic('sops-age', _, *ns, _in=priv)
+    cmd = [
+        'flux',
+        'bootstrap',
+        'gitlab',
+        f'--owner={E("GITOPS_OWNER")}',
+        f'--path={E("GITOPS_PATH")}',
+        f'--repository={E("GITOPS_REPO")}',
+        f'--hostname={host}',
+        f'--branch={E("GITOPS_BRANCH")}',
+        '--token-auth',
+    ]
+    run(cmd)
+    if s.startswith('pass:'):
+        add_to_pass(s, priv)
+
+
+class flux:
+    install = install

src/pyhk3/gitops.py

@@ -0,0 +1,17 @@
+from .assets.kubectl import T_NS, T_SECRET
+from .tools import run
+import sh
+
+
+def ensure_namespace(namespace: str):
+    try:
+        sh.kubectl('get', 'namespace', namespace)
+        return
+    except sh.ErrorReturnCode_1:
+        n = T_NS % {'namespace': namespace}
+        sh.kubectl('apply', '-f', '-', _in=n)
+
+
+def add_secret(name: str, namespace: str, data: str):
+    breakpoint()  # FIXME BREAKPOINT
+    run(f'kubectl create namespace {name}')

src/pyhk3/kubectl.py

@@ -1,9 +1,13 @@
 from ipaddress import ip_address
 from .hapi import ips
-from .tools import log, ssh, need_env as E
+from .tools import run, log, ssh, die, need_env as E
+from subprocess import Popen
 from functools import partial
 import sh
 
+from time import sleep
+import sys
+
 
 def ips_of_host(name):
     """name either ip or hostname"""
@@ -31,12 +35,36 @@ def ssh_add_no_hostkey_check(args):
             return
 
 
-def port_forward():
+def ensure_forward(_chck=False):
+    m = f'{E("NAME")}-master1'
+    try:
+        r = run('kubectl get nodes', no_fail=True, capture_output=True).decode()
+        if m in r:
+            return True
+        die('Wrong kubernetes cluster', notfound=m, output=r)
+    except Exception as _:
+        if _chck:
+            return 0
+        port_forward(nohup=True)
+        for _ in range(10):
+            sleep(1)
+            print('.', file=sys.stderr, end='')
+            if ensure_forward(_chck=True):
+                print('')
+                return
+        die('No tunnel')
+
+
+def port_forward(nohup=False):
+    log.info('Port forward', kubecfg_fwd_port=kubecfg_fwd_port)
+    c = ['htop', '-d', '50']
+    if nohup:
+        c = ['sleep', '3600']
     fwd = f'{kubecfg_fwd_port}:127.0.0.1:6443'
-    run_remote('master1', 'htop', '-d', '50', _term=True, _fwd=fwd)
+    run_remote('master1', *c, _term=True, _fwd=fwd, _nohup=nohup)
 
 
-def run_remote(name, *cmd, _fwd=None, _term=False, _fg=True):
+def run_remote(name, *cmd, _fwd=None, _term=False, _fg=True, _nohup=False):
     """ssh to servers, e.g. ssh proxy [cmd]. autovia via proxy built in."""
     log.debug('Run remote', name=name, cmd=cmd)
     ip_pub, ip = ips_of_host(name)
@@ -52,7 +80,18 @@ def run_remote(name, *cmd, _fwd=None, _term=False, _fg=True):
         args.insert(0, _fwd)
         args.insert(0, '-L')
     args.extend(list(cmd))
-    return sh.ssh(args, _fg=_fg)  # this can be redirected w/o probs
+    try:
+        if _nohup:
+            ssh_command = ' '.join(f'"{i}"' for i in args)
+            # sh.nohup not working
+            return Popen(
+                f'nohup ssh {ssh_command} > /dev/null 2> /dev/null &', shell=True
+            )
+        else:
+            r = sh.ssh(args, _fg=_fg)
+    except Exception as ex:
+        die('ssh failed', name=name, cmd=cmd, ex=ex)
+    return r
 
 
 kubecfg_fwd_port = int(E('HK_HOST_NETWORK')) + 6443

src/pyhk3/ssh.py

@@ -1,6 +1,7 @@
 import structlog
 import os
 import sys
+import sh
 import json
 import subprocess
 import importlib.util
@@ -77,6 +78,9 @@ def pyval(k, fn, dflt=None):
     return v
 
 
+env_key_on_missing = '__env_key_on_missing__'
+
+
 def env(key, dflt=None):
     v = os.environ.get(key, nil)
     if v == nil:
@@ -87,12 +91,33 @@ def env(key, dflt=None):
     if str(v).startswith('pass:'):
         x = _secrets.get(v, nil)
         if x == nil:
-            x = run(['pass', 'show', v[5:]], capture_output=True, text=True) or dflt
+            try:
+                x = pass_(key).show(v[5:], _err=[1, 2]).strip()
+            except Exception as _:
+                # special case: allows to calc and  set the value later:
+                if dflt == env_key_on_missing:
+                    return v
             _secrets[v] = x
         v = x
     return v
 
 
+def add_to_pass(key, val):
+    log.warn('Adding key to pass', n=key[5:])
+    pass_(key).insert('-m', key[5:], _in=val)
+    if not need_env(key) == val:
+        die('Failed to update pass', key=key[5:], value=val)
+
+
+def pass_(key=''):
+    p = getattr(sh, 'pass', None)
+    h = 'Install pass: https://www.passwordstore.org/'
+    h += '- or supply a wrapper, supporting show and insert [-m] methods, e.g. for reading/writing files'
+    if p is None:
+        die('pass utility not found', hint=h, required_for=key)
+    return p
+
+
 def dt(_, __, e):
     e['timestamp'] = f'{now() - T0:>4}'
     return e
@@ -112,40 +137,51 @@ def dt(_, __, e):
 log = structlog.get_logger()
 
 
-def die(msg, **kw):
+def die(msg, only_raise=False, **kw):
     log.fatal(msg, **kw)
+    if only_raise:
+        raise Exception(msg)
     sys.exit(1)
 
 
-def run(cmd, bg=False, **kw):
+def run(cmd, bg=False, no_fail=False, **kw):
+    i = kw.get('input')
+    if i is not None:
+        kw['input'] = i.encode() if isinstance(i, str) else i
     if isinstance(cmd, str):
         cmd = cmd.split()
     pipe = kw.get('pipe', '')
     pipe = pipe if not len(pipe) > 20 else f'{pipe[:20]}...'
-    log.debug('⚙️ Cmd', cmd=cmd, pipe=pipe)
+    lw = {}
+    if pipe:
+        lw['pipe'] = pipe
+    log.debug(f'⚙️ {" ".join(cmd)}', **lw)
     if bg:
         r = subprocess.Popen(cmd, start_new_session=True)
         # r.communicate()
         return r
 
     r = subprocess.run(cmd, **kw)
     if r.returncode != 0:
-        die('Command failed', cmd=cmd, returncode=r.returncode)
+        die('Command failed', cmd=cmd, returncode=r.returncode, only_raise=no_fail)
     return r.stdout.strip() if r.stdout else ''
 
 
-def need_env(k, dflt=None):
+def need_env(k, dflt=None, _home_repl=False):
     v = env(k, dflt)
     if v is None:
         die(f'Missing env var ${k}')
+    if _home_repl:
+        for k in '~', '$HOME':
+            v = v.replace(k, os.environ['HOME'])
     return v
 
 
 def ssh(ip, port=None, cmd=None, input=None, send_env=None, capture_output=True, **kw):
     cmd = cmd if cmd is not None else 'bash -s'
     port = port if port is not None else int(env('SSH_PORT', 22))
     c = f'ssh -p {port} -o StrictHostKeyChecking=accept-new -i'.split()
-    cmd = c + [need_env('FN_SSH_KEY'), f'root@{ip}', cmd]
+    cmd = c + [need_env('FN_SSH_KEY', _home_repl=True), f'root@{ip}', cmd]
     kw['capture_output'] = capture_output
     for key in send_env or []:
         cmd.insert(1, f'SendEnv={key}')