Merge pull request #607 from bitcoin-dev-project/ingress

Add ingress

.github/workflows/test.yml

@@ -78,6 +78,8 @@ jobs:
         with:
           cpus: max
           memory: 4000m
+      - name: Start minikube's loadbalancer tunnel
+        run: minikube tunnel &> /dev/null &
       - name: Download commander artifact
         uses: actions/download-artifact@v4
         with:

resources/charts/caddy/templates/ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: caddy-ingress
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+spec:
+  ingressClassName: nginx
+  rules:
+  - http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: {{ include "caddy.fullname" . }}
+            port: 
+              number: {{ .Values.port }}

src/warnet/constants.py

@@ -13,6 +13,7 @@
 
 DEFAULT_NAMESPACE = "warnet"
 LOGGING_NAMESPACE = "warnet-logging"
+INGRESS_NAMESPACE = "ingress"
 HELM_COMMAND = "helm upgrade --install --create-namespace"
 
 # Directories and files for non-python assets, e.g., helm charts, example scenarios, default configs
@@ -35,6 +36,7 @@
 NAMESPACES_CHART_LOCATION = CHARTS_DIR.joinpath("namespaces")
 FORK_OBSERVER_CHART = str(files("resources.charts").joinpath("fork-observer"))
 CADDY_CHART = str(files("resources.charts").joinpath("caddy"))
+CADDY_INGRESS_NAME = "caddy-ingress"
 
 DEFAULT_NETWORK = Path("6_node_bitcoin")
 DEFAULT_NAMESPACES = Path("two_namespaces_two_users")
@@ -98,3 +100,10 @@
     f"helm upgrade --install grafana-dashboards {CHARTS_DIR}/grafana-dashboards --namespace warnet-logging",
     f"helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values {MANIFESTS_DIR}/grafana_values.yaml",
 ]
+
+
+INGRESS_HELM_COMMANDS = [
+    "helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx",
+    "helm repo update",
+    f"helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx --namespace {INGRESS_NAMESPACE} --create-namespace",
+]

src/warnet/control.py

@@ -16,7 +16,6 @@
 from rich.table import Table
 
 from .constants import COMMANDER_CHART, LOGGING_NAMESPACE
-from .deploy import _port_stop_internal
 from .k8s import (
     get_default_namespace,
     get_mission,
@@ -140,8 +139,6 @@ def delete_pod(pod_name, namespace):
         for future in as_completed(futures):
             console.print(f"[yellow]{future.result()}[/yellow]")
 
-    # Shutdown any port forwarding
-    _port_stop_internal("caddy", namespaces[1])
     console.print("[bold yellow]Teardown process initiated for all components.[/bold yellow]")
     console.print("[bold yellow]Note: Some processes may continue in the background.[/bold yellow]")
     console.print("[bold green]Warnet teardown process completed.[/bold green]")

src/warnet/dashboard.py

@@ -1,11 +1,27 @@
 import click
 
+from .k8s import get_ingress_ip_or_host, wait_for_ingress_controller
+
 
 @click.command()
 def dashboard():
     """Open the Warnet dashboard in default browser"""
     import webbrowser
 
-    url = "http://localhost:2019"
+    wait_for_ingress_controller()
+    ip = get_ingress_ip_or_host()
+
+    if not ip:
+        click.echo("Error: Could not get the IP address of the dashboard")
+        click.echo(
+            "If you are running Minikube please run 'minikube tunnel' in a separate terminal"
+        )
+        click.echo(
+            "If you are running in the cloud, you may need to wait a short while while the load balancer is provisioned"
+        )
+        return
+
+    url = f"http://{ip}"
+
     webbrowser.open(url)
-    click.echo("warnet dashboard opened in default browser")
+    click.echo("Warnet dashboard opened in default browser")

src/warnet/deploy.py

@@ -1,4 +1,3 @@
-import os
 import subprocess
 import sys
 import tempfile
@@ -14,13 +13,14 @@
     DEFAULTS_NAMESPACE_FILE,
     FORK_OBSERVER_CHART,
     HELM_COMMAND,
+    INGRESS_HELM_COMMANDS,
     LOGGING_HELM_COMMANDS,
     LOGGING_NAMESPACE,
     NAMESPACES_CHART_LOCATION,
     NAMESPACES_FILE,
     NETWORK_FILE,
 )
-from .k8s import get_default_namespace, wait_for_caddy_ready
+from .k8s import get_default_namespace, wait_for_ingress_controller, wait_for_pod_ready
 from .process import stream_command
 
 
@@ -51,6 +51,7 @@ def deploy(directory, debug):
         deploy_network(directory, debug)
         df = deploy_fork_observer(directory, debug)
         if dl | df:
+            deploy_ingress(debug)
             deploy_caddy(directory, debug)
     elif (directory / NAMESPACES_FILE).exists():
         deploy_namespaces(directory)
@@ -118,8 +119,21 @@ def deploy_caddy(directory: Path, debug: bool):
         click.echo(f"Failed to run Helm command: {cmd}")
         return
 
-    wait_for_caddy_ready(name, namespace)
-    _port_start_internal(name, namespace)
+    wait_for_pod_ready(name, namespace)
+    click.echo("\nTo access the warnet dashboard run:\n  warnet dashboard")
+
+
+def deploy_ingress(debug: bool):
+    click.echo("Deploying ingress controller")
+
+    for command in INGRESS_HELM_COMMANDS:
+        if not stream_command(command):
+            print(f"Failed to run Helm command: {command}")
+            return False
+
+    wait_for_ingress_controller()
+
+    return True
 
 
 def deploy_fork_observer(directory: Path, debug: bool) -> bool:
@@ -279,19 +293,3 @@ def run_detached_process(command):
         subprocess.Popen(command, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True)
 
     print(f"Started detached process: {command}")
-
-
-def _port_start_internal(name, namespace):
-    click.echo("Starting port-forwarding to warnet dashboard")
-    command = f"kubectl port-forward -n {namespace} service/{name} 2019:80"
-    run_detached_process(command)
-    click.echo("Port forwarding on port 2019 started in the background.")
-    click.echo("\nTo access the warnet dashboard visit localhost:2019 or run:\n  warnet dashboard")
-
-
-def _port_stop_internal(name, namespace):
-    if is_windows():
-        os.system("taskkill /F /IM kubectl.exe")
-    else:
-        os.system(f"pkill -f 'kubectl port-forward -n {namespace} service/{name} 2019:80'")
-    click.echo("Port forwarding stopped.")

src/warnet/k8s.py

@@ -10,7 +10,13 @@
 from kubernetes.dynamic import DynamicClient
 from kubernetes.stream import stream
 
-from .constants import DEFAULT_NAMESPACE, KUBECONFIG
+from .constants import (
+    CADDY_INGRESS_NAME,
+    DEFAULT_NAMESPACE,
+    INGRESS_NAMESPACE,
+    KUBECONFIG,
+    LOGGING_NAMESPACE,
+)
 from .process import run_command, stream_command
 
 
@@ -239,7 +245,7 @@ def snapshot_bitcoin_datadir(
         print(f"An error occurred: {str(e)}")
 
 
-def wait_for_caddy_ready(name, namespace, timeout=300):
+def wait_for_pod_ready(name, namespace, timeout=300):
     sclient = get_static_client()
     w = watch.Watch()
     for event in w.stream(
@@ -250,8 +256,29 @@ def wait_for_caddy_ready(name, namespace, timeout=300):
             conditions = pod.status.conditions or []
             ready_condition = next((c for c in conditions if c.type == "Ready"), None)
             if ready_condition and ready_condition.status == "True":
-                print(f"Caddy pod {name} is ready.")
                 w.stop()
                 return True
-    print(f"Timeout waiting for Caddy pod {name} to be ready.")
+    print(f"Timeout waiting for pod {name} to be ready.")
     return False
+
+
+def wait_for_ingress_controller(timeout=300):
+    # get name of ingress controller pod
+    sclient = get_static_client()
+    pods = sclient.list_namespaced_pod(namespace=INGRESS_NAMESPACE)
+    for pod in pods.items:
+        if "ingress-nginx-controller" in pod.metadata.name:
+            return wait_for_pod_ready(pod.metadata.name, INGRESS_NAMESPACE, timeout)
+
+
+def get_ingress_ip_or_host():
+    config.load_kube_config()
+    networking_v1 = client.NetworkingV1Api()
+    try:
+        ingress = networking_v1.read_namespaced_ingress(CADDY_INGRESS_NAME, LOGGING_NAMESPACE)
+        if ingress.status.load_balancer.ingress[0].hostname:
+            return ingress.status.load_balancer.ingress[0].hostname
+        return ingress.status.load_balancer.ingress[0].ip
+    except Exception as e:
+        print(f"Error getting ingress IP: {e}")
+        return None

src/warnet/process.py

@@ -18,12 +18,14 @@ def stream_command(command: str) -> bool:
         universal_newlines=True,
     )
 
+    message = ""
     for line in iter(process.stdout.readline, ""):
+        message += line
         print(line, end="")
 
     process.stdout.close()
     return_code = process.wait()
 
     if return_code != 0:
-        raise Exception(process.stderr)
+        raise Exception(message)
     return True

test/logging_test.py

@@ -7,7 +7,7 @@
 import requests
 from test_base import TestBase
 
-GRAFANA_URL = "http://localhost:2019/grafana/"
+from warnet.k8s import get_ingress_ip_or_host
 
 
 class LoggingTest(TestBase):
@@ -29,13 +29,17 @@ def setup_network(self):
         self.log.info(self.warnet(f"deploy {self.network_dir}"))
         self.wait_for_all_tanks_status(target="running", timeout=10 * 60)
         self.wait_for_all_edges()
+        self.wait_for_predicate(lambda: get_ingress_ip_or_host())
+        ingress_ip = get_ingress_ip_or_host()
+        self.grafana_url = f"http://{ingress_ip}/grafana"
+        self.log.info(f"Grafana URL: {self.grafana_url}")
 
     def wait_for_endpoint_ready(self):
         self.log.info("Waiting for Grafana to be ready to receive API calls...")
 
         def check_endpoint():
             try:
-                response = requests.get(f"{GRAFANA_URL}login")
+                response = requests.get(f"{self.grafana_url}/login")
                 return response.status_code == 200
             except requests.RequestException:
                 return False
@@ -50,7 +54,7 @@ def make_grafana_api_request(self, ds_uid, start, metric):
             "from": f"{start}",
             "to": "now",
         }
-        reply = requests.post(f"{GRAFANA_URL}api/ds/query", json=data)
+        reply = requests.post(f"{self.grafana_url}/api/ds/query", json=data)
         if reply.status_code != 200:
             self.log.error(f"Grafana API request failed with status code {reply.status_code}")
             self.log.error(f"Response content: {reply.text}")
@@ -67,7 +71,7 @@ def test_prometheus_and_grafana(self):
         self.warnet(f"run {miner_file} --allnodes --interval=5 --mature")
         self.warnet(f"run {tx_flood_file} --interval=1")
 
-        prometheus_ds = requests.get(f"{GRAFANA_URL}api/datasources/name/Prometheus")
+        prometheus_ds = requests.get(f"{self.grafana_url}/api/datasources/name/Prometheus")
         assert prometheus_ds.status_code == 200
         prometheus_uid = prometheus_ds.json()["uid"]
         self.log.info(f"Got Prometheus data source uid from Grafana: {prometheus_uid}")
@@ -92,7 +96,7 @@ def get_five_values_for_metric(metric):
         self.wait_for_predicate(lambda: get_five_values_for_metric("txrate"))
 
         # Verify default dashboard exists
-        dbs = requests.get(f"{GRAFANA_URL}api/search").json()
+        dbs = requests.get(f"{self.grafana_url}/api/search").json()
         assert dbs[0]["title"] == "Default Warnet Dashboard"
 
 

test/services_test.py

@@ -6,6 +6,8 @@
 import requests
 from test_base import TestBase
 
+from warnet.k8s import get_ingress_ip_or_host
+
 
 class ServicesTest(TestBase):
     def __init__(self):
@@ -32,7 +34,9 @@ def check_fork_observer(self):
         # Port will be auto-forwarded by `warnet deploy`, routed through the enabled Caddy pod
 
         def call_fo_api():
-            fo_root = "http://localhost:2019/fork-observer"
+            # if on minikube remember to run `minikube tunnel` for this test to run
+            ingress_ip = get_ingress_ip_or_host()
+            fo_root = f"http://{ingress_ip}/fork-observer"
             try:
                 fo_res = requests.get(f"{fo_root}/api/networks.json")
                 network_id = fo_res.json()["networks"][0]["id"]