diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 000000000..8aea137b3 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,51 @@ +# Quick run + +## Installation + +Either install warnet via pip, or clone the source and install: + +### via pip + +You can install warnet via `pip` into a virtual environment with + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install warnet +``` + +### via cloned source + +You can install warnet from source into a virtual environment with + +```bash +git clone https://github.com/bitcoin-dev-project/warnet.git +cd warnet +python3 -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +## Running + +To get started first check you have all the necessary requirements: + +```bash +warnet setup +``` + +Then create your first network: + +```bash +# Create a new network in the current directory +warnet init + +# Or in a directory of choice +warnet new +``` + +Follow the guide to configure network variables. + +## fork-observer + +If you enabled [fork-observer](https://github.com/0xB10C/fork-observer), it will be available from the landing page at `localhost:2019`. diff --git a/resources/charts/caddy/.helmignore b/resources/charts/caddy/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/resources/charts/caddy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/resources/charts/caddy/Chart.yaml b/resources/charts/caddy/Chart.yaml new file mode 100644 index 000000000..4fbb87241 --- /dev/null +++ b/resources/charts/caddy/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: caddy-server +description: A Helm chart for Caddy + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: 0.1.0 diff --git a/resources/charts/caddy/templates/NOTES.txt b/resources/charts/caddy/templates/NOTES.txt new file mode 100644 index 000000000..9d2cc4cf0 --- /dev/null +++ b/resources/charts/caddy/templates/NOTES.txt @@ -0,0 +1 @@ +Caddy is serving your every need. diff --git a/resources/charts/caddy/templates/_helpers.tpl b/resources/charts/caddy/templates/_helpers.tpl new file mode 100644 index 000000000..7cfc3d479 --- /dev/null +++ b/resources/charts/caddy/templates/_helpers.tpl @@ -0,0 +1,57 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "caddy.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "caddy.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "caddy.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "caddy.labels" -}} +helm.sh/chart: {{ include "caddy.chart" . }} +{{ include "caddy.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "caddy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "caddy.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "caddy.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "caddy.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/resources/charts/caddy/templates/configmap.yaml b/resources/charts/caddy/templates/configmap.yaml new file mode 100644 index 000000000..a80ac8ea9 --- /dev/null +++ b/resources/charts/caddy/templates/configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "caddy.fullname" . }} + labels: + {{- include "caddy.labels" . | nindent 4 }} +data: + Caddyfile: | + {{- .Values.caddyConfig | nindent 4 }} + index: | + {{- .Values.htmlConfig | nindent 4 }} diff --git a/resources/charts/caddy/templates/pod.yaml b/resources/charts/caddy/templates/pod.yaml new file mode 100644 index 000000000..6e034934a --- /dev/null +++ b/resources/charts/caddy/templates/pod.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "caddy.fullname" . }} + labels: + {{- include "caddy.labels" . | nindent 4 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + app: {{ include "caddy.fullname" . }} +spec: + restartPolicy: "{{ .Values.restartPolicy }}" + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 4 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: web + containerPort: {{ .Values.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 8 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 8 }} + resources: + {{- toYaml .Values.resources | nindent 8 }} + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 8 }} + volumes: + {{- toYaml .Values.volumes | nindent 4 }} diff --git a/resources/charts/caddy/templates/service.yaml b/resources/charts/caddy/templates/service.yaml new file mode 100644 index 000000000..a25c46946 --- /dev/null +++ b/resources/charts/caddy/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "caddy.fullname" . }} + labels: + {{- include "caddy.labels" . | nindent 4 }} + app: {{ include "caddy.fullname" . }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.port }} + targetPort: web + protocol: TCP + name: http + selector: + {{- include "caddy.selectorLabels" . | nindent 4 }} diff --git a/resources/charts/caddy/values.yaml b/resources/charts/caddy/values.yaml new file mode 100644 index 000000000..01338e9ac --- /dev/null +++ b/resources/charts/caddy/values.yaml @@ -0,0 +1,123 @@ +# Default values for caddy. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +namespace: warnet + +restartPolicy: Always + +image: + repository: caddy + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "2.8.4" + +imagePullSecrets: [] + +nameOverride: "" + +fullnameOverride: "" + +podLabels: + app: "warnet" + mission: "proxy" + +podSecurityContext: {} +# fsGroup: 2000 + +securityContext: {} +# capabilities: +# drop: +# - ALL +# readOnlyRootFilesystem: true +# runAsNonRoot: true +# runAsUser: 1000 + +service: + type: ClusterIP + +resources: {} +# We usually recommend not to specify default resources and to leave this as a conscious +# choice for the user. This also increases chances charts run on environments with little +# resources, such as Minikube. If you do want to specify resources, uncomment the following +# lines, adjust them as necessary, and remove the curly braces after 'resources:'. +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +livenessProbe: + httpGet: + path: /live + port: 80 + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + +readinessProbe: + httpGet: + path: /ready + port: 80 + failureThreshold: 1 + periodSeconds: 1 + successThreshold: 1 + timeoutSeconds: 1 + +volumes: + - name: caddy-config + configMap: + name: caddy + items: + - key: Caddyfile + path: Caddyfile + - key: index + path: index + +volumeMounts: + - name: caddy-config + mountPath: /etc/caddy/Caddyfile + subPath: Caddyfile + - name: caddy-config + mountPath: /usr/share/caddy/index.html + subPath: index + +port: 80 + +caddyConfig: | + :80 { + respond /live 200 + respond /ready 200 + + root * /usr/share/caddy + file_server + + handle_path /fork-observer/* { + reverse_proxy fork-observer:2323 + } + + handle_path /grafana/* { + reverse_proxy loki-grafana:80 + } + + } + +htmlConfig: | + + + + + + Welcome + + +

Welcome to the Warnet dashboard

+

You can access the following services:

+ + + diff --git a/resources/charts/fork-observer/templates/NOTES.txt b/resources/charts/fork-observer/templates/NOTES.txt index 9894b7843..36a37bdc8 100644 --- a/resources/charts/fork-observer/templates/NOTES.txt +++ b/resources/charts/fork-observer/templates/NOTES.txt @@ -1,5 +1 @@ -To view forkobserver you must forward the port from the cluster to your local machine - -kubectl port-forward fork-observer 2323 - -fork-observer will then be available at web address: http://localhost:2323 \ No newline at end of file +Fork observer enabled. diff --git a/resources/manifests/grafana_values.yaml b/resources/manifests/grafana_values.yaml index 110622911..a6d16e3ab 100644 --- a/resources/manifests/grafana_values.yaml +++ b/resources/manifests/grafana_values.yaml @@ -16,6 +16,9 @@ grafana.ini: auth: disable_login_form: true disable_signout_menu: true + server: + # this is required to use Grafana behind a reverse proxy (caddy) + root_url: "%(protocol)s://%(domain)s:%(http_port)s/grafana/" auth.anonymous: enabled: true org_name: Main Org. diff --git a/resources/networks/6_node_bitcoin/network.yaml b/resources/networks/6_node_bitcoin/network.yaml index 6103d8a9b..21b05875d 100644 --- a/resources/networks/6_node_bitcoin/network.yaml +++ b/resources/networks/6_node_bitcoin/network.yaml @@ -29,4 +29,6 @@ nodes: - tank-0006 - name: tank-0006 fork_observer: - enabled: false + enabled: true +caddy: + enabled: true diff --git a/src/warnet/constants.py b/src/warnet/constants.py index bdd9dce9d..3f837328d 100644 --- a/src/warnet/constants.py +++ b/src/warnet/constants.py @@ -12,6 +12,7 @@ ] DEFAULT_NAMESPACE = "warnet" +LOGGING_NAMESPACE = "warnet-logging" HELM_COMMAND = "helm upgrade --install --create-namespace" # Directories and files for non-python assets, e.g., helm charts, example scenarios, default configs @@ -32,6 +33,9 @@ FORK_OBSERVER_CHART = str(CHARTS_DIR.joinpath("fork-observer")) COMMANDER_CHART = str(CHARTS_DIR.joinpath("commander")) NAMESPACES_CHART_LOCATION = CHARTS_DIR.joinpath("namespaces") +FORK_OBSERVER_CHART = str(files("resources.charts").joinpath("fork-observer")) +CADDY_CHART = str(files("resources.charts").joinpath("caddy")) + DEFAULT_NETWORK = Path("6_node_bitcoin") DEFAULT_NAMESPACES = Path("two_namespaces_two_users") diff --git a/src/warnet/control.py b/src/warnet/control.py index 5c35b131a..aa5991a92 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -14,7 +14,8 @@ from rich.prompt import Confirm, Prompt from rich.table import Table -from .constants import COMMANDER_CHART +from .constants import COMMANDER_CHART, LOGGING_NAMESPACE +from .deploy import _port_stop_internal from .k8s import ( get_default_namespace, get_mission, @@ -129,7 +130,7 @@ def down(): """Bring down a running warnet quickly""" console.print("[bold yellow]Bringing down the warnet...[/bold yellow]") - namespaces = [get_default_namespace(), "warnet-logging"] + namespaces = [get_default_namespace(), LOGGING_NAMESPACE] def uninstall_release(namespace, release_name): cmd = f"helm uninstall {release_name} --namespace {namespace} --wait=false" @@ -162,6 +163,8 @@ def delete_pod(pod_name, namespace): for future in as_completed(futures): console.print(f"[yellow]{future.result()}[/yellow]") + # Shutdown any port forwarding + _port_stop_internal("caddy", namespaces[1]) console.print("[bold yellow]Teardown process initiated for all components.[/bold yellow]") console.print("[bold yellow]Note: Some processes may continue in the background.[/bold yellow]") console.print("[bold green]Warnet teardown process completed.[/bold green]") diff --git a/src/warnet/dashboard.py b/src/warnet/dashboard.py new file mode 100644 index 000000000..28ac0cba6 --- /dev/null +++ b/src/warnet/dashboard.py @@ -0,0 +1,11 @@ +import click + + +@click.command() +def dashboard(): + """Open the Warnet dashboard in default browser""" + import webbrowser + + url = "http://localhost:2019" + webbrowser.open(url) + click.echo("warnet dashboard opened in default browser") diff --git a/src/warnet/deploy.py b/src/warnet/deploy.py index 7154c7b0b..0bcbd2c0f 100644 --- a/src/warnet/deploy.py +++ b/src/warnet/deploy.py @@ -1,3 +1,6 @@ +import os +import subprocess +import sys import tempfile from pathlib import Path @@ -6,16 +9,18 @@ from .constants import ( BITCOIN_CHART_LOCATION, + CADDY_CHART, DEFAULTS_FILE, DEFAULTS_NAMESPACE_FILE, FORK_OBSERVER_CHART, HELM_COMMAND, LOGGING_HELM_COMMANDS, + LOGGING_NAMESPACE, NAMESPACES_CHART_LOCATION, NAMESPACES_FILE, NETWORK_FILE, ) -from .k8s import get_default_namespace +from .k8s import get_default_namespace, wait_for_caddy_ready from .process import stream_command @@ -42,9 +47,11 @@ def deploy(directory, debug): directory = Path(directory) if (directory / NETWORK_FILE).exists(): - deploy_logging_stack(directory, debug) + dl = deploy_logging_stack(directory, debug) deploy_network(directory, debug) - deploy_fork_observer(directory, debug) + df = deploy_fork_observer(directory, debug) + if dl | df: + deploy_caddy(directory, debug) elif (directory / NAMESPACES_FILE).exists(): deploy_namespaces(directory) else: @@ -77,9 +84,9 @@ def check_logging_required(directory: Path): return False -def deploy_logging_stack(directory: Path, debug: bool): +def deploy_logging_stack(directory: Path, debug: bool) -> bool: if not check_logging_required(directory): - return + return False click.echo("Found collectLogs or metricsExport in network definition, Deploying logging stack") @@ -90,16 +97,42 @@ def deploy_logging_stack(directory: Path, debug: bool): return True -def deploy_fork_observer(directory: Path, debug: bool): +def deploy_caddy(directory: Path, debug: bool): network_file_path = directory / NETWORK_FILE with network_file_path.open() as f: network_file = yaml.safe_load(f) + namespace = LOGGING_NAMESPACE + # TODO: get this from the helm chart + name = "caddy" + # Only start if configured in the network file - if not network_file.get("fork_observer", {}).get("enabled", False): + if not network_file.get(name, {}).get("enabled", False): return - namespace = get_default_namespace() + cmd = f"{HELM_COMMAND} {name} {CADDY_CHART} --namespace {namespace}" + if debug: + cmd += " --debug" + + if not stream_command(cmd): + click.echo(f"Failed to run Helm command: {cmd}") + return + + wait_for_caddy_ready(name, namespace) + _port_start_internal(name, namespace) + + +def deploy_fork_observer(directory: Path, debug: bool) -> bool: + network_file_path = directory / NETWORK_FILE + with network_file_path.open() as f: + network_file = yaml.safe_load(f) + + # Only start if configured in the network file + if not network_file.get("fork_observer", {}).get("enabled", False): + return False + + default_namespace = get_default_namespace() + namespace = LOGGING_NAMESPACE cmd = f"{HELM_COMMAND} 'fork-observer' {FORK_OBSERVER_CHART} --namespace {namespace}" if debug: cmd += " --debug" @@ -115,7 +148,7 @@ def deploy_fork_observer(directory: Path, debug: bool): id = {i} name = "{node_name}" description = "A node. Just A node." -rpc_host = "{node_name}" +rpc_host = "{node_name}.{default_namespace}.svc" rpc_port = 18443 rpc_user = "forkobserver" rpc_password = "tabconf2024" @@ -138,7 +171,8 @@ def deploy_fork_observer(directory: Path, debug: bool): if not stream_command(cmd): click.echo(f"Failed to run Helm command: {cmd}") - return + return False + return True def deploy_network(directory: Path, debug: bool = False): @@ -221,3 +255,43 @@ def deploy_namespaces(directory: Path): finally: if temp_override_file_path.exists(): temp_override_file_path.unlink() + + +def is_windows(): + return sys.platform.startswith("win") + + +def run_detached_process(command): + if is_windows(): + # For Windows, use CREATE_NEW_PROCESS_GROUP and DETACHED_PROCESS + subprocess.Popen( + command, + shell=True, + stdin=None, + stdout=None, + stderr=None, + close_fds=True, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS, + ) + else: + # For Unix-like systems, use nohup and redirect output + command = f"nohup {command} > /dev/null 2>&1 &" + subprocess.Popen(command, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True) + + print(f"Started detached process: {command}") + + +def _port_start_internal(name, namespace): + click.echo("Starting port-forwarding to warnet dashboard") + command = f"kubectl port-forward -n {namespace} service/{name} 2019:80" + run_detached_process(command) + click.echo("Port forwarding on port 2019 started in the background.") + click.echo("\nTo access the warnet dashboard visit localhost:2019 or run:\n warnet dashboard") + + +def _port_stop_internal(name, namespace): + if is_windows(): + os.system("taskkill /F /IM kubectl.exe") + else: + os.system(f"pkill -f 'kubectl port-forward -n {namespace} service/{name} 2019:80'") + click.echo("Port forwarding stopped.") diff --git a/src/warnet/graph.py b/src/warnet/graph.py index 34b98960d..0f25b9d02 100644 --- a/src/warnet/graph.py +++ b/src/warnet/graph.py @@ -33,6 +33,8 @@ def custom_graph( datadir: Path, fork_observer: bool, fork_obs_query_interval: int, + caddy: bool, + logging: bool, ): datadir.mkdir(parents=False, exist_ok=False) # Generate network.yaml @@ -68,6 +70,9 @@ def custom_graph( "enabled": fork_observer, "configQueryInterval": fork_obs_query_interval, } + network_yaml_data["caddy"] = { + "enabled": caddy, + } with open(os.path.join(datadir, "network.yaml"), "w") as f: yaml.dump(network_yaml_data, f, default_flow_style=False) @@ -75,10 +80,13 @@ def custom_graph( # Generate node-defaults.yaml default_yaml_path = files("resources.networks").joinpath("node-defaults.yaml") with open(str(default_yaml_path)) as f: - defaults_yaml_content = f.read() + defaults_yaml_content = yaml.safe_load(f) + + # Configure logging + defaults_yaml_content["collectLogs"] = logging with open(os.path.join(datadir, "node-defaults.yaml"), "w") as f: - f.write(defaults_yaml_content) + yaml.dump(defaults_yaml_content, f, default_flow_style=False, sort_keys=False) click.echo( f"Project '{datadir}' has been created with 'network.yaml' and 'node-defaults.yaml'." @@ -171,6 +179,15 @@ def inquirer_create_network(project_path: Path): type=int, default=20, ) + + logging = click.prompt( + click.style( + "\nWould you like to enable grafana logging on the network?", fg="blue", bold=True + ), + type=bool, + default=False, + ) + caddy = fork_observer | logging custom_network_path = project_path / "networks" / net_answers["network_name"] click.secho("\nGenerating custom network...", fg="yellow", bold=True) custom_graph( @@ -180,6 +197,8 @@ def inquirer_create_network(project_path: Path): custom_network_path, fork_observer, fork_observer_query_interval, + caddy, + logging, ) return custom_network_path diff --git a/src/warnet/k8s.py b/src/warnet/k8s.py index bbe32f4ac..62a918320 100644 --- a/src/warnet/k8s.py +++ b/src/warnet/k8s.py @@ -4,7 +4,7 @@ from pathlib import Path import yaml -from kubernetes import client, config +from kubernetes import client, config, watch from kubernetes.client.models import CoreV1Event, V1PodList from kubernetes.dynamic import DynamicClient from kubernetes.stream import stream @@ -227,3 +227,21 @@ def snapshot_bitcoin_datadir( except Exception as e: print(f"An error occurred: {str(e)}") + + +def wait_for_caddy_ready(name, namespace, timeout=300): + sclient = get_static_client() + w = watch.Watch() + for event in w.stream( + sclient.list_namespaced_pod, namespace=namespace, timeout_seconds=timeout + ): + pod = event["object"] + if pod.metadata.name == name and pod.status.phase == "Running": + conditions = pod.status.conditions or [] + ready_condition = next((c for c in conditions if c.type == "Ready"), None) + if ready_condition and ready_condition.status == "True": + print(f"Caddy pod {name} is ready.") + w.stop() + return True + print(f"Timeout waiting for Caddy pod {name} to be ready.") + return False diff --git a/src/warnet/main.py b/src/warnet/main.py index 341d5757e..76893575c 100644 --- a/src/warnet/main.py +++ b/src/warnet/main.py @@ -3,6 +3,7 @@ from .admin import admin from .bitcoin import bitcoin from .control import down, logs, run, snapshot, stop +from .dashboard import dashboard from .deploy import deploy from .graph import create, graph from .image import image @@ -21,6 +22,7 @@ def cli(): cli.add_command(bitcoin) cli.add_command(deploy) cli.add_command(down) +cli.add_command(dashboard) cli.add_command(graph) cli.add_command(image) cli.add_command(init) diff --git a/src/warnet/project.py b/src/warnet/project.py index 33fb8ce47..4b54e2bd3 100644 --- a/src/warnet/project.py +++ b/src/warnet/project.py @@ -1,17 +1,14 @@ import os import platform -import random import subprocess import sys from dataclasses import dataclass from enum import Enum, auto -from importlib.resources import files from pathlib import Path from typing import Callable import click import inquirer -import yaml from .graph import inquirer_create_network from .network import copy_network_defaults, copy_scenario_defaults @@ -319,62 +316,3 @@ def init(): """Initialize a warnet project in the current directory""" current_dir = Path.cwd() new_internal(directory=current_dir, from_init=True) - - -def custom_graph( - num_nodes: int, - num_connections: int, - version: str, - datadir: Path, - fork_observer: bool, - fork_obs_query_interval: int, -): - datadir.mkdir(parents=False, exist_ok=False) - # Generate network.yaml - nodes = [] - connections = set() - - for i in range(num_nodes): - node = {"name": f"tank-{i:04d}", "connect": [], "image": {"tag": version}} - - # Add round-robin connection - next_node = (i + 1) % num_nodes - node["connect"].append(f"tank-{next_node:04d}") - connections.add((i, next_node)) - - # Add random connections - available_nodes = list(range(num_nodes)) - available_nodes.remove(i) - if next_node in available_nodes: - available_nodes.remove(next_node) - - for _ in range(min(num_connections - 1, len(available_nodes))): - random_node = random.choice(available_nodes) - # Avoid circular loops of A -> B -> A - if (random_node, i) not in connections: - node["connect"].append(f"tank-{random_node:04d}") - connections.add((i, random_node)) - available_nodes.remove(random_node) - - nodes.append(node) - - network_yaml_data = {"nodes": nodes} - network_yaml_data["fork_observer"] = { - "enabled": fork_observer, - "configQueryInterval": fork_obs_query_interval, - } - - with open(os.path.join(datadir, "network.yaml"), "w") as f: - yaml.dump(network_yaml_data, f, default_flow_style=False) - - # Generate node-defaults.yaml - default_yaml_path = files("resources.networks").joinpath("node-defaults.yaml") - with open(str(default_yaml_path)) as f: - defaults_yaml_content = f.read() - - with open(os.path.join(datadir, "node-defaults.yaml"), "w") as f: - f.write(defaults_yaml_content) - - click.echo( - f"Project '{datadir}' has been created with 'network.yaml' and 'node-defaults.yaml'." - ) diff --git a/test/data/logging/network.yaml b/test/data/logging/network.yaml index 59de12158..a06a5ea24 100644 --- a/test/data/logging/network.yaml +++ b/test/data/logging/network.yaml @@ -10,4 +10,6 @@ nodes: metrics: txrate=getchaintxstats(10)["txrate"] - name: tank-0002 connect: - - tank-0000 \ No newline at end of file + - tank-0000 +caddy: + enabled: true \ No newline at end of file diff --git a/test/data/services/network.yaml b/test/data/services/network.yaml index 6c19027a2..d523fbf97 100644 --- a/test/data/services/network.yaml +++ b/test/data/services/network.yaml @@ -33,3 +33,5 @@ nodes: rpcwhitelistdefault=0 fork_observer: enabled: true +caddy: + enabled: true diff --git a/test/graph_test.py b/test/graph_test.py index 5cc3200a6..482c555ab 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -40,6 +40,8 @@ def directory_exists(self): self.sut.sendline("") self.sut.expect("seconds", timeout=10) self.sut.sendline("") + self.sut.expect("enable grafana", timeout=10) + self.sut.sendline("") self.sut.expect("successfully", timeout=50) diff --git a/test/logging_test.py b/test/logging_test.py index f5c7134d4..218f51380 100755 --- a/test/logging_test.py +++ b/test/logging_test.py @@ -10,6 +10,8 @@ import requests from test_base import TestBase +GRAFANA_URL = "http://localhost:2019/grafana/" + class LoggingTest(TestBase): def __init__(self): @@ -60,7 +62,7 @@ def wait_for_endpoint_ready(self): def check_endpoint(): try: - response = requests.get("http://localhost:3000/login") + response = requests.get(f"{GRAFANA_URL}login") return response.status_code == 200 except requests.RequestException: return False @@ -75,7 +77,7 @@ def make_grafana_api_request(self, ds_uid, start, metric): "from": f"{start}", "to": "now", } - reply = requests.post("http://localhost:3000/api/ds/query", json=data) + reply = requests.post(f"{GRAFANA_URL}api/ds/query", json=data) if reply.status_code != 200: self.log.error(f"Grafana API request failed with status code {reply.status_code}") self.log.error(f"Response content: {reply.text}") @@ -92,7 +94,7 @@ def test_prometheus_and_grafana(self): self.warnet(f"run {miner_file} --allnodes --interval=5 --mature") self.warnet(f"run {tx_flood_file} --interval=1") - prometheus_ds = requests.get("http://localhost:3000/api/datasources/name/Prometheus") + prometheus_ds = requests.get(f"{GRAFANA_URL}api/datasources/name/Prometheus") assert prometheus_ds.status_code == 200 prometheus_uid = prometheus_ds.json()["uid"] self.log.info(f"Got Prometheus data source uid from Grafana: {prometheus_uid}") diff --git a/test/services_test.py b/test/services_test.py index 59048d3ce..a80717db9 100755 --- a/test/services_test.py +++ b/test/services_test.py @@ -2,7 +2,6 @@ import os from pathlib import Path -from subprocess import PIPE, Popen import requests from test_base import TestBase @@ -30,21 +29,14 @@ def check_fork_observer(self): self.log.info("Creating chain split") self.warnet("bitcoin rpc john createwallet miner") self.warnet("bitcoin rpc john -generate 1") - self.log.info("Forwarding port 2323...") - # Stays alive in background - self.fo_port_fwd_process = Popen( - ["kubectl", "port-forward", "fork-observer", "2323"], - stdout=PIPE, - stderr=PIPE, - bufsize=1, - universal_newlines=True, - ) + # Port will be auto-forwarded by `warnet deploy`, routed through the enabled Caddy pod def call_fo_api(): + fo_root = "http://localhost:2019/fork-observer" try: - fo_res = requests.get("http://localhost:2323/api/networks.json") + fo_res = requests.get(f"{fo_root}/api/networks.json") network_id = fo_res.json()["networks"][0]["id"] - fo_data = requests.get(f"http://localhost:2323/api/{network_id}/data.json") + fo_data = requests.get(f"{fo_root}/api/{network_id}/data.json") # fork observed! return len(fo_data.json()["header_infos"]) == 2 except Exception as e: @@ -54,7 +46,6 @@ def call_fo_api(): self.wait_for_predicate(call_fo_api) self.log.info("Fork observed!") - self.fo_port_fwd_process.terminate() if __name__ == "__main__":