diff --git a/resources/networks/6_node_bitcoin/node-defaults.yaml b/resources/networks/6_node_bitcoin/node-defaults.yaml index 8ecb0c79f..a1454d8a1 100644 --- a/resources/networks/6_node_bitcoin/node-defaults.yaml +++ b/resources/networks/6_node_bitcoin/node-defaults.yaml @@ -1,7 +1,7 @@ chain: regtest collectLogs: true -metricsExport: false +metricsExport: true resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/resources/scripts/install_logging.sh b/resources/scripts/install_logging.sh deleted file mode 100755 index 15fd1ab1e..000000000 --- a/resources/scripts/install_logging.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -e - -THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -MANIFESTS_DIR=$( cd -- "$THIS_DIR/../manifests" &> /dev/null && pwd ) - -helm repo add grafana https://grafana.github.io/helm-charts -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -helm repo update - -helm upgrade --install --namespace warnet-logging --create-namespace --values "${MANIFESTS_DIR}/loki_values.yaml" loki grafana/loki --version 5.47.2 -helm upgrade --install --namespace warnet-logging promtail grafana/promtail -helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false -helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${MANIFESTS_DIR}/grafana_values.yaml" \ No newline at end of file diff --git a/src/warnet/deploy.py b/src/warnet/deploy.py index 5417da5ae..7154c7b0b 100644 --- a/src/warnet/deploy.py +++ b/src/warnet/deploy.py @@ -10,6 +10,7 @@ DEFAULTS_NAMESPACE_FILE, FORK_OBSERVER_CHART, HELM_COMMAND, + LOGGING_HELM_COMMANDS, NAMESPACES_CHART_LOCATION, NAMESPACES_FILE, NETWORK_FILE, @@ -41,6 +42,7 @@ def deploy(directory, debug): directory = Path(directory) if (directory / NETWORK_FILE).exists(): + deploy_logging_stack(directory, debug) deploy_network(directory, debug) deploy_fork_observer(directory, debug) elif (directory / NAMESPACES_FILE).exists(): @@ -51,6 +53,43 @@ def deploy(directory, debug): ) +def check_logging_required(directory: Path): + # check if node-defaults has logging or metrics enabled + default_file_path = directory / DEFAULTS_FILE + with default_file_path.open() as f: + default_file = yaml.safe_load(f) + if default_file.get("collectLogs", False): + return True + if default_file.get("metricsExport", False): + return True + + # check to see if individual nodes have logging enabled + network_file_path = directory / NETWORK_FILE + with network_file_path.open() as f: + network_file = yaml.safe_load(f) + nodes = network_file.get("nodes", []) + for node in nodes: + if node.get("collectLogs", False): + return True + if node.get("metricsExport", False): + return True + + return False + + +def deploy_logging_stack(directory: Path, debug: bool): + if not check_logging_required(directory): + return + + click.echo("Found collectLogs or metricsExport in network definition, Deploying logging stack") + + for command in LOGGING_HELM_COMMANDS: + if not stream_command(command): + print(f"Failed to run Helm command: {command}") + return False + return True + + def deploy_fork_observer(directory: Path, debug: bool): network_file_path = directory / NETWORK_FILE with network_file_path.open() as f: diff --git a/src/warnet/network.py b/src/warnet/network.py index 6c42e0b5d..fcb5a5ee7 100644 --- a/src/warnet/network.py +++ b/src/warnet/network.py @@ -6,20 +6,10 @@ from .bitcoin import _rpc from .constants import ( - LOGGING_HELM_COMMANDS, NETWORK_DIR, SCENARIOS_DIR, ) from .k8s import get_mission -from .process import stream_command - - -def setup_logging_helm() -> bool: - for command in LOGGING_HELM_COMMANDS: - if not stream_command(command): - print(f"Failed to run Helm command: {command}") - return False - return True def copy_defaults(directory: Path, target_subdir: str, source_path: Path, exclude_list: list[str]): diff --git a/test/data/logging/node-defaults.yaml b/test/data/logging/node-defaults.yaml index 7e021cad1..b914c8bba 100644 --- a/test/data/logging/node-defaults.yaml +++ b/test/data/logging/node-defaults.yaml @@ -1,3 +1,4 @@ +collectLogs: true image: repository: bitcoindevproject/bitcoin pullPolicy: IfNotPresent diff --git a/test/logging_test.py b/test/logging_test.py index 9b8c0f9f3..f5c7134d4 100755 --- a/test/logging_test.py +++ b/test/logging_test.py @@ -5,7 +5,7 @@ import threading from datetime import datetime from pathlib import Path -from subprocess import PIPE, Popen, run +from subprocess import PIPE, Popen import requests from test_base import TestBase @@ -22,8 +22,8 @@ def __init__(self): def run_test(self): try: - self.start_logging() self.setup_network() + self.start_logging() self.test_prometheus_and_grafana() finally: if self.connect_logging_process is not None: @@ -32,10 +32,6 @@ def run_test(self): self.cleanup() def start_logging(self): - self.log.info("Running install_logging.sh") - # Block until complete - run([f"{self.scripts_dir / 'install_logging.sh'}"]) - self.log.info("Running connect_logging.sh") # Stays alive in background self.connect_logging_process = Popen( [f"{self.scripts_dir / 'connect_logging.sh'}"], @@ -51,13 +47,13 @@ def start_logging(self): ) self.connect_logging_thread.daemon = True self.connect_logging_thread.start() + self.wait_for_endpoint_ready() def setup_network(self): self.log.info("Setting up network") self.log.info(self.warnet(f"deploy {self.network_dir}")) self.wait_for_all_tanks_status(target="running", timeout=10 * 60) self.wait_for_all_edges() - self.wait_for_endpoint_ready() def wait_for_endpoint_ready(self): self.log.info("Waiting for Grafana to be ready to receive API calls...")