From 629b9c77573ae278c81bce1bcb7466d99612c3b1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 1 Oct 2024 12:39:07 +1000 Subject: [PATCH 01/12] Initial metrics server addition --- book/src/SUMMARY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 66be9cd..0ad81e7 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -1,7 +1,8 @@ # Summary * [Introduction](./intro.md) +* [Metrics](./metrics.md) * [FAQs](./faq.md) * [Protocol Developers](./developers.md) * [Contributing](./contributing.md) - * [Development Environment](./setup.md) +* [Development Environment](./setup.md) From 2d87ca39c3025e086b714cad8c220c6237c8d6e2 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 1 Oct 2024 12:42:00 +1000 Subject: [PATCH 02/12] Add metrics --- book/src/metrics.md | 52 ++ metrics/README.md | 40 ++ metrics/dashboards/dashboards_here | 0 metrics/docker-compose.yaml | 28 + metrics/grafana/Dockerfile | 4 + metrics/grafana/grafana.ini | 529 ++++++++++++++++++ .../provisioning/datasources/prometheus.yaml | 6 + metrics/prometheus/Dockerfile | 8 + metrics/prometheus/prometheus.yml | 12 + metrics/scrape-targets/README.md | 7 + .../scrape-targets-lighthouse.json | 11 + metrics/scrape-targets/scrape-targets.json | 10 + 12 files changed, 707 insertions(+) create mode 100644 book/src/metrics.md create mode 100644 metrics/README.md create mode 100644 metrics/dashboards/dashboards_here create mode 100644 metrics/docker-compose.yaml create mode 100644 metrics/grafana/Dockerfile create mode 100644 metrics/grafana/grafana.ini create mode 100644 metrics/grafana/provisioning/datasources/prometheus.yaml create mode 100644 metrics/prometheus/Dockerfile create mode 100644 metrics/prometheus/prometheus.yml create mode 100644 metrics/scrape-targets/README.md create mode 100644 metrics/scrape-targets/scrape-targets-lighthouse.json create mode 100644 metrics/scrape-targets/scrape-targets.json diff --git a/book/src/metrics.md b/book/src/metrics.md new file mode 100644 index 0000000..1c7e256 --- /dev/null +++ b/book/src/metrics.md @@ -0,0 +1,52 @@ +# Metrics + +Anchor comes pre-built with a suite of metrics for developers or users to monitor the health +and performance of their node. + +They must be enabled at runtime using the `--metrics` CLI flag. + +## Usage + +In order to run a metrics server, `docker` is required to be installed. + +Once docker is installed, a metrics server can be run locally via the following steps: + +1. Start an anchor node with `$ anchor --metrics` + - The `--metrics` flag is required for metrics. +1. Move into the metrics directory `$ cd metrics`. +1. Bring the environment up with `$ docker-compose up --build -d`. +1. Ensure that Prometheus can access your Anchor node by ensuring it is in + the `UP` state at [http://localhost:9090/targets](http://localhost:9090/targets). +1. Browse to [http://localhost:3000](http://localhost:3000) + - Username: `admin` + - Password: `changeme` +1. Import some dashboards from the `metrics/dashboards` directory in this repo: + - In the Grafana UI, go to `Dashboards` -> `Manage` -> `Import` -> `Upload .json file`. + - The `Summary.json` dashboard is a good place to start. + +## Dashboards + +A suite of dashboards can be found in `metrics/dashboard` directory. The Anchor team will +frequently update these dashboards as new metrics are introduced. + +We welcome Pull Requests for any users wishing to add their dashboards to this repository for +others to share. + +## Scrape Targets + +Prometheus periodically reads the `metrics/scrape-targets/scrape-targets.json` file. This +file tells Prometheus which endpoints to collect data from. The current file is setup to read +from Anchor on its default metrics port. You can add additional endpoints if you want to collect +metrics from other servers. + +An example is Lighthouse. You can collect metrics from Anchor and Lighthouse simultaneously if +they are both running. We have an example file `scrape-targets-lighthouse.json` which allows this. +You can replace the `scrape-targets.json` file with the contents of +`scrape-targets-lighthouse.json` if you wish to collect metrics from Anchor and Lighthouse +simultaneously. + +## Hosting Publicly + +By default Prometheus and Grafana will only bind to localhost (127.0.0.1), in +order to protect you from accidentally exposing them to the public internet. If +you would like to change this you must edit the `http_addr` in `metrics/grafana/grafana.ini`. diff --git a/metrics/README.md b/metrics/README.md new file mode 100644 index 0000000..97ddd5b --- /dev/null +++ b/metrics/README.md @@ -0,0 +1,40 @@ +# Anchor Metrics + +[![metrics.png](https://i.postimg.cc/Jh7rxtgp/metrics.png)](https://postimg.cc/4YMRN4Xc) + +Here we provides a `docker-compose` environment which scrapes metrics from Anchor +nodes using Prometheus and presents them in a browser-based Grafana GUI. + +For in-depth details on running metrics, please see the [anchor +book](https://anchor-book.sigmaprime.io). + +## Basic Usage + +1. Start an anchor node with `$ anchor --metrics` + - The `--metrics` flag is required for metrics. +1. Move in to the `metrics` directory `$ cd metrics`. +1. Bring the environment up with `$ docker-compose up --build`. +1. Ensure that Prometheus can access your Anchor node by ensuring it is in + the `UP` state at [http://localhost:9090/targets](http://localhost:9090/targets). +1. Browse to [http://localhost:3000](http://localhost:3000) + - Username: `admin` + - Password: `changeme` +1. Import some dashboards from the `dashboards` directory in this repo: + - In the Grafana UI, go to `Dashboards` -> `Manage` -> `Import` -> `Upload .json file`. + - The `Summary.json` dashboard is a good place to start. + +## Hosting Publicly + +By default Prometheus and Grafana will only bind to localhost (127.0.0.1), in +order to protect you from accidentally exposing them to the public internet. If +you would like to change this you must edit the `http_addr` in `grafana.ini`. + +## Maintenance and Contributing + +The Anchor team has a hosted version of this stack where we do the +majority of the monitoring. The dashboards in this repo are not guaranteed to +be kept updated as we add/modify metrics to Anchor. If you're having +problems, please reach out and we can update the dashboards here. + +Feel free to create your own dashboards, export them and submit them here as +PRs. diff --git a/metrics/dashboards/dashboards_here b/metrics/dashboards/dashboards_here new file mode 100644 index 0000000..e69de29 diff --git a/metrics/docker-compose.yaml b/metrics/docker-compose.yaml new file mode 100644 index 0000000..7a27eb4 --- /dev/null +++ b/metrics/docker-compose.yaml @@ -0,0 +1,28 @@ +# +# Metrics services -- provides a web GUI to monitor Lighthouse nodes. +# +version: "3.3" + +services: + prometheus: + build: + context: prometheus + volumes: + - prometheus-data:/prometheus + - type: bind + source: ./scrape-targets + target: /prometheus/targets + restart: always + network_mode: host + grafana: + build: + context: grafana + volumes: + - grafana-data:/var/lib/grafana + restart: always + network_mode: host + +volumes: + grafana-data: + prometheus-data: + targets: diff --git a/metrics/grafana/Dockerfile b/metrics/grafana/Dockerfile new file mode 100644 index 0000000..f8344e7 --- /dev/null +++ b/metrics/grafana/Dockerfile @@ -0,0 +1,4 @@ +FROM grafana/grafana + +COPY provisioning/ /etc/grafana/provisioning +COPY grafana.ini /etc/grafana/grafana.ini diff --git a/metrics/grafana/grafana.ini b/metrics/grafana/grafana.ini new file mode 100644 index 0000000..990b7ce --- /dev/null +++ b/metrics/grafana/grafana.ini @@ -0,0 +1,529 @@ +##################### Grafana Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +;app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +;instance_name = ${HOSTNAME} + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +;data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +;temp_data_lifetime = 24h + +# Directory where grafana can store logs +;logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +;plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +; provisioning = /usr/share/grafana/conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, socket) +;protocol = http + +# The ip address to bind to, empty will bind to all interfaces +http_addr = 127.0.0.1 + +# The http port to use +;http_port = 3000 + +# The public facing domain name used to access grafana from a browser +domain = lhm.sigp.io + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +root_url = https://lhm.sigp.io + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket path +;socket = + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +;type = sqlite3 +;host = 127.0.0.1:3306 +;name = grafana +;user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres" only, either "disable", "require" or "verify-full" +;ssl_mode = disable + +# For "sqlite3" only, path relative to data_path setting +;path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` +# memcache: 127.0.0.1:11211 +;connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy should wait before timing out default is 30 (seconds) +;timeout = 30 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +;send_user_header = false + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# Set to false to disable all checks to https://grafana.net +# for new vesions (grafana itself and plugins), check is used +# in some UI views to notify that grafana or plugin update exists +# This option does not cause any auto updates, nor send any information +# only a GET request to http://grafana.com to get latest versions +;check_for_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +#################################### Security #################################### +[security] +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +admin_password = changeme + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,