-
Notifications
You must be signed in to change notification settings - Fork 223
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: Enable collection of logs and metrics
- Loading branch information
Showing
8 changed files
with
325 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| # Timestamps are in seconds | ||
| from_timestamp="$(date '+%s')" | ||
| monitoring_period=900 # 15 minutes | ||
| to_timestamp="$((from_timestamp + monitoring_period))" | ||
|
|
||
| # Grafana expects microseconds, so pad timestamps with 3 zeros | ||
| metrics_url="${GRAFANA_URL}&var-filter=gh_job_id%7C%3D%7C${GH_JOB_ID}&from=${from_timestamp}000&to=${to_timestamp}000" | ||
|
|
||
| # Optionally ensure that the link displays metrics only for the shared | ||
| # network rather than mixing it with the results for private networks. | ||
| if [[ -n "${FILTER_BY_OWNER:-}" ]]; then | ||
| metrics_url="${metrics_url}&var-filter=network_owner%7C%3D%7C${FILTER_BY_OWNER}" | ||
| fi | ||
|
|
||
| echo "::notice links::metrics ${metrics_url}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| # Starts a prometheus instance in agent-mode, forwarding to a central | ||
| # instance. Intended to enable metrics collection from temporary networks running | ||
| # locally and in CI. | ||
| # | ||
| # The prometheus instance will remain running in the background and will forward | ||
| # metrics to the central instance for all tmpnet networks. | ||
| # | ||
| # To stop it: | ||
| # | ||
| # $ kill -9 `cat ~/.tmpnet/prometheus/run.pid` && rm ~/.tmpnet/prometheus/run.pid | ||
| # | ||
|
|
||
| # e.g., | ||
| # PROMETHEUS_ID=<id> PROMETHEUS_PASSWORD=<password> ./scripts/run_prometheus.sh | ||
| if ! [[ "$0" =~ scripts/run_prometheus.sh ]]; then | ||
| echo "must be run from repository root" | ||
| exit 255 | ||
| fi | ||
|
|
||
| PROMETHEUS_WORKING_DIR="${HOME}/.tmpnet/prometheus" | ||
| PIDFILE="${PROMETHEUS_WORKING_DIR}"/run.pid | ||
|
|
||
| # First check if an agent-mode prometheus is already running. A single instance can collect | ||
| # metrics from all local temporary networks. | ||
| if pgrep --pidfile="${PIDFILE}" -f 'prometheus.*enable-feature=agent' &> /dev/null; then | ||
| echo "prometheus is already running locally with --enable-feature=agent" | ||
| exit 0 | ||
| fi | ||
|
|
||
| PROMETHEUS_URL="${PROMETHEUS_URL:-https://prometheus-experimental.avax-dev.network}" | ||
| if [[ -z "${PROMETHEUS_URL}" ]]; then | ||
| echo "Please provide a value for PROMETHEUS_URL" | ||
| exit 1 | ||
| fi | ||
|
|
||
| PROMETHEUS_ID="${PROMETHEUS_ID:-}" | ||
| if [[ -z "${PROMETHEUS_ID}" ]]; then | ||
| echo "Please provide a value for PROMETHEUS_ID" | ||
| exit 1 | ||
| fi | ||
|
|
||
| PROMETHEUS_PASSWORD="${PROMETHEUS_PASSWORD:-}" | ||
| if [[ -z "${PROMETHEUS_PASSWORD}" ]]; then | ||
| echo "Plase provide a value for PROMETHEUS_PASSWORD" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # This was the LTS version when this script was written. Probably not | ||
| # much reason to update it unless something breaks since the usage | ||
| # here is only to collect metrics from temporary networks. | ||
| VERSION="2.45.3" | ||
|
|
||
| # Ensure the prometheus command is locally available | ||
| CMD=prometheus | ||
| if ! command -v "${CMD}" &> /dev/null; then | ||
| # Try to use a local version | ||
| CMD="${PWD}/bin/prometheus" | ||
| if ! command -v "${CMD}" &> /dev/null; then | ||
| echo "prometheus not found, attempting to install..." | ||
|
|
||
| # Determine the arch | ||
| if which sw_vers &> /dev/null; then | ||
| echo "on macos, only amd64 binaries are available so rosetta is required on apple silicon machines." | ||
| echo "to avoid using rosetta, install via homebrew: brew install prometheus" | ||
| DIST=darwin | ||
| else | ||
| ARCH="$(uname -i)" | ||
| if [[ "${ARCH}" != "x86_64" ]]; then | ||
| echo "on linux, only amd64 binaries are available. manual installation of prometheus is required." | ||
| exit 1 | ||
| else | ||
| DIST="linux" | ||
| fi | ||
| fi | ||
|
|
||
| # Install the specified release | ||
| PROMETHEUS_FILE="prometheus-${VERSION}.${DIST}-amd64" | ||
| URL="https://github.com/prometheus/prometheus/releases/download/v${VERSION}/${PROMETHEUS_FILE}.tar.gz" | ||
| curl -s -L "${URL}" | tar zxv -C /tmp > /dev/null | ||
| mkdir -p "$(dirname "${CMD}")" | ||
| cp /tmp/"${PROMETHEUS_FILE}/prometheus" "${CMD}" | ||
| fi | ||
| fi | ||
|
|
||
| # Configure prometheus | ||
| FILE_SD_PATH="${PROMETHEUS_WORKING_DIR}/file_sd_configs" | ||
| mkdir -p "${FILE_SD_PATH}" | ||
|
|
||
| echo "writing configuration..." | ||
| cat >"${PROMETHEUS_WORKING_DIR}"/prometheus.yaml <<EOL | ||
| # my global config | ||
| global: | ||
| # Make sure this value takes into account the network-shutdown-delay in tests/fixture/e2e/env.go | ||
| scrape_interval: 10s # Default is every 1 minute. | ||
| evaluation_interval: 10s # The default is every 1 minute. | ||
| scrape_timeout: 5s # The default is every 10s | ||
| scrape_configs: | ||
| - job_name: "avalanchego" | ||
| metrics_path: "/ext/metrics" | ||
| file_sd_configs: | ||
| - files: | ||
| - '${FILE_SD_PATH}/*.json' | ||
| remote_write: | ||
| - url: "${PROMETHEUS_URL}/api/v1/write" | ||
| basic_auth: | ||
| username: "${PROMETHEUS_ID}" | ||
| password: "${PROMETHEUS_PASSWORD}" | ||
| EOL | ||
|
|
||
| echo "starting prometheus..." | ||
| cd "${PROMETHEUS_WORKING_DIR}" | ||
| nohup "${CMD}" --config.file=prometheus.yaml --web.listen-address=localhost:0 --enable-feature=agent > prometheus.log 2>&1 & | ||
| echo $! > "${PIDFILE}" | ||
| echo "running with pid $(cat "${PIDFILE}")" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| # Starts a promtail instance to collect logs from temporary networks | ||
| # running locally and in CI. | ||
| # | ||
| # The promtail instance will remain running in the background and will forward | ||
| # logs to the central instance for all tmpnet networks. | ||
| # | ||
| # To stop it: | ||
| # | ||
| # $ kill -9 `cat ~/.tmpnet/promtail/run.pid` && rm ~/.tmpnet/promtail/run.pid | ||
| # | ||
|
|
||
| # e.g., | ||
| # LOKI_ID=<id> LOKI_PASSWORD=<password> ./scripts/run_promtail.sh | ||
| if ! [[ "$0" =~ scripts/run_promtail.sh ]]; then | ||
| echo "must be run from repository root" | ||
| exit 255 | ||
| fi | ||
|
|
||
| PROMTAIL_WORKING_DIR="${HOME}/.tmpnet/promtail" | ||
| PIDFILE="${PROMTAIL_WORKING_DIR}"/run.pid | ||
|
|
||
| # First check if promtail is already running. A single instance can | ||
| # collect logs from all local temporary networks. | ||
| if pgrep --pidfile="${PIDFILE}" &> /dev/null; then | ||
| echo "promtail is already running" | ||
| exit 0 | ||
| fi | ||
|
|
||
| LOKI_URL="${LOKI_URL:-https://loki-experimental.avax-dev.network}" | ||
| if [[ -z "${LOKI_URL}" ]]; then | ||
| echo "Please provide a value for LOKI_URL" | ||
| exit 1 | ||
| fi | ||
|
|
||
| LOKI_ID="${LOKI_ID:-}" | ||
| if [[ -z "${LOKI_ID}" ]]; then | ||
| echo "Please provide a value for LOKI_ID" | ||
| exit 1 | ||
| fi | ||
|
|
||
| LOKI_PASSWORD="${LOKI_PASSWORD:-}" | ||
| if [[ -z "${LOKI_PASSWORD}" ]]; then | ||
| echo "Plase provide a value for LOKI_PASSWORD" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Version as of this writing | ||
| VERSION="v2.9.5" | ||
|
|
||
| # Ensure the promtail command is locally available | ||
| CMD=promtail | ||
| if ! command -v "${CMD}" &> /dev/null; then | ||
| # Try to use a local version | ||
| CMD="${PWD}/bin/promtail" | ||
| if ! command -v "${CMD}" &> /dev/null; then | ||
| echo "promtail not found, attempting to install..." | ||
| # Determine the arch | ||
| if which sw_vers &> /dev/null; then | ||
| DIST="darwin-$(uname -m)" | ||
| else | ||
| ARCH="$(uname -i)" | ||
| if [[ "${ARCH}" == "aarch64" ]]; then | ||
| ARCH="arm64" | ||
| elif [[ "${ARCH}" == "x86_64" ]]; then | ||
| ARCH="amd64" | ||
| fi | ||
| DIST="linux-${ARCH}" | ||
| fi | ||
|
|
||
| # Install the specified release | ||
| PROMTAIL_FILE="promtail-${DIST}" | ||
| ZIP_PATH="/tmp/${PROMTAIL_FILE}.zip" | ||
| BIN_DIR="$(dirname "${CMD}")" | ||
| URL="https://github.com/grafana/loki/releases/download/${VERSION}/promtail-${DIST}.zip" | ||
| curl -L -o "${ZIP_PATH}" "${URL}" | ||
| unzip "${ZIP_PATH}" -d "${BIN_DIR}" | ||
| mv "${BIN_DIR}/${PROMTAIL_FILE}" "${CMD}" | ||
| fi | ||
| fi | ||
|
|
||
| # Configure promtail | ||
| FILE_SD_PATH="${PROMTAIL_WORKING_DIR}/file_sd_configs" | ||
| mkdir -p "${FILE_SD_PATH}" | ||
|
|
||
| echo "writing configuration..." | ||
| cat >"${PROMTAIL_WORKING_DIR}"/promtail.yaml <<EOL | ||
| server: | ||
| http_listen_port: 0 | ||
| grpc_listen_port: 0 | ||
| positions: | ||
| filename: "${PROMTAIL_WORKING_DIR}/positions.yaml" | ||
| client: | ||
| url: "${LOKI_URL}/api/prom/push" | ||
| basic_auth: | ||
| username: "${LOKI_ID}" | ||
| password: "${LOKI_PASSWORD}" | ||
| scrape_configs: | ||
| - job_name: "avalanchego" | ||
| file_sd_configs: | ||
| - files: | ||
| - '${FILE_SD_PATH}/*.json' | ||
| EOL | ||
|
|
||
| echo "starting promtail..." | ||
| cd "${PROMTAIL_WORKING_DIR}" | ||
| nohup "${CMD}" -config.file=promtail.yaml > promtail.log 2>&1 & | ||
| echo $! > "${PIDFILE}" | ||
| echo "running with pid $(cat "${PIDFILE}")" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.