Gathering OpenShift alerts in Logging must-gather

Gathering OpenShift alerts in Logging must-gather, based on the script in "standard" must-gather

Author: oarribas

must-gather/README.md

@@ -41,6 +41,8 @@ The `clusterlogging` and `clusterlogforwarder` resources, and also `installplans
 
 The `deployments`, `daemonsets` and `secrets` are also found under `namespaces/[namespace_name]/` and can also be seen using the [`omc`](https://github.com/gmeghnag/omc/) tool.
 
+The cluster alerts are now in the `monitoring/prometheus/` directory, and can be checked with `omc prometheus alertrule` and `oc prometheus alertrule -s firing` (for the firing ones only.
+
 Example must-gather for cluster-logging output (use `tree` for up-to-date structure):
 ```
 ├── cluster-logging
@@ -78,7 +80,11 @@ Example must-gather for cluster-logging output (use `tree` for up-to-date struct
 ├── [...]
 ├── event-filter.html
 ├── gather-debug.log
-└── namespaces
+├── monitoring                                                                                                         
+│   └── prometheus                                                                                                     
+│       ├── rules.json                                                                                                 
+│       └── rules.stderr                                                                                               
+├── namespaces
 │  ├── [namespace_name]       ## including openshift-logging
 │  │  ├── apps
 │  │  │  ├── daemonsets.yaml

must-gather/collection-scripts/gather

@@ -108,6 +108,10 @@ else
   log "UIPlugin not installed" 2>&1 | tee -a "${LOGFILE_PATH}"
 fi
 
+log "BEGIN gathering alerts ..." | tee -a "${LOGFILE_PATH}"
+${SCRIPT_DIR}/gather_monitoring "$BASE_COLLECTION_PATH" 2>&1 | tee -a "${LOGFILE_PATH}"  &
+pids+=($!)
+
 default_clo_found="$(oc -n "$LOGGING_NS" get deployment cluster-logging-operator --ignore-not-found --no-headers)"
 
 if [ "$default_clo_found" != "" ] ; then

must-gather/collection-scripts/gather_monitoring

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# based on gather_monitoring script from standard must-gather
+
+# safeguards
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# global readonly constants
+# expect base collection path as an argument
+declare -r BASE_COLLECTION_PATH=$1
+declare -r MONITORING_PATH="${BASE_COLLECTION_PATH}/monitoring"
+
+source "$(dirname "$0")"/monitoring_common.sh
+
+# init initializes global variables that need to be computed.
+# E.g. get token of the default ServiceAccount
+init() {
+  mkdir -p "${MONITORING_PATH}"
+
+  readarray -t PROM_PODS < <(
+    oc get pods -n openshift-monitoring  -l prometheus=k8s \
+      --no-headers -o custom-columns=":metadata.name"
+  )
+}
+
+# prom_get makes http GET requests to prometheus /api/v1/$object and stores
+# the stdout and stderr results
+prom_get() {
+  local object="$1"; shift
+  local path="${1:-$object}"; shift || true
+  local pod
+  pod=$(get_first_ready_prom_pod)
+
+  local result_path="$MONITORING_PATH/prometheus/$path"
+  mkdir -p "$(dirname "$result_path")"
+
+  echo "INFO: Getting ${object} from ${pod}"
+  oc exec "${pod}" \
+    -c prometheus \
+    -n openshift-monitoring \
+    -- /bin/bash -c "curl -sG http://localhost:9090/api/v1/${object}" \
+      >  "${result_path}.json" \
+      2> "${result_path}.stderr"
+}
+
+monitoring_gather(){
+  init
+
+  echo "INFO: Found ${#PROM_PODS[@]} replicas - ${PROM_PODS[*]}"
+
+  # begin gathering
+  # NOTE || true ignores failures
+
+  prom_get rules    || true
+
+  # force disk flush to ensure that all data gathered are written
+  sync
+}
+
+monitoring_gather

must-gather/collection-scripts/monitoring_common.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# based on monitoring_common.sh script from standard must-gather
+
+# safeguards
+set -o nounset
+set -o errexit
+set -o pipefail
+
+get_first_ready_prom_pod() {
+  readarray -t READY_PROM_PODS < <(
+    oc get pods -n openshift-monitoring  -l prometheus=k8s --field-selector=status.phase==Running \
+      --no-headers -o custom-columns=":metadata.name"
+  )
+  echo "${READY_PROM_PODS[0]}"
+}