+monit 新增monit

Author: Leif160519

ansible.cfg

@@ -1,6 +1,8 @@
 [defaults]
 force_valid_group_names        = ignore
 inventory                      = ./inventory
+library                        = ./modules
+action_plugins                 = ./modules/action_plugins
 roles_path                     = ./roles
 sudo_user                      = root
 remote_user                    = root

files/prometheus/alert/rules/mfs.rules

@@ -0,0 +1,49 @@
+groups:
+  - name: moosefs
+    rules:
+    - alert: "文件库降级"
+      expr: moosefs_all_chunks_matrix_chunkclass_endangered{job="node"}
+            / (moosefs_all_chunks_matrix_chunkclass_endangered{job="node"} + moosefs_all_chunks_matrix_chunkclass_stable{job="node"}) > 0.1
+      for: 1h
+      labels:
+        severity: warning
+        channels: "@linux/feishu;"
+        level: P1
+      annotations:
+        summary: "{{ $value | humanizePercentage }} 危急占比"
+
+    - alert: "文件库落盘"
+      expr: time() - moosefs_master_info_last_successful_store{job="node"} > (3600 * 2)
+      for: 2m
+      labels:
+        severity: warning
+        channels: "@linux/feishu;"
+        level: P1
+      annotations:
+        summary: "{{ $value | humanizeDuration }} 距离上次"
+
+    - alert: "文件库配额"
+      expr: |
+        (moosefs_active_quotas_current_size{job="node"} / moosefs_active_quotas_soft_size{job="node"}) <= 1 > 0.9
+        OR
+        (moosefs_active_quotas_current_size{job="node"} / moosefs_active_quotas_hard_size{job="node"}) <= 1 > 0.9
+      for: 1m
+      labels:
+        severity: info
+        channels: "@linux/feishu;"
+        level: P2
+      annotations:
+        summary: "{{ $labels.path }}: {{ $value | humanizePercentage }}({{ $labels.type }})"
+
+    - alert: "文件库配额"
+      expr: |
+        (moosefs_active_quotas_soft_size{job="node"} - moosefs_active_quotas_current_size{job="node"}) >= 0 < (2*1024*1024*1024)
+        OR
+        (moosefs_active_quotas_hard_size{job="node"} - moosefs_active_quotas_current_size{job="node"}) >= 0 < (2*1024*1024*1024)
+      for: 1m
+      labels:
+        severity: warning
+        channels: "@linux/feishu;"
+        level: P1
+      annotations:
+        summary: "{{ $labels.path }}: {{ $value | humanize1024 }}B({{ $labels.type }})"

files/prometheus/exporter/mfscli2prom.awk

@@ -0,0 +1,155 @@
+#!/usr/bin/awk -f
+#
+# usage: 
+#
+#   % /usr/bin/mfscli -ns"#" -SIM -SMU -SIG -SCS -SIC -SSC -SQU | ./mfscli2prom.awk
+#
+# moosefs >= v3.0.105
+#
+
+BEGIN {
+    FS="[#:|]"
+}
+
+function SCS() {
+    k=gensub(/\s/, "_", "g", $1);
+    printf "moosefs_%s_load{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$8;
+
+    if ($9 ~ /maintenance_off/) {
+        printf "moosefs_%s_maintenance{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} 0\n", k,$3,$4,$5,$6,$7;
+    } else {
+        printf "moosefs_%s_maintenance{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} 0\n", k,$3,$4,$5,$6,$7;
+    }
+
+    printf "moosefs_%s_regular_hdd_space_chunks{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$10;
+    printf "moosefs_%s_regular_hdd_space_used{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$11;
+    printf "moosefs_%s_regular_hdd_space_total{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$12;
+
+    printf "moosefs_%s_removal_hdd_space_chunks{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$14;
+    printf "moosefs_%s_removal_hdd_space_used{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$15;
+    printf "moosefs_%s_removal_hdd_space_total{ip=\"%s\",port=\"%s\",id=\"%s\",labels=\"%s\",version=\"%s\"} %s\n", k,$3,$4,$5,$6,$7,$16;
+
+}
+
+function is_number(x) { return x+0 == x }
+function SIG() {
+    if (!is_number($NF)) next
+
+    v=$NF;
+    $(NF--);
+
+    k=$0;
+    k=gensub(/\s+/, "_", "g", k);
+    k=gensub(/[()]/, "", "g", k);
+    k=tolower(k)
+
+    print "moosefs_"k,v
+}
+
+function SIM() {
+    k=gensub(/\s/, "_", "g", $1);
+
+    printf "moosefs_%s_metadata_version{ip=\"%s\", version=\"%s\"} %d\n", k,$3,$4,gensub(/[ ]/, "", "g", $7)
+    printf "moosefs_%s_ram_used{ip=\"%s\", version=\"%s\"} %d\n", k,$3,$4,$9
+
+    printf "moosefs_%s_cpu_used_all{ip=\"%s\", version=\"%s\"} %f\n", k,$3,$4,gensub(/(.*)%.*/, "\\1", "g", $11)
+    printf "moosefs_%s_cpu_used_sys{ip=\"%s\", version=\"%s\"} %f\n", k,$3,$4,gensub(/(.*)%.*/, "\\1", "g", $12)
+    printf "moosefs_%s_cpu_used_user{ip=\"%s\", version=\"%s\"} %f\n", k,$3,$4,gensub(/(.*)%.*/, "\\1", "g", $13)
+    printf "moosefs_%s_last_meta_save{ip=\"%s\", version=\"%s\", cksum=\"%s\"} %d\n", k,$3,$4,$NF,$14
+    printf "moosefs_%s_last_save_duration{ip=\"%s\", version=\"%s\", cksum=\"%s\"} %f\n", k,$3,$4,$NF,$15
+
+    if ($16 ~ /[Ss]aved in background/){
+        printf "moosefs_%s_last_saved_in_background{ip=\"%s\", version=\"%s\", cksum=\"%s\"} 1\n", k,$3,$4,$NF
+    } else {
+        printf "moosefs_%s_last_saved_in_background{ip=\"%s\", version=\"%s\", cksum=\"%s\"} 0\n", k,$3,$4,$NF
+    }
+}
+
+function SMU() {
+#    k=gensub(/\s/, "_", "g", $1);
+#
+#    printf "moosefs_%s_%s_used %s\n", k, gensub(/\s/, "_", "g", $3), $4
+#    printf "moosefs_%s_%s_allocated %s\n", k, gensub(/\s/, "_", "g", $3), $5
+}
+
+function SIC() {
+    k=gensub(/\s/, "_", "g", $1) "_" gensub(/\s/, "_", "g", $3);
+
+    printf "moosefs_%s %s\n", k, $5
+}
+
+function SSC() {
+    k=gensub(/\s/, "_", "g", $1);
+    C=gensub(/\s/, "", "g", $17);
+    K=gensub(/\s/, "", "g", $20);
+    A=gensub(/\s/, "", "g", $23);
+    printf "moosefs_%s_create{id=\"%s\",name=\"%s\",mode=\"%s\",can=\"%s\",labels=\"%s\"} %s\n", k,$3,$4,$6,$15,C,$16
+    printf "moosefs_%s_keep{id=\"%s\",name=\"%s\",mode=\"%s\",can=\"%s\",labels=\"%s\"} %s\n", k,$3,$4,$6,$18,K,$19
+    printf "moosefs_%s_archive{id=\"%s\",name=\"%s\",mode=\"%s\",can=\"%s\",labels=\"%s\"} %s\n", k,$3,$4,$6,$21,A,$22
+
+    printf "moosefs_%s_files_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$7
+    printf "moosefs_%s_dirs_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$8
+    printf "moosefs_%s_standard_under_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$9
+    printf "moosefs_%s_standard_exact_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$10
+    printf "moosefs_%s_standard_over_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$11
+    printf "moosefs_%s_archived_under_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$12
+    printf "moosefs_%s_archived_exact_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$13
+    printf "moosefs_%s_archived_over_total{id=\"%s\",name=\"%s\"} %d\n", k,$3,$4,$14
+}
+
+function SQU() {
+    k=gensub(/\s/, "_", "g", $1);  # keyword
+    P=gensub(/\s/, "",  "g", $3);  # path
+    SI=gensub(/\s/, "", "g", $7);  # soft inodes
+    SL=gensub(/\s/, "", "g", $8);  # soft length
+    SS=gensub(/\s/, "", "g", $9);  # soft size
+    SR=gensub(/\s/, "", "g", $10);  # soft real size
+    HI=gensub(/\s/, "", "g", $11); # hard inodes
+    HL=gensub(/\s/, "", "g", $12); # hard length
+    HS=gensub(/\s/, "", "g", $13); # hard size
+    HR=gensub(/\s/, "", "g", $14); # hard real size
+    CI=gensub(/\s/, "", "g", $15); # current inodes
+    CL=gensub(/\s/, "", "g", $16); # current length
+    CS=gensub(/\s/, "", "g", $17); # current size
+    CR=gensub(/\s/, "", "g", $18); # current real size
+
+    printf "moosefs_%s_soft_inodes{path=\"%s\"} %d\n",    k,P,SI
+    printf "moosefs_%s_soft_length{path=\"%s\"} %d\n",    k,P,SL
+    printf "moosefs_%s_soft_size{path=\"%s\"} %d\n",      k,P,SS
+    printf "moosefs_%s_soft_real{path=\"%s\"} %d\n",      k,P,SR
+
+    printf "moosefs_%s_hard_inodes{path=\"%s\"} %d\n",    k,P,HI
+    printf "moosefs_%s_hard_length{path=\"%s\"} %d\n",    k,P,HL
+    printf "moosefs_%s_hard_size{path=\"%s\"} %d\n",      k,P,HS
+    printf "moosefs_%s_hard_real{path=\"%s\"} %d\n",      k,P,HR
+
+    printf "moosefs_%s_current_inodes{path=\"%s\"} %d\n",     k,P,CI
+    printf "moosefs_%s_current_length{path=\"%s\"} %d\n",     k,P,CL
+    printf "moosefs_%s_current_size{path=\"%s\"} %d\n",       k,P,CS
+    printf "moosefs_%s_current_real{path=\"%s\"} %d\n",       k,P,CR
+}
+
+{
+    if ($0 ~ /^\s*$/) next
+
+    if ($0 ~ /^active quotas:/)
+        SQU()
+
+    if ($0 ~ /^metadata servers:/)
+        SIM()
+
+    if ($0 ~ /^memory usage detailed info:/)
+        SMU()
+
+    if ($0 ~ /^chunk servers:/)
+        SCS()
+
+    if ($0 ~ /^master info:/)
+        SIG()
+
+    if ($0 ~ /chunkclass /)
+        SIC()
+
+    if ($0 ~ /storage classes:/)
+        SSC()
+}

files/prometheus/exporter/mfsdirinfo2prom.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# 统计 / 子目录的细节
+#
+
+awk -f- <( \
+
+    cd /mnt/mfs/ && \
+    find prometheus                 -mindepth 1 -maxdepth 1 -type d -exec mfsdirinfo -i -d -f -c -l -s -r {} \; && \
+    find jenkins                    -mindepth 1 -maxdepth 1 -type d -exec mfsdirinfo -i -d -f -c -l -s -r {} \; && \
+    find backuppc                   -mindepth 1 -maxdepth 1 -type d -exec mfsdirinfo -i -d -f -c -l -s -r {} \;
+
+) <<'EOF'
+
+function mfsdirinfo(line) {
+    split(line, info)
+
+    inodes  =info[1]
+    dirs    =info[2]
+    files   =info[3]
+    chunks  =info[4]
+    len     =info[5]
+    size    =info[6]
+    real    =info[7]
+    path    =info[8]
+
+    printf "moosefs_inodes_count{path=\"%s\"} %d\n", path, inodes
+    printf "moosefs_dirs_count{path=\"%s\"} %d\n",   path, dirs
+    printf "moosefs_files_count{path=\"%s\"} %d\n",  path, files
+    printf "moosefs_chunks_count{path=\"%s\"} %d\n", path, chunks
+    printf "moosefs_length_bytes{path=\"%s\"} %d\n", path, len
+    printf "moosefs_size_bytes{path=\"%s\"} %d\n",   path, size
+    printf "moosefs_real_bytes{path=\"%s\"} %d\n",   path, real
+}
+
+{
+    mfsdirinfo($0)
+}
+
+EOF
+

group_vars/pro.mfs.master

@@ -0,0 +1,40 @@
+gens_mfs_master_monit_services__to_merge:
+  - name: mfsmaster
+    type: process
+    target: "/var/lib/mfs/.mfsmaster.lock"
+    rules: |
+      if changed pid then alert
+
+  - name: mfscgiserv
+    type: process
+    target: "/var/lib/mfs/.mfscgiserv.lock"
+    rules: |
+      if changed pid then alert
+
+  - name: metadata
+    type: file
+    target: "/var/lib/mfs/metadata.mfs.back"
+    rules: |
+      if not exist for 3 cycles then alert
+      if timestamp is older than 2 hours then alert
+
+  - name: available
+    type: program
+    target: "/bin/sh -c 'mfsdirinfo /mnt/mfs/ | tee /mnt/mfs/.watchdog_by_monit'"
+    extend: "with timeout 10 seconds"
+    rules: |
+      if status != 0 for 2 cycles then alert
+
+  - name: mfscli2prom
+    type: program
+    target: "/bin/sh -c '/usr/bin/mfscli -ns\"|\" -SIM -SMU -SIG -SCS -SIC -SSC -SQU | /opt/mfscli2prom.awk > /var/log/prometheus/mfscli.prom'"
+    extend: "with timeout 10 seconds"
+    rules: |
+      if status != 0 for 2 cycles then alert
+
+  - name: mfsdirinfo2prom
+    type: program
+    target: "/bin/sh -c '/opt/mfsdirinfo2prom.sh | /usr/bin/stdbuf -oL tee /var/log/prometheus/mfsdirinfo.prom'"
+    extend: "with timeout 10 seconds"
+    rules: |
+      if status != 0 for 2 cycles then alert

group_vars/pro.mfs.slaver

@@ -0,0 +1,23 @@
+gens_mfs_slaver_monit_services__to_merge:
+  - name: mfsmetalogger
+    type: process
+    target: "/var/lib/mfs/.mfsmetalogger.lock"
+    rules: |
+      if changed pid then alert
+      alert [email protected]
+
+  - name: changelog
+    type: file
+    target: "/var/lib/mfs/changelog_ml.0.mfs"
+    rules: |
+      if not exist for 3 cycles then alert
+      if timestamp is older than 10 minutes then alert
+      alert [email protected] on { timestamp }
+
+  - name: metadata
+    type: file
+    target: "/var/lib/mfs/metadata_ml.mfs.back"
+    rules: |
+      if not exist for 3 cycles then alert
+      if timestamp is older than 2 days then alert
+      alert [email protected] on { timestamp }

inventory/pro

@@ -23,12 +23,32 @@ minio-cluster-4
 # }}}
 
 [pro.moosefs]  # {{{ moosefs 集群
+[pro.moosefs:children]
+pro.mfs.master
+pro.mfs.slaver
+pro.mfs.chunks
+pro.mfs.client
+
 mfsmaster
 mfs-master
 mfs-metalogger
 mfs-chunkserver-1
 mfs-chunkserver-2
 mfs-chunkserver-3
+
+[pro.mfs.master]
+mfs-master
+
+[pro.mfs.slaver]
+mfs-metalogger
+
+[pro.mfs.chunks]
+mfs-chunkserver-1
+mfs-chunkserver-2
+mfs-chunkserver-3
+
+[pro.mfs.client]
+mfs-master
 # }}}
 
 [pro.redis] # {{{ redis 集群

modules/README.md

@@ -0,0 +1,15 @@
+=== action_plugins/merge_vars.py
+https://github.com/leapfrogonline/ansible-merge-vars
+
+monit 的 services，散布在不同的 group_vars 和 host_vars。
+某些资产，属于多个 group，本应该从各个 group 继承对应的 services，
+但是由于 ansible 默认的覆盖(override)行为，导致变量无法自动合并。
+
+依赖 `pip3 install ansible-merge-vars`
+
+=== mount_leif160519.py
+原生的 mount 模块的 absent 动作，不仅从 fstab 移除挂载点，
+而且还会执行 umount、以及更过分的 rm 删除挂载点目录的动作。
+
+因为不希望对系统执行 umount 和 rm 这样的危险动作，
+所以从原生模块继承并修改得到了自定义的 mount_leif160519 模块。

modules/action_plugins/merge_vars.py

@@ -0,0 +1 @@
+from ansible_merge_vars import ActionModule