Merge pull request #699 from WeBankPartners/dev

Dev

Dockerfile

@@ -49,6 +49,7 @@ RUN chmod +x $BASE_HOME/*.sh
 RUN chmod +x $PING_EXPORTER/ping_exporter
 RUN chmod +x $ARCHIVE_TOOL/archive_mysql_tool
 RUN chmod +x $DB_DATA_EXPORTER/db_data_exporter
+RUN apk add -U tzdata
 
 WORKDIR $BASE_HOME
 

build/conf/monitor.json

@@ -17,6 +17,7 @@
     "log": {
       "level": "{{MONITOR_LOG_LEVEL}}",
       "file": "logs/open-monitor.log",
+      "access_file": "logs/open-monitor-access.log",
       "archive_max_size": 64,
       "archive_max_backup": 10,
       "archive_max_day": 15,
@@ -178,5 +179,6 @@
     "local_storage_max_day": 30,
     "five_min_start_day": 90
   },
-  "process_check_list": ["ping_exporter", "agent_manager"]
+  "process_check_list": ["ping_exporter", "agent_manager"],
+  "default_admin_role": "SUPER_ADMIN"
 }

build/register.xml

@@ -5,7 +5,7 @@
     <packageDependencies>
         <packageDependency name="wecmdb" version="v1.5.0"/>
         <packageDependency name="saltstack" version="v1.9.0"/>
-        <packageDependency name="platform" version="v2.5.0"/>
+        <packageDependency name="platform" version="v2.9.0"/>
     </packageDependencies>
 
     <!-- 2.菜单注入 - 描述运行本插件包需要注入的菜单 -->
@@ -75,6 +75,7 @@
         <systemParameter name="MONITOR_ARCHIVE_MYSQL_USER" scopeType="global" defaultValue=""/>
         <systemParameter name="MONITOR_ARCHIVE_MYSQL_PWD" scopeType="global" defaultValue=""/>
         <systemParameter name="MONITOR_LOG_LEVEL" scopeType="global" defaultValue="info"/>
+        <systemParameter name="MONITOR_MAIL_DEFAULT_RECEIVER" scopeType="global" defaultValue=""/>
     </systemParameters>
 
 
@@ -91,7 +92,7 @@
 
     <!-- 6.运行资源 - 描述部署运行本插件包需要的基础资源(如主机、虚拟机、容器、数据库等) -->
     <resourceDependencies>
-        <docker imageName="open-monitor:{{PLUGIN_VERSION}}" containerName="open-monitor-{{PLUGIN_VERSION}}" portBindings="19091:19091,14241:14241,{{ALLOCATE_PORT}}:8080" volumeBindings="{{BASE_MOUNT_PATH}}/prometheus/logs:/app/monitor/prometheus/logs,{{BASE_MOUNT_PATH}}/prometheus/data:/app/monitor/prometheus/data,{{BASE_MOUNT_PATH}}/prometheus/rules:/app/monitor/prometheus/rules,{{BASE_MOUNT_PATH}}/alertmanager/logs:/app/monitor/alertmanager/logs,{{BASE_MOUNT_PATH}}/alertmanager/data:/app/monitor/alertmanager/data,{{BASE_MOUNT_PATH}}/consul/logs:/app/monitor/consul/logs,{{BASE_MOUNT_PATH}}/consul/data:/app/monitor/consul/data,{{BASE_MOUNT_PATH}}/monitor/logs:/app/monitor/monitor/logs,{{BASE_MOUNT_PATH}}/agent_deploy:/app/deploy,{{BASE_MOUNT_PATH}}/transgateway/logs:/app/monitor/transgateway/logs,{{BASE_MOUNT_PATH}}/transgateway/data:/app/monitor/transgateway/data,/etc/localtime:/etc/localtime,{{BASE_MOUNT_PATH}}/certs:/data/certs" envVariables="MONITOR_DB_HOST={{DB_HOST}},MONITOR_DB_PORT={{DB_PORT}},MONITOR_DB_SCHEMA={{DB_SCHEMA}},MONITOR_DB_USER={{DB_USER}},MONITOR_DB_PWD={{DB_PWD}},CORE_ADDR={{CORE_ADDR}},GATEWAY_URL={{GATEWAY_URL}},MONITOR_HOST_IP={{ALLOCATE_HOST}},MONITOR_CHECK_EVENT_KEY={{MONITOR_CHECK_EVENT_KEY}},MONITOR_CHECK_EVENT_TO_MAIL={{MONITOR_CHECK_EVENT_TO_MAIL}},MONITOR_CHECK_EVENT_INTERVAL_MIN={{MONITOR_CHECK_EVENT_INTERVAL_MIN}},MONITOR_ARCHIVE_ENABLE={{MONITOR_ARCHIVE_ENABLE}},MONITOR_ARCHIVE_MYSQL_HOST={{MONITOR_ARCHIVE_MYSQL_HOST}},MONITOR_ARCHIVE_MYSQL_PORT={{MONITOR_ARCHIVE_MYSQL_PORT}},MONITOR_ARCHIVE_MYSQL_USER={{MONITOR_ARCHIVE_MYSQL_USER}},MONITOR_ARCHIVE_MYSQL_PWD={{MONITOR_ARCHIVE_MYSQL_PWD}},MONITOR_LOG_LEVEL={{MONITOR_LOG_LEVEL}},JWT_SIGNING_KEY={{JWT_SIGNING_KEY}},ALARM_FIRING_CALLBACK={{MONITOR_ALARM_FIRING_CALLBACK}},ALARM_RECOVER_CALLBACK={{MONITOR_ALARM_RECOVER_CALLBACK}}"/>
+        <docker imageName="open-monitor:{{PLUGIN_VERSION}}" containerName="open-monitor-{{PLUGIN_VERSION}}" portBindings="19091:19091,14241:14241,{{ALLOCATE_PORT}}:8080" volumeBindings="{{BASE_MOUNT_PATH}}/prometheus/logs:/app/monitor/prometheus/logs,{{BASE_MOUNT_PATH}}/prometheus/data:/app/monitor/prometheus/data,{{BASE_MOUNT_PATH}}/prometheus/rules:/app/monitor/prometheus/rules,{{BASE_MOUNT_PATH}}/alertmanager/logs:/app/monitor/alertmanager/logs,{{BASE_MOUNT_PATH}}/alertmanager/data:/app/monitor/alertmanager/data,{{BASE_MOUNT_PATH}}/consul/logs:/app/monitor/consul/logs,{{BASE_MOUNT_PATH}}/consul/data:/app/monitor/consul/data,{{BASE_MOUNT_PATH}}/monitor/logs:/app/monitor/monitor/logs,{{BASE_MOUNT_PATH}}/agent_deploy:/app/deploy,{{BASE_MOUNT_PATH}}/transgateway/logs:/app/monitor/transgateway/logs,{{BASE_MOUNT_PATH}}/transgateway/data:/app/monitor/transgateway/data,/etc/localtime:/etc/localtime,{{BASE_MOUNT_PATH}}/certs:/data/certs" envVariables="MONITOR_DB_HOST={{DB_HOST}},MONITOR_DB_PORT={{DB_PORT}},MONITOR_DB_SCHEMA={{DB_SCHEMA}},MONITOR_DB_USER={{DB_USER}},MONITOR_DB_PWD={{DB_PWD}},CORE_ADDR={{CORE_ADDR}},GATEWAY_URL={{GATEWAY_URL}},MONITOR_HOST_IP={{ALLOCATE_HOST}},MONITOR_CHECK_EVENT_KEY={{MONITOR_CHECK_EVENT_KEY}},MONITOR_CHECK_EVENT_TO_MAIL={{MONITOR_CHECK_EVENT_TO_MAIL}},MONITOR_CHECK_EVENT_INTERVAL_MIN={{MONITOR_CHECK_EVENT_INTERVAL_MIN}},MONITOR_ARCHIVE_ENABLE={{MONITOR_ARCHIVE_ENABLE}},MONITOR_ARCHIVE_MYSQL_HOST={{MONITOR_ARCHIVE_MYSQL_HOST}},MONITOR_ARCHIVE_MYSQL_PORT={{MONITOR_ARCHIVE_MYSQL_PORT}},MONITOR_ARCHIVE_MYSQL_USER={{MONITOR_ARCHIVE_MYSQL_USER}},MONITOR_ARCHIVE_MYSQL_PWD={{MONITOR_ARCHIVE_MYSQL_PWD}},MONITOR_LOG_LEVEL={{MONITOR_LOG_LEVEL}},JWT_SIGNING_KEY={{JWT_SIGNING_KEY}},ALARM_FIRING_CALLBACK={{MONITOR_ALARM_FIRING_CALLBACK}},ALARM_RECOVER_CALLBACK={{MONITOR_ALARM_RECOVER_CALLBACK}},SUB_SYSTEM_CODE={{SUB_SYSTEM_CODE}},SUB_SYSTEM_KEY={{SUB_SYSTEM_KEY}}"/>
         <mysql schema="monitor" initFileName="init.sql" upgradeFileName="upgrade.sql"/>
         <s3 bucketName="wecube-agent"/>
     </resourceDependencies>

monitor-agent/archive_mysql_tool/funcs/config.go

@@ -1,6 +1,7 @@
 package funcs
 
 import (
+	"os/exec"
 	"sync"
 	"log"
 	"os"
@@ -60,6 +61,7 @@ type GlobalConfig struct {
 var (
 	config     *GlobalConfig
 	lock       = new(sync.RWMutex)
+	DefaultLocalTimeZone string
 )
 
 func Config() *GlobalConfig {
@@ -93,9 +95,25 @@ func InitConfig(cfg string) error {
 	config = &c
 	log.Println("read config file:", cfg, "successfully")
 	lock.Unlock()
+	initLocalTimeZone()
 	hostIp = "127.0.0.1"
 	if os.Getenv("MONITOR_HOST_IP") != "" {
 		hostIp = os.Getenv("MONITOR_HOST_IP")
 	}
 	return nil
 }
+
+func initLocalTimeZone()  {
+	cmdOut,err := exec.Command("/bin/sh", "-c", "date|awk '{print $5}'").Output()
+	if err != nil {
+		log.Printf("init local time zone fail,%s \n", err.Error())
+	}else{
+		cmdOutString := strings.TrimSpace(string(cmdOut))
+		if cmdOutString != "" {
+			DefaultLocalTimeZone = cmdOutString
+			log.Printf("init local time zone to %s \n", DefaultLocalTimeZone)
+		}else{
+			DefaultLocalTimeZone = "CST"
+		}
+	}
+}

monitor-agent/archive_mysql_tool/funcs/http.go

@@ -21,7 +21,7 @@ func InitHttpHandles()  {
 
 func handleCustomJob(w http.ResponseWriter,r *http.Request)  {
 	dateString := r.FormValue("date")
-	_, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", dateString))
+	_, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, dateString))
 	if err != nil {
 		returnJson(r,w,err,nil)
 	}else{
@@ -32,7 +32,7 @@ func handleCustomJob(w http.ResponseWriter,r *http.Request)  {
 
 func handleFiveMinJob(w http.ResponseWriter,r *http.Request)  {
 	dateString := r.FormValue("date")
-	t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", dateString))
+	t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, dateString))
 	if err != nil {
 		returnJson(r,w,err,nil)
 	}else{

monitor-agent/archive_mysql_tool/funcs/job.go

@@ -25,7 +25,7 @@ func StartCronJob()  {
 	}
 	jobChannelList = make(chan ArchiveActionList, Config().Prometheus.MaxHttpOpen)
 	go consumeJob()
-	t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02")))
+	t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, time.Now().Format("2006-01-02")))
 	subSecond := t.Unix()+86410-time.Now().Unix()
 	time.Sleep(time.Duration(subSecond)*time.Second)
 	c := time.NewTicker(24*time.Hour).C
@@ -49,12 +49,12 @@ func CreateJob(dateString string)  {
 	}
 	var start,end int64
 	if dateString == "" {
-		t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02")))
+		t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, time.Now().Format("2006-01-02")))
 		start = t.Unix()-86400
 		end = t.Unix()
 		dateString = time.Unix(start, 0).Format("2006-01-02")
 	}else {
-		t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", dateString))
+		t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, dateString))
 		if err != nil {
 			log.Printf("dateString validate fail,must format like 2006-01-02 \n")
 			return
@@ -179,7 +179,7 @@ func archiveAction(param ArchiveActionList)  {
 					tmpFloatList = []float64{vvv[1]}
 				}
 			}
-			if len(tmpFloatList) > 0 {
+			if len(tmpFloatList) > 0 && tmpStartTime <= v.End {
 				avg,min,max,p95 := calcData(tmpFloatList)
 				rowData = append(rowData, &ArchiveTable{Endpoint:v.Endpoint,Metric:v.Metric,Tags:tmpTagString,UnixTime:tmpStartTime-60,Avg:avg,Min:min,Max:max,P95:p95})
 			}
@@ -217,7 +217,7 @@ func ArchiveFromMysql(tableUnixTime int64)  {
 		if Config().Trans.FiveMinStartDay > 0 {
 			startDays = Config().Trans.FiveMinStartDay
 		}
-		t, _ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02")))
+		t, _ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, time.Now().Format("2006-01-02")))
 		tableUnixTime = t.Unix() - (startDays * 86400)
 	}
 	oldTableName := fmt.Sprintf("archive_%s", time.Unix(tableUnixTime, 0).Format("2006_01_02"))

monitor-agent/archive_mysql_tool/funcs/prometheus.go

@@ -1,15 +1,15 @@
 package funcs
 
 import (
-	"net/http"
-	"time"
-	"net/url"
+	"encoding/json"
 	"fmt"
-	"strings"
 	"io/ioutil"
-	"encoding/json"
-	"strconv"
+	"net/http"
+	"net/url"
 	"sort"
+	"strconv"
+	"strings"
+	"time"
 )
 
 var (

monitor-server/api/api.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/WeBankPartners/open-monitor/monitor-server/api/v1/agent"
 	"github.com/WeBankPartners/open-monitor/monitor-server/api/v1/alarm"
@@ -13,7 +14,9 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/swaggo/gin-swagger"
 	"github.com/swaggo/gin-swagger/swaggerFiles"
+	"io/ioutil"
 	"net/http"
+	"strings"
 	"time"
 )
 
@@ -213,7 +216,7 @@ func InitClusterApi()  {
 		return
 	}
 	http.Handle("/sync/config", http.HandlerFunc(alarm.SyncConfigHandle))
-	http.Handle("/sync/consul", http.HandlerFunc(alarm.SyncConsulHandle))
+	http.Handle("/sync/sd", http.HandlerFunc(alarm.SyncSdFileHandle))
 	http.ListenAndServe(fmt.Sprintf(":%s", m.Config().Cluster.HttpPort), nil)
 }
 
@@ -223,8 +226,34 @@ func InitDependenceParam()  {
 
 func httpLogHandle() gin.HandlerFunc {
 	return func(c *gin.Context) {
-		start := time.Now()
-		c.Next()
-		log.Logger.Info("request", log.String("url", c.Request.RequestURI), log.String("method", c.Request.Method), log.Int("code", c.Writer.Status()), log.String("operator", c.GetString("operatorName")), log.String("ip", c.ClientIP()), log.Int64("cost_time", time.Now().Sub(start).Milliseconds()))
+		ignoreLog := false
+		for _,v := range m.LogIgnorePath {
+			if strings.Contains(c.Request.RequestURI, v) {
+				ignoreLog = true
+				break
+			}
+		}
+		if ignoreLog {
+			c.Next()
+		}else {
+			start := time.Now()
+			var bodyBytes []byte
+			if c.Request.Method == http.MethodPost {
+				ignore := false
+				for _, v := range m.LogParamIgnorePath {
+					if strings.Contains(c.Request.RequestURI, v) {
+						ignore = true
+						break
+					}
+				}
+				if !ignore {
+					bodyBytes, _ = ioutil.ReadAll(c.Request.Body)
+					c.Request.Body.Close()
+					c.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
+				}
+			}
+			c.Next()
+			log.AccessLogger.Info("request", log.String("url", c.Request.RequestURI), log.String("method", c.Request.Method), log.Int("code", c.Writer.Status()), log.String("operator", c.GetString("operatorName")), log.String("ip", c.ClientIP()), log.Float64("cost_second", time.Now().Sub(start).Seconds()), log.String("body", string(bodyBytes)))
+		}
 	}
 }

monitor-server/api/v1/agent/deregister.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"github.com/WeBankPartners/open-monitor/monitor-server/services/other"
 	"github.com/gin-gonic/gin"
 	m "github.com/WeBankPartners/open-monitor/monitor-server/models"
 	"github.com/WeBankPartners/open-monitor/monitor-server/services/prom"
@@ -73,7 +74,7 @@ func DeregisterJob(guid string) error {
 		log.Logger.Error("Sync service discover file error", log.Error(err))
 		return err
 	}
-
+	go other.SyncConfig(0, m.SyncSdConfigDto{Guid:guid, Step:endpointObj.Step, IsRegister:false})
 	db.UpdateAgentManagerTable(m.EndpointTable{Guid:guid}, "", "", "", "", false)
 	return err
 }

monitor-server/api/v1/agent/register.go

@@ -2,6 +2,7 @@ package agent
 
 import (
 	"fmt"
+	"github.com/WeBankPartners/open-monitor/monitor-server/services/other"
 	"strings"
 	m "github.com/WeBankPartners/open-monitor/monitor-server/models"
 	mid "github.com/WeBankPartners/open-monitor/monitor-server/middleware"
@@ -115,6 +116,7 @@ func AgentRegister(param m.RegisterParamNew) (validateMessage,guid string,err er
 				log.Logger.Error("Sync service discover file error", log.Error(err))
 			}
 		}
+		go other.SyncConfig(0, m.SyncSdConfigDto{Guid:rData.endpoint.Guid, Ip:fmt.Sprintf("%s:%s", tmpIp, tmpPort), Step:rData.endpoint.Step, IsRegister:true})
 	}
 	if rData.addDefaultGroup {
 		if param.DefaultGroupName != "" {