diff --git a/Dockerfile b/Dockerfile index c1717ffdd..3f32861f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ RUN chmod +x $BASE_HOME/*.sh RUN chmod +x $PING_EXPORTER/ping_exporter RUN chmod +x $ARCHIVE_TOOL/archive_mysql_tool RUN chmod +x $DB_DATA_EXPORTER/db_data_exporter +RUN apk add -U tzdata WORKDIR $BASE_HOME diff --git a/build/conf/monitor.json b/build/conf/monitor.json index e8de90e56..5baa9f898 100644 --- a/build/conf/monitor.json +++ b/build/conf/monitor.json @@ -17,6 +17,7 @@ "log": { "level": "{{MONITOR_LOG_LEVEL}}", "file": "logs/open-monitor.log", + "access_file": "logs/open-monitor-access.log", "archive_max_size": 64, "archive_max_backup": 10, "archive_max_day": 15, @@ -178,5 +179,6 @@ "local_storage_max_day": 30, "five_min_start_day": 90 }, - "process_check_list": ["ping_exporter", "agent_manager"] + "process_check_list": ["ping_exporter", "agent_manager"], + "default_admin_role": "SUPER_ADMIN" } \ No newline at end of file diff --git a/build/register.xml b/build/register.xml index 8ee15f197..549883abf 100644 --- a/build/register.xml +++ b/build/register.xml @@ -5,7 +5,7 @@ - + @@ -75,6 +75,7 @@ + @@ -91,7 +92,7 @@ - + diff --git a/monitor-agent/archive_mysql_tool/funcs/config.go b/monitor-agent/archive_mysql_tool/funcs/config.go index 668d0083f..aee6e1616 100644 --- a/monitor-agent/archive_mysql_tool/funcs/config.go +++ b/monitor-agent/archive_mysql_tool/funcs/config.go @@ -1,6 +1,7 @@ package funcs import ( + "os/exec" "sync" "log" "os" @@ -60,6 +61,7 @@ type GlobalConfig struct { var ( config *GlobalConfig lock = new(sync.RWMutex) + DefaultLocalTimeZone string ) func Config() *GlobalConfig { @@ -93,9 +95,25 @@ func InitConfig(cfg string) error { config = &c log.Println("read config file:", cfg, "successfully") lock.Unlock() + initLocalTimeZone() hostIp = "127.0.0.1" if os.Getenv("MONITOR_HOST_IP") != "" { hostIp = os.Getenv("MONITOR_HOST_IP") } return nil } + +func initLocalTimeZone() { + cmdOut,err := exec.Command("/bin/sh", "-c", "date|awk '{print $5}'").Output() + if err != nil { + log.Printf("init local time zone fail,%s \n", err.Error()) + }else{ + cmdOutString := strings.TrimSpace(string(cmdOut)) + if cmdOutString != "" { + DefaultLocalTimeZone = cmdOutString + log.Printf("init local time zone to %s \n", DefaultLocalTimeZone) + }else{ + DefaultLocalTimeZone = "CST" + } + } +} \ No newline at end of file diff --git a/monitor-agent/archive_mysql_tool/funcs/http.go b/monitor-agent/archive_mysql_tool/funcs/http.go index 063af87ad..e98ca430a 100644 --- a/monitor-agent/archive_mysql_tool/funcs/http.go +++ b/monitor-agent/archive_mysql_tool/funcs/http.go @@ -21,7 +21,7 @@ func InitHttpHandles() { func handleCustomJob(w http.ResponseWriter,r *http.Request) { dateString := r.FormValue("date") - _, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", dateString)) + _, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, dateString)) if err != nil { returnJson(r,w,err,nil) }else{ @@ -32,7 +32,7 @@ func handleCustomJob(w http.ResponseWriter,r *http.Request) { func handleFiveMinJob(w http.ResponseWriter,r *http.Request) { dateString := r.FormValue("date") - t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", dateString)) + t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, dateString)) if err != nil { returnJson(r,w,err,nil) }else{ diff --git a/monitor-agent/archive_mysql_tool/funcs/job.go b/monitor-agent/archive_mysql_tool/funcs/job.go index caa5aef0d..bb1686874 100644 --- a/monitor-agent/archive_mysql_tool/funcs/job.go +++ b/monitor-agent/archive_mysql_tool/funcs/job.go @@ -25,7 +25,7 @@ func StartCronJob() { } jobChannelList = make(chan ArchiveActionList, Config().Prometheus.MaxHttpOpen) go consumeJob() - t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02"))) + t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, time.Now().Format("2006-01-02"))) subSecond := t.Unix()+86410-time.Now().Unix() time.Sleep(time.Duration(subSecond)*time.Second) c := time.NewTicker(24*time.Hour).C @@ -49,12 +49,12 @@ func CreateJob(dateString string) { } var start,end int64 if dateString == "" { - t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02"))) + t,_ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, time.Now().Format("2006-01-02"))) start = t.Unix()-86400 end = t.Unix() dateString = time.Unix(start, 0).Format("2006-01-02") }else { - t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", dateString)) + t, err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, dateString)) if err != nil { log.Printf("dateString validate fail,must format like 2006-01-02 \n") return @@ -179,7 +179,7 @@ func archiveAction(param ArchiveActionList) { tmpFloatList = []float64{vvv[1]} } } - if len(tmpFloatList) > 0 { + if len(tmpFloatList) > 0 && tmpStartTime <= v.End { avg,min,max,p95 := calcData(tmpFloatList) rowData = append(rowData, &ArchiveTable{Endpoint:v.Endpoint,Metric:v.Metric,Tags:tmpTagString,UnixTime:tmpStartTime-60,Avg:avg,Min:min,Max:max,P95:p95}) } @@ -217,7 +217,7 @@ func ArchiveFromMysql(tableUnixTime int64) { if Config().Trans.FiveMinStartDay > 0 { startDays = Config().Trans.FiveMinStartDay } - t, _ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02"))) + t, _ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+DefaultLocalTimeZone, time.Now().Format("2006-01-02"))) tableUnixTime = t.Unix() - (startDays * 86400) } oldTableName := fmt.Sprintf("archive_%s", time.Unix(tableUnixTime, 0).Format("2006_01_02")) diff --git a/monitor-agent/archive_mysql_tool/funcs/prometheus.go b/monitor-agent/archive_mysql_tool/funcs/prometheus.go index f5276f9fb..64fcbdc7b 100644 --- a/monitor-agent/archive_mysql_tool/funcs/prometheus.go +++ b/monitor-agent/archive_mysql_tool/funcs/prometheus.go @@ -1,15 +1,15 @@ package funcs import ( - "net/http" - "time" - "net/url" + "encoding/json" "fmt" - "strings" "io/ioutil" - "encoding/json" - "strconv" + "net/http" + "net/url" "sort" + "strconv" + "strings" + "time" ) var ( diff --git a/monitor-server/api/api.go b/monitor-server/api/api.go index 45d17b58f..d58b6cfef 100644 --- a/monitor-server/api/api.go +++ b/monitor-server/api/api.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "fmt" "github.com/WeBankPartners/open-monitor/monitor-server/api/v1/agent" "github.com/WeBankPartners/open-monitor/monitor-server/api/v1/alarm" @@ -13,7 +14,9 @@ import ( "github.com/gin-gonic/gin" "github.com/swaggo/gin-swagger" "github.com/swaggo/gin-swagger/swaggerFiles" + "io/ioutil" "net/http" + "strings" "time" ) @@ -213,7 +216,7 @@ func InitClusterApi() { return } http.Handle("/sync/config", http.HandlerFunc(alarm.SyncConfigHandle)) - http.Handle("/sync/consul", http.HandlerFunc(alarm.SyncConsulHandle)) + http.Handle("/sync/sd", http.HandlerFunc(alarm.SyncSdFileHandle)) http.ListenAndServe(fmt.Sprintf(":%s", m.Config().Cluster.HttpPort), nil) } @@ -223,8 +226,34 @@ func InitDependenceParam() { func httpLogHandle() gin.HandlerFunc { return func(c *gin.Context) { - start := time.Now() - c.Next() - log.Logger.Info("request", log.String("url", c.Request.RequestURI), log.String("method", c.Request.Method), log.Int("code", c.Writer.Status()), log.String("operator", c.GetString("operatorName")), log.String("ip", c.ClientIP()), log.Int64("cost_time", time.Now().Sub(start).Milliseconds())) + ignoreLog := false + for _,v := range m.LogIgnorePath { + if strings.Contains(c.Request.RequestURI, v) { + ignoreLog = true + break + } + } + if ignoreLog { + c.Next() + }else { + start := time.Now() + var bodyBytes []byte + if c.Request.Method == http.MethodPost { + ignore := false + for _, v := range m.LogParamIgnorePath { + if strings.Contains(c.Request.RequestURI, v) { + ignore = true + break + } + } + if !ignore { + bodyBytes, _ = ioutil.ReadAll(c.Request.Body) + c.Request.Body.Close() + c.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + } + } + c.Next() + log.AccessLogger.Info("request", log.String("url", c.Request.RequestURI), log.String("method", c.Request.Method), log.Int("code", c.Writer.Status()), log.String("operator", c.GetString("operatorName")), log.String("ip", c.ClientIP()), log.Float64("cost_second", time.Now().Sub(start).Seconds()), log.String("body", string(bodyBytes))) + } } } \ No newline at end of file diff --git a/monitor-server/api/v1/agent/deregister.go b/monitor-server/api/v1/agent/deregister.go index bbb50104e..1fedad39e 100644 --- a/monitor-server/api/v1/agent/deregister.go +++ b/monitor-server/api/v1/agent/deregister.go @@ -1,6 +1,7 @@ package agent import ( + "github.com/WeBankPartners/open-monitor/monitor-server/services/other" "github.com/gin-gonic/gin" m "github.com/WeBankPartners/open-monitor/monitor-server/models" "github.com/WeBankPartners/open-monitor/monitor-server/services/prom" @@ -73,7 +74,7 @@ func DeregisterJob(guid string) error { log.Logger.Error("Sync service discover file error", log.Error(err)) return err } - + go other.SyncConfig(0, m.SyncSdConfigDto{Guid:guid, Step:endpointObj.Step, IsRegister:false}) db.UpdateAgentManagerTable(m.EndpointTable{Guid:guid}, "", "", "", "", false) return err } diff --git a/monitor-server/api/v1/agent/register.go b/monitor-server/api/v1/agent/register.go index d99854145..7b5545420 100644 --- a/monitor-server/api/v1/agent/register.go +++ b/monitor-server/api/v1/agent/register.go @@ -2,6 +2,7 @@ package agent import ( "fmt" + "github.com/WeBankPartners/open-monitor/monitor-server/services/other" "strings" m "github.com/WeBankPartners/open-monitor/monitor-server/models" mid "github.com/WeBankPartners/open-monitor/monitor-server/middleware" @@ -115,6 +116,7 @@ func AgentRegister(param m.RegisterParamNew) (validateMessage,guid string,err er log.Logger.Error("Sync service discover file error", log.Error(err)) } } + go other.SyncConfig(0, m.SyncSdConfigDto{Guid:rData.endpoint.Guid, Ip:fmt.Sprintf("%s:%s", tmpIp, tmpPort), Step:rData.endpoint.Step, IsRegister:true}) } if rData.addDefaultGroup { if param.DefaultGroupName != "" { diff --git a/monitor-server/api/v1/alarm/alarm.go b/monitor-server/api/v1/alarm/alarm.go index 726e04d47..2b4d56eb5 100644 --- a/monitor-server/api/v1/alarm/alarm.go +++ b/monitor-server/api/v1/alarm/alarm.go @@ -1,20 +1,19 @@ package alarm import ( - "github.com/gin-gonic/gin" - m "github.com/WeBankPartners/open-monitor/monitor-server/models" - mid "github.com/WeBankPartners/open-monitor/monitor-server/middleware" - "strconv" + "encoding/json" "fmt" + mid "github.com/WeBankPartners/open-monitor/monitor-server/middleware" + "github.com/WeBankPartners/open-monitor/monitor-server/middleware/log" + m "github.com/WeBankPartners/open-monitor/monitor-server/models" "github.com/WeBankPartners/open-monitor/monitor-server/services/db" - "strings" - "time" - "github.com/WeBankPartners/open-monitor/monitor-server/services/other" + "github.com/gin-gonic/gin" "io/ioutil" - "encoding/json" - "sort" - "github.com/WeBankPartners/open-monitor/monitor-server/middleware/log" "net/http" + "sort" + "strconv" + "strings" + "time" ) func AcceptAlertMsg(c *gin.Context) { @@ -25,17 +24,17 @@ func AcceptAlertMsg(c *gin.Context) { mid.ReturnSuccess(c) } log.Logger.Debug("accept", log.JsonObj("body", param)) - var alarms []*m.AlarmTable + var alarms []*m.AlarmHandleObj for _,v := range param.Alerts { if v.Labels["instance"] == "127.0.0.1:8300" { continue } - log.Logger.Debug("Accept alert msg", log.JsonObj("alert", v)) var tmpValue float64 var tmpAlarms m.AlarmProblemList var tmpTags string var sortTagList m.DefaultSortList - tmpAlarm := m.AlarmTable{Status: v.Status} + tmpAlarm := m.AlarmHandleObj{} + tmpAlarm.Status = v.Status for labelKey,labelValue := range v.Labels { sortTagList = append(sortTagList, &m.DefaultSortObj{Key:labelKey, Value:labelValue}) } @@ -114,6 +113,8 @@ func AcceptAlertMsg(c *gin.Context) { tmpAlarm.SLast = strategyObj.Last tmpAlarm.SPriority = strategyObj.Priority tmpAlarm.Content = v.Annotations["description"] + tmpAlarm.NotifyEnable = strategyObj.NotifyEnable + tmpAlarm.NotifyDelay = strategyObj.NotifyDelay tmpSummaryMsg := strings.Split(v.Annotations["summary"], "__") var tmpEndpointIp string if len(tmpSummaryMsg) == 4 { @@ -152,7 +153,6 @@ func AcceptAlertMsg(c *gin.Context) { continue } } - //tmpAlarmQuery := m.AlarmTable{Endpoint: tmpAlarm.Endpoint, StrategyId: tmpAlarm.StrategyId, Tags:tmpAlarm.Tags, SCond:tmpAlarm.SCond, SLast:tmpAlarm.SLast} tmpAlarmQuery := m.AlarmTable{Endpoint: tmpAlarm.Endpoint, StrategyId: tmpAlarm.StrategyId, Tags:tmpAlarm.Tags} _, tmpAlarms = db.GetAlarms(tmpAlarmQuery, 1, false, false) } @@ -183,7 +183,12 @@ func AcceptAlertMsg(c *gin.Context) { continue } if tmpOperation == "resolve" { - tmpAlarm = m.AlarmTable{Id:tmpAlarms[0].Id, Endpoint:tmpAlarms[0].Endpoint, StrategyId:tmpAlarms[0].StrategyId, Status:"ok", EndValue:tmpValue, End:time.Now()} + tmpAlarm.Id = tmpAlarms[0].Id + tmpAlarm.Endpoint = tmpAlarms[0].Endpoint + tmpAlarm.StrategyId = tmpAlarms[0].StrategyId + tmpAlarm.Status = "ok" + tmpAlarm.EndValue = tmpValue + tmpAlarm.End = time.Now() }else if tmpOperation == "add" { tmpAlarm.StartValue = tmpValue tmpAlarm.Start = time.Now() @@ -196,26 +201,11 @@ func AcceptAlertMsg(c *gin.Context) { mid.ReturnUpdateTableError(c, "alarm", err) return } - if m.Config().Alert.Enable { - for _,v := range alarms { - var sao m.SendAlertObj - accept := db.GetMailByStrategy(v.StrategyId) - if len(accept) == 0 { - continue - } - sao.Accept = accept - sao.Subject = fmt.Sprintf("[%s][%s] Endpoint:%s Metric:%s", v.Status, v.SPriority, v.Endpoint, v.SMetric) - sao.Content = fmt.Sprintf("Endpoint:%s \r\nStatus:%s\r\nMetric:%s\r\nEvent:%.3f%s\r\nLast:%s\r\nPriority:%s\r\nNote:%s\r\nTime:%s",v.Endpoint,v.Status,v.SMetric,v.StartValue,v.SCond,v.SLast,v.SPriority,v.Content,v.Start.Format(m.DatetimeFormat)) - other.SendSmtpMail(sao) - } - } - if m.CoreUrl != "" { - for _, v := range alarms { - notifyErr := db.NotifyCoreEvent(v.Endpoint, v.StrategyId, 0, 0) - if notifyErr != nil { - log.Logger.Error("notify core event fail", log.Error(notifyErr)) - } + for _,v := range alarms { + if v.NotifyEnable == 0 { + continue } + go db.NotifyAlarm(v) } mid.ReturnSuccess(c) }else{ @@ -357,6 +347,9 @@ func OpenAlarmApi(c *gin.Context) { var requestObj m.OpenAlarmObj requestObj.AlertInfo = c.PostForm("alert_info") requestObj.AlertIp = c.PostForm("alert_ip") + if requestObj.AlertIp == "" { + requestObj.AlertIp = c.ClientIP() + } requestObj.AlertLevel = c.PostForm("alert_level") requestObj.AlertObj = c.PostForm("alert_obj") requestObj.AlertTitle = c.PostForm("alert_title") @@ -374,6 +367,11 @@ func OpenAlarmApi(c *gin.Context) { c.JSON(http.StatusOK, m.OpenAlarmResponse{ResultCode:0, ResultMsg:"success"}) }else { if err := c.ShouldBindJSON(¶m); err == nil { + for _,v := range param.AlertList { + if v.AlertIp == "" { + v.AlertIp = c.ClientIP() + } + } err = db.SaveOpenAlarm(param) if err != nil { c.JSON(http.StatusOK, m.OpenAlarmResponse{ResultCode:-1, ResultMsg:err.Error()}) diff --git a/monitor-server/api/v1/alarm/strategy.go b/monitor-server/api/v1/alarm/strategy.go index 814a45055..2c86de6a3 100644 --- a/monitor-server/api/v1/alarm/strategy.go +++ b/monitor-server/api/v1/alarm/strategy.go @@ -59,7 +59,7 @@ func AddStrategy(c *gin.Context) { mid.ReturnValidateError(c, "grp_id and endpoint_id can not be provided at the same time") return } - err,tplObj := db.AddTpl(param.GrpId, param.EndpointId, "") + err,tplObj := db.AddTpl(param.GrpId, param.EndpointId, mid.GetOperateUser(c)) if err != nil { mid.ReturnUpdateTableError(c, "tpl", err) return @@ -67,6 +67,8 @@ func AddStrategy(c *gin.Context) { param.TplId = tplObj.Id } strategyObj := m.StrategyTable{TplId:param.TplId,Metric:param.Metric,Expr:param.Expr,Cond:param.Cond,Last:param.Last,Priority:param.Priority,Content:param.Content} + strategyObj.NotifyEnable = param.NotifyEnable + strategyObj.NotifyDelay = param.NotifyDelay err = db.UpdateStrategy(&m.UpdateStrategy{Strategy:[]*m.StrategyTable{&strategyObj}, Operation:"insert"}) if err != nil { mid.ReturnUpdateTableError(c, "strategy", err) @@ -90,6 +92,11 @@ func EditStrategy(c *gin.Context) { mid.ReturnParamEmptyError(c, "strategy_id") return } + _,strategy := db.GetStrategyTable(param.StrategyId) + if strategy.TplId <= 0 { + mid.ReturnHandleError(c, "template for this strategy is empty", nil) + return + } // check param param.Expr = strings.Replace(param.Expr, "'", "", -1) param.Content = strings.Replace(param.Content, "'", "", -1) @@ -98,14 +105,13 @@ func EditStrategy(c *gin.Context) { mid.ReturnValidateError(c, "cond or last illegal") return } - strategyObj := m.StrategyTable{Id:param.StrategyId,Metric:param.Metric,Expr:param.Expr,Cond:param.Cond,Last:param.Last,Priority:param.Priority,Content:param.Content} + strategyObj := m.StrategyTable{Id:param.StrategyId,TplId:strategy.TplId,Metric:param.Metric,Expr:param.Expr,Cond:param.Cond,Last:param.Last,Priority:param.Priority,Content:param.Content,NotifyEnable: param.NotifyEnable,NotifyDelay: param.NotifyDelay} err = db.UpdateStrategy(&m.UpdateStrategy{Strategy:[]*m.StrategyTable{&strategyObj}, Operation:"update"}) if err != nil { mid.ReturnUpdateTableError(c, "strategy", err) return } - _,strategy := db.GetStrategyTable(param.StrategyId) - db.UpdateTpl(strategy.TplId, "") + db.UpdateTpl(strategy.TplId, mid.GetOperateUser(c)) err = SaveConfigFile(strategy.TplId, false) if err != nil { mid.ReturnHandleError(c, "save alert rules file failed", err) @@ -191,7 +197,7 @@ func SaveConfigFile(tplId int, fromCluster bool) error { return err } if !fromCluster { - go other.SyncConfig(tplId, m.SyncConsulDto{}) + go other.SyncConfig(tplId, m.SyncSdConfigDto{}) } return nil } @@ -476,4 +482,48 @@ func SyncConsulHandle(w http.ResponseWriter,r *http.Request) { } response.Code = 200 response.Message = "Success" +} + +func SyncSdFileHandle(w http.ResponseWriter,r *http.Request) { + log.Logger.Info("start to sync sd config") + var response mid.RespJson + w.Header().Set("Content-Type", "application/json") + defer w.Write([]byte(fmt.Sprintf("{\"Code\":%d,\"Message\":\"%s\",\"Data\":\"%v\"}", response.Code,response.Message,response.Data))) + var param m.SyncSdConfigDto + b,_ := ioutil.ReadAll(r.Body) + err := json.Unmarshal(b, ¶m) + if err != nil { + response.Code = 401 + response.Message = "Param json format fail" + response.Data = err + return + } + if param.Guid == "" { + response.Code = 401 + response.Message = "Guid is empty" + return + } + if param.IsRegister { + stepList := prom.AddSdEndpoint(m.ServiceDiscoverFileObj{Guid: param.Guid, Address: param.Ip, Step: param.Step}) + for _,tmpStep := range stepList { + err = prom.SyncSdConfigFile(tmpStep) + if err != nil { + log.Logger.Error("Sync service discover file error", log.Error(err)) + } + } + }else{ + prom.DeleteSdEndpoint(param.Guid) + err = prom.SyncSdConfigFile(param.Step) + if err != nil { + log.Logger.Error("Sync service discover file error", log.Error(err)) + } + } + if err != nil { + response.Code = 500 + response.Message = "Sync consul fail" + response.Data = err + return + } + response.Code = 200 + response.Message = "Success" } \ No newline at end of file diff --git a/monitor-server/api/v1/dashboard/dashboard.go b/monitor-server/api/v1/dashboard/dashboard.go index 34f0e0f8a..73c6e6552 100644 --- a/monitor-server/api/v1/dashboard/dashboard.go +++ b/monitor-server/api/v1/dashboard/dashboard.go @@ -417,12 +417,12 @@ func GetChart(c *gin.Context) { var subStartSecond,subEndSecond int64 // validate config time if paramConfig[0].CompareFirstStart != "" && paramConfig[0].CompareFirstEnd != "" { - st,err := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 00:00:00 CST", paramConfig[0].CompareFirstStart)) + st,err := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 00:00:00 "+m.DefaultLocalTimeZone, paramConfig[0].CompareFirstStart)) if err != nil { mid.ReturnParamTypeError(c, "compare_first_start", "2006-01-02") return } - et,err := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 23:59:59 CST", paramConfig[0].CompareFirstEnd)) + et,err := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 23:59:59 "+m.DefaultLocalTimeZone, paramConfig[0].CompareFirstEnd)) if err != nil { mid.ReturnParamTypeError(c, "compare_first_end", "2006-01-02") return @@ -530,8 +530,8 @@ func GetChart(c *gin.Context) { } querys = append(querys, m.QueryMonitorData{Start: query.Start, End: query.End, PromQ: tmpPromQl, Legend: tmpLegend, Metric: []string{v}, Endpoint: []string{tmpParamConfig.Endpoint}, CompareLegend:compareLegend, SameEndpoint:sameEndpoint, Step:step}) if paramConfig[0].CompareSecondStart != "" && paramConfig[0].CompareSecondEnd != "" { - st,sErr := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 00:00:00 CST", paramConfig[0].CompareSecondStart)) - et,eErr := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 23:59:59 CST", paramConfig[0].CompareSecondEnd)) + st,sErr := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 00:00:00 "+m.DefaultLocalTimeZone, paramConfig[0].CompareSecondStart)) + et,eErr := time.Parse(m.DateFormatWithZone, fmt.Sprintf("%s 23:59:59 "+m.DefaultLocalTimeZone, paramConfig[0].CompareSecondEnd)) stTimestamp := st.Unix() etTimestamp := et.Unix() if sErr == nil && eErr == nil { diff --git a/monitor-server/api/v1/user/auth.go b/monitor-server/api/v1/user/auth.go index b0534135b..597c52559 100644 --- a/monitor-server/api/v1/user/auth.go +++ b/monitor-server/api/v1/user/auth.go @@ -165,6 +165,16 @@ func GetUserMsg(c *gin.Context) { mid.ReturnSuccessData(c, userObj) } +type pluginInterfaceResultObj struct { + ResultCode string `json:"resultCode"` + ResultMessage string `json:"resultMessage"` + Results pluginInterfaceResultOutput `json:"results"` +} + +type pluginInterfaceResultOutput struct { + Outputs []string `json:"outputs"` +} + func AuthRequired() gin.HandlerFunc { return func(c *gin.Context) { if strings.Contains(c.Request.RequestURI, "/export/ping/source") { @@ -179,7 +189,26 @@ func AuthRequired() gin.HandlerFunc { c.Abort() } else { c.Set("operatorName", coreToken.User) - c.Next() + c.Set("operatorRoles", coreToken.Roles) + // plugin interface call + if strings.Contains(c.Request.RequestURI, "/agent/export/") || strings.Contains(c.Request.RequestURI, "/dashboard/system/") { + isSystemCall := false + for _,v := range coreToken.Roles { + if v == m.SystemRole { + if coreToken.User == m.PlatformUser { + isSystemCall = true + } + break + } + } + if !isSystemCall { + c.JSON(http.StatusOK, pluginInterfaceResultObj{ResultCode:"1", ResultMessage:"Token authority validate fail", Results:pluginInterfaceResultOutput{Outputs: []string{}}}) + }else { + c.Next() + } + }else { + c.Next() + } } } else { mid.ReturnTokenError(c) diff --git a/monitor-server/conf/default.json b/monitor-server/conf/default.json index 2f3b4ea8e..0bfec8663 100644 --- a/monitor-server/conf/default.json +++ b/monitor-server/conf/default.json @@ -17,6 +17,7 @@ "log": { "level": "debug", "file": "logs/open-monitor.log", + "access_file": "logs/open-monitor-access.log", "archive_max_size": 64, "archive_max_backup": 10, "archive_max_day": 15, @@ -178,5 +179,6 @@ "local_storage_max_day": 30, "five_min_start_day": 90 }, - "process_check_list": ["ping_exporter", "agent_manager"] + "process_check_list": ["ping_exporter", "agent_manager"], + "default_admin_role": "SUPER_ADMIN" } \ No newline at end of file diff --git a/monitor-server/conf/i18n/en.json b/monitor-server/conf/i18n/en.json index 6c2b81837..232a25aa3 100644 --- a/monitor-server/conf/i18n/en.json +++ b/monitor-server/conf/i18n/en.json @@ -11,5 +11,6 @@ "delete_table_data_error": "Failed to delete db table %t where %k=%v", "handle_error": "Failed to handle request, details: %s", "password_error": "Password is wrong.", - "token_error": "Invalid session token" + "token_error": "Invalid session token", + "token_authority_error": "Token authority validate fail" } diff --git a/monitor-server/conf/i18n/zh-cn.json b/monitor-server/conf/i18n/zh-cn.json index 61cef0966..5a9b1c235 100644 --- a/monitor-server/conf/i18n/zh-cn.json +++ b/monitor-server/conf/i18n/zh-cn.json @@ -11,5 +11,6 @@ "delete_table_data_error": "用条件 %k=%v 删除数据库表 %t 失败", "handle_error": "请求处理失败,详情:%s", "password_error": "密码错误", - "token_error": "请求认证失败" + "token_error": "请求认证失败", + "token_authority_error": "鉴权失败,权限不足" } \ No newline at end of file diff --git a/monitor-server/main.go b/monitor-server/main.go index bd2504fbd..77a66d9b4 100644 --- a/monitor-server/main.go +++ b/monitor-server/main.go @@ -26,7 +26,7 @@ func main() { flag.Parse() m.InitConfig(*cfgFile) //middleware.InitMonitorLog() - log.InitArchiveZapLogger() + log.InitLogger() db.InitDbConn() if m.Config().Http.Session.Enable { middleware.InitSession() diff --git a/monitor-server/middleware/log/zap.go b/monitor-server/middleware/log/zap.go index 3f50edaa5..a5b6fe359 100644 --- a/monitor-server/middleware/log/zap.go +++ b/monitor-server/middleware/log/zap.go @@ -1,23 +1,28 @@ package log import ( + "encoding/json" + m "github.com/WeBankPartners/open-monitor/monitor-server/models" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "time" "gopkg.in/natefinch/lumberjack.v2" - "os" - m "github.com/WeBankPartners/open-monitor/monitor-server/models" "strings" - "encoding/json" + "time" ) var ( Logger *zap.Logger + AccessLogger *zap.Logger levelStringList = []string{"debug","info","warn","error"} LogLevel string ) -func InitArchiveZapLogger() { +func InitLogger() { + Logger = InitArchiveZapLogger(m.Config().Http.Log.File) + AccessLogger = InitArchiveZapLogger(m.Config().Http.Log.AccessFile) +} + +func InitArchiveZapLogger(filePath string) *zap.Logger { LogLevel = strings.ToLower(m.Config().Http.Log.Level) var level int for i,v := range levelStringList { @@ -29,7 +34,7 @@ func InitArchiveZapLogger() { zapLevel := zap.NewAtomicLevel() zapLevel.SetLevel(zapcore.Level(level)) hook := lumberjack.Logger{ - Filename: m.Config().Http.Log.File, + Filename: filePath, MaxSize: m.Config().Http.Log.ArchiveMaxSize, MaxBackups: m.Config().Http.Log.ArchiveMaxBackup, MaxAge: m.Config().Http.Log.ArchiveMaxDay, @@ -50,9 +55,10 @@ func InitArchiveZapLogger() { EncodeDuration: zapcore.SecondsDurationEncoder, EncodeCaller: zapcore.ShortCallerEncoder, } - core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout),zapcore.AddSync(&hook)), zapLevel) - Logger = zap.New(core, zap.AddCaller(), zap.Development()) - Logger.Info("success init zap log !!") + core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.NewMultiWriteSyncer(zapcore.AddSync(&hook)), zapLevel) + logger := zap.New(core, zap.AddCaller(), zap.Development()) + logger.Info("success init zap log !!") + return logger } func Error(err error) zap.Field { diff --git a/monitor-server/middleware/response.go b/monitor-server/middleware/response.go index ee6322cac..a342d240e 100644 --- a/monitor-server/middleware/response.go +++ b/monitor-server/middleware/response.go @@ -100,4 +100,8 @@ func ReturnPasswordError(c *gin.Context) { func ReturnTokenError(c *gin.Context) { ReturnError(c, 401, GetMessageMap(c).TokenError, nil) +} + +func ReturnTokenAuthorityError(c *gin.Context) { + ReturnError(c, 403, GetMessageMap(c).TokenAuthorityError, nil) } \ No newline at end of file diff --git a/monitor-server/middleware/validate.go b/monitor-server/middleware/validate.go index 1061297d6..1d886216b 100644 --- a/monitor-server/middleware/validate.go +++ b/monitor-server/middleware/validate.go @@ -7,14 +7,19 @@ import ( "reflect" "fmt" "regexp" + "sync" ) -var invalidData = []string{"select", "insert", "update", "alter", "delete", "drop", "truncate", "show"} -var regCond = regexp.MustCompile(`^([<=|>=|!=|==|<|>]*)-?\d+(\.\d+)?$`) -var regLast = regexp.MustCompile(`^\d+[s|m|h]$`) -var regPath = regexp.MustCompile(`^\/([\w|\.|\-]+\/?)+$`) -var regNormal = regexp.MustCompile(`^[\w|\.|\-|\~|\!|\@|\#|\$|\%|\^|\[|\]|\{|\}|\(|\)|\,|\s]+$`) -var regIp = regexp.MustCompile(`^((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))$`) +var ( + invalidData = []string{"select", "insert", "update", "alter", "delete", "drop", "truncate", "show"} + regCond = regexp.MustCompile(`^([<=|>=|!=|==|<|>]*)-?\d+(\.\d+)?$`) + regLast = regexp.MustCompile(`^\d+[s|m|h]$`) + regPath = regexp.MustCompile(`^\/([\w|\.|\-]+\/?)+$`) + regNormal = regexp.MustCompile(`^[\w|\.|\-|\~|\!|\@|\#|\$|\%|\^|\[|\]|\{|\}|\(|\)|\,|\s]+$`) + regIp = regexp.MustCompile(`^((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))$`) + roleEndpointMap []map[string]int + roleEndpointLock sync.RWMutex +) func ValidateGet(c *gin.Context) { isOk := true @@ -104,4 +109,16 @@ func IsIllegalNormalInput(str string) bool { func IsIllegalIp(str string) bool { return !regIp.MatchString(str) +} + +func InitRoleEndpointMap() { + +} + +func CheckRoleEndpointOwner(roles []string) { + +} + +func UpdateRoleEndpointMap() { + } \ No newline at end of file diff --git a/monitor-server/models/alarm.go b/monitor-server/models/alarm.go index 13d6c3324..495f02294 100644 --- a/monitor-server/models/alarm.go +++ b/monitor-server/models/alarm.go @@ -38,6 +38,8 @@ type StrategyTable struct { Priority string `json:"priority" binding:"required"` Content string `json:"content" binding:"required"` ConfigType string `json:"config_type"` + NotifyEnable int `json:"notify_enable"` + NotifyDelay int `json:"notify_delay"` } type AlarmTable struct { @@ -58,6 +60,12 @@ type AlarmTable struct { End time.Time `json:"end"` } +type AlarmHandleObj struct { + AlarmTable + NotifyEnable int `json:"notify_enable"` + NotifyDelay int `json:"notify_delay"` +} + type AlarmProblemQuery struct { Id int `json:"id"` StrategyId int `json:"strategy_id"` @@ -192,6 +200,8 @@ type TplStrategyTable struct { Last string `json:"last" binding:"required"` Priority string `json:"priority" binding:"required"` Content string `json:"content" binding:"required"` + NotifyEnable int `json:"notify_enable"` + NotifyDelay int `json:"notify_delay"` } type UpdateStrategy struct { @@ -338,6 +348,15 @@ type SyncConsulDto struct { Interval int `json:"interval"` } +type SyncSdConfigDto struct { + IsRegister bool `json:"is_register"` + Guid string `json:"guid"` + Ip string `json:"ip"` + Port string `json:"port"` + Tags []string `json:"tags"` + Step int `json:"step"` +} + type EndpointHttpTable struct { Id int `json:"id"` EndpointGuid string `json:"endpoint_guid"` diff --git a/monitor-server/models/config.go b/monitor-server/models/config.go index d8f89ddb2..f3dc28b74 100644 --- a/monitor-server/models/config.go +++ b/monitor-server/models/config.go @@ -1,6 +1,8 @@ package models import ( + "os/exec" + "strings" "sync" "log" "github.com/toolkits/file" @@ -11,6 +13,7 @@ import ( type LogConfig struct { Level string `json:"level"` File string `json:"file"` + AccessFile string `json:"access_file"` ArchiveMaxSize int `json:"archive_max_size"` ArchiveMaxBackup int `json:"archive_max_backup"` ArchiveMaxDay int `json:"archive_max_day"` @@ -169,6 +172,7 @@ type GlobalConfig struct { SdFile SdFileConfig `json:"sd_file"` ArchiveMysql ArchiveMysqlConfig `json:"archive_mysql"` ProcessCheckList []string `json:"process_check_list"` + DefaultAdminRole string `json:"default_admin_role"` } var ( @@ -179,6 +183,10 @@ var ( CoreJwtKey string FiringCallback string RecoverCallback string + SubSystemCode string + SubSystemKey string + DefaultMailReceiver []string + DefaultLocalTimeZone string ) func Config() *GlobalConfig { @@ -218,11 +226,37 @@ func InitConfig(cfg string) { for _,v :=range config.Dependence { if v.Name == "core" { CoreUrl = v.Server + if strings.HasSuffix(CoreUrl, "/") { + CoreUrl = CoreUrl[:len(CoreUrl)-1] + } break } } CoreJwtKey = DecryptRsa(os.Getenv("JWT_SIGNING_KEY")) + SubSystemCode = os.Getenv("SUB_SYSTEM_CODE") + SubSystemKey = os.Getenv("SUB_SYSTEM_KEY") FiringCallback = os.Getenv("ALARM_FIRING_CALLBACK") RecoverCallback = os.Getenv("ALARM_RECOVER_CALLBACK") log.Println("read config file:", cfg, "successfully") + if CoreUrl != "" && SubSystemCode != "" && SubSystemKey != "" { + InitCoreToken() + }else{ + log.Printf("Init core token fail,coreUrl & subSystemCode & subSystemKey can not empty") + } + initLocalTimeZone() +} + +func initLocalTimeZone() { + cmdOut,err := exec.Command("/bin/sh", "-c", "date|awk '{print $5}'").Output() + if err != nil { + log.Printf("init local time zone fail,%s \n", err.Error()) + }else{ + cmdOutString := strings.TrimSpace(string(cmdOut)) + if cmdOutString != "" { + DefaultLocalTimeZone = cmdOutString + log.Printf("init local time zone to %s \n", DefaultLocalTimeZone) + }else{ + DefaultLocalTimeZone = "CST" + } + } } \ No newline at end of file diff --git a/monitor-server/models/const.go b/monitor-server/models/const.go index 91663aa72..52085b233 100644 --- a/monitor-server/models/const.go +++ b/monitor-server/models/const.go @@ -5,6 +5,11 @@ const ( DatetimeFormat = `2006-01-02 15:04:05` DateFormatWithZone = `2006-01-02 15:04:05 MST` Version = `1.0.0` - TmpCoreToken = `Bearer eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJTWVNfTU9OSVRPUiIsImlhdCI6MTU5MDExNzM5NSwidHlwZSI6ImFjY2Vzc1Rva2VuIiwiY2xpZW50VHlwZSI6IlNVQl9TWVNURU0iLCJleHAiOjE3NDU2MzczOTUsImF1dGhvcml0eSI6IltTVUJfU1lTVEVNXSJ9.soBixGZNfZKJTsm-augwGsu-fOoPuTYvZNc_VlxS8oAcZsRn4-ccSjEeAvVAso-7y0dxvzz5c_gw9iUE9LVK2w` - + SystemRole = `SUB_SYSTEM` + PlatformUser = `SYS_PLATFORM` ) + +var ( + LogIgnorePath = []string{"/monitor/webhook", "export/ping/source"} + LogParamIgnorePath = []string{"/dashboard/newchart", "/dashboard/pie/chart", "/problem/query", "/problem/history"} +) \ No newline at end of file diff --git a/monitor-server/models/error.go b/monitor-server/models/error.go index e06a24d7f..f87f70ecc 100644 --- a/monitor-server/models/error.go +++ b/monitor-server/models/error.go @@ -21,4 +21,5 @@ type ErrorMessageObj struct{ PasswordError string `json:"password_error"` TokenError string `json:"token_error"` + TokenAuthorityError string `json:"token_authority_error"` } diff --git a/monitor-server/models/rsa.go b/monitor-server/models/rsa.go index 9ad07ee40..2851debd3 100644 --- a/monitor-server/models/rsa.go +++ b/monitor-server/models/rsa.go @@ -1,6 +1,8 @@ package models import ( + "fmt" + "math/big" "strings" "encoding/base64" "io/ioutil" @@ -17,7 +19,7 @@ func DecryptRsa(inputString string) string { } inputString = inputString[4:] result := inputString - inputBytes,err := base64.RawStdEncoding.DecodeString(inputString) + inputBytes,err := base64.StdEncoding.DecodeString(inputString) if err != nil { log.Printf("Input string format to base64 fail,%s \n", err.Error()) return inputString @@ -43,3 +45,67 @@ func DecryptRsa(inputString string) string { result = string(decodeBytes) return result } + +func RSAEncryptByPrivate(orgidata []byte,privatekey string) ([]byte, error) { + decodeBytes, err := base64.StdEncoding.DecodeString(privatekey) + if err != nil { + return nil, fmt.Errorf("RSASign private key is bad") + } + + privInterface, err := x509.ParsePKCS8PrivateKey(decodeBytes) + if err != nil { + return nil, err + } + + priv := privInterface.(*rsa.PrivateKey) + + k := (priv.N.BitLen() + 7) / 8 + tLen := len(orgidata) + em := make([]byte, k) + em[1] = 1 + for i := 2; i < k-tLen-1; i++ { + em[i] = 0xff + } + copy(em[k-tLen:k], orgidata) + c := new(big.Int).SetBytes(em) + if c.Cmp(priv.N) > 0 { + return nil, nil + } + var m *big.Int + var ir *big.Int + if priv.Precomputed.Dp == nil { + m = new(big.Int).Exp(c, priv.D, priv.N) + } else { + // We have the precalculated values needed for the CRT. + m = new(big.Int).Exp(c, priv.Precomputed.Dp, priv.Primes[0]) + m2 := new(big.Int).Exp(c, priv.Precomputed.Dq, priv.Primes[1]) + m.Sub(m, m2) + if m.Sign() < 0 { + m.Add(m, priv.Primes[0]) + } + m.Mul(m, priv.Precomputed.Qinv) + m.Mod(m, priv.Primes[0]) + m.Mul(m, priv.Primes[1]) + m.Add(m, m2) + + for i, values := range priv.Precomputed.CRTValues { + prime := priv.Primes[2+i] + m2.Exp(c, values.Exp, prime) + m2.Sub(m2, m) + m2.Mul(m2, values.Coeff) + m2.Mod(m2, prime) + if m2.Sign() < 0 { + m2.Add(m2, prime) + } + m2.Mul(m2, values.R) + m.Add(m, m2) + } + } + + if ir != nil { + // Unblind. + m.Mul(m, ir) + m.Mod(m, priv.N) + } + return m.Bytes(), nil +} \ No newline at end of file diff --git a/monitor-server/models/user.go b/monitor-server/models/user.go index 79def6d48..7a707d449 100644 --- a/monitor-server/models/user.go +++ b/monitor-server/models/user.go @@ -132,4 +132,42 @@ type CoreJwtToken struct { User string `json:"user"` Expire int64 `json:"expire"` Roles []string `json:"roles"` +} + +type RequestCoreVariableDto struct { + Filters []*CoreVariableFilter `json:"filters"` + Pageable CoreVariablePage `json:"pageable"` + Paging bool `json:"paging"` +} + +type CoreVariableFilter struct { + Name string `json:"name"` + Operator string `json:"operator"` + Value string `json:"value"` +} + +type CoreVariablePage struct { + PageSize int `json:"pageSize"` + StartIndex int `json:"startIndex"` +} + +type RequestCoreVariableResult struct { + Data CoreVariableResultData `json:"data"` + Message string `json:"message"` + Status string `json:"status"` +} + +type CoreVariableResultData struct { + Contents []*CoreVariableResultContentObj `json:"contents"` +} + +type CoreVariableResultContentObj struct { + DefaultValue string `json:"defaultValue"` + Id string `json:"id"` + Name string `json:"name"` + PackageName string `json:"packageName"` + Scope string `json:"scope"` + Source string `json:"source"` + Status string `json:"status"` + Value string `json:"value"` } \ No newline at end of file diff --git a/monitor-server/models/wecube.go b/monitor-server/models/wecube.go new file mode 100644 index 000000000..d21d1b111 --- /dev/null +++ b/monitor-server/models/wecube.go @@ -0,0 +1,147 @@ +package models + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "strconv" + "strings" + "time" +) + +var ( + coreRefreshToken string + coreRefreshTokenExpTime time.Time + coreRequestToken string + coreRequestTokenExpTime time.Time + requestCoreNonce = "monitor" +) + +type requestToken struct { + Password string `json:"password"` + Username string `json:"username"` + Nonce string `json:"nonce"` + ClientType string `json:"clientType"` +} + +type responseObj struct { + Status string `json:"status"` + Message string `json:"message"` + Data []*responseDataObj `json:"data"` +} + +type responseDataObj struct { + Expiration string `json:"expiration"` + Token string `json:"token"` + TokenType string `json:"tokenType"` +} + +func refreshToken() error { + req,err := http.NewRequest(http.MethodGet, CoreUrl + "/auth/v1/api/token", strings.NewReader("")) + if err != nil { + return fmt.Errorf("http new request fail,%s ", err.Error()) + } + req.Header.Set("Authorization", coreRefreshToken) + resp,err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("http response fail,%s ", err.Error()) + } + var respObj responseObj + bodyBytes,_ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + err = json.Unmarshal(bodyBytes, &respObj) + if err != nil { + return fmt.Errorf("http response body json unmarshal fail,%s ", err.Error()) + } + for _,v := range respObj.Data { + if len(v.Expiration) > 10 { + v.Expiration = v.Expiration[:10] + } + expInt,_ := strconv.ParseInt(v.Expiration, 10, 64) + if v.TokenType == "refreshToken" { + coreRefreshToken = "Bearer " + v.Token + coreRefreshTokenExpTime = time.Unix(expInt, 0) + } + if v.TokenType == "accessToken" { + coreRequestToken = "Bearer " + v.Token + coreRequestTokenExpTime = time.Unix(expInt, 0) + } + } + return nil +} + +func requestCoreToken(rsaKey string) error { + encryptBytes,err := RSAEncryptByPrivate([]byte(fmt.Sprintf("%s:%s", SubSystemCode, requestCoreNonce)), rsaKey) + encryptString := base64.StdEncoding.EncodeToString(encryptBytes) + if err != nil { + return err + } + postParam := requestToken{Username: SubSystemCode, Nonce: requestCoreNonce, ClientType: "SUB_SYSTEM", Password: encryptString} + postBytes,_ := json.Marshal(postParam) + fmt.Printf("param: %s \n", string(postBytes)) + req,err := http.NewRequest(http.MethodPost, CoreUrl + "/auth/v1/api/login", bytes.NewReader(postBytes)) + if err != nil { + return fmt.Errorf("http new request fail,%s ", err.Error()) + } + resp,err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("http response fail, %s ", err.Error()) + } + var respObj responseObj + bodyBytes,_ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + err = json.Unmarshal(bodyBytes, &respObj) + if err != nil { + return fmt.Errorf("http response body read fail,%s ", err.Error()) + } + for _,v := range respObj.Data { + if len(v.Expiration) > 10 { + v.Expiration = v.Expiration[:10] + } + expInt,_ := strconv.ParseInt(v.Expiration, 10, 64) + if v.TokenType == "refreshToken" { + coreRefreshToken = "Bearer " + v.Token + coreRefreshTokenExpTime = time.Unix(expInt, 0) + } + if v.TokenType == "accessToken" { + coreRequestToken = "Bearer " + v.Token + coreRequestTokenExpTime = time.Unix(expInt, 0) + } + } + return nil +} + +func InitCoreToken() { + err := requestCoreToken(SubSystemKey) + if err != nil { + log.Printf("Init core token fail,error: %s ", err.Error()) + }else{ + log.Println("Init core token success") + } +} + +func GetCoreToken() string { + if CoreUrl == "" { + return "" + } + if coreRequestTokenExpTime.Unix() > time.Now().Unix() && coreRequestToken != "" { + return coreRequestToken + } + if coreRefreshTokenExpTime.Unix() > time.Now().Unix() && coreRefreshToken != "" { + err := refreshToken() + if err != nil { + log.Printf("Refresh token fail,%s ", err.Error()) + }else{ + return coreRequestToken + } + } + err := requestCoreToken(SubSystemKey) + if err != nil { + log.Printf("Try to init core token fail,%s ", err.Error()) + } + return coreRefreshToken +} \ No newline at end of file diff --git a/monitor-server/services/db/alarm.go b/monitor-server/services/db/alarm.go index 7d8b4a6c5..24695ecc6 100644 --- a/monitor-server/services/db/alarm.go +++ b/monitor-server/services/db/alarm.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "github.com/WeBankPartners/open-monitor/monitor-server/services/other" "sort" "strconv" "strings" @@ -258,7 +259,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { grpIds += ") OR" } var tpls []*m.TplStrategyTable - sql := `SELECT t1.id tpl_id,t1.grp_id,t1.endpoint_id,t2.id strategy_id,t2.metric,t2.expr,t2.cond,t2.last,t2.priority,t2.content + sql := `SELECT t1.id tpl_id,t1.grp_id,t1.endpoint_id,t2.id strategy_id,t2.metric,t2.expr,t2.cond,t2.last,t2.priority,t2.content,t2.notify_enable,t2.notify_delay FROM tpl t1 LEFT JOIN strategy t2 ON t1.id=t2.tpl_id WHERE (` + grpIds + ` endpoint_id=?) order by t1.endpoint_id,t1.id,t2.id` err = x.SQL(sql, query.SearchId).Find(&tpls) if err != nil { @@ -279,7 +280,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { if i == 0 { tmpTplId = v.TplId if v.StrategyId > 0 { - tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content}) + tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content, NotifyEnable: v.NotifyEnable, NotifyDelay: v.NotifyDelay}) } } else { if v.TplId != tmpTplId { @@ -303,7 +304,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { tmpStrategys = []*m.StrategyTable{} } if v.StrategyId > 0 { - tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content}) + tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content, NotifyEnable: v.NotifyEnable, NotifyDelay: v.NotifyDelay}) } } } @@ -335,7 +336,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { tmpParentId := grps[0].Parent for i := 0; i < 10; i++ { parentGrp := getGrpParent(tmpParentId) - sql := `SELECT t1.id tpl_id,t1.grp_id,t1.endpoint_id,t2.id strategy_id,t2.metric,t2.expr,t2.cond,t2.last,t2.priority,t2.content + sql := `SELECT t1.id tpl_id,t1.grp_id,t1.endpoint_id,t2.id strategy_id,t2.metric,t2.expr,t2.cond,t2.last,t2.priority,t2.content,t2.notify_enable,t2.notify_delay FROM tpl t1 LEFT JOIN strategy t2 ON t1.id=t2.tpl_id WHERE t1.grp_id=? ORDER BY t2.id` parentTpls = []*m.TplStrategyTable{} x.SQL(sql, parentGrp.Id).Find(&parentTpls) @@ -346,7 +347,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { if ignoreLogMonitor && v.Metric == "log_monitor" { continue } - tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content}) + tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content, NotifyEnable: v.NotifyEnable, NotifyDelay: v.NotifyDelay}) } } result = append(result, &m.TplObj{TplId: parentTpls[0].TplId, ObjId: parentGrp.Id, ObjName: parentGrp.Name, ObjType: "grp", Operation: false, Strategy: tmpStrategys}) @@ -371,7 +372,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { } result = newResult } - sql := `SELECT t1.id tpl_id,t1.grp_id,t1.endpoint_id,t2.id strategy_id,t2.metric,t2.expr,t2.cond,t2.last,t2.priority,t2.content + sql := `SELECT t1.id tpl_id,t1.grp_id,t1.endpoint_id,t2.id strategy_id,t2.metric,t2.expr,t2.cond,t2.last,t2.priority,t2.content,t2.notify_enable,t2.notify_delay FROM tpl t1 LEFT JOIN strategy t2 ON t1.id=t2.tpl_id WHERE t1.grp_id=? ORDER BY t2.id` err = x.SQL(sql, query.SearchId).Find(&tpls) if err != nil { @@ -385,7 +386,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { if ignoreLogMonitor && v.Metric == "log_monitor" { continue } - tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content}) + tmpStrategys = append(tmpStrategys, &m.StrategyTable{Id: v.StrategyId, TplId: v.TplId, Metric: v.Metric, Expr: v.Expr, Cond: v.Cond, Last: v.Last, Priority: v.Priority, Content: v.Content, NotifyEnable: v.NotifyEnable, NotifyDelay: v.NotifyDelay}) } } result = append(result, &m.TplObj{TplId: tpls[0].TplId, ObjId: tpls[0].GrpId, ObjName: grps[0].Name, ObjType: "grp", Operation: true, Strategy: tmpStrategys}) @@ -403,7 +404,7 @@ func GetStrategys(query *m.TplQuery, ignoreLogMonitor bool) error { func UpdateStrategy(obj *m.UpdateStrategy) error { var actions []*Action for _, v := range obj.Strategy { - action := Classify(*v, obj.Operation, "strategy", false) + action := Classify(*v, obj.Operation, "strategy", true) if action.Sql != "" { actions = append(actions, &action) } @@ -650,7 +651,7 @@ func GetAlarms(query m.AlarmTable, limit int, extLogMonitor, extOpenAlarm bool) return err, sortResult } -func UpdateAlarms(alarms []*m.AlarmTable) error { +func UpdateAlarms(alarms []*m.AlarmHandleObj) error { if len(alarms) == 0 { return nil } @@ -662,7 +663,7 @@ func UpdateAlarms(alarms []*m.AlarmTable) error { _, cErr = x.Exec(action.Sql, v.Status, v.EndValue, v.End.Format(m.DatetimeFormat), v.Id) } else { action.Sql = "INSERT INTO alarm(strategy_id,endpoint,status,s_metric,s_expr,s_cond,s_last,s_priority,content,start_value,start,tags) VALUE (?,?,?,?,?,?,?,?,?,?,?,?)" - _, cErr = x.Exec(action.Sql, v.StrategyId, v.Endpoint, v.Status, v.SMetric, v.SExpr, v.SCond, v.SLast, v.SPriority, v.Content, v.StartValue, time.Now().Format(m.DatetimeFormat), v.Tags) + _, cErr = x.Exec(action.Sql, v.StrategyId, v.Endpoint, v.Status, v.SMetric, v.SExpr, v.SCond, v.SLast, v.SPriority, v.Content, v.StartValue, v.Start.Format(m.DatetimeFormat), v.Tags) } if cErr != nil { log.Logger.Error("Update alarm fail", log.Error(cErr)) @@ -1052,7 +1053,10 @@ func SaveOpenAlarm(param m.OpenAlarmRequest) error { var err error var alertLevel, subSystemId int for _, v := range param.AlertList { - var customAlarmId int + if len(v.AlertInfo) > 1024 { + v.AlertInfo = v.AlertInfo[:1024] + } + var customAlarmId int var query []*m.OpenAlarmObj x.SQL("SELECT * FROM alarm_custom WHERE alert_title=? AND closed=0", v.AlertTitle).Find(&query) if v.AlertLevel == "0" { @@ -1088,7 +1092,7 @@ func SaveOpenAlarm(param m.OpenAlarmRequest) error { customAlarmId = vv.Id } } - if v.UseUmgPolicy != "1" && v.AlertReciver != "" && customAlarmId > 0 { + if v.UseUmgPolicy != "1" && customAlarmId > 0 { sendMailErr := NotifyCoreEvent("", 0, 0, customAlarmId) if sendMailErr != nil { log.Logger.Error("Send custom alarm mail event fail", log.Error(sendMailErr)) @@ -1263,23 +1267,6 @@ func QueryAlarmBySql(sql string, params []interface{}) (err error, result m.Alar v.Content = strings.ReplaceAll(v.Content, "\n", "
") } } - //if len(logMonitorStrategyIds) > 0 { - // var logMonitorQuery []*m.LogMonitorTable - // x.SQL("SELECT * FROM log_monitor WHERE strategy_id IN (" + strings.Join(logMonitorStrategyIds, ",") + ")").Find(&logMonitorQuery) - // if len(logMonitorQuery) > 0 { - // for _, v := range alarmQuery { - // if v.SMetric == "log_monitor" { - // for _, vv := range logMonitorQuery { - // if v.StrategyId == vv.StrategyId { - // v.Path = vv.Path - // v.Keyword = vv.Keyword - // break - // } - // } - // } - // } - // } - //} metricMap := make(map[string]int) for _, v := range alarmQuery { if v.SPriority == "high" { @@ -1329,3 +1316,54 @@ func QueryHistoryAlarm(param m.QueryHistoryAlarmParam) (err error, result m.Alar err, result = QueryAlarmBySql(sql, []interface{}{}) return err, result } + +func NotifyAlarm(alarmObj *m.AlarmHandleObj) { + if alarmObj.NotifyDelay > 0 { + var alarmRows []*m.AlarmTable + abortNotify := false + if alarmObj.Id > 0 { + x.SQL("select * from alarm where id=?", alarmObj.Id).Find(&alarmRows) + if len(alarmRows) > 0 { + if (alarmRows[0].End.Unix()-alarmRows[0].Start.Unix()) <= int64(alarmObj.NotifyDelay) { + log.Logger.Info("Abort recover alarm notify", log.Int("id", alarmObj.Id), log.String("start",alarmRows[0].Start.Format(m.DatetimeFormat)),log.String("end",alarmRows[0].End.Format(m.DatetimeFormat))) + abortNotify = true + } + } + } else { + time.Sleep(time.Duration(alarmObj.NotifyDelay) * time.Second) + x.SQL("select * from alarm where strategy_id=? and endpoint=? and start=?", alarmObj.StrategyId, alarmObj.Endpoint, alarmObj.Start.Format(m.DatetimeFormat)).Find(&alarmRows) + if len(alarmRows) > 0 { + if alarmRows[0].Status == "ok" { + log.Logger.Info("Abort firing alarm notify", log.String("endpoint",alarmObj.Endpoint), log.Int("strategyId",alarmObj.StrategyId), log.String("start",alarmObj.Start.Format(m.DatetimeFormat))) + abortNotify = true + } + } + } + if abortNotify { + return + } + } + if m.CoreUrl != "" { + notifyErr := NotifyCoreEvent(alarmObj.Endpoint, alarmObj.StrategyId, 0, 0) + if notifyErr != nil { + log.Logger.Error("notify core event fail", log.Error(notifyErr)) + } + }else{ + if m.Config().Alert.Enable { + var sao m.SendAlertObj + accept := GetMailByStrategy(alarmObj.StrategyId) + if len(accept) == 0 { + return + } + timeString := alarmObj.Start.Format(m.DatetimeFormat) + if alarmObj.Status == "ok" { + timeString = alarmObj.End.Format(m.DatetimeFormat) + } + sao.Accept = accept + sao.Subject = fmt.Sprintf("[%s][%s] Endpoint:%s Metric:%s", alarmObj.Status, alarmObj.SPriority, alarmObj.Endpoint, alarmObj.SMetric) + sao.Content = fmt.Sprintf("Endpoint:%s \r\nStatus:%s\r\nMetric:%s\r\nEvent:%.3f%s\r\nLast:%s\r\nPriority:%s\r\nNote:%s\r\nTime:%s",alarmObj.Endpoint,alarmObj.Status,alarmObj.SMetric,alarmObj.StartValue,alarmObj.SCond,alarmObj.SLast,alarmObj.SPriority,alarmObj.Content,timeString) + other.SendSmtpMail(sao) + } + } + +} \ No newline at end of file diff --git a/monitor-server/services/db/alert_window.go b/monitor-server/services/db/alert_window.go index 57f0e19c9..d3d0880ef 100644 --- a/monitor-server/services/db/alert_window.go +++ b/monitor-server/services/db/alert_window.go @@ -59,12 +59,12 @@ func CheckEndpointActiveAlert(endpoint string) bool { nTime := time.Now() for _,v := range tableData { if strings.Contains(v.Weekday, "All") || strings.Contains(v.Weekday, time.Now().Weekday().String()) { - startTime,err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s %s:00 CST", nTime.Format("2006-01-02"), v.Start)) + startTime,err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s %s:00 "+m.DefaultLocalTimeZone, nTime.Format("2006-01-02"), v.Start)) if err != nil { log.Logger.Error("Check endpoint is in active alert window error", log.String("start", v.Start), log.Error(err)) continue } - endTime,err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s %s:00 CST", nTime.Format("2006-01-02"), v.End)) + endTime,err := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s %s:00 "+m.DefaultLocalTimeZone, nTime.Format("2006-01-02"), v.End)) if err != nil { log.Logger.Error("Check endpoint is in active alert window error", log.String("end", v.End), log.Error(err)) continue diff --git a/monitor-server/services/db/cron.go b/monitor-server/services/db/cron.go index e07843235..a58cda22b 100644 --- a/monitor-server/services/db/cron.go +++ b/monitor-server/services/db/cron.go @@ -44,21 +44,21 @@ func StartCheckCron() { var timeSubValue,sleepWaitTime int64 switch intervalMin { case 1: - timeStartValue = fmt.Sprintf("%s:00 CST", time.Now().Format("2006-01-02 15:04")) + timeStartValue = fmt.Sprintf("%s:00 "+m.DefaultLocalTimeZone, time.Now().Format("2006-01-02 15:04")) timeSubValue=60 case 10: tmpTimeString := time.Now().Format("2006-01-02 15:04") - timeStartValue = fmt.Sprintf("%s0:00 CST", tmpTimeString[:len(tmpTimeString)-1]) + timeStartValue = fmt.Sprintf("%s0:00 "+m.DefaultLocalTimeZone, tmpTimeString[:len(tmpTimeString)-1]) timeSubValue=600 case 30: - timeStartValue = fmt.Sprintf("%s:00:00 CST", time.Now().Format("2006-01-02 15")) + timeStartValue = fmt.Sprintf("%s:00:00 "+m.DefaultLocalTimeZone, time.Now().Format("2006-01-02 15")) timeSubValue=1800 case 60: - timeStartValue = fmt.Sprintf("%s:00:00 CST", time.Now().Format("2006-01-02 15")) + timeStartValue = fmt.Sprintf("%s:00:00 "+m.DefaultLocalTimeZone, time.Now().Format("2006-01-02 15")) timeSubValue=3600 default: if intervalMin%60==0 && intervalMin/60>1 { - timeStartValue = fmt.Sprintf("%s:00:00 CST", time.Now().Format("2006-01-02 15")) + timeStartValue = fmt.Sprintf("%s:00:00 "+m.DefaultLocalTimeZone, time.Now().Format("2006-01-02 15")) timeSubValue=3600 }else{ timeSubValue = 0 @@ -104,7 +104,7 @@ func DoCheckProgress() error { log.Logger.Info("Notify request data", log.String("eventSeqNo",requestParam.EventSeqNo),log.String("operationKey",requestParam.OperationKey),log.String("operationData",requestParam.OperationData)) b, _ := json.Marshal(requestParam) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/platform/v1/operation-events", m.CoreUrl), strings.NewReader(string(b))) - request.Header.Set("Authorization", m.TmpCoreToken) + request.Header.Set("Authorization", m.GetCoreToken()) request.Header.Set("Content-Type", "application/json") if err != nil { log.Logger.Error("Notify core event new request fail", log.Error(err)) @@ -180,7 +180,7 @@ func CheckLogKeyword() { return } if len(logMonitorTable) == 0 { - log.Logger.Info("Check log keyword config empty") + log.Logger.Debug("Check log keyword config empty") return } var alarmTable []*m.AlarmTable diff --git a/monitor-server/services/db/dashboard.go b/monitor-server/services/db/dashboard.go index 0d8c2daa1..75d612d09 100644 --- a/monitor-server/services/db/dashboard.go +++ b/monitor-server/services/db/dashboard.go @@ -522,7 +522,7 @@ func getDateStringList(start,end int64) []string { dateList = append(dateList, time.Unix(cursorTime, 0).Format("2006_01_02")) cursorTime += 86400 } - t,_ := time.Parse("2006_01_02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Unix(cursorTime, 0).Format("2006_01_02"))) + t,_ := time.Parse("2006_01_02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+m.DefaultLocalTimeZone, time.Unix(cursorTime, 0).Format("2006_01_02"))) if end > t.Unix()+60 { dateList = append(dateList, time.Unix(cursorTime, 0).Format("2006_01_02")) } @@ -541,7 +541,7 @@ func queryArchiveTables(endpoint,metric,tag,agg string,dateList []string,query * if i == 0 { tmpStart = query.Start }else{ - tmpT,err := time.Parse("2006_01_02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", v)) + tmpT,err := time.Parse("2006_01_02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+m.DefaultLocalTimeZone, v)) if err == nil { tmpStart = tmpT.Unix() }else{ @@ -551,7 +551,7 @@ func queryArchiveTables(endpoint,metric,tag,agg string,dateList []string,query * if i == len(v)-1 { tmpEnd = query.End }else{ - tmpT,err := time.Parse("2006_01_02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", v)) + tmpT,err := time.Parse("2006_01_02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+m.DefaultLocalTimeZone, v)) if err == nil { tmpEnd = tmpT.Unix()+86400 }else{ diff --git a/monitor-server/services/db/entity.go b/monitor-server/services/db/entity.go index 3b9ae9224..567c75387 100644 --- a/monitor-server/services/db/entity.go +++ b/monitor-server/services/db/entity.go @@ -29,7 +29,7 @@ func getCoreProcessKey() string { log.Logger.Error("Get core process key new request fail", log.Error(err)) return "" } - request.Header.Set("Authorization", m.TmpCoreToken) + request.Header.Set("Authorization", m.GetCoreToken()) res,err := ctxhttp.Do(context.Background(), http.DefaultClient, request) if err != nil { log.Logger.Error("Get core process key ctxhttp request fail", log.Error(err)) @@ -63,7 +63,7 @@ func GetCoreEventList() (result m.CoreProcessResult,err error) { log.Logger.Error("Get core process key new request fail", log.Error(err)) return result,err } - request.Header.Set("Authorization", m.TmpCoreToken) + request.Header.Set("Authorization", m.GetCoreToken()) res,err := ctxhttp.Do(context.Background(), http.DefaultClient, request) if err != nil { log.Logger.Error("Get core process key ctxhttp request fail", log.Error(err)) @@ -174,7 +174,7 @@ func NotifyCoreEvent(endpoint string,strategyId int,alarmId int,customAlarmId in log.Logger.Info(fmt.Sprintf("notify request data --> eventSeqNo:%s operationKey:%s operationData:%s", requestParam.EventSeqNo, requestParam.OperationKey, requestParam.OperationData)) b, _ := json.Marshal(requestParam) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/platform/v1/operation-events", m.CoreUrl), strings.NewReader(string(b))) - request.Header.Set("Authorization", m.TmpCoreToken) + request.Header.Set("Authorization", m.GetCoreToken()) request.Header.Set("Content-Type", "application/json") if err != nil { log.Logger.Error("Notify core event new request fail", log.Error(err)) @@ -297,8 +297,11 @@ func GetAlarmEvent(alarmType,inputGuid string,id int) (result m.AlarmEntityObj,e toRole = append(toRole, v.Name) } } + if len(toMail) == 0 { + toMail = m.DefaultMailReceiver + } result.To = strings.Join(toMail, ",") - result.ToMail = strings.Join(toMail, ",") + result.ToMail = result.To result.ToPhone = strings.Join(toPhone, ",") result.ToRole = strings.Join(toRole, ",") result.Subject = fmt.Sprintf("[%s][%s] Endpoint:%s Metric:%s", alarms[0].Status, alarms[0].SPriority, alarms[0].Endpoint, alarms[0].SMetric) @@ -316,8 +319,12 @@ func getCustomAlarmEvent(id int) (result m.AlarmEntityObj,err error) { err = fmt.Errorf("can not find any custom alarm with id:%d", id) return result,err } - result.To = customAlarms[0].AlertReciver - result.ToMail = customAlarms[0].AlertReciver + if !strings.Contains(customAlarms[0].AlertReciver, "@") { + result.To = strings.Join(m.DefaultMailReceiver, ",") + }else { + result.To = customAlarms[0].AlertReciver + } + result.ToMail = result.To alarmStatus := "firing" if customAlarms[0].Closed == 1 { alarmStatus = "ok" diff --git a/monitor-server/services/db/kubernetes.go b/monitor-server/services/db/kubernetes.go index 4e421d55c..a5830f73f 100644 --- a/monitor-server/services/db/kubernetes.go +++ b/monitor-server/services/db/kubernetes.go @@ -123,8 +123,8 @@ func SyncKubernetesConfig() error { return nil } -func StartCronSyncKubernetesPod() { - t := time.NewTicker(time.Second*60).C +func StartCronSyncKubernetesPod(interval int) { + t := time.NewTicker(time.Duration(interval*10)*time.Second).C for { <- t go syncPodToEndpoint() diff --git a/monitor-server/services/db/user.go b/monitor-server/services/db/user.go index 3135f9d52..c78e2f6d6 100644 --- a/monitor-server/services/db/user.go +++ b/monitor-server/services/db/user.go @@ -1,6 +1,7 @@ package db import ( + "bytes" m "github.com/WeBankPartners/open-monitor/monitor-server/models" "fmt" "strings" @@ -264,17 +265,18 @@ func StartCronJob() { if m.Config().CronJob.Interval > 30 { intervalSec = m.Config().CronJob.Interval } - go StartSyncCoreRoleJob(intervalSec) + go StartSyncCoreJob(intervalSec) go prom.StartCheckPrometheusJob(intervalSec) go prom.StartCheckProcessList(intervalSec) - go StartCronSyncKubernetesPod() + go StartCronSyncKubernetesPod(intervalSec) } -func StartSyncCoreRoleJob(interval int) { +func StartSyncCoreJob(interval int) { // Sync core role - t := time.NewTicker(time.Second*time.Duration(interval)).C + t := time.NewTicker(time.Second*time.Duration(interval*5)).C for { - go SyncCoreRole() + //go SyncCoreRole() + go SyncCoreSystemVariable() <- t } } @@ -288,7 +290,7 @@ func SyncCoreRole() { log.Logger.Error("Get core role key new request fail", log.Error(err)) return } - request.Header.Set("Authorization", m.TmpCoreToken) + request.Header.Set("Authorization", m.GetCoreToken()) res,err := ctxhttp.Do(context.Background(), http.DefaultClient, request) if err != nil { log.Logger.Error("Get core role key ctxhttp request fail", log.Error(err)) @@ -296,12 +298,17 @@ func SyncCoreRole() { } defer res.Body.Close() b,_ := ioutil.ReadAll(res.Body) + log.Logger.Info("Get core role response", log.String("body", string(b))) var result m.CoreRoleDto err = json.Unmarshal(b, &result) if err != nil { log.Logger.Error("Get core role key json unmarshal result", log.Error(err)) return } + if len(result.Data) == 0 { + log.Logger.Warn("Get core role key fail with no data") + return + } var tableData,insertData,updateData,deleteData []*m.RoleTable x.SQL("SELECT id,name,display_name FROM role").Find(&tableData) for _,v := range result.Data { @@ -353,6 +360,56 @@ func SyncCoreRole() { } } +func SyncCoreSystemVariable() { + if m.CoreUrl == "" { + return + } + var param m.RequestCoreVariableDto + var filters []*m.CoreVariableFilter + param.Paging = true + param.Pageable = m.CoreVariablePage{PageSize: 10, StartIndex: 0} + filters = append(filters, &m.CoreVariableFilter{Name: "name", Operator: "contains", Value: "MONITOR_MAIL_DEFAULT_RECEIVER"}) + filters = append(filters, &m.CoreVariableFilter{Name: "status", Operator: "eq", Value: "active"}) + param.Filters = filters + postBytes,_ := json.Marshal(param) + request,err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/platform/v1/system-variables/retrieve", m.CoreUrl), bytes.NewReader(postBytes)) + if err != nil { + log.Logger.Error("Get core system variable new request fail", log.Error(err)) + return + } + request.Header.Set("Authorization", m.GetCoreToken()) + request.Header.Set("Content-Type", "application/json") + res,err := ctxhttp.Do(context.Background(), http.DefaultClient, request) + if err != nil { + log.Logger.Error("Get core system variable ctxhttp request fail", log.Error(err)) + return + } + defer res.Body.Close() + b,_ := ioutil.ReadAll(res.Body) + var result m.RequestCoreVariableResult + log.Logger.Info("Get core system variable response", log.String("body", string(b))) + err = json.Unmarshal(b, &result) + if err != nil { + log.Logger.Error("Get core system variable json unmarshal result", log.Error(err)) + return + } + if result.Message != "Success" { + log.Logger.Error("Get core system variable fail", log.JsonObj("response", result)) + }else{ + if len(result.Data.Contents) > 0 { + if strings.Join(m.DefaultMailReceiver, ",") != result.Data.Contents[0].Value { + log.Logger.Info("Get core system variable success", log.String("name", "MONITOR_MAIL_DEFAULT_RECEIVER"), log.String("value", result.Data.Contents[0].Value)) + } + m.DefaultMailReceiver = []string{} + for _,v := range strings.Split(result.Data.Contents[0].Value, ",") { + if v != "" { + m.DefaultMailReceiver = append(m.DefaultMailReceiver, v) + } + } + } + } +} + func UpdateRoleUser(param m.UpdateRoleUserDto) error { var roleUserTable []*m.RelRoleUserTable err := x.SQL("SELECT user_id FROM rel_role_user WHERE role_id=?", param.RoleId).Find(&roleUserTable) diff --git a/monitor-server/services/other/sync.go b/monitor-server/services/other/sync.go index 2e9d1cc91..a7798f5a7 100644 --- a/monitor-server/services/other/sync.go +++ b/monitor-server/services/other/sync.go @@ -18,8 +18,7 @@ var clusterList []string var selfIp string var timeoutCheck int64 -func SyncConfig(tplId int, param m.SyncConsulDto) { - log.Logger.Info(fmt.Sprintf("Start sync config: id->%d param.guid->%s param.is_register->%v", tplId, param.Guid, param.IsRegister)) +func SyncConfig(tplId int, param m.SyncSdConfigDto) { if !m.Config().Cluster.Enable { return } @@ -29,6 +28,7 @@ func SyncConfig(tplId int, param m.SyncConsulDto) { return } }else{ + log.Logger.Info(fmt.Sprintf("Start sync config: id->%d param.guid->%s param.is_register->%v", tplId, param.Guid, param.IsRegister)) if len(m.Config().Cluster.ServerList) == 0 { log.Logger.Warn("Config cluster server list is empty, return") return @@ -73,15 +73,15 @@ func SyncConfig(tplId int, param m.SyncConsulDto) { } } -func requestClusterSync(tplId int,address string,param m.SyncConsulDto) bool { - log.Logger.Debug(fmt.Sprintf("Request sync: tplid->%d address->%s", tplId, address)) +func requestClusterSync(tplId int,address string,param m.SyncSdConfigDto) bool { + log.Logger.Info(fmt.Sprintf("Request sync: tplid->%d address->%s", tplId, address)) url := fmt.Sprintf("http://%s", address) var req *http.Request if tplId > 0 { req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/sync/config?id=%d", url, tplId), strings.NewReader("")) }else{ postData,_ := json.Marshal(param) - req,_ = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/sync/consul", url), strings.NewReader(string(postData))) + req,_ = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/sync/sd", url), strings.NewReader(string(postData))) } req.Header.Set("X-Auth-Token", "default-token-used-in-server-side") resp,err := ctxhttp.Do(context.Background(), http.DefaultClient, req) @@ -116,7 +116,7 @@ func getCoreContainerHost() (result coreHostDto,err error) { log.Logger.Error("Get core hosts key new request fail", log.Error(err)) return result,err } - request.Header.Set("Authorization", m.TmpCoreToken) + request.Header.Set("Authorization", m.GetCoreToken()) res,err := ctxhttp.Do(context.Background(), http.DefaultClient, request) if err != nil { log.Logger.Error("Get core hosts key ctxhttp request fail", log.Error(err)) diff --git a/monitor-server/services/prom/agent_monitor.go b/monitor-server/services/prom/agent_monitor.go index 319d0211d..f6303ee45 100644 --- a/monitor-server/services/prom/agent_monitor.go +++ b/monitor-server/services/prom/agent_monitor.go @@ -7,10 +7,16 @@ import ( "strings" "io/ioutil" m "github.com/WeBankPartners/open-monitor/monitor-server/models" + "sync" "time" "github.com/WeBankPartners/open-monitor/monitor-server/middleware/log" ) +var ( + AgentManagerInitFlag = false + AgentManagerLock = new(sync.RWMutex) +) + type agentManagerRequest struct { Guid string `json:"guid"` Exporter string `json:"exporter"` @@ -28,6 +34,11 @@ type agentManagerResponse struct { } func DeployAgent(agentType,instance,bin,ip,port,user,pwd,url,configFile string) (address string,err error) { + if !AgentManagerInitFlag { + time.Sleep(1*time.Second) + AgentManagerLock.RLock() + AgentManagerLock.RUnlock() + } var param agentManagerRequest param.Guid = fmt.Sprintf("%s_%s_%s", instance, ip, agentType) param.Exporter = bin @@ -71,6 +82,7 @@ func StopAgent(agentType,instance,ip,url string) error { func InitAgentManager(param []*m.AgentManagerTable, url string) { count := 0 + AgentManagerLock.Lock() for { time.Sleep(30*time.Second) resp, err := requestAgentMonitor(param, url, "init") @@ -89,11 +101,13 @@ func InitAgentManager(param []*m.AgentManagerTable, url string) { break } } + AgentManagerLock.Unlock() + AgentManagerInitFlag = true } func StartSyncAgentManagerJob(param []*m.AgentManagerTable, url string) { intervalSecond := 86400 - timeStartValue, _ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 CST", time.Now().Format("2006-01-02"))) + timeStartValue, _ := time.Parse("2006-01-02 15:04:05 MST", fmt.Sprintf("%s 00:00:00 "+m.DefaultLocalTimeZone, time.Now().Format("2006-01-02"))) time.Sleep(time.Duration(timeStartValue.Unix()+86400-time.Now().Unix()) * time.Second) t := time.NewTicker(time.Duration(intervalSecond) * time.Second).C for { diff --git a/monitor-server/services/prom/consul.go b/monitor-server/services/prom/consul.go index a00366a2e..e57209f43 100644 --- a/monitor-server/services/prom/consul.go +++ b/monitor-server/services/prom/consul.go @@ -58,7 +58,7 @@ func RegisteConsul(guid,ip,port string, tags []string, interval int, fromCluster return fmt.Errorf("consul response %s", string(body)) } if !fromCluster { - go other.SyncConfig(0, m.SyncConsulDto{Guid:guid, Ip:ip, Port:port, Tags:tags, Interval:interval, IsRegister:true}) + go other.SyncConfig(0, m.SyncSdConfigDto{Guid:guid, Ip:ip, Port:port, Tags:tags, Step:interval, IsRegister:true}) } return nil } @@ -87,7 +87,7 @@ func DeregisteConsul(guid string, fromCluster bool) error { return fmt.Errorf("consul response %s", string(body)) } if !fromCluster { - go other.SyncConfig(0, m.SyncConsulDto{Guid:guid, IsRegister:false}) + go other.SyncConfig(0, m.SyncSdConfigDto{Guid:guid, IsRegister:false}) } return nil } diff --git a/monitor-ui/src/assets/locale/lang/en.json b/monitor-ui/src/assets/locale/lang/en.json index 055a3a46f..c4ba74dd7 100755 --- a/monitor-ui/src/assets/locale/lang/en.json +++ b/monitor-ui/src/assets/locale/lang/en.json @@ -206,5 +206,7 @@ "rule": "Rule", "addMetricConfig": "Add Metric", "addStringMap": "Add Mapping", - "operationDoc": "Operation Document" + "operationDoc": "Operation Document", + "sendAlarm": "Send Alarm", + "delay": "Delay" } diff --git a/monitor-ui/src/assets/locale/lang/zh-CN.json b/monitor-ui/src/assets/locale/lang/zh-CN.json index 82f6cfb6c..7e1a5aef7 100755 --- a/monitor-ui/src/assets/locale/lang/zh-CN.json +++ b/monitor-ui/src/assets/locale/lang/zh-CN.json @@ -206,5 +206,7 @@ "rule": "规则", "addMetricConfig": "新增指标配置", "addStringMap": "新增映射", - "operationDoc": "操作文档" + "operationDoc": "操作文档", + "sendAlarm": "告警发送", + "delay": "延时" } diff --git a/monitor-ui/src/views/alarm-history.vue b/monitor-ui/src/views/alarm-history.vue index 76effb8b6..b13e5402d 100644 --- a/monitor-ui/src/views/alarm-history.vue +++ b/monitor-ui/src/views/alarm-history.vue @@ -121,7 +121,7 @@ export default { return { startDate: '', endDate: '', - filter:'all', + filter:'start', filterList: [ {label: 'all', value: 'all'}, {label: 'start', value: 'start'} diff --git a/monitor-ui/src/views/alarm-management.vue b/monitor-ui/src/views/alarm-management.vue index 514625d28..87688bfdf 100644 --- a/monitor-ui/src/views/alarm-management.vue +++ b/monitor-ui/src/views/alarm-management.vue @@ -3,12 +3,12 @@ @@ -96,27 +96,29 @@