Skip to content

Commit

Permalink
Fix metrics handling if reloading (SIGHUP)
Browse files Browse the repository at this point in the history 
Signed-off-by: Markus Blaschke <[email protected]>
  • Loading branch information
@mblaschke
mblaschke committed Jun 29, 2020
1 parent da02e31 commit d1785c5
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 84 deletions.
8 changes: 4 additions & 4 deletions funcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ import (
func fileGetAbsolutePath(path string) (string, os.FileInfo) {
ret, err := filepath.Abs(path)
if err != nil {
log.Fatalf("Invalid file: %v", err)
log.Fatalf("invalid file: %v", err)
}

f, err := os.Lstat(ret)
if err != nil {
log.Fatalf("File stats failed: %v", err)
log.Fatalf("file stats failed: %v", err)
}

return ret, f
Expand Down Expand Up @@ -80,10 +80,10 @@ func checkIfFileIsValid(f os.FileInfo, path string) bool {
if f.Mode().Perm()&0022 == 0 {
return true
} else {
log.Infof("Ignoring file with wrong modes (not xx22) %s\n", path)
log.Infof("ignoring file with wrong modes (not xx22) %s\n", path)
}
} else {
log.Infof("Ignoring non regular file %s\n", path)
log.Infof("ignoring non regular file %s\n", path)
}

return false
Expand Down
31 changes: 17 additions & 14 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,9 @@ func initArgParser() []string {
// Log error object as message
func logFatalErrorAndExit(err error, exitCode int) {
if err != nil {
log.Errorf("ERROR: %s\n", err)
log.Errorln(err)
} else {
log.Errorf("ERROR: Unknown fatal error")
log.Errorf("Unknown fatal error")
}
os.Exit(exitCode)
}
Expand All @@ -128,7 +128,7 @@ func findFilesInPaths(pathlist []string, callback func(os.FileInfo, string)) {
log.Fatal(err)
}
} else {
log.Infof("Path %s does not exists\n", path)
log.Infof("path %s does not exists\n", path)
}
}
}
Expand All @@ -138,7 +138,7 @@ func findExecutabesInPathes(pathlist []string, callback func(os.FileInfo, string
if f.Mode().IsRegular() && (f.Mode().Perm()&0100 != 0) {
callback(f, path)
} else {
log.Infof("Ignoring non exectuable file %s\n", path)
log.Infof("ignoring non exectuable file %s\n", path)
}
})
}
Expand Down Expand Up @@ -207,7 +207,7 @@ func parseCrontab(path string, username string) []CrontabEntry {
}

if err != nil {
log.Fatalf("Parser read err: %v", err)
log.Fatalf("parser read err: %v", err)
}

crontabEntries := parser.Parse()
Expand Down Expand Up @@ -254,7 +254,7 @@ func collectCrontabs(args []string) []CrontabEntry {

ret = append(ret, includeRunPartsDirectory(cronSpec, cronPath)...)
} else {
log.Infof("Ignoring --run-parts because of missing time spec: %s\n", runPart)
log.Infof("ignoring --run-parts because of missing time spec: %s\n", runPart)
}
}
}
Expand Down Expand Up @@ -395,37 +395,40 @@ func main() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP)

log.Infof("Starting %s version %s (%s; %s) ", Name, gitTag, gitCommit, runtime.Version())
log.Infof("starting %s version %s (%s; %s) ", Name, gitTag, gitCommit, runtime.Version())

// check if user switching is possible (have to be root)
opts.EnableUserSwitching = true
currentUser, err := user.Current()
if err != nil || currentUser.Uid != "0" {
if opts.AllowUnprivileged {
log.Errorf("WARNING: go-crond is NOT running as root, disabling user switching")
log.Warnln("go-crond is NOT running as root, disabling user switching")
opts.EnableUserSwitching = false
} else {
log.Errorf("ERROR: go-crond is NOT running as root, add option --allow-unprivileged if this is ok")
log.Errorln("go-crond is NOT running as root, add option --allow-unprivileged if this is ok")
os.Exit(1)
}
}

// get current path
confDir, err := os.Getwd()
if err != nil {
log.Fatalf("Could not get current path: %v", err)
log.Fatalf("could not get current path: %v", err)
}

// daemon mode
initMetrics()
log.Infof("starting http server on %s", opts.ServerBind)
startHttpServer()

// endless daemon-reload loop
for {
resetMetrics()

// change to initial directory for fetching crontabs
err = os.Chdir(confDir)
if err != nil {
log.Fatalf("Cannot switch to path %s: %v", confDir, err)
log.Fatalf("cannot switch to path %s: %v", confDir, err)
}

// create new cron runner
Expand All @@ -435,7 +438,7 @@ func main() {
// chdir to root to prevent relative path errors
err = os.Chdir("/")
if err != nil {
log.Fatalf("Cannot switch to path %s: %v", confDir, err)
log.Fatalf("cannot switch to path %s: %v", confDir, err)
}

// start new cron runner
Expand Down Expand Up @@ -475,10 +478,10 @@ func registerRunnerShutdown(runner *Runner) {
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
s := <-c
log.Infof("Got signal: %v", s)
log.Infof("got signal: %v", s)
runner.Stop()

log.Infof("Terminated")
log.Infof("terminated")
os.Exit(1)
}()
}
66 changes: 66 additions & 0 deletions metrics.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package main

import "github.com/prometheus/client_golang/prometheus"

var (
prometheusMetricTask *prometheus.GaugeVec
prometheusMetricTaskRunCount *prometheus.CounterVec
prometheusMetricTaskRunResult *prometheus.GaugeVec
prometheusMetricTaskRunTime *prometheus.GaugeVec
prometheusMetricTaskRunDuration *prometheus.GaugeVec
)

func initMetrics() {
prometheusMetricTask = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_info",
Help: "gocrond task info",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(prometheusMetricTask)

prometheusMetricTaskRunCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gocrond_task_run_count",
Help: "gocrond task run count",
},
[]string{"cronSpec", "cronUser", "cronCommand", "result"},
)
prometheus.MustRegister(prometheusMetricTaskRunCount)

prometheusMetricTaskRunResult = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_run_result",
Help: "gocrond task run result",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(prometheusMetricTaskRunResult)

prometheusMetricTaskRunTime = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_run_time",
Help: "gocrond task last run time",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(prometheusMetricTaskRunTime)

prometheusMetricTaskRunDuration = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_run_duration",
Help: "gocrond task last run duration",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(prometheusMetricTaskRunDuration)
}

func resetMetrics() {
prometheusMetricTask.Reset()
prometheusMetricTaskRunCount.Reset()
prometheusMetricTaskRunResult.Reset()
prometheusMetricTaskRunTime.Reset()
prometheusMetricTaskRunDuration.Reset()
}
78 changes: 12 additions & 66 deletions runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,66 +16,12 @@ import (

type Runner struct {
cron *cron.Cron

prometheus struct {
task *prometheus.GaugeVec
taskRunCount *prometheus.CounterVec
taskRunResult *prometheus.GaugeVec
taskRunTime *prometheus.GaugeVec
taskRunDuration *prometheus.GaugeVec
}
}

func NewRunner() *Runner {
r := &Runner{
cron: cron.New(),
}

r.prometheus.task = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_info",
Help: "gocrond task info",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(r.prometheus.task)

r.prometheus.taskRunCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gocrond_task_run_count",
Help: "gocrond task run count",
},
[]string{"cronSpec", "cronUser", "cronCommand", "result"},
)
prometheus.MustRegister(r.prometheus.taskRunCount)

r.prometheus.taskRunResult = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_run_result",
Help: "gocrond task run result",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(r.prometheus.taskRunResult)

r.prometheus.taskRunTime = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_run_time",
Help: "gocrond task last run time",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(r.prometheus.taskRunTime)

r.prometheus.taskRunDuration = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gocrond_task_run_duration",
Help: "gocrond task last run duration",
},
[]string{"cronSpec", "cronUser", "cronCommand"},
)
prometheus.MustRegister(r.prometheus.taskRunDuration)

return r
}

Expand All @@ -93,10 +39,10 @@ func (r *Runner) Add(cronjob CrontabEntry) error {
}))

if err != nil {
r.prometheus.task.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
log.WithFields(LogCronjobToFields(cronjob)).Errorf("cronjob failed adding:%v", err)
} else {
r.prometheus.task.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
log.WithFields(LogCronjobToFields(cronjob)).Infof("cronjob added")
}

Expand Down Expand Up @@ -143,10 +89,10 @@ func (r *Runner) AddWithUser(cronjob CrontabEntry) error {
}))

if err != nil {
r.prometheus.task.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
log.WithFields(LogCronjobToFields(cronjob)).Errorf("cronjob failed adding: %v", err)
} else {
r.prometheus.task.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
log.WithFields(LogCronjobToFields(cronjob)).Infof("cronjob added")
}

Expand All @@ -160,14 +106,14 @@ func (r *Runner) Len() int {

// Start runner
func (r *Runner) Start() {
log.Infof("Start runner with %d jobs\n", r.Len())
log.Infof("start runner with %d jobs\n", r.Len())
r.cron.Start()
}

// Stop runner
func (r *Runner) Stop() {
r.cron.Stop()
log.Infof("Stop runner")
log.Infof("stop runner")
}

// Execute crontab command
Expand Down Expand Up @@ -197,18 +143,18 @@ func (r *Runner) cmdFunc(cronjob CrontabEntry, cmdCallback func(*exec.Cmd) bool)

elapsed := time.Since(start)

r.prometheus.taskRunDuration.With(r.cronjobToPrometheusLabels(cronjob)).Set(elapsed.Seconds())
r.prometheus.taskRunTime.With(r.cronjobToPrometheusLabels(cronjob)).SetToCurrentTime()
prometheusMetricTaskRunDuration.With(r.cronjobToPrometheusLabels(cronjob)).Set(elapsed.Seconds())
prometheusMetricTaskRunTime.With(r.cronjobToPrometheusLabels(cronjob)).SetToCurrentTime()

logFields := LogCronjobToFields(cronjob)
if err != nil {
r.prometheus.taskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "error"})).Inc()
r.prometheus.taskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "error"})).Inc()
prometheusMetricTaskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
logFields["result"] = "error"
log.WithFields(logFields).Errorln(string(cmdStdout))
} else {
r.prometheus.taskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "success"})).Inc()
r.prometheus.taskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "success"})).Inc()
prometheusMetricTaskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
logFields["result"] = "success"
log.WithFields(logFields).Debugln(string(cmdStdout))
}
Expand Down

0 comments on commit d1785c5

Please sign in to comment.