diff --git a/README.md b/README.md index 6e87a24..fe36cec 100644 --- a/README.md +++ b/README.md @@ -139,3 +139,37 @@ go-crond exposes [Prometheus][] metrics on `:8080/metrics` if enabled. | `gocrond_task_run_duration` | Duration of last exec | [Prometheus]: https://prometheus.io/ + +##jsonlog in ElasticSearch +##Logstash + +An example of our production filter for logstash +``` +logstashPipeline: + logstash.conf: | + input { + beats { + port => "5044" + } + } + filter { + json { + skip_on_invalid_json => true + source => "message" + target => "json-data" + add_tag => [ "_message_json_parsed" ] + } + if [kubernetes][container][name] =~ ".*-cron" { + mutate { + rename => { "[json-data][level]" => "[json-data][level_name]" } + gsub => [ "[json-data][msg]", "\\\\n", "\n" ] + remove_field => [ "message" ] + } + date { + match => [ "[json-data][started_at]", "ISO8601" ] + } + } + ... +``` +##Grafana +![Image alt](https://raw.githubusercontent.com/promzeus/go-crond/master/grafana-dashboard/go-crond.png) \ No newline at end of file diff --git a/grafana-dashboard/go-crond.json b/grafana-dashboard/go-crond.json new file mode 100644 index 0000000..98c2128 --- /dev/null +++ b/grafana-dashboard/go-crond.json @@ -0,0 +1,279 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "iteration": 1683293128446, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "gocrond_task_run_duration{container=~\"$container\",cronCommand=~\"$cronCommand\"}", + "instant": false, + "interval": "", + "legendFormat": "{{cronCommand}}", + "refId": "A" + } + ], + "title": "Task run duration", + "transparent": true, + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "max" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "gocrond_task_run_result{container=~\"$container\",cronCommand=~\"$cronCommand\"}", + "instant": false, + "interval": "", + "legendFormat": "{{cronCommand}}", + "refId": "A" + } + ], + "title": "Success", + "transparent": true, + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "prod-dd-cron" + ], + "value": [ + "prod-dd-cron" + ] + }, + "definition": "label_values(gocrond_task_info, container)", + "hide": 0, + "includeAll": false, + "multi": true, + "name": "container", + "options": [], + "query": { + "query": "label_values(gocrond_task_info, container)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(gocrond_task_run_duration, cronCommand)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cronCommand", + "options": [], + "query": { + "query": "label_values(gocrond_task_run_duration, cronCommand)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "go-crond", + "uid": "10ensUs4k", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/grafana-dashboard/go-crond.png b/grafana-dashboard/go-crond.png new file mode 100644 index 0000000..20b5bec Binary files /dev/null and b/grafana-dashboard/go-crond.png differ diff --git a/logger.go b/logger.go index 9065ef6..fb14149 100644 --- a/logger.go +++ b/logger.go @@ -11,5 +11,6 @@ func LogCronjobToFields(cronjob CrontabEntry) log.Fields { "command": cronjob.Command, "crontab": cronjob.CrontabPath, "shell": cronjob.Shell, + "pid": cronjob.Pid, } } diff --git a/parser.go b/parser.go index 2dc404a..82dd5f0 100644 --- a/parser.go +++ b/parser.go @@ -32,6 +32,7 @@ type CrontabEntry struct { Spec string User string Command string + Pid int Env []string Shell string CrontabPath string diff --git a/runner.go b/runner.go index f24e5fc..81b71d9 100644 --- a/runner.go +++ b/runner.go @@ -34,7 +34,7 @@ func NewRunner() *Runner { func (r *Runner) Add(cronjob CrontabEntry) error { _, err := r.cron.AddFunc(cronjob.Spec, r.cmdFunc(cronjob, func(execCmd *exec.Cmd) bool { // before exec callback - log.WithFields(LogCronjobToFields(cronjob)).Infof("executing") + // log.WithFields(LogCronjobToFields(cronjob)).Infof("executing") return true })) @@ -53,7 +53,7 @@ func (r *Runner) Add(cronjob CrontabEntry) error { func (r *Runner) AddWithUser(cronjob CrontabEntry) error { _, err := r.cron.AddFunc(cronjob.Spec, r.cmdFunc(cronjob, func(execCmd *exec.Cmd) bool { // before exec callback - log.WithFields(LogCronjobToFields(cronjob)).Debugf("executing") + // log.WithFields(LogCronjobToFields(cronjob)).Debugf("executing") // lookup username u, err := user.Lookup(cronjob.User) @@ -142,10 +142,17 @@ func (r *Runner) cmdFunc(cronjob CrontabEntry, cmdCallback func(*exec.Cmd) bool) logFields := LogCronjobToFields(cronjob) logFields["elapsed_s"] = elapsed.Seconds() + logFields["started_at"] = start.Format(time.RFC3339) + if execCmd.ProcessState != nil { logFields["exitCode"] = execCmd.ProcessState.ExitCode() } + // add pid to log fields + if execCmd.Process != nil { + logFields["pid"] = execCmd.Process.Pid + } + if err != nil { prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "error"})).Inc() prometheusMetricTaskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(0) @@ -156,9 +163,11 @@ func (r *Runner) cmdFunc(cronjob CrontabEntry, cmdCallback func(*exec.Cmd) bool) logFields["result"] = "success" } - log.WithFields(logFields).Info("finished") + // log.WithFields(logFields).Info("finished") if len(cmdStdout) > 0 { - log.Debugln(string(cmdStdout)) + log.WithFields(logFields).Info(string(cmdStdout)) + } else { + log.WithFields(logFields).Info("Command has finished executing without any output to stdout.") } } }