diff --git a/logstash/map.jinja b/logstash/map.jinja index a458da5..edb5832 100644 --- a/logstash/map.jinja +++ b/logstash/map.jinja @@ -1,4 +1,4 @@ -{% set logstash = salt['grains.filter_by']({ +{%- set logstash = salt['grains.filter_by']({ 'Debian': { 'use_upstream_repo': True, 'pkg': 'logstash', @@ -14,52 +14,134 @@ 'indent': 4 } }, merge=salt['pillar.get']('logstash')) %} - -{%- macro output_indented(col, string) %} -{{ string|indent(col, true) -}} -{%- endmacro %} - -{% macro format_logstash_config(plugin_type, plugin_array) %} -{%- set col = 0 %} -{{- output_indented(col, plugin_type + ' {') }} -{%- set col = col + logstash.indent %} -{%- for plugin in plugin_array %} - {%- if plugin.cond is defined %} - {{- output_indented(col, (plugin.cond + " {")) }} - {%- set col = col + logstash.indent %} - {%- endif %} - {{- output_indented(col, (plugin.plugin_name + " {")) }} +{#- +Format Logstash Config Files #} +{%- macro format_logstash_config(plugin_type, plugin_array) %} + {%- set col = 0 %} + {{- output_indented(col, plugin_type + ' {') }} {%- set col = col + logstash.indent %} - {%- for key, value in plugin.items() %} - {%- if key == 'plugin_name' or key == 'cond' %} {#pass#} - {%- elif value is sameas True or value is sameas False %} - {{- output_indented(col, (key + ' => ' + value|string|lower)) }} - {%- elif value is string or value is number %} - {{- output_indented(col, (key + ' => "' + value|string + '"')) }} - {%- elif value is mapping %} - {{- output_indented(col, (key + ' => {')) }} - {%- set col = col + logstash.indent %} - {%- for attr_key, attr_value in value.items() %} - {{- output_indented(col, (attr_key + ' => "' + attr_value + '"')) }} + {%- for plugin in plugin_array %} + {{- format_plugin(col, plugin) }} + {%- endfor %} + {%- set col = col - logstash.indent %} + {{- output_indented(col, '}') }} +{%- endmacro %} +{#- +Format Plugins #} +{% macro format_plugin(col, plugin) %} + {#- + Format Conditionals #} + {%- if plugin.plugin_name == 'conditional' %} + {{- format_if(col, plugin) }} + {%- if plugin.elseif is defined %} + {%- for plugin in plugin.elseif %} + {{- format_elseif(col, plugin) }} {%- endfor %} - {%- set col = col - logstash.indent %} - {{- output_indented(col, '}') }} - {%- elif value is iterable %} - {{- output_indented(col, key + ' => [') }} + {%- endif %} + {%- if plugin.else is defined %} + {{- format_else(col, plugin) }} + {%- endif %} + {#- + Format Plugins #} + {%- else %} + {%- if plugin.cond is defined %} {# Support old style conditionals #} + {{- output_indented(col, (plugin.cond + " {")) }} {%- set col = col + logstash.indent %} - {%- for item in value %} - {{- output_indented(col, '"' + item + '"') }}{%- if loop.last == false %},{%- endif -%} - {%- endfor %} + {%- endif %} + {{- output_indented(col, (plugin.plugin_name + ' {')) }} + {%- set col = col + logstash.indent %} + {%- for key, value in plugin.items() %} + {%- if key == 'plugin_name' or key == 'cond' %} {#pass#} + {#- + Format Boolean Values #} + {%- elif value is sameas True or value is sameas False %} + {{- output_indented(col, (key + ' => ' + value|string|lower)) }} + {#- + Format Strings and Numbers #} + {%- elif value is string or value is number %} + {{- output_indented(col, (key + ' => "' + value|string + '"')) }} + {#- + Format Mappings #} + {%- elif value is mapping %} + {{- output_indented(col, (key + ' => {')) }} + {%- set col = col + logstash.indent %} + {%- for attr_key, attr_value in value.items() %} + {{- output_indented(col, (attr_key + ' => "' + attr_value + '"')) }} + {%- endfor %} + {%- set col = col - logstash.indent %} + {{- output_indented(col, '}') }} + {#- + Format Lists #} + {%- elif value is iterable %} + {{- output_indented(col, key + ' => [') }} + {%- set col = col + logstash.indent %} + {%- for item in value %} + {{- output_indented(col, '"' + item + '"') }}{%- if loop.last == false %},{%- endif -%} + {%- endfor %} + {%- set col = col - logstash.indent %} + {{- output_indented(col, ']') }} + {%- endif %} + {%- endfor %} + {%- if plugin.cond is defined %} {# Support old style conditionals #} {%- set col = col - logstash.indent %} - {{- output_indented(col, ']') }} + {{- output_indented(col, "}") }} {%- endif %} - {%- endfor %} - {%- set col = col - logstash.indent %} - {{- output_indented(col, '}') }} - {%- if plugin.cond is defined %} {%- set col = col - logstash.indent %} {{- output_indented(col, '}') }} {%- endif %} +{%- endmacro %} +{#- +Format If #} +{%- macro format_if(col, plugin) %} + {{- output_indented(col, 'if ' + plugin.if.cond + ' {') }} + {%- set col = col + logstash.indent %} + {%- for p in plugin.if.then %} + {{- format_plugin(col, p) }} {%- endfor %} -} + {%- set col = col - logstash.indent %} + {{- output_indented(col, '}') }} +{%- endmacro %} +{#- +Format Else If #} +{%- macro format_elseif(col, plugin) %} + {{- output_indented(col, 'elseif ' + plugin.cond + ' {') }} + {%- set col = col + logstash.indent %} + {%- for p in plugin.then %} + {{- format_plugin(col, p) }} + {%- endfor %} + {%- set col = col - logstash.indent %} + {{- output_indented(col, '}') }} +{%- endmacro %} +{#- +Format Else #} +{%- macro format_else(col, plugin) %} + {{- output_indented(col, 'else {') }} + {%- set col = col + logstash.indent %} + {%- for p in plugin.else.then %} + {{- format_plugin(col, p) }} + {%- endfor %} + {%- set col = col - logstash.indent %} + {{- output_indented(col, '}') }} +{%- endmacro %} +{#- +Format Conditionals #} +{%- macro format_conditional(col, plugin, verb) %} + {%- if verb == 'if' %} + {{- output_indented(col, verb + ' ' + plugin[verb].cond + ' {') }} + {%- elif verb == 'elseif' %} + {{- output_indented(col, verb + ' ' + plugin.cond + ' {') }} + {%- elif verb == 'else' %} + {{- output_indented(col, verb + ' {') }} + {%- endif %} + {%- set col = col + logstash.indent %} + {%- for p in plugin[verb].then %} + {{- format_plugin(col, p) }} + {%- endfor %} + {%- set col = col - logstash.indent %} + {{- output_indented(col, '}') }} +{%- endmacro %} +{#- +Output Indented #} +{%- macro output_indented(col, string) %} +{{ string|indent(col, true) -}} {%- endmacro %} diff --git a/pillar.example b/pillar.example index f5fffc1..75f063b 100644 --- a/pillar.example +++ b/pillar.example @@ -1,39 +1,54 @@ ---- logstash: + pkgstate: latest + indent: 2 inputs: - - - plugin_name: file + - plugin_name: file path: - /var/log/syslog - /var/log/auth.log type: syslog filters: - - - plugin_name: grok - cond: 'if [type] == "syslog"' - match: - message: '%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}' - add_field: - received_at: '%{@timestamp}' - received_from: '%{host}' - - - plugin_name: grok - cond: 'else if [type] == "nginx"' - match: - message: '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{URIPATHPARAM:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) \"(?:%{URI:referrer}|-)\" %{QS:agent}' - add_field: - received_at: '%{@timestamp}' - received_from: '%{host}' - - - plugin_name: date + - plugin_name: conditional + if: + cond: '[type] == "syslog"' + then: + - plugin_name: grok + match: + message: '%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}' + add_field: + received_at: '%{@timestamp}' + received_from: '%{host}' + - plugin_name: date match: - 'syslog_timestamp' - 'MMM d HH:mm:ss' - 'MMM dd HH:mm:ss' outputs: - - - plugin_name: lumberjack + - plugin_name: lumberjack hosts: - logs.example.com port: 5000 - ssl_certificate: /etc/ssl/certs/lumberjack.crt \ No newline at end of file + ssl_certificate: /etc/ssl/certs/lumberjack.crt + - plugin_name: conditional + if: + cond: '[type] == "syslog"' + then: + - plugin_name: conditional + if: + cond: '"CRITICAL" in [syslog_message]' + then: + - plugin_name: email + to: 'critical@example.com' + elseif: + - cond: '"ERROR" in [syslog_message]' + then: + - plugin_name: email + to: 'error@example.com' + - cond: '"WARN" in [syslog_message]' + then: + - plugin_name: email + to: 'warning@example.com' + else: + then: + - plugin_name: email + to: 'other@example.com'