From a05c4eb289cfa11f712202d00aed1f47314d8e5f Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 17:22:37 -0400 Subject: [PATCH 01/13] adding in logsearch job --- jobs/ingestor_logsearch/monit | 5 + jobs/ingestor_logsearch/spec | 213 ++++++++++++++++++ .../templates/bin/ingestor_syslog | 112 +++++++++ .../templates/bin/post-start.erb | 36 +++ .../templates/bin/pre-start | 22 ++ .../templates/config/bpm.yml.erb | 14 ++ .../templates/config/ca.erb | 5 + .../config/filters_override.conf.erb | 9 + .../templates/config/filters_post.conf.erb | 22 ++ .../templates/config/filters_pre.conf.erb | 120 ++++++++++ .../templates/config/ingestor-crt.erb | 1 + .../templates/config/ingestor-pem.erb | 1 + .../config/input_and_output.conf.erb | 60 +++++ .../templates/config/jvm.options.erb | 96 ++++++++ .../templates/config/logstash.yml.erb | 9 + .../templates/config/syslog_tls.crt.erb | 1 + .../templates/config/syslog_tls.key.erb | 1 + packages/logsearch-config/packaging | 17 ++ packages/logsearch-config/spec | 9 + src/logsearch-config/.gitignore | 6 + src/logsearch-config/.rspec | 2 + src/logsearch-config/.ruby-version | 1 + src/logsearch-config/Gemfile | 6 + src/logsearch-config/Gemfile.lock | 32 +++ src/logsearch-config/README.md | 1 + src/logsearch-config/Rakefile | 62 +++++ src/logsearch-config/bin/build | 11 + .../logstash-filters/cluster_monitor.conf.erb | 3 + .../src/logstash-filters/default.conf.erb | 9 + .../src/logstash-filters/deployment.conf | 11 + .../logstash-filters/deployment_lookup.yml | 9 + .../if_it_looks_like_json.conf | 75 ++++++ .../logstash-filters/snippets/bosh_nats.conf | 93 ++++++++ .../logstash-filters/snippets/haproxy.conf | 30 +++ .../src/logstash-filters/snippets/metric.conf | 16 ++ .../snippets/monitor_filter.conf | 29 +++ .../snippets/redact_passwords.conf | 24 ++ .../snippets/syslog_standard.conf | 87 +++++++ .../src/logstash-filters/timecop.conf | 20 ++ src/logsearch-config/target/.gitkeep | 0 src/logsearch-config/vendor/.gitkeep | 0 41 files changed, 1280 insertions(+) create mode 100644 jobs/ingestor_logsearch/monit create mode 100644 jobs/ingestor_logsearch/spec create mode 100644 jobs/ingestor_logsearch/templates/bin/ingestor_syslog create mode 100644 jobs/ingestor_logsearch/templates/bin/post-start.erb create mode 100644 jobs/ingestor_logsearch/templates/bin/pre-start create mode 100644 jobs/ingestor_logsearch/templates/config/bpm.yml.erb create mode 100644 jobs/ingestor_logsearch/templates/config/ca.erb create mode 100644 jobs/ingestor_logsearch/templates/config/filters_override.conf.erb create mode 100644 jobs/ingestor_logsearch/templates/config/filters_post.conf.erb create mode 100644 jobs/ingestor_logsearch/templates/config/filters_pre.conf.erb create mode 100644 jobs/ingestor_logsearch/templates/config/ingestor-crt.erb create mode 100644 jobs/ingestor_logsearch/templates/config/ingestor-pem.erb create mode 100644 jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb create mode 100644 jobs/ingestor_logsearch/templates/config/jvm.options.erb create mode 100644 jobs/ingestor_logsearch/templates/config/logstash.yml.erb create mode 100644 jobs/ingestor_logsearch/templates/config/syslog_tls.crt.erb create mode 100644 jobs/ingestor_logsearch/templates/config/syslog_tls.key.erb create mode 100644 packages/logsearch-config/packaging create mode 100644 packages/logsearch-config/spec create mode 100644 src/logsearch-config/.gitignore create mode 100644 src/logsearch-config/.rspec create mode 100644 src/logsearch-config/.ruby-version create mode 100644 src/logsearch-config/Gemfile create mode 100644 src/logsearch-config/Gemfile.lock create mode 100644 src/logsearch-config/README.md create mode 100644 src/logsearch-config/Rakefile create mode 100755 src/logsearch-config/bin/build create mode 100644 src/logsearch-config/src/logstash-filters/cluster_monitor.conf.erb create mode 100644 src/logsearch-config/src/logstash-filters/default.conf.erb create mode 100644 src/logsearch-config/src/logstash-filters/deployment.conf create mode 100644 src/logsearch-config/src/logstash-filters/deployment_lookup.yml create mode 100644 src/logsearch-config/src/logstash-filters/if_it_looks_like_json.conf create mode 100644 src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf create mode 100644 src/logsearch-config/src/logstash-filters/snippets/haproxy.conf create mode 100644 src/logsearch-config/src/logstash-filters/snippets/metric.conf create mode 100644 src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf create mode 100644 src/logsearch-config/src/logstash-filters/snippets/redact_passwords.conf create mode 100644 src/logsearch-config/src/logstash-filters/snippets/syslog_standard.conf create mode 100644 src/logsearch-config/src/logstash-filters/timecop.conf create mode 100644 src/logsearch-config/target/.gitkeep create mode 100644 src/logsearch-config/vendor/.gitkeep diff --git a/jobs/ingestor_logsearch/monit b/jobs/ingestor_logsearch/monit new file mode 100644 index 00000000..6e421fa7 --- /dev/null +++ b/jobs/ingestor_logsearch/monit @@ -0,0 +1,5 @@ +check process ingestor_logsearch + with pidfile /var/vcap/sys/run/bpm/ingestor_syslog/ingestor_syslog.pid + start program "/var/vcap/jobs/bpm/bin/bpm start ingestor_syslog" + stop program "/var/vcap/jobs/bpm/bin/bpm stop ingestor_syslog" + group vcap diff --git a/jobs/ingestor_logsearch/spec b/jobs/ingestor_logsearch/spec new file mode 100644 index 00000000..34876d65 --- /dev/null +++ b/jobs/ingestor_logsearch/spec @@ -0,0 +1,213 @@ +--- +name: ingestor_logsearch + +description: | + This job runs Logstash process which ingests data by standard Syslog protocol, make some processing, + then forward it to opensearch cluster + +packages: +- logstash +- logsearch-config +- cf-logstash-filters +- openjdk-17 + +templates: + bin/ingestor_syslog: bin/ingestor_syslog.sh + bin/pre-start: bin/pre-start + bin/post-start.erb: bin/post-start + config/bpm.yml.erb: config/bpm.yml + config/input_and_output.conf.erb: config/input_and_output.conf + config/filters_pre.conf.erb: config/filters_pre.conf + config/filters_post.conf.erb: config/filters_post.conf + config/filters_override.conf.erb: config/filters_override.conf + config/syslog_tls.crt.erb: config/syslog_tls.crt + config/syslog_tls.key.erb: config/syslog_tls.key + config/logstash.yml.erb: config/logstash.yml + config/jvm.options.erb: config/jvm.options + config/ingestor-crt.erb: config/ssl/ingestor.crt + config/ingestor-pem.erb: config/ssl/ingestor.pem + config/ca.erb: config/ssl/opensearch.ca + +provides: +- name: ingestor + type: ingestor + properties: + - logstash_ingestor.syslog.port + - logstash_ingestor.syslog.transport + - logstash_ingestor.syslog_tls.port + - logstash_ingestor.relp.port +- name: syslog_forwarder + type: syslog_forwarder + properties: + - logstash_ingestor.syslog.port +- name: ingestor_inputs + type: ingestor_inputs + properties: + - logstash_parser.inputs +consumes: +- name: opensearch + type: opensearch + optional: true + +properties: + logstash.ssl_client_authentication: + description: Controls the servers behavior in regard to requesting a certificate from client connections + default: required + logstash.heap_size: + description: sets jvm heap sized + logstash.heap_percentage: + description: The percentage value used in the calculation to set the heap size. + default: 46 + logstash.jvm_options: + description: additional jvm options + default: [] + logstash.metadata_level: + description: "Whether to include additional metadata throughout the event lifecycle. NONE = disabled, DEBUG = fully enabled" + default: "NONE" + logstash.log_level: + description: The default logging level (e.g. WARN, DEBUG, INFO) + default: info + logstash.plugins: + description: "Array of hashes describing logstash plugins to install" + example: + - {name: logstash-output-cloudwatchlogs, version: 2.0.0} + default: [] + + logstash.ecs_compatibility: + description: Whether to enable ECS compatibility for geoip filters. See https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-ecs_compatibility + default: "disabled" + + logstash.env: + description: "a list of arbitrary key-value pairs to be passed on as process environment variables. eg: FOO: 123" + default: [] + + logstash.queue.type: + description: Internal queuing model, "memory" for legacy in-memory based queuing and "persisted" for disk-based acked queueing. + default: persisted + logstash.queue.page_capacity: + description: The page data files size. The queue data consists of append-only data files separated into pages. + default: 250mb + logstash.queue.max_events: + description: The maximum number of unread events in the queue. + default: 0 + logstash.queue.max_bytes: + description: The total capacity of the queue in number of bytes. + default: 1024mb + logstash.queue.checkpoint.acks: + description: The maximum number of acked events before forcing a checkpoint. + default: 1024 + logstash.queue.checkpoint.writes: + description: The maximum number of written events before forcing a checkpoint. + default: 1024 + logstash.queue.checkpoint.interval: + description: The interval in milliseconds when a checkpoint is forced on the head page. + default: 1000 + + logstash_ingestor.filters: + description: Filters to execute on the ingestors + default: "" + + logstash_ingestor.syslog.port: + description: Port to listen for syslog messages + logstash_ingestor.syslog.transport: + description: Transport protocol to use + default: "tcp" + logstash_ingestor.syslog.use_keepalive: + description: Instruct the socket to use TCP keep alives + + logstash_ingestor.health.disable_post_start: + description: Skip post-start health checks? + default: false + logstash_ingestor.health.interval: + description: Logstash syslog health check interval (seconds) + default: 5 + logstash_ingestor.health.timeout: + description: Logstash syslog health check number of attempts (seconds) + default: 300 + + logstash_ingestor.syslog_tls.port: + description: Port to listen for syslog-TLS messages (omit to disable) + logstash_ingestor.syslog_tls.ssl_cert: + description: Syslog-TLS SSL certificate (file contents, not a path) - required if logstash_ingestor.syslog_tls.port set + logstash_ingestor.syslog_tls.ssl_key: + description: Syslog-TLS SSL key (file contents, not a path) - required if logstash_ingestor.syslog_tls.port set + logstash_ingestor.syslog_tls.skip_ssl_validation: + description: Verify the identity of the other end of the SSL connection against the CA. + default: false + logstash_ingestor.syslog_tls.use_keepalive: + description: Instruct the socket to use TCP keep alives + + logstash_ingestor.relp.port: + description: Port to listen for RELP messages + default: 2514 + + logstash_parser.debug: + description: Debug level logging + default: false + logstash_parser.message_max_size: + description: "Maximum log message length. Anything larger is truncated (TODO: move this to ingestor?)" + default: 1048576 + logstash_parser.filters: + description: "The configuration to embed into the logstash filters section. Can either be a set of parsing rules as a string or a list of hashes in the form of [{name: path_to_parsing_rules.conf}]" + default: '' + logstash_parser.deployment_dictionary: + description: "A list of files concatenated into one deployment dictionary file. Each file contains a hash of job name-deployment name keypairs for @source.deployment lookup." + default: [ "/var/vcap/packages/logsearch-config/deployment_lookup.yml" ] + logstash_parser.inputs: + description: | + A list of input plugins, with a hash of options for each of them. Please refer to example below. + example: + inputs: + - plugin: rabbitmq + options: + host: 192.168.1.1 + user: logsearch + password: c1oudbunny + logstash_parser.outputs: + description: | + A list of output plugins, with a hash of options for each of them. Please refer to example below. + example: + inputs: + - plugin: mongodb + options: + uri: 192.168.1.1 + database: logsearch + collection: logs + default: [ { plugin: "opensearch", options: {} } ] + logstash_parser.workers: + description: "The number of worker threads that logstash should use (default: auto = one per CPU)" + default: auto + logstash_parser.opensearch.idle_flush_time: + description: "How frequently to flush events if the output queue is not full." + logstash_parser.opensearch.document_id: + description: "Use a specific, dynamic ID rather than an auto-generated identifier." + default: ~ + logstash_parser.opensearch.index: + description: "The specific, dynamic index name to write events to." + default: "logstash-%{+YYYY.MM.dd}" + logstash_parser.opensearch.index_type: + description: "The specific, dynamic index type name to write events to." + default: "%{@type}" + logstash_parser.opensearch.routing: + description: "The routing to be used when indexing a document." + logstash_parser.opensearch.ssl.certificate: + description: Node certificate for communication between logstash_parser and Opensearch + logstash_parser.opensearch.ssl.private_key: + description: Private key for communication between logstash_parser and Opensearch + logstash_parser.opensearch.data_hosts: + description: The list of opensearch data node IPs + logstash_parser.opensearch.verification_mode: + description: the verification mode, can be full or none + default: "full" + logstash_parser.timecop.reject_greater_than_hours: + description: "Logs with timestamps greater than this many hours in the future won't be parsed and will get tagged with fail/timecop" + default: 1 + logstash_parser.timecop.reject_less_than_hours: + description: "Logs with timestamps less than this many hours in the past won't be parsed and will get tagged with fail/timecop" + default: 24 + logstash_parser.enable_json_filter: + description: "Toggles the if_it_looks_like_json.conf filter rule" + default: false + logstash_parser.wait_for_templates: + description: "A list of index templates that need to be present in opensearch before the process starts" + default: ["index_template"] diff --git a/jobs/ingestor_logsearch/templates/bin/ingestor_syslog b/jobs/ingestor_logsearch/templates/bin/ingestor_syslog new file mode 100644 index 00000000..c04d70a0 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/bin/ingestor_syslog @@ -0,0 +1,112 @@ +#!/bin/bash + +set -e # exit immediately if a simple command exits with a non-zero status +set -u # report the usage of uninitialized variables + +# Setup env vars and folders for the webapp_ctl script +#source /var/vcap/jobs/ingestor_syslog/helpers/ctl_setup.sh 'ingestor_syslog' +JOB_NAME=ingestor_logsearch +export LOG_DIR=/var/vcap/sys/log/$JOB_NAME +export STORE_DIR=/var/vcap/store/$JOB_NAME +export JOB_DIR=/var/vcap/jobs/$JOB_NAME +source /var/vcap/packages/openjdk-17/bosh/runtime.env + +<% + es_host = nil + if_p("logstash_parser.opensearch.data_hosts") { |hosts| es_host = hosts.first } + unless es_host + es_host = link("opensearch").instances.first.address + end +%> + +function wait_for_template { + local template_name="$1" + local MASTER_URL="<%= es_host %>:9200" + + set +e + while true; do + echo "Waiting for index template to be uploaded: $template_name" + curl \ + --key ${JOB_DIR}/ssl/ingestor.key \ + --cert ${JOB_DIR}/ssl/ingestor.crt \ + --cacert ${JOB_DIR}/ssl/opensearch.ca \ + -I -f -i "$MASTER_URL"/_template/$template_name > /dev/null 2>&1 + [ $? ] && break + sleep 5 + done + set -e + echo "Found $template_name" +} + +export PORT=${PORT:-5000} +export LANG=en_US.UTF-8 +<% if 'auto' == p('logstash_parser.workers') %> +# 1 logstash worker / CPU core +export LOGSTASH_WORKERS=`grep -c ^processor /proc/cpuinfo` +<% else %> +export LOGSTASH_WORKERS=<%= p('logstash_parser.workers') %> +<% end %> +export TIMECOP_REJECT_GREATER_THAN_HOURS=<%= p('logstash_parser.timecop.reject_greater_than_hours') %> +export TIMECOP_REJECT_LESS_THAN_HOURS=<%= p('logstash_parser.timecop.reject_less_than_hours') %> +export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * <%= p("logstash.heap_percentage") %> ) / 100 ))K +<% if_p('logstash.heap_size') do |heap_size| %> +HEAP_SIZE=<%= heap_size %> +<% end %> +<% p("logstash.env").each do |env| %> +export <%= env.keys[0] %>="<%= env.values[0] %>" +<% end %> + +<% p("logstash_parser.wait_for_templates").each do |template| %> +wait_for_template "<%= template %>" +<% end %> + +export LS_JAVA_OPTS="-Xms$HEAP_SIZE -Xmx$HEAP_SIZE -DPID=$$" + +# construct a complete config file from all the fragments +cat ${JOB_DIR}/config/input_and_output.conf > ${JOB_DIR}/config/logstash.conf + +# append deployment dictionary files +<% p('logstash_parser.deployment_dictionary').each do |dictionary_path| %> + cat "<%= dictionary_path %>" >> ${JOB_DIR}/config/deployment_lookup.yml +<% end %> + +echo "filter {" >> ${JOB_DIR}/config/logstash.conf + +cat ${JOB_DIR}/config/filters_pre.conf >> ${JOB_DIR}/config/logstash.conf +cat /var/vcap/packages/logsearch-config/logstash-filters-default.conf >> ${JOB_DIR}/config/logstash.conf + +<% if p('logstash_parser.filters').is_a? Array %> + <% p('logstash_parser.filters').each do |filter| %> + <% if filter.key? 'path' %> + cat "<%= filter['path'] %>" >> ${JOB_DIR}/config/logstash.conf + <% elsif !filter.key? 'content' %> + <% _, path = filter.first %> + cat "<%= path %>" >> ${JOB_DIR}/config/logstash.conf + <% end %> + <% end %> +<% end %> +cat ${JOB_DIR}/config/filters_override.conf >> ${JOB_DIR}/config/logstash.conf + +cat ${JOB_DIR}/config/filters_post.conf >> ${JOB_DIR}/config/logstash.conf +<% if p('logstash_parser.enable_json_filter') %> + cat /var/vcap/packages/logsearch-filters/if_it_looks_like_json.conf >> ${JOB_DIR}/config/logstash.conf +<% end %> +cat /var/vcap/packages/logsearch-filters/timecop.conf >> ${JOB_DIR}/config/logstash.conf +cat /var/vcap/packages/logsearch-filters/deployment.conf >> ${JOB_DIR}/config/logstash.conf + +echo "} #close filters" >> ${JOB_DIR}/config/logstash.conf + +# clear persistent queue if the upgrade failed last run +if cat $LOG_DIR/$JOB_NAME.stdout.log | grep -a 'QueueUpgrade - Logstash was unable to upgrade your persistent queue data' >/dev/null ; then + mkdir ${STORE_DIR}/oldqueue.$$ + mv ${STORE_DIR}/queue ${STORE_DIR}/.lock ${STORE_DIR}/dead_letter_queue ${STORE_DIR}/uuid ${STORE_DIR}/oldqueue.$$ + mv $LOG_DIR/$JOB_NAME.stdout.log $LOG_DIR/$JOB_NAME.stdout.log.old +fi + +/var/vcap/packages/logstash/bin/logstash \ + --path.data ${STORE_DIR} \ + --path.config ${JOB_DIR}/config/logstash.conf \ + --path.settings ${JOB_DIR}/config \ + --pipeline.ecs_compatibility <%= p("logstash.ecs_compatibility") %> \ + --pipeline.workers ${LOGSTASH_WORKERS} \ + --log.format=json --log.level=<%= p("logstash.log_level") %> diff --git a/jobs/ingestor_logsearch/templates/bin/post-start.erb b/jobs/ingestor_logsearch/templates/bin/post-start.erb new file mode 100644 index 00000000..a7a995aa --- /dev/null +++ b/jobs/ingestor_logsearch/templates/bin/post-start.erb @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e +set -x + +<% if !p("logstash_ingestor.health.disable_post_start") %> + +echo "Waiting <%= p("logstash_ingestor.health.timeout") %>s for syslog to accept connections..." +elapsed=0 + +<% if_p("logstash_ingestor.syslog.port") do |port| %> +until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ] +do + nc -vz localhost <%= port %> 2>&1 && break + elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>] + sleep <%= p("logstash_ingestor.health.interval") %> +done +<% end %> + +<% if_p("logstash_ingestor.syslog_tls.port") do |tls_port| %> +until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ] +do + nc -vz localhost <%= tls_port %> 2>&1 && break + elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>] + sleep <%= p("logstash_ingestor.health.interval") %> +done +<% end %> + +if [ "$elapsed" -ge "<%= p("logstash_ingestor.health.timeout") %>" ]; then + echo "ERROR: Cannot connect to syslog. Exiting..." + exit 1 +fi + +<% end %> + +exit 0 diff --git a/jobs/ingestor_logsearch/templates/bin/pre-start b/jobs/ingestor_logsearch/templates/bin/pre-start new file mode 100644 index 00000000..c184598d --- /dev/null +++ b/jobs/ingestor_logsearch/templates/bin/pre-start @@ -0,0 +1,22 @@ +#!/bin/bash +source /var/vcap/packages/openjdk-17/bosh/runtime.env + +export JOB_NAME=ingestor_logsearch +export JOB_DIR=/var/vcap/jobs/$JOB_NAME + +<% p("logstash.plugins").each do |plugin| %> +/var/vcap/packages/logstash/bin/logstash-plugin install \ + <%= plugin.except("name").map { |key, value| "--#{key}=#{value}" }.join(" ") %> \ + <%= plugin["name"] %> +<% end %> + +<% if_link('opensearch') do |ingestor_syslog| %> +openssl pkcs8 -v1 "PBE-SHA1-3DES" \ +-in "${JOB_DIR}/config/ssl/ingestor.pem" -topk8 \ +-out "${JOB_DIR}/config/ssl/ingestor.key" -nocrypt +chmod 600 ${JOB_DIR}/config/ssl/ingestor.key +<% end %> + +if [ -d ${JOB_DIR}/config/ssl ]; then + chown -R vcap:vcap ${JOB_DIR}/config/ssl +fi \ No newline at end of file diff --git a/jobs/ingestor_logsearch/templates/config/bpm.yml.erb b/jobs/ingestor_logsearch/templates/config/bpm.yml.erb new file mode 100644 index 00000000..2eb59548 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/bpm.yml.erb @@ -0,0 +1,14 @@ +processes: + - name: ingestor_syslog + hooks: + pre_start: /var/vcap/jobs/ingestor_syslog/bin/pre-start + executable: /var/vcap/jobs/ingestor_syslog/bin/ingestor_syslog.sh + ephemeral_disk: true + persistent_disk: true + additional_volumes: + - path: /var/vcap/sys/tmp/ingestor_syslog + writable: true + allow_executions: true + - path: /var/vcap/jobs/ingestor_syslog/config + writable: true + - path: /var/vcap/jobs/deployment_lookup_config/config diff --git a/jobs/ingestor_logsearch/templates/config/ca.erb b/jobs/ingestor_logsearch/templates/config/ca.erb new file mode 100644 index 00000000..d6ab20c7 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/ca.erb @@ -0,0 +1,5 @@ +<% if_link("opensearch") do |opensearch_config| %> +<% opensearch_config.if_p('opensearch.node.ssl.ca') do %> +<%= opensearch_config.p('opensearch.node.ssl.ca', '') %> +<% end %> +<% end %> diff --git a/jobs/ingestor_logsearch/templates/config/filters_override.conf.erb b/jobs/ingestor_logsearch/templates/config/filters_override.conf.erb new file mode 100644 index 00000000..9a6c8c28 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/filters_override.conf.erb @@ -0,0 +1,9 @@ +<% if p('logstash_parser.filters').is_a? Array %> + <% p('logstash_parser.filters').each do |filter| %> + <% if filter.key? 'content' %> + <%= filter['content'] %> + <% end %> + <% end %> +<% elsif p('logstash_parser.filters') != '' %> + <%= p('logstash_parser.filters') %> +<% end %> diff --git a/jobs/ingestor_logsearch/templates/config/filters_post.conf.erb b/jobs/ingestor_logsearch/templates/config/filters_post.conf.erb new file mode 100644 index 00000000..4570fcc2 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/filters_post.conf.erb @@ -0,0 +1,22 @@ + + <% if 'DEBUG' == p('logstash.metadata_level') %> + ruby { + code => " + n = Time.now + event.set('@parser[duration]', (1000 * (n - Time.parse(event.get('@parser[timestamp]')))).to_i) + event.set('@timer[ingested_to_parsed]', (1000 * (n - Time.parse(event.get('@ingestor[timestamp]')))).to_i) if event.get('@ingestor[timestamp]') + timestamp = event.get('@timestamp') + if timestamp.instance_of? Time + event.set('@timer[emit_to_ingested]', (1000 * (Time.parse(event.get('@ingestor[timestamp]')) - timestamp)).to_i) if event.get('@ingestor[timestamp]') + event.set('@timer[emit_to_parsed]', (1000 * (n - timestamp)).to_i) + end + " + } + <% end %> + + if ![@type] { + mutate { + add_field => [ "@type", "unknown" ] + } + } + diff --git a/jobs/ingestor_logsearch/templates/config/filters_pre.conf.erb b/jobs/ingestor_logsearch/templates/config/filters_pre.conf.erb new file mode 100644 index 00000000..eead03e7 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/filters_pre.conf.erb @@ -0,0 +1,120 @@ + # drop empty logs (eg: monit connection healthcheck) + ruby { + code => 'event.cancel if event.get("message") == "\u0000"' + } + + #When behind haproxy, this is always the IP of the haproxy, not the IP of the actual host sending the data. + #Remove it to avoid confusion + mutate { + remove_field => [ "host" ] + } + <% if 'DEBUG' == p('logstash.metadata_level') %> + mutate { + add_field => { "[@ingestor][service]" => "syslog" } + add_field => { "[@ingestor][job]" => "<%= name %>/<%= index %>" } + } + ruby { + code => ' + ingestor = event.get("@ingestor") + ingestor["timestamp"] = Time.now.iso8601(3) + event.set("@ingestor", ingestor) + ' + } + <% end %> + + # + # Injected custom filters below from logstash_inject.filters manifest property + # + <%= p('logstash_ingestor.filters') %> + + + <% if 'DEBUG' == p('logstash.metadata_level') %> + mutate { + add_field => [ "@parser[job]", "<%= name %>/<%= index %>" ] + } + + ruby { + code => "event.set('@parser[timestamp]', Time.now.iso8601(3))" + } + <% end %> + + # + # Default type to _logstash_input + # + + alter { + coalesce => [ + "type", "%{_logstash_input}", "%{_type}" + ] + } + + # + # rewrite our defined globals + # + + if [type] == 'redis' or [type] == 'redis-input' { + mutate { + remove_field => [ 'type' ] + } + } + + if [type] != '' { + mutate { + rename => [ "type", "@type" ] + } + } + + if [message] != '' { + mutate { + rename => [ "message", "@message" ] + } + } else if [message] == '' and [@message] !~ /^.+$/ { + drop { } + } + + # + # avoid bad interpolations, like `%{type}` when its missing + # + + if [@type] == "" { + mutate { + add_field => [ "@type", "unknown" ] + } + } + + # + # ignore particularly useless lines + # + + if [@message] =~ /^\s*$/ or [@message] =~ /^#.*$/ { + drop { } + } + + # + # trim excessively long messages + # + + ruby { + code => ' + max_line_length = <%= p("logstash_parser.message_max_size") %> + message = event.get("@message") + if message && message.length > max_line_length + event.set("@message", message[0, max_line_length]) + tags = event.get("tags") + tags ||= [] << "_groktrimmed" + event.set("tags", tags) + end + ' + } + + # + # trim excess whitespace + # + + mutate { + strip => [ "@message" ] + } + + # + # Additional filter types from deployment manifest + # diff --git a/jobs/ingestor_logsearch/templates/config/ingestor-crt.erb b/jobs/ingestor_logsearch/templates/config/ingestor-crt.erb new file mode 100644 index 00000000..47aa2091 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/ingestor-crt.erb @@ -0,0 +1 @@ +<%= p('logstash_parser.opensearch.ssl.certificate', '') %> diff --git a/jobs/ingestor_logsearch/templates/config/ingestor-pem.erb b/jobs/ingestor_logsearch/templates/config/ingestor-pem.erb new file mode 100644 index 00000000..dfa14786 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/ingestor-pem.erb @@ -0,0 +1 @@ +<%= p('logstash_parser.opensearch.ssl.private_key', '') %> diff --git a/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb b/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb new file mode 100644 index 00000000..df476ca5 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb @@ -0,0 +1,60 @@ +input +{ + <% p("logstash_parser.inputs", []).each do | input | %> + <%= input["plugin"] %> { + <% input["options"].each do | k, v | %> + <%= k %> => <%= v.inspect %> + <% end %> + } + <% end %> +} + +<% + es_output_hosts = nil + if_p("logstash_parser.opensearch.data_hosts") { |hosts| es_output_hosts = hosts.map { |ip| "https://#{ip}:9200" }.join(',') } + unless es_output_hosts + es_output_hosts = link("opensearch").address + end +%> + +output { + <% if p("logstash_parser.debug") %> + stdout { + codec => "json" + } + <% end %> + + <% p('logstash_parser.outputs').each do | output | %> + if [@index_type] == "app" { + <%= output['plugin'] %> { + <% if 'opensearch' == output['plugin'] %> + <% + options = { + "hosts" => [ es_output_hosts ], + "index" => p('logstash_parser.opensearch.index'), + "manage_template" => false, + "ssl_certificate_verification" => true, + "tls_certificate" => "/var/vcap/jobs/ingestor_syslog/config/ssl/ingestor.crt", + "tls_key" => "/var/vcap/jobs/ingestor_syslog/config/ssl/ingestor.key", + "ssl" => true, + "cacert" => "/var/vcap/jobs/ingestor_syslog/config/ssl/opensearch.ca" + } + if p('logstash_parser.opensearch.idle_flush_time', nil) + options['idle_flush_time'] = p('logstash_parser.opensearch.idle_flush_time') + end + if p('logstash_parser.opensearch.document_id', nil) + options['document_id'] = p('logstash_parser.opensearch.document_id') + end + if p('logstash_parser.opensearch.routing', nil) + options['routing'] = p('logstash_parser.opensearch.routing') + end + output['options'] = options.merge(output['options']) + %> + <% end %> + <% output['options'].each do | k, v | %> + <%= k %> => <%= v.inspect %> + <% end %> + } + } + <% end %> +} diff --git a/jobs/ingestor_logsearch/templates/config/jvm.options.erb b/jobs/ingestor_logsearch/templates/config/jvm.options.erb new file mode 100644 index 00000000..9ebf2e93 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/jvm.options.erb @@ -0,0 +1,96 @@ +################################################################ +# logstash base config +# this is the stuff from Elastic. We should update this whenever we update logstash +################################################################ +## JVM configuration + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms1g +-Xmx1g + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +8-13:-XX:+UseConcMarkSweepGC +8-13:-XX:CMSInitiatingOccupancyFraction=75 +8-13:-XX:+UseCMSInitiatingOccupancyOnly + +## Locale +# Set the locale language +#-Duser.language=en + +# Set the locale country +#-Duser.country=US + +# Set the locale variant, if any +#-Duser.variant= + +## basic + +# set the I/O temp directory +#-Djava.io.tmpdir=$HOME + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +#-Djna.nosys=true + +# Turn on JRuby invokedynamic +-Djruby.compile.invokedynamic=true +# Force Compilation +-Djruby.jit.threshold=0 +# Make sure joni regexp interruptability is enabled +-Djruby.regexp.interruptible=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof + +## GC logging +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${LS_GC_LOG_FILE} + +# Entropy source for randomness +-Djava.security.egd=file:/dev/urandom + +# Copy the logging context from parent threads to children +-Dlog4j2.isThreadContextMapInheritable=true + +17-:--add-opens java.base/sun.nio.ch=ALL-UNNAMED +17-:--add-opens java.base/java.io=ALL-UNNAMED +################################################################ +# end logstash base config. +# everything after this is our customizations +################################################################ + +-Djava.io.tmpdir=/var/vcap/sys/tmp/ingestor_syslog + +<% p('logstash.jvm_options').each do |opt| %><%= opt %><% end %> diff --git a/jobs/ingestor_logsearch/templates/config/logstash.yml.erb b/jobs/ingestor_logsearch/templates/config/logstash.yml.erb new file mode 100644 index 00000000..d4df7031 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/logstash.yml.erb @@ -0,0 +1,9 @@ +# ------------ Queuing Settings -------------- + +queue.type: <%= p("logstash.queue.type") %> +queue.page_capacity: <%= p("logstash.queue.page_capacity") %> +queue.max_events: <%= p("logstash.queue.max_events") %> +queue.max_bytes: <%= p("logstash.queue.max_bytes") %> +queue.checkpoint.acks: <%= p("logstash.queue.checkpoint.acks") %> +queue.checkpoint.writes: <%= p("logstash.queue.checkpoint.writes") %> +queue.checkpoint.interval: <%= p("logstash.queue.checkpoint.interval") %> diff --git a/jobs/ingestor_logsearch/templates/config/syslog_tls.crt.erb b/jobs/ingestor_logsearch/templates/config/syslog_tls.crt.erb new file mode 100644 index 00000000..38ff81cf --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/syslog_tls.crt.erb @@ -0,0 +1 @@ +<% if_p("logstash_ingestor.syslog_tls.ssl_cert") do | ssl | %><%= ssl %><% end %> \ No newline at end of file diff --git a/jobs/ingestor_logsearch/templates/config/syslog_tls.key.erb b/jobs/ingestor_logsearch/templates/config/syslog_tls.key.erb new file mode 100644 index 00000000..1db62a33 --- /dev/null +++ b/jobs/ingestor_logsearch/templates/config/syslog_tls.key.erb @@ -0,0 +1 @@ +<% if_p("logstash_ingestor.syslog_tls.ssl_key") do | ssl | %><%= ssl %><% end %> \ No newline at end of file diff --git a/packages/logsearch-config/packaging b/packages/logsearch-config/packaging new file mode 100644 index 00000000..4850ba14 --- /dev/null +++ b/packages/logsearch-config/packaging @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e -x + +RAKE_VERSION=$(find ruby -maxdepth 1 -name 'rake-*' | sed 's/ruby\/rake-\(.*\)\.gem/\1/' | head -1) + +# shellcheck disable=1090 +source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/ruby-3.1/bosh/compile.env" + +gem install "ruby/rake-${RAKE_VERSION}.gem" --local +logsearch-config/bin/build + +cp logsearch-config/target/* "${BOSH_INSTALL_TARGET}" +cp logsearch-config/src/logstash-filters/if_it_looks_like_json.conf "${BOSH_INSTALL_TARGET}" +cp logsearch-config/src/logstash-filters/timecop.conf "${BOSH_INSTALL_TARGET}" +cp logsearch-config/src/logstash-filters/deployment.conf "${BOSH_INSTALL_TARGET}" +cp logsearch-config/src/logstash-filters/deployment_lookup.yml "${BOSH_INSTALL_TARGET}" diff --git a/packages/logsearch-config/spec b/packages/logsearch-config/spec new file mode 100644 index 00000000..132f5526 --- /dev/null +++ b/packages/logsearch-config/spec @@ -0,0 +1,9 @@ +--- +name: logsearch-config + +dependencies: + - ruby-3.1 + +files: + - logsearch-config/**/* + - ruby/rake-12.3.2.gem diff --git a/src/logsearch-config/.gitignore b/src/logsearch-config/.gitignore new file mode 100644 index 00000000..7f7e1651 --- /dev/null +++ b/src/logsearch-config/.gitignore @@ -0,0 +1,6 @@ +vendor/logstash +vendor/logstash* +target/*.conf +target/*.html +target/parsing_rules.html +spec/logstash-templates/target diff --git a/src/logsearch-config/.rspec b/src/logsearch-config/.rspec new file mode 100644 index 00000000..8c18f1ab --- /dev/null +++ b/src/logsearch-config/.rspec @@ -0,0 +1,2 @@ +--format documentation +--color diff --git a/src/logsearch-config/.ruby-version b/src/logsearch-config/.ruby-version new file mode 100644 index 00000000..0117b3e1 --- /dev/null +++ b/src/logsearch-config/.ruby-version @@ -0,0 +1 @@ +jruby-9.4.7.0 diff --git a/src/logsearch-config/Gemfile b/src/logsearch-config/Gemfile new file mode 100644 index 00000000..97285479 --- /dev/null +++ b/src/logsearch-config/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true +source "https://rubygems.org" + +gem 'rspec' +gem 'bosh-template' +gem 'rake' diff --git a/src/logsearch-config/Gemfile.lock b/src/logsearch-config/Gemfile.lock new file mode 100644 index 00000000..0900f2af --- /dev/null +++ b/src/logsearch-config/Gemfile.lock @@ -0,0 +1,32 @@ +GEM + remote: https://rubygems.org/ + specs: + bosh-template (2.4.0) + semi_semantic (~> 1.2.0) + diff-lcs (1.5.1) + rake (13.2.1) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + semi_semantic (1.2.0) + +PLATFORMS + universal-java-1.8 + +DEPENDENCIES + bosh-template + rake + rspec + +BUNDLED WITH + 2.3.26 diff --git a/src/logsearch-config/README.md b/src/logsearch-config/README.md new file mode 100644 index 00000000..76f369b4 --- /dev/null +++ b/src/logsearch-config/README.md @@ -0,0 +1 @@ +# Logsearch-config diff --git a/src/logsearch-config/Rakefile b/src/logsearch-config/Rakefile new file mode 100644 index 00000000..88f4e0c7 --- /dev/null +++ b/src/logsearch-config/Rakefile @@ -0,0 +1,62 @@ +require 'erb' +require 'yaml' +require 'json' + +task :clean do + FileUtils.mkdir_p('target') + FileUtils.rm_rf(Dir.glob('target/*')) +end + +desc "Builds logstash filters" +task :build => :clean do + puts "===> Building ..." + compile_erb 'src/logstash-filters/default.conf.erb', 'target/logstash-filters-default.conf' + compile_erb 'src/logstash-filters/cluster_monitor.conf.erb', 'target/logstash-filters-monitor.conf' + + puts "===> Artifacts:" + puts `find target` +end + +desc "Runs all tests" +task default: [ + :compile_templates +] + + +def compile_erb(source_file, dest_file) + if File.extname(source_file) == '.erb' + output = ERB.new(File.read(source_file)).result(binding) + File.write(dest_file, output) + else + cp source_file, dest_file + end +end + +task :clean_templates do + FileUtils.mkdir_p('spec/logstash-templates/target') + FileUtils.rm_rf(Dir.glob('spec/logstash-templates/target/*')) +end + +desc "Compile bosh templates for tests" +task :compile_templates => :clean_templates do + puts "===> Compiling Bosh templates for tests ..." + + FileList.new("spec/logstash-templates/*/compile.yml").each do |compile_file| + template = YAML.load_file(compile_file)['template']; + + template['testcase'].each do |template_testcase| + compile_bosh_template '../../' + template['path'], 'spec/logstash-templates/' + template_testcase['config'], 'spec/logstash-templates/' + template_testcase['destination'] + end + end + + puts "===> Compiled Templates:" + puts `find spec/logstash-templates/target` +end + + +def compile_bosh_template(template_file, config_file, dest_file) + + config = YAML.load(File.read(config_file)).to_json + `bosh-template #{template_file} --context '#{config}' > #{dest_file}` + +end diff --git a/src/logsearch-config/bin/build b/src/logsearch-config/bin/build new file mode 100755 index 00000000..745db56e --- /dev/null +++ b/src/logsearch-config/bin/build @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +BASE_DIR=$(cd $SCRIPT_DIR/.. ; pwd) + +cd $BASE_DIR + +rake build + diff --git a/src/logsearch-config/src/logstash-filters/cluster_monitor.conf.erb b/src/logsearch-config/src/logstash-filters/cluster_monitor.conf.erb new file mode 100644 index 00000000..135d526b --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/cluster_monitor.conf.erb @@ -0,0 +1,3 @@ +<%= File.read('src/logstash-filters/snippets/haproxy.conf').gsub(/^/, ' ') %> +<%= File.read('src/logstash-filters/snippets/metric.conf').gsub(/^/, ' ') %> +<%= File.read('src/logstash-filters/snippets/monitor_filter.conf').gsub(/^/, ' ') %> diff --git a/src/logsearch-config/src/logstash-filters/default.conf.erb b/src/logsearch-config/src/logstash-filters/default.conf.erb new file mode 100644 index 00000000..1beaf2dc --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/default.conf.erb @@ -0,0 +1,9 @@ +<%= File.read('src/logstash-filters/snippets/redact_passwords.conf') %> + +if [@type] in ["syslog", "relp"] { + <%= File.read('src/logstash-filters/snippets/syslog_standard.conf').gsub(/^/, ' ') %> +} + +if [syslog_program] == "nats_to_syslog" { + <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> +} diff --git a/src/logsearch-config/src/logstash-filters/deployment.conf b/src/logsearch-config/src/logstash-filters/deployment.conf new file mode 100644 index 00000000..e8957e67 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/deployment.conf @@ -0,0 +1,11 @@ +if ![@source][deployment] and [@source][job] { + translate { + field => "[@source][job]" + dictionary_path => "/var/vcap/jobs/ingestor_syslog/config/deployment_lookup.yml" + fallback => "Unknown" + destination => "[@source][deployment]" + exact => true + regex => true + add_tag => "auto_deployment" + } +} diff --git a/src/logsearch-config/src/logstash-filters/deployment_lookup.yml b/src/logsearch-config/src/logstash-filters/deployment_lookup.yml new file mode 100644 index 00000000..2a2f1ed5 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/deployment_lookup.yml @@ -0,0 +1,9 @@ +elasticsearch_data.*: logsearch +elasticsearch_master.*: logsearch +firehose-to-syslog.*: logsearch +ingestor.*: logsearch +ingestor-bosh-nats.*: logsearch +kibana.*: logsearch +maintenance.*: logsearch +monitor.*: logsearch +ls-router.*: logsearch diff --git a/src/logsearch-config/src/logstash-filters/if_it_looks_like_json.conf b/src/logsearch-config/src/logstash-filters/if_it_looks_like_json.conf new file mode 100644 index 00000000..e15c83ea --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/if_it_looks_like_json.conf @@ -0,0 +1,75 @@ +#If it looks like JSON, parse it as JSON +if [@message] =~ /^\{/ { + # convert timestamp (if present) to string + grok { + match => { "@message" => "(?:\"timestamp\":(%{NUMBER:time}|\"%{NUMBER:time}\"))" } + tag_on_failure => [ "no_timestamp_found" ] + } + + mutate { + convert => { "time" => "string" } + } + + json { + source => "@message" + target => "parsed_json_data" + add_tag => "json/auto_detect" + remove_field => ["@message"] + } + + if "_jsonparsefailure" in [tags] { + + mutate { + remove_tag => [ "_jsonparsefailure" ] + } + + } else { + + if "no_timestamp_found" not in [tags] { + date { + match => [ "[parsed_json_data][timestamp]", "ISO8601", "UNIX" ] + add_tag => "json/hoist_@timestamp" + } + # Capture nanoseconds as @timestamp_ns from UNIX timestamps with nanosecond precision - eg, from 1458655387.327962286 store @timestamp_ns=962286 + ruby { + code => "timestamp = event.get('time'); event.set('@timestamp_ns', timestamp.split(/\.\d{3}/).last.to_i) if timestamp =~ /\d+\.\d+/" + } + mutate { + remove_field => ["time"] + } + } else { + mutate { + remove_tag => [ "no_timestamp_found" ] + } + } + + # + # Put the parsed_json_data into a top level field named after the @source.service|@source.component|@source.program|syslog_program + # + if [@source][service] { + mutate { add_field => { "parsed_json_key" => "%{[@source][service]}" } } + } else if [@source][program] { + mutate { add_field => { "parsed_json_key" => "%{[@source][program]}" } } + } else if [@source][component] { + mutate { add_field => { "parsed_json_key" => "%{[@source][component]}" } } + } else if [syslog_program] { + mutate { + add_field => { "parsed_json_key" => "%{syslog_program}"} + } + } else { + mutate { + add_field => { "parsed_json_key" => "unknown_source" } + } + } + + mutate { + gsub => [ + "parsed_json_key", "[\s/\\?#-\.]", "_" + ] + } + ruby { + code => "event.set(event.get('parsed_json_key').to_s.downcase, LogStash::Util.normalize(event.get('parsed_json_data')))" + remove_field => [ "parsed_json_key", "parsed_json_data" ] + } + } +} diff --git a/src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf b/src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf new file mode 100644 index 00000000..28fc1e31 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf @@ -0,0 +1,93 @@ +# Parse BOSH NATS logs +if [syslog_program] == "nats_to_syslog" { + json { + source => "@message" + target => "NATS" + add_field => { "[@level]" => "INFO" } + add_tag => ["NATS"] + remove_field => ["@message"] + } + + if "_jsonparsefailure" in [tags] { + mutate { + add_tag => "fail/bosh_nats/json" + remove_tag => "_jsonparsefailure" + } + } else { + + json { + source => "[NATS][Data]" + target => "[NATS][Data]" + } + + if "_jsonparsefailure" in [tags] { + mutate { + add_tag => "fail/bosh_nats/Data/json" + remove_tag => "_jsonparsefailure" + } + } else { + if [NATS][Subject] =~ /hm\.agent\.heartbeat.*/ { + mutate { + add_field => { "[@source][vm]" => "%{[NATS][Data][job]}/%{[NATS][Data][index]}" } + add_tag => ["hm_agent_heartbeat"] + } + mutate { + rename => { "[NATS][Data][job]" => "[@source][job]" } + rename => { "[NATS][Data][index]" => "[@source][index]" } + } + + mutate { + convert => { "[NATS][Data][vitals][cpu][sys]" => "float" } + convert => { "[NATS][Data][vitals][cpu][user]" => "float" } + convert => { "[NATS][Data][vitals][cpu][wait]" => "float" } + convert => { "[NATS][Data][vitals][disk][ephemeral][inode_percent]" => "float" } + convert => { "[NATS][Data][vitals][disk][ephemeral][percent]" => "float" } + convert => { "[NATS][Data][vitals][disk][system][inode_percent]" => "float" } + convert => { "[NATS][Data][vitals][disk][system][percent]" => "float" } + convert => { "[NATS][Data][vitals][mem][kb]" => "float" } + convert => { "[NATS][Data][vitals][mem][percent]" => "float" } + convert => { "[NATS][Data][vitals][swap][kb]" => "float" } + convert => { "[NATS][Data][vitals][swap][percent]" => "float" } + } + if [NATS][Data][vitals][disk][persistent] { + mutate { + convert => { "[NATS][Data][vitals][disk][persistent][inode_percent]" => "float" } + convert => { "[NATS][Data][vitals][disk][persistent][percent]" => "float" } + } + } + ruby { + code => ' + vitals = event.get("NATS")["Data"]["vitals"].merge( {"load" => { + "avg01" => event.get("NATS")["Data"]["vitals"]["load"][0].to_f, + "avg05" => event.get("NATS")["Data"]["vitals"]["load"][1].to_f, + "avg15" => event.get("NATS")["Data"]["vitals"]["load"][2].to_f, + }}) + data = event.get("NATS")["Data"].merge({"vitals" => vitals}) + nats = event.get("NATS").merge({"Data" => data}) + event.set("NATS", nats) + ' + } + } else if [NATS][Subject] =~ /hm\.(director|agent)\.alert.*/ { + mutate { + add_tag => "hm_alert" + } + date { + match => [ "[NATS][Data][created_at]", "UNIX" ] + tag_on_failure => "fail/NATS/hm_alert/date" + remove_field => "[NATS][Data][created_at]" + } + translate { + field => "[NATS][Data][severity]" + destination => "[@level]" + override => true + dictionary => [ + "1", "FATAL", + "2", "FATAL", + "3", "ERROR", + "4", "WARN" ] + fallback => "INFO" + } + } + } + } +} diff --git a/src/logsearch-config/src/logstash-filters/snippets/haproxy.conf b/src/logsearch-config/src/logstash-filters/snippets/haproxy.conf new file mode 100644 index 00000000..391f6d6f --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/snippets/haproxy.conf @@ -0,0 +1,30 @@ +if [syslog_program] == "ls-router" { + grok { + match => { "@message" => "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{INT:[haproxy][time_queue_ms]:int}/%{INT:[haproxy][time_backend_connect_ms]:int}/%{NOTSPACE:[haproxy][time_duration_ms]:int} %{NOTSPACE:[haproxy][bytes_read]:int} %{NOTSPACE:[haproxy][termination_state]} %{INT:[haproxy][actconn]:int}/%{INT:[haproxy][feconn]:int}/%{INT:[haproxy][beconn]:int}/%{INT:[haproxy][srvconn]:int}/%{NOTSPACE:[haproxy][retries]:int} %{INT:[haproxy][srv_queue]:int}/%{INT:[haproxy][backend_queue]:int}" } + match => { "@message" => "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]}:%{SPACE}%{GREEDYDATA:[haproxy][message]}" } + match => { "@message" => "%{GREEDYDATA:[haproxy][message]}" } + add_tag => [ "haproxy" ] + remove_field => ["@message"] + tag_on_failure => "fail/haproxy/grok" + } + + date { + match => [ "[haproxy][accept_date]", "dd/MMM/YYYY:HH:mm:ss.SSS" ] + target => "[haproxy][accept_date]" + } + +#Add some helpful descripions for the termination state + translate { + dictionary => [ + "CD", "Session unexpectedly aborted by client", + "cD", "Client-side timeout expired", + "sD", "Server-side timeout expired" + ] + field => "[haproxy][termination_state]" + destination => "@message" + } + + mutate { + add_field => [ "[@source][job]", "ls-router" ] + } +} diff --git a/src/logsearch-config/src/logstash-filters/snippets/metric.conf b/src/logsearch-config/src/logstash-filters/snippets/metric.conf new file mode 100644 index 00000000..f6a5d997 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/snippets/metric.conf @@ -0,0 +1,16 @@ +if [@type] == "metric" { + grok { + match => [ "@message", "%{NOTSPACE:[metric][name]} %{NUMBER:[metric][value]:float} %{INT:[metric][timestamp]}" ] + tag_on_failure => [ "fail/metric" ] + add_tag => [ "metric" ] + remove_tag => "raw" + remove_field => [ "@message" ] + } + + if "metric" in [tags] { + date { + match => [ "[metric][timestamp]", "UNIX" ] + remove_field => "[metric][timestamp]" + } + } +} diff --git a/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf b/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf new file mode 100644 index 00000000..f4e1dde4 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf @@ -0,0 +1,29 @@ +mutate { + rename => { + "[syslog_sd_params][job]" => "[@source][job]" + "[syslog_sd_params][index]" => "[@source][index]" + } + + add_field => { + "[@source][program]" => "%{syslog_program}" + } + + convert => { + "[@source][index]" => "integer" + } +} + +if ![@source][deployment] { + translate { + field => "[@source][job]" + destination => "[@source][deployment]" + regex => true + exact => true + dictionary_path => "/var/vcap/jobs/ingestor_syslog/config/deployment_lookup.yml" + fallback => "Unknown" + } +} + +if [@source][deployment] != "logsearch" { + drop {} +} diff --git a/src/logsearch-config/src/logstash-filters/snippets/redact_passwords.conf b/src/logsearch-config/src/logstash-filters/snippets/redact_passwords.conf new file mode 100644 index 00000000..51f81fda --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/snippets/redact_passwords.conf @@ -0,0 +1,24 @@ +if [@message] =~ "AWS_ACCESS_KEY_ID" { + mutate { + add_tag => ["_redacted"] + gsub => [ + "@message", "AWS_ACCESS_KEY_ID=(.{3}).{17}", "AWS_ACCESS_KEY_ID=\1******" + ] + } +} + +if [@message] =~ "AWS_SECRET_ACCESS_KEY" { + mutate { + add_tag => ["_redacted"] + gsub => [ + "@message", "AWS_SECRET_ACCESS_KEY=(.{3}).{37}", "AWS_SECRET_ACCESS_KEY=\1******" + ] + } +} + +if "_redacted" in [tags] { + mutate { + remove_tag => [ "_redacted" ] + add_tag => ["redacted"] + } +} diff --git a/src/logsearch-config/src/logstash-filters/snippets/syslog_standard.conf b/src/logsearch-config/src/logstash-filters/snippets/syslog_standard.conf new file mode 100644 index 00000000..d42e9fd1 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/snippets/syslog_standard.conf @@ -0,0 +1,87 @@ +# syslog/relp + +grok { + match => { "@message" => "(?:%{INT:syslog6587_msglen} )?<%{POSINT:syslog_pri}>(%{SPACE})?(?:%{NONNEGINT:syslog5424_ver} )?(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:syslog_timestamp}) %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?(:)? %{GREEDYDATA:syslog_message}" } + match => { "@message" => "<%{POSINT:syslog_pri}>(%{SPACE})?%{SYSLOGTIMESTAMP:syslog_timestamp} %{DATA:syslog_program}\[%{POSINT:syslog_pid}\]: %{GREEDYDATA:syslog_message}" } + add_tag => [ "syslog_standard" ] + add_field => { "@raw" => "%{@message}"} + tag_on_failure => ["fail/syslog_standard/_grokparsefailure"] +} + +if !("fail/syslog_standard/_grokparsefailure" in [tags]) { + syslog_pri { } + + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ] + timezone => "UTC" + remove_field => "syslog_timestamp" + } + + # Populate @source.host + mutate { + add_field => [ "[@source][host]", "%{syslog_hostname}" ] + } + + mutate { + convert => [ "syslog5424_ver", "integer" ] + convert => [ "syslog6587_msglen", "integer" ] + } + + if [syslog5424_ver] == 1 { + grok { + # I don't think this is rfc5424-legal because it says SD *must* exist and message *may* exist. + # However, this makes parsing compatible with common syslog implementations. + match => [ "syslog_message", "(?:%{DATA:syslog_procid}|\-) (?:%{DATA:syslog_msgid}|\-)(?: %{SYSLOG5424SD:syslog_sd}| \-)? %{GREEDYDATA:syslog_message}" ] + overwrite => [ + "syslog_message" + ] + tag_on_failure => [ "fail/syslog_standard/_grokparsefailure-syslog_standard-5424" ] + } + + # structured-data + if [syslog_sd] { + grok { + match => [ "syslog_sd", "\[%{DATA:syslog_sd_id} (?[^\]]+)\]" ] + remove_field => [ + "syslog_sd" + ] + tag_on_failure => [ "fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sds" ] + } + + if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { + # convert the the key-value pairs + kv { + source => "syslog_sd_params_raw" + target => "syslog_sd_params" + remove_field => [ + "syslog_sd_params_raw" + ] + } + # When an additional host is specified in the sd_params, promote syslog_hostname to @shipper.host + # and replace @source.host with sd_params.host + if [syslog_sd_params][host] { + mutate { + add_field => { "[@shipper][host]" => "%{[syslog_hostname]}" } + replace => { "[@source][host]" => "%{[syslog_sd_params][host]}" } + } + } + + if [syslog_sd_params][type] { + # when the syslog params include a type, prepare the message for parsing by additional downstream parsing rules: + # - Change the @type - this triggers downstream parsing rules + # - @message_body = 'unparsed' message body that will be parsed by downstream @type rules + mutate { + replace => { "@type" => "%{syslog_sd_params[type]}" } + } + + } + } + } + } + + # @message should contain the remaining unparsed text + mutate { + rename => { "syslog_message" => "@message" } + } + +} diff --git a/src/logsearch-config/src/logstash-filters/timecop.conf b/src/logsearch-config/src/logstash-filters/timecop.conf new file mode 100644 index 00000000..1acda4e8 --- /dev/null +++ b/src/logsearch-config/src/logstash-filters/timecop.conf @@ -0,0 +1,20 @@ +# Unparse logs with @timestamps outside the configured acceptable range +ruby { + code => ' + max_hours = (ENV["TIMECOP_REJECT_GREATER_THAN_HOURS"] || 1).to_i + min_hours = (ENV["TIMECOP_REJECT_LESS_THAN_HOURS"] || 24).to_i + max_timestamp = Time.now + 60 * 60 * max_hours + min_timestamp = Time.new - 60 * 60 * min_hours + + timestamp = event.get("@timestamp").time + if timestamp > max_timestamp || timestamp < min_timestamp + event.overwrite( LogStash::Event.new( { + "tags" => event.get("tags") << "fail/timecop", + "invalid_fields" => { "@timestamp" => timestamp }, + "@raw" => event.get("@raw"), + "@source" => event.get("@source"), + "@shipper" => event.get("@shipper") + } ) ) + end + ' +} diff --git a/src/logsearch-config/target/.gitkeep b/src/logsearch-config/target/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/logsearch-config/vendor/.gitkeep b/src/logsearch-config/vendor/.gitkeep new file mode 100644 index 00000000..e69de29b From 4dd564293eea3f17309ffd32acd2bc9d2fda3f15 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 17:34:32 -0400 Subject: [PATCH 02/13] updating ruby version --- packages/logsearch-config/spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/logsearch-config/spec b/packages/logsearch-config/spec index 132f5526..bf5eee9e 100644 --- a/packages/logsearch-config/spec +++ b/packages/logsearch-config/spec @@ -6,4 +6,4 @@ dependencies: files: - logsearch-config/**/* - - ruby/rake-12.3.2.gem + - ruby/rake-13.0.6.gem From 120ef7d065388ac9b433b6750f07580109fe7f25 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 17:40:54 -0400 Subject: [PATCH 03/13] removing extra --- jobs/ingestor_logsearch/spec | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/jobs/ingestor_logsearch/spec b/jobs/ingestor_logsearch/spec index 34876d65..2cdf30d2 100644 --- a/jobs/ingestor_logsearch/spec +++ b/jobs/ingestor_logsearch/spec @@ -29,21 +29,13 @@ templates: config/ca.erb: config/ssl/opensearch.ca provides: -- name: ingestor - type: ingestor +- name: ingestor_logsearch + type: ingestor_logsearch properties: - logstash_ingestor.syslog.port - logstash_ingestor.syslog.transport - logstash_ingestor.syslog_tls.port - logstash_ingestor.relp.port -- name: syslog_forwarder - type: syslog_forwarder - properties: - - logstash_ingestor.syslog.port -- name: ingestor_inputs - type: ingestor_inputs - properties: - - logstash_parser.inputs consumes: - name: opensearch type: opensearch From d4cb026af3121389f613d6a305be76ce5f9376f1 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 18:16:42 -0400 Subject: [PATCH 04/13] removing extra --- jobs/ingestor_logsearch/monit | 6 +++--- jobs/ingestor_logsearch/spec | 2 +- .../bin/{ingestor_syslog => ingestor_logsearch} | 7 +++---- jobs/ingestor_logsearch/templates/bin/pre-start | 2 +- jobs/ingestor_logsearch/templates/config/bpm.yml.erb | 10 +++++----- .../templates/config/jvm.options.erb | 4 ++-- 6 files changed, 15 insertions(+), 16 deletions(-) rename jobs/ingestor_logsearch/templates/bin/{ingestor_syslog => ingestor_logsearch} (91%) diff --git a/jobs/ingestor_logsearch/monit b/jobs/ingestor_logsearch/monit index 6e421fa7..8e8263e7 100644 --- a/jobs/ingestor_logsearch/monit +++ b/jobs/ingestor_logsearch/monit @@ -1,5 +1,5 @@ check process ingestor_logsearch - with pidfile /var/vcap/sys/run/bpm/ingestor_syslog/ingestor_syslog.pid - start program "/var/vcap/jobs/bpm/bin/bpm start ingestor_syslog" - stop program "/var/vcap/jobs/bpm/bin/bpm stop ingestor_syslog" + with pidfile /var/vcap/sys/run/bpm/ingestor_logsearch/ingestor_logsearch.pid + start program "/var/vcap/jobs/bpm/bin/bpm start ingestor_logsearch" + stop program "/var/vcap/jobs/bpm/bin/bpm stop ingestor_logsearch" group vcap diff --git a/jobs/ingestor_logsearch/spec b/jobs/ingestor_logsearch/spec index 2cdf30d2..d202d594 100644 --- a/jobs/ingestor_logsearch/spec +++ b/jobs/ingestor_logsearch/spec @@ -12,7 +12,7 @@ packages: - openjdk-17 templates: - bin/ingestor_syslog: bin/ingestor_syslog.sh + bin/ingestor_logsearch: bin/ingestor_logsearch.sh bin/pre-start: bin/pre-start bin/post-start.erb: bin/post-start config/bpm.yml.erb: config/bpm.yml diff --git a/jobs/ingestor_logsearch/templates/bin/ingestor_syslog b/jobs/ingestor_logsearch/templates/bin/ingestor_logsearch similarity index 91% rename from jobs/ingestor_logsearch/templates/bin/ingestor_syslog rename to jobs/ingestor_logsearch/templates/bin/ingestor_logsearch index c04d70a0..1795dd70 100644 --- a/jobs/ingestor_logsearch/templates/bin/ingestor_syslog +++ b/jobs/ingestor_logsearch/templates/bin/ingestor_logsearch @@ -4,7 +4,6 @@ set -e # exit immediately if a simple command exits with a non-zero status set -u # report the usage of uninitialized variables # Setup env vars and folders for the webapp_ctl script -#source /var/vcap/jobs/ingestor_syslog/helpers/ctl_setup.sh 'ingestor_syslog' JOB_NAME=ingestor_logsearch export LOG_DIR=/var/vcap/sys/log/$JOB_NAME export STORE_DIR=/var/vcap/store/$JOB_NAME @@ -89,10 +88,10 @@ cat ${JOB_DIR}/config/filters_override.conf >> ${JOB_DIR}/config/logstash.conf cat ${JOB_DIR}/config/filters_post.conf >> ${JOB_DIR}/config/logstash.conf <% if p('logstash_parser.enable_json_filter') %> - cat /var/vcap/packages/logsearch-filters/if_it_looks_like_json.conf >> ${JOB_DIR}/config/logstash.conf + cat /var/vcap/packages/logsearch-config/if_it_looks_like_json.conf >> ${JOB_DIR}/config/logstash.conf <% end %> -cat /var/vcap/packages/logsearch-filters/timecop.conf >> ${JOB_DIR}/config/logstash.conf -cat /var/vcap/packages/logsearch-filters/deployment.conf >> ${JOB_DIR}/config/logstash.conf +cat /var/vcap/packages/logsearch-config/timecop.conf >> ${JOB_DIR}/config/logstash.conf +cat /var/vcap/packages/logsearch-config/deployment.conf >> ${JOB_DIR}/config/logstash.conf echo "} #close filters" >> ${JOB_DIR}/config/logstash.conf diff --git a/jobs/ingestor_logsearch/templates/bin/pre-start b/jobs/ingestor_logsearch/templates/bin/pre-start index c184598d..5c967920 100644 --- a/jobs/ingestor_logsearch/templates/bin/pre-start +++ b/jobs/ingestor_logsearch/templates/bin/pre-start @@ -10,7 +10,7 @@ export JOB_DIR=/var/vcap/jobs/$JOB_NAME <%= plugin["name"] %> <% end %> -<% if_link('opensearch') do |ingestor_syslog| %> +<% if_link('opensearch') do |ingestor_logsearch| %> openssl pkcs8 -v1 "PBE-SHA1-3DES" \ -in "${JOB_DIR}/config/ssl/ingestor.pem" -topk8 \ -out "${JOB_DIR}/config/ssl/ingestor.key" -nocrypt diff --git a/jobs/ingestor_logsearch/templates/config/bpm.yml.erb b/jobs/ingestor_logsearch/templates/config/bpm.yml.erb index 2eb59548..f9ffacab 100644 --- a/jobs/ingestor_logsearch/templates/config/bpm.yml.erb +++ b/jobs/ingestor_logsearch/templates/config/bpm.yml.erb @@ -1,14 +1,14 @@ processes: - - name: ingestor_syslog + - name: ingestor_logsearch hooks: - pre_start: /var/vcap/jobs/ingestor_syslog/bin/pre-start - executable: /var/vcap/jobs/ingestor_syslog/bin/ingestor_syslog.sh + pre_start: /var/vcap/jobs/ingestor_logsearch/bin/pre-start + executable: /var/vcap/jobs/ingestor_logsearch/bin/ingestor_logsearch.sh ephemeral_disk: true persistent_disk: true additional_volumes: - - path: /var/vcap/sys/tmp/ingestor_syslog + - path: /var/vcap/sys/tmp/ingestor_logsearch writable: true allow_executions: true - - path: /var/vcap/jobs/ingestor_syslog/config + - path: /var/vcap/jobs/ingestor_logsearch/config writable: true - path: /var/vcap/jobs/deployment_lookup_config/config diff --git a/jobs/ingestor_logsearch/templates/config/jvm.options.erb b/jobs/ingestor_logsearch/templates/config/jvm.options.erb index 9ebf2e93..7f1d75dd 100644 --- a/jobs/ingestor_logsearch/templates/config/jvm.options.erb +++ b/jobs/ingestor_logsearch/templates/config/jvm.options.erb @@ -87,10 +87,10 @@ 17-:--add-opens java.base/sun.nio.ch=ALL-UNNAMED 17-:--add-opens java.base/java.io=ALL-UNNAMED ################################################################ -# end logstash base config. +# end logstash base config. # everything after this is our customizations ################################################################ --Djava.io.tmpdir=/var/vcap/sys/tmp/ingestor_syslog +-Djava.io.tmpdir=/var/vcap/sys/tmp/ingestor_logsearch <% p('logstash.jvm_options').each do |opt| %><%= opt %><% end %> From d14c34de463d2e5896f6dd602c326e43ca4b2aa5 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 19:59:52 -0400 Subject: [PATCH 05/13] no post start --- .../templates/bin/post-start.erb | 36 ------------------- 1 file changed, 36 deletions(-) delete mode 100644 jobs/ingestor_logsearch/templates/bin/post-start.erb diff --git a/jobs/ingestor_logsearch/templates/bin/post-start.erb b/jobs/ingestor_logsearch/templates/bin/post-start.erb deleted file mode 100644 index a7a995aa..00000000 --- a/jobs/ingestor_logsearch/templates/bin/post-start.erb +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -e -set -x - -<% if !p("logstash_ingestor.health.disable_post_start") %> - -echo "Waiting <%= p("logstash_ingestor.health.timeout") %>s for syslog to accept connections..." -elapsed=0 - -<% if_p("logstash_ingestor.syslog.port") do |port| %> -until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ] -do - nc -vz localhost <%= port %> 2>&1 && break - elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>] - sleep <%= p("logstash_ingestor.health.interval") %> -done -<% end %> - -<% if_p("logstash_ingestor.syslog_tls.port") do |tls_port| %> -until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ] -do - nc -vz localhost <%= tls_port %> 2>&1 && break - elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>] - sleep <%= p("logstash_ingestor.health.interval") %> -done -<% end %> - -if [ "$elapsed" -ge "<%= p("logstash_ingestor.health.timeout") %>" ]; then - echo "ERROR: Cannot connect to syslog. Exiting..." - exit 1 -fi - -<% end %> - -exit 0 From bdd456f400dc88dc566390e69d07a4b9f5e10d96 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 20:01:32 -0400 Subject: [PATCH 06/13] no post start --- jobs/ingestor_logsearch/spec | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/ingestor_logsearch/spec b/jobs/ingestor_logsearch/spec index d202d594..b8433e9b 100644 --- a/jobs/ingestor_logsearch/spec +++ b/jobs/ingestor_logsearch/spec @@ -14,7 +14,6 @@ packages: templates: bin/ingestor_logsearch: bin/ingestor_logsearch.sh bin/pre-start: bin/pre-start - bin/post-start.erb: bin/post-start config/bpm.yml.erb: config/bpm.yml config/input_and_output.conf.erb: config/input_and_output.conf config/filters_pre.conf.erb: config/filters_pre.conf From 93a8577d4bf749f1862442b0cd302f68afbf31eb Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 20:09:31 -0400 Subject: [PATCH 07/13] fixing deployment location --- src/logsearch-config/src/logstash-filters/deployment.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logsearch-config/src/logstash-filters/deployment.conf b/src/logsearch-config/src/logstash-filters/deployment.conf index e8957e67..6d037641 100644 --- a/src/logsearch-config/src/logstash-filters/deployment.conf +++ b/src/logsearch-config/src/logstash-filters/deployment.conf @@ -1,7 +1,7 @@ if ![@source][deployment] and [@source][job] { translate { field => "[@source][job]" - dictionary_path => "/var/vcap/jobs/ingestor_syslog/config/deployment_lookup.yml" + dictionary_path => "/var/vcap/jobs/ingestor_logsearch/config/deployment_lookup.yml" fallback => "Unknown" destination => "[@source][deployment]" exact => true From 44f3cd4d6c8f4f6cb3cd0f88b95d6d5f9d1233f8 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 20:10:34 -0400 Subject: [PATCH 08/13] fixing deployment location --- .../src/logstash-filters/snippets/monitor_filter.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf b/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf index f4e1dde4..a7cf7dcf 100644 --- a/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf +++ b/src/logsearch-config/src/logstash-filters/snippets/monitor_filter.conf @@ -19,7 +19,7 @@ if ![@source][deployment] { destination => "[@source][deployment]" regex => true exact => true - dictionary_path => "/var/vcap/jobs/ingestor_syslog/config/deployment_lookup.yml" + dictionary_path => "/var/vcap/jobs/ingestor_logsearch/config/deployment_lookup.yml" fallback => "Unknown" } } From f22c837e263958b788bb518613d868cb3157cf99 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 7 Oct 2024 20:31:18 -0400 Subject: [PATCH 09/13] fixing deployment location --- .../templates/config/input_and_output.conf.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb b/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb index df476ca5..20be4e0e 100644 --- a/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_logsearch/templates/config/input_and_output.conf.erb @@ -34,10 +34,10 @@ output { "index" => p('logstash_parser.opensearch.index'), "manage_template" => false, "ssl_certificate_verification" => true, - "tls_certificate" => "/var/vcap/jobs/ingestor_syslog/config/ssl/ingestor.crt", - "tls_key" => "/var/vcap/jobs/ingestor_syslog/config/ssl/ingestor.key", + "tls_certificate" => "/var/vcap/jobs/ingestor_logsearch/config/ssl/ingestor.crt", + "tls_key" => "/var/vcap/jobs/ingestor_logsearch/config/ssl/ingestor.key", "ssl" => true, - "cacert" => "/var/vcap/jobs/ingestor_syslog/config/ssl/opensearch.ca" + "cacert" => "/var/vcap/jobs/ingestor_logsearch/config/ssl/opensearch.ca" } if p('logstash_parser.opensearch.idle_flush_time', nil) options['idle_flush_time'] = p('logstash_parser.opensearch.idle_flush_time') From 2e6f90bfdd6620192d9fe3516bfc714f5df6a318 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 8 Oct 2024 10:13:35 -0400 Subject: [PATCH 10/13] adding in the logsearch features --- jobs/ingestor_logsearch/spec | 2 +- packages/logsearch-filters/packaging | 14 + packages/logsearch-filters/spec | 11 + src/logsearch-filters/.gitignore | 2 + src/logsearch-filters/.rspec | 2 + src/logsearch-filters/.ruby-version | 1 + src/logsearch-filters/Gemfile | 29 ++ src/logsearch-filters/Gemfile.lock | 204 ++++++++ src/logsearch-filters/Rakefile | 70 +++ src/logsearch-filters/bin/build | 10 + .../spec/helpers/app_helper.rb | 163 +++++++ .../spec/helpers/filters_helper.rb | 78 +++ .../spec/helpers/platform_helper.rb | 110 +++++ .../logstash-filters/integration/app_spec.rb | 357 ++++++++++++++ .../integration/platform_spec.rb | 375 +++++++++++++++ .../snippets/app_containermetric_spec.rb | 91 ++++ .../snippets/app_counterevent_spec.rb | 47 ++ .../snippets/app_error_spec.rb | 47 ++ .../snippets/app_http_spec.rb | 91 ++++ .../snippets/app_logmessage_app_spec.rb | 241 ++++++++++ .../snippets/app_logmessage_rtr_spec.rb | 452 ++++++++++++++++++ .../snippets/app_logmessage_spec.rb | 129 +++++ .../logstash-filters/snippets/app_spec.rb | 301 ++++++++++++ .../snippets/app_valuemetric_spec.rb | 47 ++ .../platform_cloud_controller_ng_spec.rb | 95 ++++ .../snippets/platform_haproxy_spec.rb | 171 +++++++ .../snippets/platform_spec.rb | 161 +++++++ .../snippets/platform_uaa_spec.rb | 199 ++++++++ .../snippets/platform_vcap_spec.rb | 192 ++++++++ .../logstash-filters/snippets/setup_spec.rb | 98 ++++ .../snippets/teardown_spec.rb | 204 ++++++++ src/logsearch-filters/spec/spec_helper.rb | 15 + .../src/logstash-filters/default.conf.erb | 33 ++ .../snippets/app-containermetric.conf | 27 ++ .../snippets/app-counterevent.conf | 12 + .../logstash-filters/snippets/app-error.conf | 12 + .../logstash-filters/snippets/app-http.conf | 24 + .../snippets/app-logmessage-app.conf | 82 ++++ .../snippets/app-logmessage-rtr.conf | 112 +++++ .../snippets/app-logmessage.conf | 35 ++ .../snippets/app-valuemetric.conf | 12 + .../src/logstash-filters/snippets/app.conf | 140 ++++++ .../platform-cloud_controller_ng.conf | 17 + .../snippets/platform-gorouter.conf | 35 ++ .../snippets/platform-haproxy.conf | 48 ++ .../snippets/platform-uaa.conf | 71 +++ .../snippets/platform-vcap.conf | 53 ++ .../logstash-filters/snippets/platform.conf | 49 ++ .../src/logstash-filters/snippets/setup.conf | 38 ++ .../logstash-filters/snippets/teardown.conf | 77 +++ .../test/logstash-filters/it_app_helper.rb | 156 ++++++ .../logstash-filters/it_platform_helper.rb | 108 +++++ .../snippets/platform-spec.rb | 109 +++++ 53 files changed, 5258 insertions(+), 1 deletion(-) create mode 100644 packages/logsearch-filters/packaging create mode 100644 packages/logsearch-filters/spec create mode 100644 src/logsearch-filters/.gitignore create mode 100644 src/logsearch-filters/.rspec create mode 100644 src/logsearch-filters/.ruby-version create mode 100644 src/logsearch-filters/Gemfile create mode 100644 src/logsearch-filters/Gemfile.lock create mode 100644 src/logsearch-filters/Rakefile create mode 100755 src/logsearch-filters/bin/build create mode 100644 src/logsearch-filters/spec/helpers/app_helper.rb create mode 100644 src/logsearch-filters/spec/helpers/filters_helper.rb create mode 100644 src/logsearch-filters/spec/helpers/platform_helper.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/integration/app_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/integration/platform_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_containermetric_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_counterevent_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_error_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_http_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_app_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_rtr_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/app_valuemetric_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/platform_cloud_controller_ng_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/platform_haproxy_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/platform_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/platform_uaa_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/platform_vcap_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/setup_spec.rb create mode 100644 src/logsearch-filters/spec/logstash-filters/snippets/teardown_spec.rb create mode 100644 src/logsearch-filters/spec/spec_helper.rb create mode 100644 src/logsearch-filters/src/logstash-filters/default.conf.erb create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-containermetric.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-counterevent.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-error.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-http.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-logmessage.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app-valuemetric.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/app.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/setup.conf create mode 100644 src/logsearch-filters/src/logstash-filters/snippets/teardown.conf create mode 100644 src/logsearch-filters/test/logstash-filters/it_app_helper.rb create mode 100644 src/logsearch-filters/test/logstash-filters/it_platform_helper.rb create mode 100644 src/logsearch-filters/test/logstash-filters/snippets/platform-spec.rb diff --git a/jobs/ingestor_logsearch/spec b/jobs/ingestor_logsearch/spec index b8433e9b..038ef78a 100644 --- a/jobs/ingestor_logsearch/spec +++ b/jobs/ingestor_logsearch/spec @@ -8,7 +8,7 @@ description: | packages: - logstash - logsearch-config -- cf-logstash-filters +- logsearch-filters - openjdk-17 templates: diff --git a/packages/logsearch-filters/packaging b/packages/logsearch-filters/packaging new file mode 100644 index 00000000..cd023afb --- /dev/null +++ b/packages/logsearch-filters/packaging @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e -x + +RAKE_VERSION=$(find ruby -maxdepth 1 -name 'rake-*' | sed 's/ruby\/rake-\(.*\)\.gem/\1/' | head -1) + +# shellcheck disable=1090 +source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/ruby-3.1/bosh/compile.env" + +gem install "ruby/rake-${RAKE_VERSION}.gem" --local --no-ri --no-rdoc + +logsearch-filters/bin/build + +cp logsearch-filters/target/* "${BOSH_INSTALL_TARGET}" diff --git a/packages/logsearch-filters/spec b/packages/logsearch-filters/spec new file mode 100644 index 00000000..7fe00293 --- /dev/null +++ b/packages/logsearch-filters/spec @@ -0,0 +1,11 @@ +--- +name: logsearch-filters + +dependencies: + - ruby-2.4-r5 + +files: + - logsearch-filters/* + - logsearch-filters/bin/* + - logsearch-filters/src/logstash-filters/**/* + - ruby/rake-12.3.2.gem diff --git a/src/logsearch-filters/.gitignore b/src/logsearch-filters/.gitignore new file mode 100644 index 00000000..dfd165ff --- /dev/null +++ b/src/logsearch-filters/.gitignore @@ -0,0 +1,2 @@ +target/logstash-filters-default.conf +target/logs-template.json diff --git a/src/logsearch-filters/.rspec b/src/logsearch-filters/.rspec new file mode 100644 index 00000000..8c18f1ab --- /dev/null +++ b/src/logsearch-filters/.rspec @@ -0,0 +1,2 @@ +--format documentation +--color diff --git a/src/logsearch-filters/.ruby-version b/src/logsearch-filters/.ruby-version new file mode 100644 index 00000000..cd53e881 --- /dev/null +++ b/src/logsearch-filters/.ruby-version @@ -0,0 +1 @@ +jruby-9.1.5.0 diff --git a/src/logsearch-filters/Gemfile b/src/logsearch-filters/Gemfile new file mode 100644 index 00000000..ca771547 --- /dev/null +++ b/src/logsearch-filters/Gemfile @@ -0,0 +1,29 @@ +# frozen_string_literal: true +source "https://rubygems.org" + +gem 'rspec' +gem 'bosh-template' +gem 'rake' +gem 'yaml-lint' + +# Versions matching logsearch release v5.0.0 +gem 'logstash-core', '5.0.0', require: true +gem 'logstash-filter-mutate', '3.1.3' +gem 'logstash-filter-grok', '3.2.3' +gem 'logstash-filter-syslog_pri', '3.0.2' +gem 'logstash-filter-date', '3.0.3' +gem 'logstash-filter-geoip', '4.0.3' +gem 'logstash-filter-kv', '3.1.1' +gem 'logstash-filter-json', '3.0.2' +gem 'logstash-filter-ruby', '3.0.2' +gem 'logstash-filter-drop', '3.0.2' +gem 'logstash-filter-translate' +gem 'logstash-filter-alter' +gem 'logstash-output-elasticsearch', '5.2.0' +gem 'logstash-output-stdout', '3.1.0' +gem 'logstash-output-syslog' +gem 'logstash-output-mongodb' +gem 'logstash-input-redis', '3.1.1' +gem 'logstash-codec-plain', '3.0.2' +gem 'logstash-input-file', '4.0.0' +gem 'logstash-input-syslog', '3.1.1' diff --git a/src/logsearch-filters/Gemfile.lock b/src/logsearch-filters/Gemfile.lock new file mode 100644 index 00000000..1c6337ae --- /dev/null +++ b/src/logsearch-filters/Gemfile.lock @@ -0,0 +1,204 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.1) + public_suffix (~> 2.0, >= 2.0.2) + bosh-template (1.3262.24.0) + semi_semantic (~> 1.2.0) + bson (3.2.6-java) + cabin (0.9.0) + chronic_duration (0.10.6) + numerizer (~> 0.1.1) + clamp (0.6.5) + coderay (1.1.1) + concurrent-ruby (1.0.0-java) + diff-lcs (1.3) + ffi (1.9.18-java) + filesize (0.0.4) + filewatch (0.9.0) + gems (0.8.3) + i18n (0.6.9) + jar-dependencies (0.3.11) + jls-grok (0.11.4) + cabin (>= 0.6.0) + jrjackson (0.4.2-java) + jrmonitor (0.4.2) + jruby-openssl (0.9.16-java) + logstash-codec-json (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-codec-line (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-codec-multiline (3.0.3) + jls-grok (~> 0.11.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-patterns-core + logstash-codec-plain (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-core (5.0.0-java) + chronic_duration (= 0.10.6) + clamp (~> 0.6.5) + concurrent-ruby (= 1.0.0) + filesize (= 0.0.4) + gems (~> 0.8.3) + i18n (= 0.6.9) + jar-dependencies (~> 0.3.4) + jrjackson (~> 0.4.0) + jrmonitor (~> 0.4.2) + jruby-openssl (= 0.9.16) + logstash-core-event-java (= 5.0.0) + minitar (~> 0.5.4) + pry (~> 0.10.1) + puma (">= 3.12.2) + rubyzip (>= 1.3.0) + sinatra (~> 1.4, >= 1.4.6) + stud (~> 0.0.19) + thread_safe (~> 0.3.5) + treetop (< 1.5.0) + logstash-core-event-java (5.0.0-java) + jar-dependencies + ruby-maven (~> 3.3.9) + logstash-core-plugin-api (2.1.17-java) + logstash-core (= 5.0.0) + logstash-filter-alter (3.0.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-date (3.0.3) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-drop (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-geoip (4.0.3-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-grok (3.2.3) + jls-grok (~> 0.11.3) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-patterns-core + logstash-filter-json (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-kv (3.1.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-mutate (3.1.3) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-ruby (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-date + logstash-filter-syslog_pri (3.0.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-translate (3.0.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-input-file (4.0.0) + addressable + filewatch (~> 0.8, >= 0.8.1) + logstash-codec-multiline (~> 3.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-input-redis (3.1.1) + logstash-codec-json + logstash-core-plugin-api (>= 1.60, <= 2.99) + redis + logstash-input-syslog (3.1.1) + concurrent-ruby + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-date + logstash-filter-grok + stud (>= 0.0.22, < 0.1.0) + thread_safe + logstash-output-elasticsearch (5.2.0-java) + cabin (~> 0.6) + logstash-core-plugin-api (>= 1.60, <= 2.99) + manticore (>= 0.5.4, < 1.0.0) + stud (~> 0.0, >= 0.0.17) + logstash-output-mongodb (3.0.1) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + mongo (~> 2.0.6) + logstash-output-stdout (3.1.0) + logstash-codec-line + logstash-core-plugin-api (>= 1.60.1, < 2.99) + logstash-output-syslog (3.0.1) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-patterns-core (4.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + manticore (0.6.1-java) + method_source (0.8.2) + minitar (0.5.4) + mongo (2.0.6) + bson (~> 3.0) + numerizer (0.1.1) + polyglot (0.3.5) + pry (0.10.4-java) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) + spoon (~> 0.0) + public_suffix (2.0.5) + puma (">= 3.12.2) + rack (>= 1.6.12) + rack-protection (1.5.3) + rack + rake (>= 12.3.3) + redis (3.3.3) + rspec (3.5.0) + rspec-core (~> 3.5.0) + rspec-expectations (~> 3.5.0) + rspec-mocks (~> 3.5.0) + rspec-core (3.5.4) + rspec-support (~> 3.5.0) + rspec-expectations (3.5.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.5.0) + rspec-mocks (3.5.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.5.0) + rspec-support (3.5.0) + ruby-maven (3.3.12) + ruby-maven-libs (~> 3.3.9) + ruby-maven-libs (3.3.9) + rubyzip (>= 1.3.0) + semi_semantic (1.2.0) + sinatra (1.4.8) + rack (>= 1.6.12) + rack-protection (~> 1.4) + tilt (>= 1.3, < 3) + slop (3.6.0) + spoon (0.0.6) + ffi + stud (0.0.22) + thread_safe (0.3.6-java) + tilt (2.0.7) + treetop (1.4.15) + polyglot + polyglot (>= 0.3.1) + yaml-lint (0.0.9) + +PLATFORMS + java + +DEPENDENCIES + bosh-template + logstash-codec-plain (= 3.0.2) + logstash-core (= 5.0.0) + logstash-filter-alter + logstash-filter-date (= 3.0.3) + logstash-filter-drop (= 3.0.2) + logstash-filter-geoip (= 4.0.3) + logstash-filter-grok (= 3.2.3) + logstash-filter-json (= 3.0.2) + logstash-filter-kv (= 3.1.1) + logstash-filter-mutate (= 3.1.3) + logstash-filter-ruby (= 3.0.2) + logstash-filter-syslog_pri (= 3.0.2) + logstash-filter-translate + logstash-input-file (= 4.0.0) + logstash-input-redis (= 3.1.1) + logstash-input-syslog (= 3.1.1) + logstash-output-elasticsearch (= 5.2.0) + logstash-output-mongodb + logstash-output-stdout (= 3.1.0) + logstash-output-syslog + rake + rspec + yaml-lint + +BUNDLED WITH + 1.14.6 diff --git a/src/logsearch-filters/Rakefile b/src/logsearch-filters/Rakefile new file mode 100644 index 00000000..0eb60521 --- /dev/null +++ b/src/logsearch-filters/Rakefile @@ -0,0 +1,70 @@ +require 'erb' +require 'rake' + +# Clean tasks +task :clean do + FileUtils.mkdir_p('target') + FileUtils.rm_rf(Dir.glob('target/*')) +end + +# Build tasks +desc "Builds Logstash filters" +task :build => :clean do + build_erb 'src/logstash-filters/default.conf.erb', 'target/logstash-filters-default.conf' +end + +def build_erb(source_erb_file, target_file) + puts "===> Building ..." + compile_erb source_erb_file, target_file + + puts "===> Artifacts:" + puts `find target` +end + +# Test tasks +desc "Runs unit tests against filters" +task :unit, [:rspec_files] => :build do |t, args| + args.with_defaults(:rspec_files => "$(find spec -name *_spec.rb | grep -v integration)") + puts "===> Testing ..." + sh %Q[ bundle exec rspec #{args[:rspec_files]} --colour ] +end + +desc "Runs integration tests against filters" +task :integration, [:rspec_files] => :build do |t, args| + args.with_defaults(:rspec_files => "$(find spec -name *_spec.rb | grep integration)") + puts "===> Integration Testing ..." + sh %Q[ bundle exec rspec #{args[:rspec_files]} --colour ] +end + +desc "Runs YAML validation tests" +task :yaml, [:yaml_files] => :build do |t, args| + args.with_defaults(:yaml_files => "$(find ../.. -path ../../src -prune -o -name '*.yml' -print)") + puts "===> Validating YAML data ..." + sh %Q[ bundle exec yaml-lint #{args[:yaml_files]} ] +end + +task default: [ + :unit, + :integration, + :yaml +] + +def unescape_embedded_doublequote(str) + str.gsub("_eQT_", '\\\\\\\\\"') +end + +def unescape_embedded_newline(str) + str.gsub('_eLF_', '\\\\\\\\n') +end + +def compile_erb(source_file, dest_file) + if File.extname(source_file) == '.erb' + output = ERB.new(File.read(source_file)).result(binding) + output = unescape_embedded_doublequote(output) + output = unescape_embedded_newline(output) + File.write(dest_file, output) + puts "ERB #{source_file} -> #{dest_file}" + else + cp source_file, dest_file + end +end diff --git a/src/logsearch-filters/bin/build b/src/logsearch-filters/bin/build new file mode 100755 index 00000000..7ee707d3 --- /dev/null +++ b/src/logsearch-filters/bin/build @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +BASE_DIR=$(cd $SCRIPT_DIR/.. ; pwd) + +cd $BASE_DIR + +rake build diff --git a/src/logsearch-filters/spec/helpers/app_helper.rb b/src/logsearch-filters/spec/helpers/app_helper.rb new file mode 100644 index 00000000..cd97703f --- /dev/null +++ b/src/logsearch-filters/spec/helpers/app_helper.rb @@ -0,0 +1,163 @@ +# encoding: utf-8 + +## -- Setup methods +$app_event_dummy = { + "@type" => "syslog", + "syslog_program" => "doppler", + "syslog_pri" => 6, + "syslog_severity_code" => 3, # error + "host" => "bed08922-4734-4d62-9eba-3291aed1b8ce", + "@message" => "Dummy message"} + +$envelope_fields = { + "cf_origin" => "firehose", + "deployment" => "cf-full", + "ip" => "192.168.111.32", + "job" => "runner_z1", + "job_index" => "4abc5def", + "origin" => "MetronAgent", + "time" => "2016-08-16T22:46:24Z" +} + +$app_data_fields = { + "cf_app_id" => "31b928ee-4110-4e7b-996c-334c5d7ac2ac", + "cf_app_name" => "loggenerator", + "cf_org_id" => "9887ad0a-f9f7-449e-8982-76307bd17239", + "cf_org_name" => "admin", + "cf_space_id" => "59cf41f2-3a1d-42db-88e7-9540b02945e8", + "cf_space_name" => "demo" +} +module Helpers + module AppHelper + + + ## -- Verification methods + def verify_app_general_fields (metadata_index, type, source_type, message, level) + + + # no app parsing error + it "sets tags" do + expect(parsed_results.get("tags")).not_to include "fail/cloudfoundry/app/json" + expect(parsed_results.get("tags")).to include "app" + end + + it { expect(parsed_results.get("@type")).to eq type } + + it { expect(parsed_results.get("@index_type")).to eq "app" } + it { expect(parsed_results.get("@metadata")["index"]).to eq metadata_index } + + it { expect(parsed_results.get("@input")).to eq "syslog" } + + it { expect(parsed_results.get("@shipper")["priority"]).to eq 6 } + it { expect(parsed_results.get("@shipper")["name"]).to eq "doppler_syslog" } + + it "sets @source fields" do + expect(parsed_results.get("@source")["deployment"]).to eq "cf-full" + expect(parsed_results.get("@source")["host"]).to eq "192.168.111.32" + expect(parsed_results.get("@source")["job"]).to eq "runner_z1" + expect(parsed_results.get("@source")["job_index"]).to eq "4abc5def" + expect(parsed_results.get("@source")["component"]).to eq "MetronAgent" + expect(parsed_results.get("@source")["type"]).to eq source_type + end + + it { expect(parsed_results.get("@message")).to eq message } + it { expect(parsed_results.get("@level")).to eq level } + + # verify no (default) dynamic JSON fields + it { expect(parsed_results.get("parsed_json_field")).to be_nil } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + + end + + def verify_app_cf_fields (app_instance) + + it "sets @cf fields" do + expect(parsed_results.get("@cf")["app"]).to eq "loggenerator" + expect(parsed_results.get("@cf")["app_id"]).to eq "31b928ee-4110-4e7b-996c-334c5d7ac2ac" + expect(parsed_results.get("@cf")["app_instance"]).to eq app_instance + expect(parsed_results.get("@cf")["space"]).to eq "demo" + expect(parsed_results.get("@cf")["space_id"]).to eq "59cf41f2-3a1d-42db-88e7-9540b02945e8" + expect(parsed_results.get("@cf")["org"]).to eq "admin" + expect(parsed_results.get("@cf")["org_id"]).to eq "9887ad0a-f9f7-449e-8982-76307bd17239" + end + + end + + ## -- Special cases + def verify_parsing_logmessage_app_CF_versions(level, msg, expected_level, expected_message, &block) + context "in (Diego CF)" do + verify_parsing_logmessage_app(true, # Diego + level, msg, expected_level, expected_message, &block) + end + + context "in (Dea CF)" do + verify_parsing_logmessage_app(false, # Dea + level, msg, expected_level, expected_message, &block) + end + end + + def verify_parsing_logmessage_app(isDiego, level, msg, expected_level, expected_message, &block) + sample_fields = {"source_type" => isDiego ? "APP" : "App", # Diego/Dea specific + "source_instance" => "99", + "message_type" => "OUT", + "timestamp" => 1471387745714800488, + "level" => level, + "msg" => "Dummy msg"} + + sample_fields["msg"] = msg + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event("LogMessage", true, sample_fields) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app-admin-demo", "LogMessage", "APP", + expected_message, expected_level) + + verify_app_cf_fields(99) + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include("logmessage", "logmessage-app") } + it { expect(parsed_results.get("logmessage")["message_type"]).to eq "OUT" } + + # additional verifications + describe("", &block) + + end + end + + private + def append_fields_from_hash(hash) + result = "" + hash.each do |key, value| + result += append_field(key, value) + end + result + end + + def append_field(field_name, field_value) + '"' + field_name + '":' + (field_value.is_a?(String) ? + '"' + field_value + '"' : field_value.to_s) + ',' + end + + def construct_event(event_type, is_include_app_data, event_fields_hash) + + # envelope + result = '{' + + append_field("event_type", event_type) + + append_fields_from_hash($envelope_fields) + + # app data + if is_include_app_data + result += append_fields_from_hash($app_data_fields) + end + + # event fields + result += append_fields_from_hash(event_fields_hash) + + result = result[0...-1] + '}' # cut last comma (,) + + end + end +end + diff --git a/src/logsearch-filters/spec/helpers/filters_helper.rb b/src/logsearch-filters/spec/helpers/filters_helper.rb new file mode 100644 index 00000000..4945f08d --- /dev/null +++ b/src/logsearch-filters/spec/helpers/filters_helper.rb @@ -0,0 +1,78 @@ +# encoding: utf-8 +require 'json' +require 'logstash-core/logstash-core' +require 'logstash/util/loggable' +require 'logstash/filters/alter' +require 'logstash/pipeline' + +module LogStash::Environment + # running the grok code outside a logstash package means + # LOGSTASH_HOME will not be defined, so let's set it here + # before requiring the grok filter + unless self.const_defined?(:LOGSTASH_HOME) + LOGSTASH_HOME = File.expand_path("../../../", __FILE__) + end + + # also :pattern_path method must exist so we define it too + unless self.method_defined?(:pattern_path) + def pattern_path(path) + ::File.join(LOGSTASH_HOME, "patterns", path) + end + end +end + +module Helpers + module FilterHelper + class LogStashPipeline + class << self + def instance=(instance) + @the_pipeline = instance + end + + def instance + @the_pipeline + end + + private :new + end + end + + def load_filters(filters) + pipeline = LogStash::Pipeline.new(filters) + pipeline.instance_eval { @filters.each(&:register) } + + LogStashPipeline.instance = pipeline + end + + def when_parsing_log(sample_event, &block) + name = "" + if sample_event.is_a?(String) + name = sample_event + sample_event = { '@type' => 'syslog', '@message' => sample_event } + else + name = LogStash::Json.dump(sample_event) + end + + name = name[0..200] + "..." if name.length > 200 + + describe "[\"#{name}\"]" do + + before(:all) do + results = [] + # filter call the block on all filtered events, included new events added by the filter + event = LogStash::Event.new(sample_event) + LogStashPipeline.instance.filter(event) { |filtered_event| results << filtered_event } + # flush makes sure to empty any buffered events in the filter + LogStashPipeline.instance.flush_filters(:final => true) { |flushed_event| results << flushed_event } + + @parsed_results = results.select { |e| !e.cancelled? } + end + + subject(:parsed_results) { @parsed_results.first } + + describe("", &block) + end + + end + end +end diff --git a/src/logsearch-filters/spec/helpers/platform_helper.rb b/src/logsearch-filters/spec/helpers/platform_helper.rb new file mode 100644 index 00000000..0b6d300d --- /dev/null +++ b/src/logsearch-filters/spec/helpers/platform_helper.rb @@ -0,0 +1,110 @@ +# encoding: utf-8 + +class MessagePayload + attr_accessor :deployment, :job, :message_text +end + +class MessagePayloadBuilder + attr_accessor :message_payload + + def initialize + @message_payload = MessagePayload.new + end + + def build + @message_payload + end + + def job(job) + @message_payload.job = job + self + end + + def deployment(deployment) + @message_payload.deployment = deployment + self + end + + def message_text(message_text) + @message_payload.message_text = message_text + self + end +end + +module Helpers + module PlatformHelper + def construct_cf_message__metronagent_format (message_payload) + '[job='+ message_payload.job + ' index=555] ' + message_payload.message_text + end + + def construct_cf_message__syslogrelease_format (message_payload) + '[bosh instance='+ message_payload.deployment + '/' + message_payload.job + '/5abc6def7ghi] ' + message_payload.message_text + end + + # -- Verify methods -- + def verify_platform_cf_fields__metronagent_format (expected_shipper, expected_component, expected_job, + expected_type, expected_tags, + expected_message, expected_level) + + verify_platform_cf_fields(expected_shipper, expected_job, expected_component, expected_type, expected_tags, + expected_message, expected_level) + + it { expect(parsed_results.get("@source")["index"]).to eq 555 } + it { expect(parsed_results.get("@source")["vm"]).to eq expected_job + '/555' } + it { expect(parsed_results.get("@source")["job_index"]).to be_nil } + it { expect(parsed_results.get("@source")["deployment"]).to be_nil } + + end + + def verify_platform_cf_fields__syslogrelease_format (expected_shipper, expected_deployment, expected_component, expected_job, + expected_type, expected_tags, + expected_message, expected_level) + + verify_platform_cf_fields(expected_shipper, expected_job, expected_component, expected_type, expected_tags, + expected_message, expected_level); + + it { expect(parsed_results.get("@source")["job_index"]).to eq "5abc6def7ghi" } + it { expect(parsed_results.get("@source")["index"]).to be_nil } + it { expect(parsed_results.get("@source")["vm"]).to be_nil } + it { expect(parsed_results.get("@source")["deployment"]).to eq expected_deployment } + + end + + ## helper methods + + def verify_platform_cf_fields (expected_shipper, expected_job, expected_component, + expected_type, expected_tags, + expected_message, expected_level) + + verify_platform_fields(expected_shipper, expected_component, expected_type, expected_tags, + expected_message, expected_level) + + # verify CF format parsing + it { expect(parsed_results.get("tags")).not_to include "fail/cloudfoundry/platform/grok" } + it { expect(parsed_results.get("@source")["type"]).to eq "cf" } + it { expect(parsed_results.get("@source")["job"]).to eq expected_job } + end + + def verify_platform_fields (expected_shipper, expected_component, expected_type, expected_tags, + expected_message, expected_level) + + # fields + it { expect(parsed_results.get("@message")).to eq expected_message } + it { expect(parsed_results.get("@level")).to eq expected_level } + + it { expect(parsed_results.get("@index_type")).to eq "platform" } + it { expect(parsed_results.get("@metadata")["index"]).to eq "platform" } + it { expect(parsed_results.get("@input")).to eq "relp" } + it { expect(parsed_results.get("@shipper")["priority"]).to eq 14 } + it { expect(parsed_results.get("@shipper")["name"]).to eq expected_shipper } + it { expect(parsed_results.get("@source")["host"]).to eq "192.168.111.24" } + it { expect(parsed_results.get("@source")["component"]).to eq expected_component } + it { expect(parsed_results.get("@type")).to eq expected_type } + it { expect(parsed_results.get("tags")).to eq expected_tags } + + # verify no (default) dynamic JSON fields + it { expect(parsed_results.get("parsed_json_field")).to be_nil } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + end + end +end diff --git a/src/logsearch-filters/spec/logstash-filters/integration/app_spec.rb b/src/logsearch-filters/spec/logstash-filters/integration/app_spec.rb new file mode 100644 index 00000000..da512859 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/integration/app_spec.rb @@ -0,0 +1,357 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "App IT" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("target/logstash-filters-default.conf")} # NOTE: we use already built config here + } + CONFIG + + end + + describe "#fields when event is" do + + describe "LogMessage (APP)" do + # NOTE: below tests include two checks - one for Diego, another for Dea + + context "(unknown msg format)" do + + verify_parsing_logmessage_app_CF_versions( + "warn", "Some text msg", # unknown msg format + "WARN", "Some text msg") do + + # verify format-specific fields + it { expect(parsed_results.get("tags")).to include "unknown_msg_format" } + + end + end + + context "(JSON)" do + + verify_parsing_logmessage_app_CF_versions( + "warn", "{\\\"timestamp\\\":\\\"2016-07-15 13:20:16.954\\\",\\\"level\\\":\\\"ERROR\\\"," + + "\\\"thread\\\":\\\"main\\\",\\\"logger\\\":\\\"com.abc.LogGenerator\\\"," + + "\\\"message\\\":\\\"Some json msg\\\"}", # JSON msg + "ERROR", "Some json msg") do + + # verify format-specific fields + it { expect(parsed_results.get("tags")).not_to include "unknown_msg_format" } + + it "sets [app] fields from JSON msg" do + expect(parsed_results.get("app")["timestamp"]).to eq "2016-07-15 13:20:16.954" + expect(parsed_results.get("app")["thread"]).to eq "main" + expect(parsed_results.get("app")["logger"]).to eq "com.abc.LogGenerator" + end + + end + end + + context "([CONTAINER] log)" do + + verify_parsing_logmessage_app_CF_versions( + # [CONTAINER] log + "warn", "[CONTAINER] org.apache.catalina.startup.Catalina INFO Server startup in 9775 ms", + "INFO", "Server startup in 9775 ms") do + + # verify format-specific fields + it { expect(parsed_results.get("tags")).not_to include "unknown_msg_format" } + it { expect(parsed_results.get("app")["logger"]).to eq "[CONTAINER] org.apache.catalina.startup.Catalina" } + + end + end + + context "(Logback status log)" do + + verify_parsing_logmessage_app_CF_versions( + # Logback status log + "warn", "16:41:17,033 |-DEBUG in ch.qos.logback.classic.joran.action.RootLoggerAction - Setting level of ROOT logger to WARN", + "DEBUG", "Setting level of ROOT logger to WARN") do + + # verify format-specific fields + it { expect(parsed_results.get("tags")).not_to include "unknown_msg_format" } + it { expect(parsed_results.get("app")["logger"]).to eq "ch.qos.logback.classic.joran.action.RootLoggerAction" } + + end + end + + end + + describe "LogMessage (RTR)" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event("LogMessage", true, + {"source_type" => "RTR", # RTR + "source_instance" => "99", + "message_type" => "OUT", + "timestamp" => 1471387745714800488, + "level" => "debug", + # RTR message + "msg" => 'parser.64.78.234.207.xip.io - [2017-03-16T13:28:25.166+0000] \"GET / HTTP/1.1\" ' + + '200 0 1677 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36\" ' + + '\"10.2.9.104:60079\" \"10.2.32.71:61010\" x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"https\" ' + + 'vcap_request_id:\"f322dd76-aacf-422e-49fb-c73bc46ce45b\" response_time:0.001602684 app_id:\"27c02dec-80ce-4af6-94c5-2b51848edae9\" app_index:\"1\"\\\n'}) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app-admin-demo", "LogMessage", "RTR", + # RTR message + '200 GET / (1.602 ms)', "INFO") + + verify_app_cf_fields(99) + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include("logmessage", "logmessage-rtr") } + it { expect(parsed_results.get("tags")).not_to include("fail/cloudfoundry/app-rtr/grok") } + + it { expect(parsed_results.get("logmessage")["message_type"]).to eq "OUT" } + + it "sets [rtr] fields" do + expect(parsed_results.get("rtr")["hostname"]).to eq "parser.64.78.234.207.xip.io" + expect(parsed_results.get("rtr")["timestamp"]).to eq "2017-03-16T13:28:25.166+0000" + expect(parsed_results.get("rtr_time")).to be_nil + expect(parsed_results.get("rtr")["verb"]).to eq "GET" + expect(parsed_results.get("rtr")["path"]).to eq "/" + expect(parsed_results.get("rtr")["http_spec"]).to eq "HTTP/1.1" + expect(parsed_results.get("rtr")["status"]).to eq 200 + expect(parsed_results.get("rtr")["request_bytes_received"]).to eq 0 + expect(parsed_results.get("rtr")["body_bytes_sent"]).to eq 1677 + expect(parsed_results.get("rtr")["referer"]).to eq "-" + expect(parsed_results.get("rtr")["http_user_agent"]).to eq "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36" + expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq ["82.209.244.50", "192.168.111.21"] + expect(parsed_results.get("rtr")["x_forwarded_proto"]).to eq "https" + expect(parsed_results.get("rtr")["vcap_request_id"]).to eq "f322dd76-aacf-422e-49fb-c73bc46ce45b" + expect(parsed_results.get("rtr")["src"]["host"]).to eq "10.2.9.104" + expect(parsed_results.get("rtr")["src"]["port"]).to eq 60079 + expect(parsed_results.get("rtr")["dst"]["host"]).to eq "10.2.32.71" + expect(parsed_results.get("rtr")["dst"]["port"]).to eq 61010 + expect(parsed_results.get("rtr")["app"]["id"]).to eq "27c02dec-80ce-4af6-94c5-2b51848edae9" + expect(parsed_results.get("rtr")["app"]["index"]).to eq 1 + # calculated values + expect(parsed_results.get("rtr")["remote_addr"]).to eq "82.209.244.50" + expect(parsed_results.get("rtr")["response_time_ms"]).to eq 1.602 + end + + it "sets geoip for [rtr][remote_addr]" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + + describe "LogMessage (other)" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event("LogMessage", true, + {"source_type" => "CELL", # neither APP, nor RTR + "source_instance" => "99", + "message_type" => "OUT", + "timestamp" => 1471387745714800488, + "level" => "debug", + "msg" => "Container became healthy"}) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app-admin-demo", "LogMessage", "CELL", + "Container became healthy", "DEBUG") + + verify_app_cf_fields(99) + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include "logmessage" } + it { expect(parsed_results.get("logmessage")["message_type"]).to eq "OUT" } + + end + end + + describe "CounterEvent" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "CounterEvent", false, + {"name" => "MessageAggregator.uncategorizedEvents", + "delta" => 15, + "total" => 29043, + "level" => "info", + "msg" => ""}) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app", "CounterEvent", "COUNT", + "MessageAggregator.uncategorizedEvents (delta=15, total=29043)", "INFO") + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include "counterevent" } + + it "sets [counterevent] fields" do + expect(parsed_results.get("counterevent")["name"]).to eq "MessageAggregator.uncategorizedEvents" + expect(parsed_results.get("counterevent")["delta"]).to eq 15 + expect(parsed_results.get("counterevent")["total"]).to eq 29043 + end + + end + end + + describe "ContainerMetric" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "ContainerMetric", false, + {"cpu_percentage" => 99, + "disk_bytes" => 134524928, + "memory_bytes" => 142368768, + "level" => "info", + "msg" => ""}) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app", "ContainerMetric", "CONTAINER", + "cpu=99, memory=142368768, disk=134524928", "INFO") + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include "containermetric" } + + it "sets [containermetric] fields" do + expect(parsed_results.get("containermetric")["cpu_percentage"]).to eq 99 + expect(parsed_results.get("containermetric")["disk_bytes"]).to eq 134524928 + expect(parsed_results.get("containermetric")["memory_bytes"]).to eq 142368768 + end + + end + end + + describe "ValueMetric" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "ValueMetric", false, + {"name" => "numGoRoutines", + "value" => 58, + "unit" => "count", + "level" => "info", + "msg" => ""}) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app", "ValueMetric", "METRIC", + "numGoRoutines = 58 (count)", "INFO") + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include "valuemetric" } + + it "sets [valuemetric] fields" do + expect(parsed_results.get("valuemetric")["name"]).to eq "numGoRoutines" + expect(parsed_results.get("valuemetric")["value"]).to eq 58 + expect(parsed_results.get("valuemetric")["unit"]).to eq "count" + end + + end + end + + describe "Error" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "Error", false, + {"source" => "uaa", + "code" => 4, + "level" => "info", + "msg" => "Error message"}) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app", "Error", "ERR", + "Error message", "INFO") + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include "error" } + + it "sets [error] fields" do + expect(parsed_results.get("error")["source"]).to eq "uaa" + expect(parsed_results.get("error")["code"]).to eq 4 + end + + end + end + + describe "HttpStartStop" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "HttpStartStop", false, + {"content_length" => 38, + "duration_ms" => 6, + "instance_id" => "1b1fc66f-9aca-47b1-796c-d9632b23f1b3", + "instance_index" => 2, + "method" => "GET", + "peer_type" => "Server", + "remote_addr" => "192.168.111.11:42801", + "request_id" => "aa694b2c-6e26-4688-4b88-4574aa4e95a5", + "start_timestamp" => 1471387748611165439, + "status_code" => 200, + "stop_timestamp" => 1471387748618073991, + "uri" => "http://192.168.111.11/internal/v3/bulk/task_states", + "user_agent" => "Go-http-client/1.1", + "level" => "info", + "msg" => ""}) + + + when_parsing_log(sample_event) do + + verify_app_general_fields("app", "HttpStartStop", "HTTP", + "200 GET http://192.168.111.11/internal/v3/bulk/task_states (6 ms)", "INFO") + + # verify event-specific fields + it { expect(parsed_results.get("tags")).to include "http" } + + it "sets [httpstartstop] fields" do + expect(parsed_results.get("httpstartstop")["content_length"]).to eq 38 + expect(parsed_results.get("httpstartstop")["duration_ms"]).to eq 6 + expect(parsed_results.get("httpstartstop")["instance_id"]).to eq "1b1fc66f-9aca-47b1-796c-d9632b23f1b3" + expect(parsed_results.get("httpstartstop")["instance_index"]).to eq 2 + expect(parsed_results.get("httpstartstop")["method"]).to eq "GET" + expect(parsed_results.get("httpstartstop")["peer_type"]).to eq "Server" + expect(parsed_results.get("httpstartstop")["remote_addr"]).to eq "192.168.111.11:42801" + expect(parsed_results.get("httpstartstop")["request_id"]).to eq "aa694b2c-6e26-4688-4b88-4574aa4e95a5" + expect(parsed_results.get("httpstartstop")["status_code"]).to eq 200 + expect(parsed_results.get("httpstartstop")["stop_timestamp"]).to eq 1471387748618073991 + expect(parsed_results.get("httpstartstop")["uri"]).to eq "http://192.168.111.11/internal/v3/bulk/task_states" + expect(parsed_results.get("httpstartstop")["user_agent"]).to eq "Go-http-client/1.1" + end + + end + + end + + end + + + # -- Special cases + describe "drops useless LogMessage-APP event" do + + context "(drop)" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "LogMessage", true, + {"source_type" => "APP", "source_instance" => "99", + "message_type" => "OUT", "timestamp" => 1471387745714800488, + "level" => "info", "msg" => ""}) # LogMEssage-App with empty msg => useless + + when_parsing_log(sample_event) do + it { expect(parsed_results).to be_nil } # drop event + end + end + + context "(keep)" do + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event( "SomeOtherEvent", true, + {"timestamp" => 1471387745714800488, "level" => "info", + "msg" => ""}) # some event with empty msg => still useful + + when_parsing_log(sample_event) do + it { expect(parsed_results).not_to be_nil } # keeps event + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/integration/platform_spec.rb b/src/logsearch-filters/spec/logstash-filters/integration/platform_spec.rb new file mode 100644 index 00000000..c0fc91bc --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/integration/platform_spec.rb @@ -0,0 +1,375 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "Platform IT" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("target/logstash-filters-default.conf")} # NOTE: we use already built config here + } + CONFIG + end + + # init event (dummy) + platform_event_dummy = {"@type" => "relp", + "syslog_pri" => 14, + "syslog_severity_code" => 3, # ERROR + "host" => "192.168.111.24", + "syslog_program" => "Dummy program", + "@message" => "Dummy message"} + + describe "when CF (metron agent) and format is" do + + context "vcap (plain text)" do + + message_payload = MessagePayloadBuilder.new + .job("nfs_z1") + .message_text('Some vcap plain text message') # plain text message + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__metronagent_format(message_payload) + sample_event["syslog_program"] = "vcap.consul-agent" # vcap + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__metronagent_format("vcap.consul-agent_relp", "consul-agent", "nfs_z1", + "vcap", ["platform", "cf", "vcap"], "Some vcap plain text message", "ERROR") + + it { expect(parsed_results.get("consul_agent")).to be_nil } # no json fields + + end + end + + context "vcap (json)" do + + message_payload = MessagePayloadBuilder.new + .job("nfs_z1") + .message_text('{"timestamp":1467852972.554088,"source":"NatsStreamForwarder", ' + + '"log_level":"info","message":"router.register", ' + + '"data":{"nats_message": "{\"uris\":[\"redis-broker.64.78.234.207.xip.io\"],\"host\":\"192.168.111.201\",\"port\":80}",' + + '"reply_inbox":"_INBOX.7e93f2a1d5115844163cc930b5"}}') + .build() # JSON message + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__metronagent_format(message_payload) + sample_event["syslog_program"] = "vcap.consul-agent" # vcap + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__metronagent_format("vcap.consul-agent_relp", "consul-agent", "nfs_z1", + "vcap", ["platform", "cf", "vcap"], "router.register", "INFO") + + # json fields + it "sets fields from JSON" do + expect(parsed_results.get("consul_agent")).not_to be_nil + expect(parsed_results.get("consul_agent")["timestamp"].to_f).to eq 1467852972.554088 + expect(parsed_results.get("consul_agent")["source"]).to eq "NatsStreamForwarder" + expect(parsed_results.get("consul_agent")["data"]["nats_message"]).to eq "{\"uris\":[\"redis-broker.64.78.234.207.xip.io\"],\"host\":\"192.168.111.201\",\"port\":80}" + expect(parsed_results.get("consul_agent")["data"]["reply_inbox"]).to eq "_INBOX.7e93f2a1d5115844163cc930b5" + end + + end + end + + context "haproxy" do + message_payload = MessagePayloadBuilder.new + .job("ha_proxy_z1") + .message_text('64.78.155.208:60677 [06/Jul/2016:13:59:57.770] https-in~ http-routers/node0 59841/0/0/157/60000 200 144206 reqC respC ---- 3/4/1/2/0 5/6 {reqHeaders} {respHeaders} "GET /v2/apps?inline-relations-depth=2 HTTP/1.1"') + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__metronagent_format(message_payload) + sample_event["syslog_program"] = "haproxy" # haproxy + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__metronagent_format("haproxy_relp", "haproxy", "ha_proxy_z1", + "haproxy", ["platform", "cf", "haproxy"], "GET /v2/apps?inline-relations-depth=2 HTTP/1.1", "INFO") + + # haproxy fields + it "sets [haproxy] fields from grok" do + expect(parsed_results.get("haproxy")["client_ip"]).to eq "64.78.155.208" + expect(parsed_results.get("haproxy")["client_port"]).to eq 60677 + expect(parsed_results.get("haproxy")["accept_date"]).to eq "06/Jul/2016:13:59:57.770" + expect(parsed_results.get("haproxy")["frontend_name"]).to eq "https-in~" + expect(parsed_results.get("haproxy")["backend_name"]).to eq "http-routers" + expect(parsed_results.get("haproxy")["server_name"]).to eq "node0" + expect(parsed_results.get("haproxy")["time_request"]).to eq 59841 + expect(parsed_results.get("haproxy")["time_queue"]).to eq 0 + expect(parsed_results.get("haproxy")["time_backend_connect"]).to eq 0 + expect(parsed_results.get("haproxy")["time_backend_response"]).to eq 157 + expect(parsed_results.get("haproxy")["time_duration"]).to eq 60000 + expect(parsed_results.get("haproxy")["http_status_code"]).to eq 200 + expect(parsed_results.get("haproxy")["bytes_read"]).to eq 144206 + expect(parsed_results.get("haproxy")["captured_request_cookie"]).to eq "reqC" + expect(parsed_results.get("haproxy")["captured_response_cookie"]).to eq "respC" + expect(parsed_results.get("haproxy")["termination_state"]).to eq "----" + expect(parsed_results.get("haproxy")["actconn"]).to eq 3 + expect(parsed_results.get("haproxy")["feconn"]).to eq 4 + expect(parsed_results.get("haproxy")["beconn"]).to eq 1 + expect(parsed_results.get("haproxy")["srvconn"]).to eq 2 + expect(parsed_results.get("haproxy")["retries"]).to eq 0 + expect(parsed_results.get("haproxy")["srv_queue"]).to eq 5 + expect(parsed_results.get("haproxy")["backend_queue"]).to eq 6 + expect(parsed_results.get("haproxy")["captured_request_headers"]).to eq "reqHeaders" + expect(parsed_results.get("haproxy")["captured_response_headers"]).to eq "respHeaders" + expect(parsed_results.get("haproxy")["http_request"]).to eq "GET /v2/apps?inline-relations-depth=2 HTTP/1.1" + expect(parsed_results.get("haproxy")["http_request_verb"]).to eq "GET" + end + + end + end + + context "uaa" do + message_payload = MessagePayloadBuilder.new + .job("uaa_z0") + .message_text('[2016-07-05 04:02:18.245] uaa - 15178 [http-bio-8080-exec-14] .... DEBUG --- FilterChainProxy: /healthz has an empty filter list') + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__metronagent_format(message_payload) + sample_event["syslog_program"] = "vcap.uaa" # uaa + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__metronagent_format("vcap.uaa_relp", "uaa", "uaa_z0", + "uaa", ["platform", "cf", "uaa"], + "/healthz has an empty filter list", "DEBUG") + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-05 04:02:18.245" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-14" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "FilterChainProxy" + end + + end + end + + context "uaa (Audit)" do + message_payload = MessagePayloadBuilder.new + .job("uaa_z0") + .message_text('[2016-07-05 04:02:18.245] uaa - 15178 [http-bio-8080-exec-14] .... INFO --- Audit: ClientAuthenticationSuccess (\'Client authentication success\'): principal=cf, origin=[remoteAddress=64.78.155.208, clientId=cf], identityZoneId=[uaa]') + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__metronagent_format(message_payload) + sample_event["syslog_program"] = "vcap.uaa" # uaa + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__metronagent_format("vcap.uaa_relp", "uaa", "uaa_z0", + "uaa-audit", ["platform", "cf", "uaa", "audit"], + "ClientAuthenticationSuccess ('Client authentication success')", "INFO") + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-05 04:02:18.245" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-14" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "Audit" + end + + it "sets [uaa][audit] fields" do + expect(parsed_results.get("uaa")["audit"]["type"]).to eq "ClientAuthenticationSuccess" + expect(parsed_results.get("uaa")["audit"]["data"]).to eq "Client authentication success" + expect(parsed_results.get("uaa")["audit"]["principal"]).to eq "cf" + expect(parsed_results.get("uaa")["audit"]["origin"]).to eq ["remoteAddress=64.78.155.208", "clientId=cf"] + expect(parsed_results.get("uaa")["audit"]["identity_zone_id"]).to eq "uaa" + expect(parsed_results.get("uaa")["audit"]["remote_address"]).to eq "64.78.155.208" + end + + it "sets geoip for remoteAddress" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "64.78.155.208" + end + + end + end + + end + + describe "when CF (syslog release) and format is" do + + context "vcap (plain text)" do + + message_payload = MessagePayloadBuilder.new + .deployment("cf_full") + .job("nfs_z1") + .message_text('Some vcap plain text message') # plain text message + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__syslogrelease_format(message_payload) + sample_event["syslog_program"] = "vcap.consul-agent" # vcap + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__syslogrelease_format("vcap.consul-agent_relp", "cf_full", "consul-agent", "nfs_z1", + "vcap", ["platform", "cf", "vcap"], "Some vcap plain text message", "ERROR") + + it { expect(parsed_results.get("consul_agent")).to be_nil } # no json fields + + end + end + + context "vcap (json)" do + + message_payload = MessagePayloadBuilder.new + .deployment("cf_full") + .job("nfs_z1") + .message_text('{"timestamp":1467852972.554088,"source":"NatsStreamForwarder", ' + + '"log_level":"info","message":"router.register", ' + + '"data":{"nats_message": "{\"uris\":[\"redis-broker.64.78.234.207.xip.io\"],\"host\":\"192.168.111.201\",\"port\":80}",' + + '"reply_inbox":"_INBOX.7e93f2a1d5115844163cc930b5"}}') + .build() # JSON message + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__syslogrelease_format(message_payload) + sample_event["syslog_program"] = "vcap.consul-agent" # vcap + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__syslogrelease_format("vcap.consul-agent_relp", "cf_full", "consul-agent", "nfs_z1", + "vcap", ["platform", "cf", "vcap"], "router.register", "INFO") + + # json fields + it "sets fields from JSON" do + expect(parsed_results.get("consul_agent")).not_to be_nil + expect(parsed_results.get("consul_agent")["timestamp"].to_f).to eq 1467852972.554088 + expect(parsed_results.get("consul_agent")["source"]).to eq "NatsStreamForwarder" + expect(parsed_results.get("consul_agent")["data"]["nats_message"]).to eq "{\"uris\":[\"redis-broker.64.78.234.207.xip.io\"],\"host\":\"192.168.111.201\",\"port\":80}" + expect(parsed_results.get("consul_agent")["data"]["reply_inbox"]).to eq "_INBOX.7e93f2a1d5115844163cc930b5" + end + + end + end + + context "haproxy" do + message_payload = MessagePayloadBuilder.new + .deployment("cf_full") + .job("ha_proxy_z1") + .message_text('64.78.155.208:60677 [06/Jul/2016:13:59:57.770] https-in~ http-routers/node0 59841/0/0/157/60000 200 144206 reqC respC ---- 3/4/1/2/0 5/6 {reqHeaders} {respHeaders} "GET /v2/apps?inline-relations-depth=2 HTTP/1.1"') + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__syslogrelease_format(message_payload) + sample_event["syslog_program"] = "haproxy" # haproxy + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__syslogrelease_format("haproxy_relp", "cf_full", "haproxy", "ha_proxy_z1", + "haproxy", ["platform", "cf", "haproxy"], "GET /v2/apps?inline-relations-depth=2 HTTP/1.1", "INFO") + + # haproxy fields + it "sets [haproxy] fields from grok" do + expect(parsed_results.get("haproxy")["client_ip"]).to eq "64.78.155.208" + expect(parsed_results.get("haproxy")["client_port"]).to eq 60677 + expect(parsed_results.get("haproxy")["accept_date"]).to eq "06/Jul/2016:13:59:57.770" + expect(parsed_results.get("haproxy")["frontend_name"]).to eq "https-in~" + expect(parsed_results.get("haproxy")["backend_name"]).to eq "http-routers" + expect(parsed_results.get("haproxy")["server_name"]).to eq "node0" + expect(parsed_results.get("haproxy")["time_request"]).to eq 59841 + expect(parsed_results.get("haproxy")["time_queue"]).to eq 0 + expect(parsed_results.get("haproxy")["time_backend_connect"]).to eq 0 + expect(parsed_results.get("haproxy")["time_backend_response"]).to eq 157 + expect(parsed_results.get("haproxy")["time_duration"]).to eq 60000 + expect(parsed_results.get("haproxy")["http_status_code"]).to eq 200 + expect(parsed_results.get("haproxy")["bytes_read"]).to eq 144206 + expect(parsed_results.get("haproxy")["captured_request_cookie"]).to eq "reqC" + expect(parsed_results.get("haproxy")["captured_response_cookie"]).to eq "respC" + expect(parsed_results.get("haproxy")["termination_state"]).to eq "----" + expect(parsed_results.get("haproxy")["actconn"]).to eq 3 + expect(parsed_results.get("haproxy")["feconn"]).to eq 4 + expect(parsed_results.get("haproxy")["beconn"]).to eq 1 + expect(parsed_results.get("haproxy")["srvconn"]).to eq 2 + expect(parsed_results.get("haproxy")["retries"]).to eq 0 + expect(parsed_results.get("haproxy")["srv_queue"]).to eq 5 + expect(parsed_results.get("haproxy")["backend_queue"]).to eq 6 + expect(parsed_results.get("haproxy")["captured_request_headers"]).to eq "reqHeaders" + expect(parsed_results.get("haproxy")["captured_response_headers"]).to eq "respHeaders" + expect(parsed_results.get("haproxy")["http_request"]).to eq "GET /v2/apps?inline-relations-depth=2 HTTP/1.1" + expect(parsed_results.get("haproxy")["http_request_verb"]).to eq "GET" + end + + end + end + + context "uaa" do + message_payload = MessagePayloadBuilder.new + .deployment("cf_full") + .job("uaa_z0") + .message_text('[2016-07-05 04:02:18.245] uaa - 15178 [http-bio-8080-exec-14] .... DEBUG --- FilterChainProxy: /healthz has an empty filter list') + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__syslogrelease_format(message_payload) + sample_event["syslog_program"] = "vcap.uaa" # uaa + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__syslogrelease_format("vcap.uaa_relp", "cf_full", "uaa", "uaa_z0", + "uaa", ["platform", "cf", "uaa"], + "/healthz has an empty filter list", "DEBUG") + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-05 04:02:18.245" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-14" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "FilterChainProxy" + end + + end + end + + context "uaa (Audit)" do + message_payload = MessagePayloadBuilder.new + .deployment("cf_full") + .job("uaa_z0") + .message_text('[2016-07-05 04:02:18.245] uaa - 15178 [http-bio-8080-exec-14] .... INFO --- Audit: ClientAuthenticationSuccess (\'Client authentication success\'): principal=cf, origin=[remoteAddress=64.78.155.208, clientId=cf], identityZoneId=[uaa]') + .build() + sample_event = platform_event_dummy.clone + sample_event["@message"] = construct_cf_message__syslogrelease_format(message_payload) + sample_event["syslog_program"] = "vcap.uaa" # uaa + + when_parsing_log(sample_event) do + + verify_platform_cf_fields__syslogrelease_format("vcap.uaa_relp", "cf_full", "uaa", "uaa_z0", + "uaa-audit", ["platform", "cf", "uaa", "audit"], + "ClientAuthenticationSuccess ('Client authentication success')", "INFO") + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-05 04:02:18.245" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-14" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "Audit" + end + + it "sets [uaa][audit] fields" do + expect(parsed_results.get("uaa")["audit"]["type"]).to eq "ClientAuthenticationSuccess" + expect(parsed_results.get("uaa")["audit"]["data"]).to eq "Client authentication success" + expect(parsed_results.get("uaa")["audit"]["principal"]).to eq "cf" + expect(parsed_results.get("uaa")["audit"]["origin"]).to eq ["remoteAddress=64.78.155.208", "clientId=cf"] + expect(parsed_results.get("uaa")["audit"]["identity_zone_id"]).to eq "uaa" + expect(parsed_results.get("uaa")["audit"]["remote_address"]).to eq "64.78.155.208" + end + + it "sets geoip for remoteAddress" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "64.78.155.208" + end + + end + end + + end + + describe "when not CF" do + + sample_event = platform_event_dummy.clone + sample_event["@message"] = "Some message" # not CF + + when_parsing_log(sample_event) do + + verify_platform_fields("Dummy program_relp", "Dummy program", + "relp", ["platform", "fail/cloudfoundry/platform/grok"], "Some message", "ERROR") + + it { expect(parsed_results.get("@source")["type"]).to eq "system" } + + end + end + +end + diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_containermetric_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_containermetric_spec.rb new file mode 100644 index 00000000..9375c083 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_containermetric_spec.rb @@ -0,0 +1,91 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-containermetric.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-containermetric.conf")} + } + CONFIG + end + + describe "#if failed" do + when_parsing_log( + "@type" => "some type", # bad type + "parsed_json_field" => { "field" => "value" }, + "@cf" => { "app_id" => "abc" }, + "@message" => "some message" + ) do + + # tag is NOT set + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + # -- general case + describe "#fields" do + when_parsing_log( + "@type" => "ContainerMetric", + "parsed_json_field" => { "instance_index" => 5, "cpu_percentage" => 123, "memory_bytes" => 456, "disk_bytes" => 789 }, + "@cf" => { "app_id" => "abc" }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("tags")).to eq ["containermetric"] } + + it { expect(parsed_results.get("@cf")["app_id"]).to eq "abc" } # keeps unchanged + it { expect(parsed_results.get("@cf")["app_instance"]).to eq 5 } + it { expect(parsed_results.get("parsed_json_field")["instance_index"]).to be_nil } + + it "keeps containermetric fields" do + expect(parsed_results.get("parsed_json_field")["cpu_percentage"]).to eq 123 + expect(parsed_results.get("parsed_json_field")["memory_bytes"]).to eq 456 + expect(parsed_results.get("parsed_json_field")["disk_bytes"]).to eq 789 + end + + it { expect(parsed_results.get("@message")).to eq "cpu=123, memory=456, disk=789" } + + it { expect(parsed_results.get("@type")).to eq "ContainerMetric" } # keeps unchanged + + end + end + + # -- special cases + describe "[@cf][app_instance] skipped" do + + context "when empty app_id" do + when_parsing_log( + "@type" => "ContainerMetric", + "parsed_json_field" => { "instance_index" => 5, "cpu_percentage" => 123, "memory_bytes" => 456, "disk_bytes" => 789 }, + "@cf" => { "app_id" => "" }, # empty + "@message" => "some message" + ) do + + it { expect(parsed_results.get("@cf")["app_id"]).to be_nil } # removes empty field + it { expect(parsed_results.get("@cf")["app_instance"]).to be_nil } # doesn't set app_instance + it { expect(parsed_results.get("parsed_json_field")["instance_index"]).to be_nil } # removes unnecessary field + + end + end + + context "when missing app_id" do + when_parsing_log( + "@type" => "ContainerMetric", + "parsed_json_field" => { "instance_index" => 5, "cpu_percentage" => 123, "memory_bytes" => 456, "disk_bytes" => 789 }, + "@cf" => { "some_field" => "" }, # missing app_id + "@message" => "some message" + ) do + + it { expect(parsed_results.get("@cf")["app_id"]).to be_nil } # missing + it { expect(parsed_results.get("@cf")["app_instance"]).to be_nil } # doesn't set app_instance + it { expect(parsed_results.get("parsed_json_field")["instance_index"]).to be_nil } # removes unnecessary field + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_counterevent_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_counterevent_spec.rb new file mode 100644 index 00000000..b54fe06b --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_counterevent_spec.rb @@ -0,0 +1,47 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-counterevent.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-counterevent.conf")} + } + CONFIG + end + + describe "#if failed" do + when_parsing_log( + "@type" => "some type", # bad type + "parsed_json_field" => { "field" => "value" }, + "@message" => "some message" + ) do + + # tag is NOT set + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + # -- general case + describe "#fields" do + when_parsing_log( + "@type" => "CounterEvent", + "parsed_json_field" => { "name" => "abc", "delta" => 123, "total" => 456 }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("tags")).to eq ["counterevent"] } + + it { expect(parsed_results.get("@message")).to eq "abc (delta=123, total=456)" } + it { expect(parsed_results.get("parsed_json_field")["name"]).to eq "abc" } + it { expect(parsed_results.get("parsed_json_field")["delta"]).to eq 123 } + it { expect(parsed_results.get("parsed_json_field")["total"]).to eq 456 } + + it { expect(parsed_results.get("@type")).to eq "CounterEvent" } # keeps unchanged + + end + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_error_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_error_spec.rb new file mode 100644 index 00000000..82ceaf32 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_error_spec.rb @@ -0,0 +1,47 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-error.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-error.conf")} + } + CONFIG + end + + describe "#if failed" do + when_parsing_log( + "@type" => "some type", # bad type + "parsed_json_field" => { "field" => "value" }, + "@message" => "some message" + ) do + + # tag is NOT set + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + # -- general case + describe "#fields" do + when_parsing_log( + "@type" => "Error", + "parsed_json_field" => { "source" => "abc", "code" => "def", "message" => "Some error message" }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("tags")).to eq ["error"] } + + it { expect(parsed_results.get("@message")).to eq "Some error message" } + it { expect(parsed_results.get("parsed_json_field")["message"]).to be_nil } + it { expect(parsed_results.get("parsed_json_field")["source"]).to eq "abc" } + it { expect(parsed_results.get("parsed_json_field")["code"]).to eq "def" } + + it { expect(parsed_results.get("@type")).to eq "Error" } # keeps unchanged + + end + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_http_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_http_spec.rb new file mode 100644 index 00000000..f9885ac9 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_http_spec.rb @@ -0,0 +1,91 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-http.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-http.conf")} + } + CONFIG + end + + describe "#if failed" do + when_parsing_log( + "@type" => "some type", # bad type + "parsed_json_field" => { "field" => "value" }, + "@message" => "some message" + ) do + + # tag is NOT set + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + # -- general case + describe "#fields" do + + when_parsing_log( + "@type" => "HttpStartStop", + "parsed_json_field" => { "method" => "PUT", "status_code" => 200, "uri" => "/some/uri", + "duration_ms" => 300, + "peer_type" => "Client", + "instance_id" => "abc", "instance_index" => 5 }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("tags")).to eq ["http"] } + + it { expect(parsed_results.get("@message")).to eq "200 PUT /some/uri (300 ms)" } # constructed + + # keeps fields + it { expect(parsed_results.get("parsed_json_field")["method"]).to eq "PUT" } + it { expect(parsed_results.get("parsed_json_field")["peer_type"]).to eq "Client" } + it { expect(parsed_results.get("parsed_json_field")["status_code"]).to eq 200 } + it { expect(parsed_results.get("parsed_json_field")["uri"]).to eq "/some/uri" } + it { expect(parsed_results.get("parsed_json_field")["duration_ms"]).to eq 300 } + it { expect(parsed_results.get("parsed_json_field")["instance_id"]).to eq "abc" } + it { expect(parsed_results.get("parsed_json_field")["instance_index"]).to eq 5 } + it { expect(parsed_results.get("@type")).to eq "HttpStartStop" } + + end + + end + + # -- special cases + describe "[instance_id] and [instance_index] skipped" do + + context "when empty instance_id" do + when_parsing_log( + "@type" => "HttpStartStop", + "parsed_json_field" => { "method" => 1, "uri" => "/some/uri", "peer_type" => 1, + "instance_id" => "", # empty + "instance_index" => 0 }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("parsed_json_field")["instance_id"]).to be_nil } # removes unnecessary field + it { expect(parsed_results.get("parsed_json_field")["instance_index"]).to be_nil } # removes unnecessary field + + end + end + + context "when missing instance_id" do + when_parsing_log( + "@type" => "HttpStartStop", + "parsed_json_field" => { "method" => 1, "uri" => "/some/uri", "peer_type" => 1, + "instance_index" => 0 }, # missing instance_id + "@message" => "some message" + ) do + + it { expect(parsed_results.get("parsed_json_field")["instance_id"]).to be_nil } # removes unnecessary field + it { expect(parsed_results.get("parsed_json_field")["instance_index"]).to be_nil } # removes unnecessary field + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_app_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_app_spec.rb new file mode 100644 index 00000000..f8f30f85 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_app_spec.rb @@ -0,0 +1,241 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-logmessage-app.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-logmessage-app.conf")} + } + CONFIG + end + + describe "#if succeeds" do + + context "simple [@source][type]" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => {"type" => "APP"}, # simple APP value + "@level" => "INFO", + "@message" => "Some message here" + ) do + + it { expect(parsed_results.get("tags")).to include "logmessage-app" } + + end + end + + context "(composite [@source][type])" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => {"type" => "APP"}, + "@level" => "INFO", + "@message" => "Some message here" + ) do + + it { expect(parsed_results.get("tags")).to include "logmessage-app" } + + end + end + end + + describe "#if failed" do + + context "(bad @type)" do + when_parsing_log( + "@type" => "Some type", # bad value + "@source" => {"type" => "APP"}, + "@level" => "INFO", + "@message" => "Some message of wrong format" + ) do + + # no tags => 'if' condition has failed + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + context "(bad [@source][type])" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => {"type" => "Some value"}, # bad value + "@level" => "INFO", + "@message" => "Some message of wrong format" + ) do + + # no log tags => 'if' condition has failed + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + context "(lowercase [@source][type])" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => {"type" => "App"}, # lowercase - bad value + "@level" => "INFO", + "@message" => "Some message of wrong format" + ) do + + # no log tags => 'if' condition has failed + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + end + + # -- general case + describe "#fields when message is" do + + describe "JSON format" do + context "(general)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + # JSON format (general) + "@message" => "{\"timestamp\":\"2016-07-15 13:20:16.954\",\"level\":\"INFO\",\"thread\":\"main\",\"logger\":\"com.abc.LogGenerator\",\"message\":\"Some message\"}" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-app"] } # no fail tag + + # fields + it "sets @message" do + expect(parsed_results.get("@message")).to eq "Some message" + expect(parsed_results.get("app")["message"]).to be_nil + end + + it "sets @level" do + expect(parsed_results.get("@level")).to eq "INFO" + expect(parsed_results.get("app")["level"]).to be_nil + end + + it "sets fields from JSON" do + expect(parsed_results.get("app")["timestamp"]).to eq "2016-07-15 13:20:16.954" + expect(parsed_results.get("app")["thread"]).to eq "main" + expect(parsed_results.get("app")["logger"]).to eq "com.abc.LogGenerator" + end + + end + end + + context "(with exception)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + # JSON format (with exception) + "@message" => "{\"message\":\"Some error\", \"exception\":\"Some exception\"}" + ) do + + it "appends exception to message" do + expect(parsed_results.get("@message")).to eq "Some error +Some exception" + expect(parsed_results.get("app")).to be_empty + end + + end + end + + context "(invalid)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + "@message" => "{\"message\":\"Some message\", }" # invalid JSON + ) do + + # unknown_message_format + it { expect(parsed_results.get("tags")).to eq ["logmessage-app", "unknown_msg_format"] } + + it { expect(parsed_results.get("@message")).to eq "{\"message\":\"Some message\", }" } # keeps unchanged + it { expect(parsed_results.get("@level")).to eq "SOME LEVEL" } # keeps unchanged + + end + end + + context "(empty)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + "@message" => "{}" # empty JSON + ) do + + # unknown_message_format tag + it { expect(parsed_results.get("tags")).to eq ["logmessage-app", "unknown_msg_format"] } + + it { expect(parsed_results.get("@message")).to eq "{}" } # keeps unchanged + it { expect(parsed_results.get("@level")).to eq "SOME LEVEL" } # keeps unchanged + + end + end + + end + + describe "[CONTAINER] log" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + # [CONTAINER] log + "@message" => "[CONTAINER] org.apache.catalina.startup.Catalina INFO Server startup in 9775 ms" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-app"] } # no unknown_msg_format tag + + it "sets fields from 'grok'" do + expect(parsed_results.get("@message")).to eq "Server startup in 9775 ms" + expect(parsed_results.get("@level")).to eq "INFO" + expect(parsed_results.get("app")["logger"]).to eq "[CONTAINER] org.apache.catalina.startup.Catalina" + end + + end + end + + describe "Logback status log" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + # Logback status log + "@message" => "16:41:17,033 |-DEBUG in ch.qos.logback.classic.joran.action.RootLoggerAction - Setting level of ROOT logger to WARN" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-app"] } # no unknown_msg_format tag + + it "sets fields from 'grok'" do + expect(parsed_results.get("@message")).to eq "Setting level of ROOT logger to WARN" + expect(parsed_results.get("@level")).to eq "DEBUG" + expect(parsed_results.get("app")["logger"]).to eq "ch.qos.logback.classic.joran.action.RootLoggerAction" + end + + end + end + + describe "unknown format" do + + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "APP" }, + "@level" => "SOME LEVEL", + "@message" => "Some Message" # unknown format + ) do + + # unknown format + it { expect(parsed_results.get("tags")).to eq ["logmessage-app", "unknown_msg_format"] } + + it { expect(parsed_results.get("@message")).to eq "Some Message" } # keeps unchanged + it { expect(parsed_results.get("@level")).to eq "SOME LEVEL" } # keeps unchanged + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_rtr_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_rtr_spec.rb new file mode 100644 index 00000000..36145f02 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_rtr_spec.rb @@ -0,0 +1,452 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-logmessage-rtr.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-logmessage-rtr.conf")} + } + CONFIG + end + + # -- general case + describe "#fields when message is" do + + context "RTR format" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /some/http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr"] } # no fail tag + + # fields + it { expect(parsed_results.get("@message")).to eq "200 GET /some/http (5.328 ms)" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + it "sets [rtr] fields" do + expect(parsed_results.get("rtr")["hostname"]).to eq "parser.64.78.234.207.xip.io" + expect(parsed_results.get("rtr")["timestamp"]).to eq "15/07/2016:09:26:25 +0000" + expect(parsed_results.get("rtr_time")).to be_nil + expect(parsed_results.get("rtr")["verb"]).to eq "GET" + expect(parsed_results.get("rtr")["path"]).to eq "/some/http" + expect(parsed_results.get("rtr")["http_spec"]).to eq "HTTP/1.1" + expect(parsed_results.get("rtr")["status"]).to eq 200 + expect(parsed_results.get("rtr")["request_bytes_received"]).to eq 0 + expect(parsed_results.get("rtr")["body_bytes_sent"]).to eq 1413 + expect(parsed_results.get("rtr")["referer"]).to eq "-" + expect(parsed_results.get("rtr")["http_user_agent"]).to eq "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" + expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq ["82.209.244.50", "192.168.111.21"] + expect(parsed_results.get("rtr")["x_forwarded_proto"]).to eq "http" + expect(parsed_results.get("rtr")["vcap_request_id"]).to eq "831e54f1-f09f-4971-6856-9fdd502d4ae3" + # calculated values + expect(parsed_results.get("rtr")["remote_addr"]).to eq "82.209.244.50" + expect(parsed_results.get("rtr")["response_time_ms"]).to eq 5.328 + end + + it "sets geoip for [rtr][remote_addr]" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + + context "RTR format (cf-release v250+)" do + + context "" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format - quoted requestRemoteAddr and destIPandPort + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /some/http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" \"192.168.111.21:35826\" \"192.861.111.12:33456\" x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr"] } # no fail tag + + # fields + it { expect(parsed_results.get("@message")).to eq "200 GET /some/http (5.328 ms)" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + it "sets [rtr] fields" do + expect(parsed_results.get("rtr")["hostname"]).to eq "parser.64.78.234.207.xip.io" + expect(parsed_results.get("rtr")["timestamp"]).to eq "15/07/2016:09:26:25 +0000" + expect(parsed_results.get("rtr_time")).to be_nil + expect(parsed_results.get("rtr")["verb"]).to eq "GET" + expect(parsed_results.get("rtr")["path"]).to eq "/some/http" + expect(parsed_results.get("rtr")["http_spec"]).to eq "HTTP/1.1" + expect(parsed_results.get("rtr")["status"]).to eq 200 + expect(parsed_results.get("rtr")["request_bytes_received"]).to eq 0 + expect(parsed_results.get("rtr")["body_bytes_sent"]).to eq 1413 + expect(parsed_results.get("rtr")["referer"]).to eq "-" + expect(parsed_results.get("rtr")["http_user_agent"]).to eq "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" + expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq ["82.209.244.50", "192.168.111.21"] + expect(parsed_results.get("rtr")["x_forwarded_proto"]).to eq "http" + expect(parsed_results.get("rtr")["vcap_request_id"]).to eq "831e54f1-f09f-4971-6856-9fdd502d4ae3" + # calculated values + expect(parsed_results.get("rtr")["remote_addr"]).to eq "82.209.244.50" + expect(parsed_results.get("rtr")["response_time_ms"]).to eq 5.328 + end + + it "sets geoip for [rtr][remote_addr]" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + end + + context "RTR format (cf-release v252+)" do + + context "" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format - quoted requestRemoteAddr and destIPandPort + "@message" => "parser.64.78.234.207.xip.io - [2017-03-16T13:28:25.166+0000] \"GET / HTTP/1.1\" 200 0 1677 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36\" \"10.2.9.104:60079\" \"10.2.32.71:61010\" x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"https\" vcap_request_id:\"f322dd76-aacf-422e-49fb-c73bc46ce45b\" response_time:0.001602684 app_id:\"27c02dec-80ce-4af6-94c5-2b51848edae9\" app_index:\"1\"\n" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr"] } # no fail tag + + # fields + it { expect(parsed_results.get("@message")).to eq "200 GET / (1.602 ms)" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + it "sets [rtr] fields" do + expect(parsed_results.get("rtr")["hostname"]).to eq "parser.64.78.234.207.xip.io" + expect(parsed_results.get("rtr")["timestamp"]).to eq "2017-03-16T13:28:25.166+0000" + expect(parsed_results.get("rtr_time")).to be_nil + expect(parsed_results.get("rtr")["verb"]).to eq "GET" + expect(parsed_results.get("rtr")["path"]).to eq "/" + expect(parsed_results.get("rtr")["http_spec"]).to eq "HTTP/1.1" + expect(parsed_results.get("rtr")["status"]).to eq 200 + expect(parsed_results.get("rtr")["request_bytes_received"]).to eq 0 + expect(parsed_results.get("rtr")["body_bytes_sent"]).to eq 1677 + expect(parsed_results.get("rtr")["referer"]).to eq "-" + expect(parsed_results.get("rtr")["http_user_agent"]).to eq "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36" + expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq ["82.209.244.50", "192.168.111.21"] + expect(parsed_results.get("rtr")["x_forwarded_proto"]).to eq "https" + expect(parsed_results.get("rtr")["vcap_request_id"]).to eq "f322dd76-aacf-422e-49fb-c73bc46ce45b" + expect(parsed_results.get("rtr")["src"]["host"]).to eq "10.2.9.104" + expect(parsed_results.get("rtr")["src"]["port"]).to eq 60079 + expect(parsed_results.get("rtr")["dst"]["host"]).to eq "10.2.32.71" + expect(parsed_results.get("rtr")["dst"]["port"]).to eq 61010 + expect(parsed_results.get("rtr")["app"]["id"]).to eq "27c02dec-80ce-4af6-94c5-2b51848edae9" + expect(parsed_results.get("rtr")["app"]["index"]).to eq 1 + # calculated values + expect(parsed_results.get("rtr")["remote_addr"]).to eq "82.209.244.50" + expect(parsed_results.get("rtr")["response_time_ms"]).to eq 1.602 + end + + it "sets geoip for [rtr][remote_addr]" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + + context "empty requestRemoteAddr and destIPandPort" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format - quoted requestRemoteAddr and destIPandPort + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /some/http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" \"-\" \"-\" x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr"] } # no fail tag + + end + end + + end + + context "RTR format (cf-deployment v12.17.0+)" do + + context "" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format - quoted requestRemoteAddr and destIPandPort + "@message" => "parser.64.78.234.207.xip.io - [2017-03-16T13:28:25.166+0000] \"GET / HTTP/1.1\" 200 0 1677 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36\" \"10.2.9.104:60079\" \"10.2.32.71:61010\" x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"https\" vcap_request_id:\"f322dd76-aacf-422e-49fb-c73bc46ce45b\" response_time:0.001602684 gorouter_time:0.000163 app_time:0.024988 app_id:\"27c02dec-80ce-4af6-94c5-2b51848edae9\" app_index:\"1\"\n" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr"] } # no fail tag + + # fields + it { expect(parsed_results.get("@message")).to eq "200 GET / (1.602 ms)" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + it "sets [rtr] fields" do + expect(parsed_results.get("rtr")["hostname"]).to eq "parser.64.78.234.207.xip.io" + expect(parsed_results.get("rtr")["timestamp"]).to eq "2017-03-16T13:28:25.166+0000" + expect(parsed_results.get("rtr_time")).to be_nil + expect(parsed_results.get("rtr")["verb"]).to eq "GET" + expect(parsed_results.get("rtr")["path"]).to eq "/" + expect(parsed_results.get("rtr")["http_spec"]).to eq "HTTP/1.1" + expect(parsed_results.get("rtr")["status"]).to eq 200 + expect(parsed_results.get("rtr")["request_bytes_received"]).to eq 0 + expect(parsed_results.get("rtr")["body_bytes_sent"]).to eq 1677 + expect(parsed_results.get("rtr")["referer"]).to eq "-" + expect(parsed_results.get("rtr")["http_user_agent"]).to eq "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36" + expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq ["82.209.244.50", "192.168.111.21"] + expect(parsed_results.get("rtr")["x_forwarded_proto"]).to eq "https" + expect(parsed_results.get("rtr")["vcap_request_id"]).to eq "f322dd76-aacf-422e-49fb-c73bc46ce45b" + expect(parsed_results.get("rtr")["src"]["host"]).to eq "10.2.9.104" + expect(parsed_results.get("rtr")["src"]["port"]).to eq 60079 + expect(parsed_results.get("rtr")["dst"]["host"]).to eq "10.2.32.71" + expect(parsed_results.get("rtr")["dst"]["port"]).to eq 61010 + expect(parsed_results.get("rtr")["app"]["id"]).to eq "27c02dec-80ce-4af6-94c5-2b51848edae9" + expect(parsed_results.get("rtr")["app"]["index"]).to eq 1 + # calculated values + expect(parsed_results.get("rtr")["remote_addr"]).to eq "82.209.244.50" + expect(parsed_results.get("rtr")["response_time_ms"]).to eq 1.602 + expect(parsed_results.get("rtr")["gorouter_time_ms"]).to_eq 0.000163 + expect(parsed_results.get("rtr")["app_time_ms"]).to_eq 0.024988 + end + + it "sets geoip for [rtr][remote_addr]" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + + context "empty requestRemoteAddr and destIPandPort" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format - quoted requestRemoteAddr and destIPandPort + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /some/http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" \"-\" \"-\" x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr"] } # no fail tag + + end + end + + end + + context "bad format" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => {"type" => "RTR"}, + "@level" => "SOME LEVEL", + "@message" => "Some message of wrong format" # bad format + ) do + + # get parsing error + it { expect(parsed_results.get("tags")).to eq ["logmessage-rtr", "fail/cloudfoundry/app-rtr/grok"] } + + # fields + it { expect(parsed_results.get("@message")).to eq "Some message of wrong format" } # keeps unchanged + it { expect(parsed_results.get("@level")).to eq "SOME LEVEL" } # keeps unchanged + + it { expect(parsed_results.get("rtr")).to be_nil } + it { expect(parsed_results.get("geoip")).to be_nil } + + end + end + + end + + # -- special cases + describe "when HTTP status" do + + context "<400 (INFO @level)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" " + + "200" + # HTTP status <400 + " 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it { expect(parsed_results.get("@level")).to eq "INFO" } + + end + end + + context "=400 (ERROR @level)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" " + + "400" + # HTTP status =400 + " 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it { expect(parsed_results.get("@level")).to eq "WARN" } + + end + end + + context ">400 (ERROR @level)" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" " + + "401" + # HTTP status >400 + " 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it { expect(parsed_results.get("@level")).to eq "WARN" } + + end + end + + end + + describe "when [rtr][x_forwarded_for]" do + + context "contains quotes & whitespaces" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 " + + "x_forwarded_for:\"\" 82.209.244.50 \", 192.168.111.21 \"" + # contains quotes & whitespaces + " x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it "removes quotes and whitespaces and split" do + expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq ["82.209.244.50", "192.168.111.21"] + end + + end + end + + context "blank value" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 " + + "x_forwarded_for:\" \"" + # blank value + " x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it { expect(parsed_results.get("rtr")["x_forwarded_for"]).to eq [] } # empty + + end + end + + end + + describe "when [rtr][remote_addr]" do + + context "has ip format" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 " + + "x_forwarded_for:\"82.209.244.50\"" + # ip format + " x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it { expect(parsed_results.get("rtr")["remote_addr"]).to eq "82.209.244.50" } + + it "sets geoip for [rtr][remote_addr]" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + + context "has bad format" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 " + + "x_forwarded_for:\"bad_format, 82.209.244.50\"" + # bad format + " x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.005328859 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + + it { expect(parsed_results.get("rtr")["remote_addr"]).to eq "bad_format" } + + end + end + + end + + describe "when NOT rtr case" do + + context "(bad @type)" do + when_parsing_log( + "@type" => "Some type", # bad value + "@source" => {"type" => "RTR"}, + "@level" => "INFO", + "@message" => "Some message of wrong format" + ) do + + # no rtr tags => 'if' condition has failed + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + context "(bad [@source][type])" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => {"type" => "Bad value"}, # bad value + "@level" => "INFO", + "@message" => "Some message of wrong format" + ) do + + # no rtr tags => 'if' condition has failed + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + end + + describe "when [rtr][response_time_sec] has only a few digits" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /some/http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:0.5 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + it { expect(parsed_results.get("rtr")["response_time_ms"]).to eq 500 } + end + end + + describe "when [rtr][response_time_sec] is an integer" do + when_parsing_log( + "@type" => "LogMessage", + "@source" => { "type" => "RTR" }, + "@level" => "SOME LEVEL", + # rtr format + "@message" => "parser.64.78.234.207.xip.io - [15/07/2016:09:26:25 +0000] \"GET /some/http HTTP/1.1\" 200 0 1413 \"-\" \"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 192.168.111.21:35826 x_forwarded_for:\"82.209.244.50, 192.168.111.21\" x_forwarded_proto:\"http\" vcap_request_id:\"831e54f1-f09f-4971-6856-9fdd502d4ae3\" response_time:5 app_id:7ae227a6-6ad1-46d4-bfb9-6e60d7796bb5\n" + ) do + it { expect(parsed_results.get("rtr")["response_time_ms"]).to eq 5000 } + end + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_spec.rb new file mode 100644 index 00000000..4e99c290 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_logmessage_spec.rb @@ -0,0 +1,129 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-logmessage.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-logmessage.conf")} + } + CONFIG + end + + describe "#if failed" do + when_parsing_log( + "@type" => "some type", # bad type + "parsed_json_field" => { "source_type" => "TestType", "source_instance" => "5", "message_type" => "OUT" }, + "@cf" => { "app_id" => "abc" }, + "@message" => "some message" + ) do + + # tag is NOT set + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + # -- general case + describe "#fields" do + when_parsing_log( + "@type" => "LogMessage", + "parsed_json_field" => { "source_type" => "TestType", "source_instance" => "5", "message_type" => "OUT" }, + "@cf" => { "app_id" => "abc", "space" => "def" }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("tags")).to eq ["logmessage"] } + + it { expect(parsed_results.get("@source")["type"]).to eq "TESTTYPE" } # uppercased + + it { expect(parsed_results.get("@cf")["app_id"]).to eq "abc" } # keeps unchanged + it { expect(parsed_results.get("@cf")["app_instance"]).to eq 5 } # converted to int + it { expect(parsed_results.get("parsed_json_field")["source_instance"]).to be_nil } + + it { expect(parsed_results.get("parsed_json_field")["message_type"]).to eq "OUT" } # keeps json fields + it { expect(parsed_results.get("@cf")["space"]).to eq "def" } # keeps @cf fields + + it { expect(parsed_results.get("@message")).to eq "some message" } + + end + end + + # -- special cases + describe "drop/keep event" do + + context "when @message is useless (empty) - drop" do + when_parsing_log( + "@type" => "LogMessage", + "@message" => "" # empty message + ) do + + # useless event was dropped + it { expect(parsed_results).to be_nil} + + end + end + + context "when @message is useless (blank) - drop" do + when_parsing_log( + "@type" => "LogMessage", + "@message" => " " # blank message + ) do + + # useless event was dropped + it { expect(parsed_results).to be_nil} + + end + end + + context "when @message is just missing - keep" do + when_parsing_log( + "@type" => "LogMessage" + # no @message field at all + ) do + + # event was NOT dropped + it { expect(parsed_results).not_to be_nil} + + end + end + + end + + describe "[@cf][app_instance] skipped" do + + context "when empty app_id" do + when_parsing_log( + "@type" => "LogMessage", + "parsed_json_field" => { "source_type" => "TestType", "source_instance" => "5", "message_type" => "OUT" }, + "@cf" => { "app_id" => ""}, # empty app_id + "@message" => "some message" + ) do + + it { expect(parsed_results.get("@cf")["app_id"]).to be_nil } # removes empty field + it { expect(parsed_results.get("@cf")["app_instance"]).to be_nil } # doesn't set app_instance + it { expect(parsed_results.get("parsed_json_field")["source_instance"]).to be_nil } # removes unnecessary field + + + end + end + + context "when missing app_id" do + when_parsing_log( + "@type" => "LogMessage", + "parsed_json_field" => { "source_type" => "TestType", "source_instance" => "5", "message_type" => "OUT" }, + "@cf" => { "field" => "value" }, # missing app_id + "@message" => "some message" + ) do + + it { expect(parsed_results.get("@cf")["app_id"]).to be_nil } # missing + it { expect(parsed_results.get("@cf")["app_instance"]).to be_nil } # doesn't set app_instance + it { expect(parsed_results.get("parsed_json_field")["source_instance"]).to be_nil } # removes unnecessary field + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_spec.rb new file mode 100644 index 00000000..f9a6cf1e --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_spec.rb @@ -0,0 +1,301 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@index_type" => "app", # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(parsed_results.get("tags")).to include "app" } + + end + end + + describe "failed" do + when_parsing_log( + "@index_type" => "some value", # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(parsed_results.get("tags")).to be_nil } + + it { expect(parsed_results.get("@index_type")).to eq "some value" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when json" do + + context "succeeded" do + when_parsing_log( + "@index_type" => "app", + "@metadata" => {"index" => "app"}, + # valid JSON (LogMessage event) + "@message" => "{\"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_org_name\":\"admin\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"cf_space_name\":\"demo\", \"deployment\":\"cf-full\", \"event_type\":\"LogMessage\", \"job_index\":\"abc123\",\"ip\":\"192.168.111.35\", \"job\":\"runner_z1\", \"level\":\"info\",\"message_type\":\"OUT\",\"msg\":\"Some Message\",\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).not_to include "fail/cloudfoundry/app/json" } + it { expect(parsed_results.get("tags")).to include "app" } + + # fields + + it { expect(parsed_results.get("parsed_json_field")["time"]).to be_nil } + it { expect(parsed_results.get("parsed_json_field")["timestamp"]).to be_nil } + + it { expect(parsed_results.get("tags")).not_to include('_rubyexception') } + + it { expect(parsed_results.get("@message")).to eq "Some Message" } + it { expect(parsed_results.get("@level")).to eq "info" } + + it "sets @source fields" do + expect(parsed_results.get("@source")["type"]).to eq "LOG" + expect(parsed_results.get("@source")["component"]).to eq "dea_logging_agent" + expect(parsed_results.get("@source")["job_index"]).to eq "abc123" + expect(parsed_results.get("@source")["job"]).to eq "runner_z1" + expect(parsed_results.get("@source")["host"]).to eq "192.168.111.35" + expect(parsed_results.get("@source")["deployment"]).to eq "cf-full" + end + + it { expect(parsed_results.get("@type")).to eq "LogMessage" } + + it "sets @cf fields" do + expect(parsed_results.get("@cf")["app"]).to eq "loggenerator" + expect(parsed_results.get("@cf")["app_id"]).to eq "31b928ee-4110-4e7b-996c-334c5d7ac2ac" + expect(parsed_results.get("@cf")["space"]).to eq "demo" + expect(parsed_results.get("@cf")["space_id"]).to eq "59cf41f2-3a1d-42db-88e7-9540b02945e8" + expect(parsed_results.get("@cf")["org"]).to eq "admin" + expect(parsed_results.get("@cf")["org_id"]).to eq "9887ad0a-f9f7-449e-8982-76307bd17239" + end + + it { expect(parsed_results.get("parsed_json_field")["message_type"]).to eq "OUT" } + it { expect(parsed_results.get("parsed_json_field_name")).to eq "LogMessage" } # @type + + it { expect(parsed_results.get("@index_type")).to eq "app" } # keeps unchanged + it { expect(parsed_results.get("@metadata")["index"]).to eq "app-admin-demo" } + + it { expect(parsed_results.get("@timestamp").to_i).to eq LogStash::Timestamp.at(1467972040).to_i } + + end + end + + context "failed" do + when_parsing_log( + "@index_type" => "app", + # invalid JSON + "@message" => "Some message that is invalid json" + ) do + + # parsing error + it { expect(parsed_results.get("tags")).to include "fail/cloudfoundry/app/json" } + it { expect(parsed_results.get("tags")).to include "app" } + + it { expect(parsed_results.get("@message")).to eq "Some message that is invalid json" } # keeps unchanged + + # no json fields set + it { expect(parsed_results.get("@source")).to be_nil } + it { expect(parsed_results.get("@cf")).to be_nil } + it { expect(parsed_results.get("@type")).to be_nil } + it { expect(parsed_results.get("parsed_json_field")).to be_nil } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + + end + end + + end + + # -- special cases + + describe "sets the timestamp correctly" do + context "when start_timestamp is set" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"start_timestamp\":1467972040073786262}" + ) do + it { expect(parsed_results.get("@timestamp").to_i).to eq LogStash::Timestamp.at(1467972040).to_i } + end + end + + context "when both start_timestamp and timestamp are set it uses timestamp" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"start_timestamp\":1467972040073786262,\"timestamp\":1467972050073786262}" + ) do + it { expect(parsed_results.get("@timestamp").to_i).to eq LogStash::Timestamp.at(1467972050).to_i } + end + end + end + + describe "mutates 'msg'" do + + context "when it contains unicode (null)" do + when_parsing_log( + "@index_type" => "app", + "@message" => "{\"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_org_name\":\"admin\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"cf_space_name\":\"demo\",\"event_type\":\"LogMessage\",\"level\":\"info\",\"message_type\":\"OUT\"," + + "\"msg\":\"\\u0000\\u0000Some Message\"," + # contains unicode \u0000 + "\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + # message (unicode characters were removed) + it { expect(parsed_results.get("@message")).to eq "Some Message" } + + end + end + + context "when it contains unicode (new line)" do + when_parsing_log( + "@index_type" => "app", + "@message" => "{\"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_org_name\":\"admin\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"cf_space_name\":\"demo\",\"event_type\":\"LogMessage\",\"level\":\"info\",\"message_type\":\"OUT\"," + + "\"msg\":\"Some Message\\u2028New line\"," + # contains unicode \u2028 + "\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + # message (unicode characters were replaced with \n) + it { expect(parsed_results.get("@message")).to eq "Some Message +New line" } + + end + end + + end + + describe "sets @type" do + + context "when event_type is missing" do + when_parsing_log( + "@index_type" => "app", + # event_type property is missing from JSON + "@message" => "{\"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_org_name\":\"admin\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"cf_space_name\":\"demo\",\"level\":\"info\",\"message_type\":\"OUT\",\"msg\":\"Some Message\",\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + it { expect(parsed_results.get("@type")).to eq "UnknownEvent" } # is set to default value + + end + end + + context "when event_type is passed" do + when_parsing_log( + "@index_type" => "app", + # event_type is passed + "@message" => "{\"event_type\":\"some event type value\", \"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_org_name\":\"admin\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"cf_space_name\":\"demo\",\"level\":\"info\",\"message_type\":\"OUT\",\"msg\":\"Some Message\",\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + it { expect(parsed_results.get("@type")).to eq "some event type value" } # is set from event_type + + end + end + + end + + describe "sets [@source][type] when event is" do + + context "LogMessage" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"LogMessage\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "LOG" } + end + end + + context "Error" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"Error\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "ERR" } + end + end + + context "ContainerMetric" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"ContainerMetric\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "CONTAINER" } + end + end + + context "ValueMetric" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"ValueMetric\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "METRIC" } + end + end + + context "CounterEvent" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"CounterEvent\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "COUNT" } + end + end + + context "HttpStartStop" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"HttpStartStop\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "HTTP" } + end + end + + context "Unknown" do + when_parsing_log( "@index_type" => "app", + "@message" => "{\"event_type\":\"Some unknown event type\", \"msg\":\"some message\"}" ) do + + it { expect(parsed_results.get("@source")["type"]).to eq "NA" } + end + end + + end + + describe "sets index name" do + + context "when no cf_org_name" do + when_parsing_log( + "@index_type" => "app", + "@metadata" => {"index" => "app"}, + # cf_org_name property is missing from JSON + "@message" => "{\"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"cf_space_name\":\"demo\",\"event_type\":\"LogMessage\",\"level\":\"info\",\"message_type\":\"OUT\",\"msg\":\"Some Message\",\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + # index name doesn't include neither org nor space + it { expect(parsed_results.get("@cf")["org"]).to be_nil } + it { expect(parsed_results.get("@metadata")["index"]).to eq "app" } + + end + end + + context "when no cf_space_name" do + when_parsing_log( + "@index_type" => "app", + "@metadata" => {"index" => "app"}, + # cf_space_name property is missing from JSON + "@message" => "{\"cf_app_id\":\"31b928ee-4110-4e7b-996c-334c5d7ac2ac\",\"cf_app_name\":\"loggenerator\",\"cf_org_id\":\"9887ad0a-f9f7-449e-8982-76307bd17239\",\"cf_org_name\":\"admin\",\"cf_origin\":\"firehose\",\"cf_space_id\":\"59cf41f2-3a1d-42db-88e7-9540b02945e8\",\"event_type\":\"LogMessage\",\"level\":\"info\",\"message_type\":\"OUT\",\"msg\":\"Some Message\",\"origin\":\"dea_logging_agent\",\"source_instance\":\"0\",\"source_type\":\"APP\",\"time\":\"2016-07-08T10:00:40Z\",\"timestamp\":1467972040073786262}" + ) do + + # index name includes org + it { expect(parsed_results.get("@cf")["space"]).to be_nil } + it { expect(parsed_results.get("@metadata")["index"]).to eq "app-admin" } + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/app_valuemetric_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/app_valuemetric_spec.rb new file mode 100644 index 00000000..9f790fab --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/app_valuemetric_spec.rb @@ -0,0 +1,47 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "app-valuemetric.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/app-valuemetric.conf")} + } + CONFIG + end + + describe "#if failed" do + when_parsing_log( + "@type" => "some type", # bad type + "parsed_json_field" => { "field" => "value" }, + "@message" => "some message" + ) do + + # tag is NOT set + it { expect(parsed_results.get("tags")).to be_nil } + + end + end + + # -- general case + describe "#fields" do + when_parsing_log( + "@type" => "ValueMetric", + "parsed_json_field" => { "name" => "abc", "value" => 123.456, "unit" => "def" }, + "@message" => "some message" + ) do + + it { expect(parsed_results.get("tags")).to eq ["valuemetric"] } + + it { expect(parsed_results.get("@message")).to eq "abc = 123.456 (def)" } + it { expect(parsed_results.get("parsed_json_field")["name"]).to eq "abc" } + it { expect(parsed_results.get("parsed_json_field")["value"]).to eq 123.456 } + it { expect(parsed_results.get("parsed_json_field")["unit"]).to eq "def" } + + it { expect(parsed_results.get("@type")).to eq "ValueMetric" } # keeps unchanged + + end + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/platform_cloud_controller_ng_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/platform_cloud_controller_ng_spec.rb new file mode 100644 index 00000000..d00993e6 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/platform_cloud_controller_ng_spec.rb @@ -0,0 +1,95 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "platform-cloud_controller_ng.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/platform-cloud_controller_ng.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@source" => {"component" => "cloud_controller_ng"}, # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(parsed_results.get("tags")).to include "cloud_controller_ng" } + + end + end + + describe "failed" do + when_parsing_log( + "@source" => {"component" => "some value"}, # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(parsed_results.get("tags")).to be_nil } + + it { expect(parsed_results.get("@type")).to be_nil } # keeps unchanged + it { expect(parsed_results.get("@source")["component"]).to eq "some value" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when message is" do + context "request format" do + when_parsing_log( + "@source" => {"component" => "cloud_controller_ng"}, + # cloud_controller_ng event + "@message" => '10.10.128.8 - [17/Aug/2019:11:59:22 +0000] "GET /healthz HTTP/1.1" 200 248 "-" "ccng_monit_http_healthcheck" 10.10.128.8 vcap_request_id:f7c00c99-052c-41ec-9c7d-89c2826dcc60 response_time:0.008' + ) do + + it { expect(parsed_results.get("tags")).to eq ["cloud_controller_ng"] } # cloud_controller_ng tag, no fail tag + + it { expect(parsed_results.get("@type")).to eq "cloud_controller_ng" } + it { expect(parsed_results.get("@source")["component"]).to eq "cloud_controller_ng" } # keeps unchanged + + it "sets [cloud_controller_ng] fields from grok" do + expect(parsed_results.get("Request_Method")).to eq "GET" + expect(parsed_results.get("Request_Host")).to eq "10.10.128.8" + expect(parsed_results.get("Request_URL")).to eq "/healthz" + expect(parsed_results.get("Request_Protocol")).to eq "HTTP/1.1" + expect(parsed_results.get("Status_Code")).to eq 200 + expect(parsed_results.get("Bytes_Received")).to eq 248 + expect(parsed_results.get("Referer")).to eq "-" + expect(parsed_results.get("User_Agent")).to eq "ccng_monit_http_healthcheck" + expect(parsed_results.get("Backend_Address")).to eq "10.10.128.8" + expect(parsed_results.get("X_Vcap_Request_ID")).to eq "f7c00c99-052c-41ec-9c7d-89c2826dcc60" + expect(parsed_results.get("Response_Time")).to eq "0.008" + end + + end + end + + context "Unknown format" do + when_parsing_log( + "@source" => {"component" => "cloud_controller_ng"}, + "@message" => "Some message" + ) do + + # parsing error + it { expect(parsed_results.get("tags")).to eq ["cloud_controller_ng", "fail/cloudfoundry/platform-cloud_controller_ng/grok"] } # no fail tag + + it { expect(parsed_results.get("@type")).to eq "cloud_controller_ng" } + it { expect(parsed_results.get("@source")["component"]).to eq "cloud_controller_ng" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/platform_haproxy_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/platform_haproxy_spec.rb new file mode 100644 index 00000000..9c9a416b --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/platform_haproxy_spec.rb @@ -0,0 +1,171 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "platform-haproxy.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/platform-haproxy.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@source" => {"component" => "haproxy"}, # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(parsed_results.get("tags")).to include "haproxy" } + + end + end + + describe "failed" do + when_parsing_log( + "@source" => {"component" => "some value"}, # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(parsed_results.get("tags")).to be_nil } + + it { expect(parsed_results.get("@type")).to be_nil } # keeps unchanged + it { expect(parsed_results.get("@source")["component"]).to eq "some value" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when message is" do + context "Http format" do + when_parsing_log( + "@source" => {"component" => "haproxy"}, + # http format + "@message" => "64.78.155.208:60677 [06/Jul/2016:13:59:57.770] https-in~ http-routers/node0 59841/0/0/157/60000 200 144206 reqC respC ---- 3/4/1/2/0 5/6 {reqHeaders} {respHeaders} \"GET /v2/apps?inline-relations-depth=2 HTTP/1.1\"" + ) do + + it { expect(parsed_results.get("tags")).to eq ["haproxy"] } # haproxy tag, no fail tag + + it { expect(parsed_results.get("@type")).to eq "haproxy" } + it { expect(parsed_results.get("@source")["component"]).to eq "haproxy" } # keeps unchanged + + it "sets [haproxy] fields from grok" do + expect(parsed_results.get("haproxy")["client_ip"]).to eq "64.78.155.208" + expect(parsed_results.get("haproxy")["client_port"]).to eq 60677 + expect(parsed_results.get("haproxy")["accept_date"]).to eq "06/Jul/2016:13:59:57.770" + expect(parsed_results.get("haproxy")["frontend_name"]).to eq "https-in~" + expect(parsed_results.get("haproxy")["backend_name"]).to eq "http-routers" + expect(parsed_results.get("haproxy")["server_name"]).to eq "node0" + expect(parsed_results.get("haproxy")["time_request"]).to eq 59841 + expect(parsed_results.get("haproxy")["time_queue"]).to eq 0 + expect(parsed_results.get("haproxy")["time_backend_connect"]).to eq 0 + expect(parsed_results.get("haproxy")["time_backend_response"]).to eq 157 + expect(parsed_results.get("haproxy")["time_duration"]).to eq 60000 + expect(parsed_results.get("haproxy")["http_status_code"]).to eq 200 + expect(parsed_results.get("haproxy")["bytes_read"]).to eq 144206 + expect(parsed_results.get("haproxy")["captured_request_cookie"]).to eq "reqC" + expect(parsed_results.get("haproxy")["captured_response_cookie"]).to eq "respC" + expect(parsed_results.get("haproxy")["termination_state"]).to eq "----" + expect(parsed_results.get("haproxy")["actconn"]).to eq 3 + expect(parsed_results.get("haproxy")["feconn"]).to eq 4 + expect(parsed_results.get("haproxy")["beconn"]).to eq 1 + expect(parsed_results.get("haproxy")["srvconn"]).to eq 2 + expect(parsed_results.get("haproxy")["retries"]).to eq 0 + expect(parsed_results.get("haproxy")["srv_queue"]).to eq 5 + expect(parsed_results.get("haproxy")["backend_queue"]).to eq 6 + expect(parsed_results.get("haproxy")["captured_request_headers"]).to eq "reqHeaders" + expect(parsed_results.get("haproxy")["captured_response_headers"]).to eq "respHeaders" + expect(parsed_results.get("haproxy")["http_request"]).to eq "GET /v2/apps?inline-relations-depth=2 HTTP/1.1" + expect(parsed_results.get("haproxy")["http_request_verb"]).to eq "GET" + end + + it { expect(parsed_results.get("@message")).to eq "GET /v2/apps?inline-relations-depth=2 HTTP/1.1" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + end + end + + context "Http format ()" do + when_parsing_log( + "@source" => {"component" => "haproxy"}, + # http format + "@message" => "64.78.155.208:60677 [06/Jul/2016:13:59:57.770] https-in~ http-routers/node0 59841/0/0/157/60000 " + + "200 144206 reqC respC ---- 3/4/1/2/0 5/6 {reqHeaders} {respHeaders} \"\"" # + ) do + + it "sets [haproxy][request] to " do + expect(parsed_results.get("haproxy")["http_request"]).to eq "" + expect(parsed_results.get("haproxy")["http_request_verb"]).to be_nil + end + + end + end + + context "Http format (ERROR level)" do + when_parsing_log( + "@source" => {"component" => "haproxy"}, + "@message" => "64.78.155.208:60677 [06/Jul/2016:13:59:57.770] https-in~ http-routers/node0 59841/0/0/157/60000 " + + "400" + # http status = 400 => ERROR level + " 144206 reqC respC ---- 3/4/1/2/0 5/6 {reqHeaders} {respHeaders} \"GET /v2/apps?inline-relations-depth=2 HTTP/1.1\"" + ) do + + # @level is set based on [haproxy][http_status_code] + it { expect(parsed_results.get("@level")).to eq "ERROR" } + + end + end + + context "Error log format" do + when_parsing_log( + "@source" => {"component" => "haproxy"}, + # error log + "@message" => "216.218.206.68:36743 [06/Jul/2016:07:16:34.605] https-in/1: SSL handshake failure" + ) do + + it { expect(parsed_results.get("tags")).to eq ["haproxy"] } # haproxy tag, no fail tag + + it { expect(parsed_results.get("@type")).to eq "haproxy" } + it { expect(parsed_results.get("@source")["component"]).to eq "haproxy" } # keeps unchanged + + it "sets [haproxy] fields from grok" do + expect(parsed_results.get("haproxy")["client_ip"]).to eq "216.218.206.68" + expect(parsed_results.get("haproxy")["client_port"]).to eq 36743 + expect(parsed_results.get("haproxy")["accept_date"]).to eq "06/Jul/2016:07:16:34.605" + expect(parsed_results.get("haproxy")["frontend_name"]).to eq "https-in" + expect(parsed_results.get("haproxy")["bind_name"]).to eq "1" + end + + it { expect(parsed_results.get("@message")).to eq "SSL handshake failure" } + + end + end + + context "Unknown format" do + when_parsing_log( + "@source" => {"component" => "haproxy"}, + # error log + "@message" => "Some message" + ) do + + # parsing error + it { expect(parsed_results.get("tags")).to eq ["haproxy", "fail/cloudfoundry/platform-haproxy/grok"] } # no fail tag + + it { expect(parsed_results.get("@type")).to eq "haproxy" } + it { expect(parsed_results.get("@source")["component"]).to eq "haproxy" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + it { expect(parsed_results.get("haproxy")).to be_nil } + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/platform_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/platform_spec.rb new file mode 100644 index 00000000..a7c267f4 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/platform_spec.rb @@ -0,0 +1,161 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "platform.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/platform.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@index_type" => "platform", # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(parsed_results.get("tags")).to include "platform" } + + end + end + + describe "failed" do + when_parsing_log( + "@index_type" => "some value", # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(parsed_results.get("tags")).to be_nil } + + it { expect(parsed_results.get("@index_type")).to eq "some value" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when message is" do + + context "CF format (metron agent)" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "[job=nfs_z1 index=0] Some message" # CF metron agent format + ) do + + it { expect(parsed_results.get("tags")).to eq ["platform", "cf"] } # no fail tag + + it { expect(parsed_results.get("@source")["type"]).to eq "cf" } + it { expect(parsed_results.get("@type")).to eq "cf" } + + it "sets grok fields" do + expect(parsed_results.get("@message")).to eq "Some message" + expect(parsed_results.get("@source")["job"]).to eq "nfs_z1" + expect(parsed_results.get("@source")["index"]).to eq 0 + end + + end + end + + context "CF format (syslog release)" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "[bosh instance=cf_full/nfs_z1/abcdefg123] Some message" # CF syslog release format + ) do + + it { expect(parsed_results.get("tags")).to eq ["platform", "cf"] } # no fail tag + + it { expect(parsed_results.get("@source")["type"]).to eq "cf" } + it { expect(parsed_results.get("@type")).to eq "cf" } + + it "sets grok fields" do + expect(parsed_results.get("@message")).to eq "Some message" + expect(parsed_results.get("@source")["deployment"]).to eq "cf_full" + expect(parsed_results.get("@source")["job"]).to eq "nfs_z1" + expect(parsed_results.get("@source")["job_index"]).to eq "abcdefg123" + end + + end + end + + + context "RFC 5424 format and enterprise number is CF" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "Some message", + "syslog_sd_id" => "instance@47450", + "syslog_sd_params" => { + "az" => "az1", + "deployment" => "deployment1", + "director" => "director1", + "group" => "group1", + "id" => "id1", + }, + ) do + + it { expect(parsed_results.get("tags")).to eq ["platform", "cf"] } # no fail tag + + it { expect(parsed_results.get("@source")["type"]).to eq "cf" } + it { expect(parsed_results.get("@type")).to eq "cf" } + + it "sets the common fields" do + expect(parsed_results.get("@message")).to eq "Some message" + expect(parsed_results.get("@source")["az"]).to eq "az1" + expect(parsed_results.get("@source")["deployment"]).to eq "deployment1" + expect(parsed_results.get("@source")["director"]).to eq "director1" + expect(parsed_results.get("@source")["id"]).to eq "id1" + expect(parsed_results.get("@source")["job"]).to eq "group1" + end + end + end + + context "empty BOSH director name" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "Some message", + "syslog_sd_id" => "instance@47450", + "syslog_sd_params" => { + "director" => "", + }, + ) do + + it { expect(parsed_results.get("tags")).to eq ["platform", "cf"] } # no fail tag + + it { expect(parsed_results.get("@source")["type"]).to eq "cf" } + it { expect(parsed_results.get("@type")).to eq "cf" } + + it "sets the common fields" do + expect(parsed_results.get("@source")["director"]).to eq "" + end + end + end + + context "not CF format" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "Some message that fails grok" # bad format + ) do + + # get parsing error + it { expect(parsed_results.get("tags")).to eq ["platform", "fail/cloudfoundry/platform/grok"] } + + # no fields set + it { expect(parsed_results.get("@message")).to eq "Some message that fails grok" } # keeps the same + it { expect(parsed_results.get("@index_type")).to eq "platform" } # keeps the same + it { expect(parsed_results.get("@source")["type"]).to eq "system" } + it { expect(parsed_results.get("@type")).to be_nil } + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/platform_uaa_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/platform_uaa_spec.rb new file mode 100644 index 00000000..b08838c8 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/platform_uaa_spec.rb @@ -0,0 +1,199 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "platform-uaa.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/platform-uaa.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@source" => {"component" => "vcap.uaa"}, # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(parsed_results.get("tags")).to include "uaa" } + + end + end + + describe "failed" do + when_parsing_log( + "@source" => {"component" => "some value"}, # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(parsed_results.get("tags")).to be_nil } + + it { expect(parsed_results.get("@type")).to be_nil } # keeps unchanged + it { expect(parsed_results.get("@source")["component"]).to eq "some value" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when message is" do + + context "UAA" do + context "(general event)" do + when_parsing_log( + "@source" => {"component" => "vcap.uaa"}, + # general UAA event + "@message" => "[2016-07-05 04:02:18.245] uaa - 15178 [http-bio-8080-exec-14] .... DEBUG --- FilterChainProxy: /healthz has an empty filter list" + ) do + + it { expect(parsed_results.get("tags")).to eq ["uaa"] } # uaa tag, no fail tag + it { expect(parsed_results.get("@type")).to eq "uaa" } + it { expect(parsed_results.get("@source")["component"]).to eq "uaa" } + + it { expect(parsed_results.get("@message")).to eq "/healthz has an empty filter list" } + it { expect(parsed_results.get("@level")).to eq "DEBUG" } + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-05 04:02:18.245" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-14" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "FilterChainProxy" + end + + end + end + + context "(bad format)" do + when_parsing_log( + "@source" => {"component" => "vcap.uaa"}, + "@message" => "Some message" # bad format + ) do + + # get parsing error + it { expect(parsed_results.get("tags")).to eq ["uaa", "fail/cloudfoundry/platform-uaa/grok"] } + it { expect(parsed_results.get("@type")).to eq "uaa" } + it { expect(parsed_results.get("@source")["component"]).to eq "uaa" } + + it { expect(parsed_results.get("@message")).to eq "Some message" } # the same as before parsing + + it { expect(parsed_results.get("uaa")).to be_nil } + + end + end + end + + context "UAA Audit" do + context "(general event)" do + when_parsing_log( + "@source" => {"component" => "vcap.uaa"}, + # general UAA event + "@message" => "[2016-07-05 04:02:18.245] uaa - 15178 [http-bio-8080-exec-14] .... INFO --- Audit: ClientAuthenticationSuccess ('Client authentication success'): principal=cf, origin=[remoteAddress=64.78.155.208, clientId=cf], identityZoneId=[uaa]" + ) do + + it { expect(parsed_results.get("tags")).to eq ["uaa", "audit"] } # uaa tag, audit tag, no fail tag + it { expect(parsed_results.get("@type")).to eq "uaa-audit" } + it { expect(parsed_results.get("@source")["component"]).to eq "uaa" } + + it { expect(parsed_results.get("@message")).to eq "ClientAuthenticationSuccess ('Client authentication success')" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-05 04:02:18.245" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-14" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "Audit" + end + + it "sets [uaa][audit] fields" do + expect(parsed_results.get("uaa")["audit"]["type"]).to eq "ClientAuthenticationSuccess" + expect(parsed_results.get("uaa")["audit"]["data"]).to eq "Client authentication success" + expect(parsed_results.get("uaa")["audit"]["principal"]).to eq "cf" + expect(parsed_results.get("uaa")["audit"]["origin"]).to eq ["remoteAddress=64.78.155.208", "clientId=cf"] + expect(parsed_results.get("uaa")["audit"]["identity_zone_id"]).to eq "uaa" + expect(parsed_results.get("uaa")["audit"]["remote_address"]).to eq "64.78.155.208" + end + + it "sets geoip for remoteAddress" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "64.78.155.208" + end + + end + end + + context "(PrincipalAuthFailure event)" do + when_parsing_log( + "@source" => {"component" => "vcap.uaa"}, + # PrincipalAuthFailure event + "@message" => "[2016-07-06 09:18:43.397] uaa - 15178 [http-bio-8080-exec-6] .... INFO --- Audit: " + + "PrincipalAuthenticationFailure ('null'): principal=admin, origin=[82.209.244.50], identityZoneId=[uaa]" + ) do + + it { expect(parsed_results.get("tags")).to eq ["uaa", "audit"] } # uaa tag, audit tag, no fail tag + it { expect(parsed_results.get("@type")).to eq "uaa-audit" } + it { expect(parsed_results.get("@source")["component"]).to eq "uaa" } + + it { expect(parsed_results.get("@message")).to eq "PrincipalAuthenticationFailure ('null')" } + it { expect(parsed_results.get("@level")).to eq "INFO" } + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-06 09:18:43.397" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-6" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "Audit" + end + + it "sets [uaa][audit] fields" do + expect(parsed_results.get("uaa")["audit"]["type"]).to eq "PrincipalAuthenticationFailure" + expect(parsed_results.get("uaa")["audit"]["data"]).to eq "null" + expect(parsed_results.get("uaa")["audit"]["principal"]).to eq "admin" + expect(parsed_results.get("uaa")["audit"]["origin"]).to eq ["82.209.244.50"] + expect(parsed_results.get("uaa")["audit"]["identity_zone_id"]).to eq "uaa" + expect(parsed_results.get("uaa")["audit"]["remote_address"]).to eq "82.209.244.50" + end + + it "sets geoip for remoteAddress" do + expect(parsed_results.get("geoip")).not_to be_nil + expect(parsed_results.get("geoip")["ip"]).to eq "82.209.244.50" + end + + end + end + + context "(bad format)" do + when_parsing_log( + "@source" => {"component" => "vcap.uaa"}, + "@message" => "[2016-07-06 09:18:43.397] uaa - 15178 [http-bio-8080-exec-6] .... INFO --- Audit: Some message" # bad format + ) do + + # get parsing error + it { expect(parsed_results.get("tags")).to eq ["uaa", "audit", "fail/cloudfoundry/platform-uaa/audit/grok"] } + it { expect(parsed_results.get("@type")).to eq "uaa-audit" } + it { expect(parsed_results.get("@source")["component"]).to eq "uaa" } + + it { expect(parsed_results.get("@message")).to eq "Some message" } + + it "sets [uaa] fields" do + expect(parsed_results.get("uaa")["timestamp"]).to eq "2016-07-06 09:18:43.397" + expect(parsed_results.get("uaa")["thread"]).to eq "http-bio-8080-exec-6" + expect(parsed_results.get("uaa")["pid"]).to eq 15178 + expect(parsed_results.get("uaa")["log_category"]).to eq "Audit" + end + + it { expect(parsed_results.get("uaa")["audit"]).to be_nil } + + end + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/platform_vcap_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/platform_vcap_spec.rb new file mode 100644 index 00000000..2e759096 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/platform_vcap_spec.rb @@ -0,0 +1,192 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "platform-vcap.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/platform-vcap.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@source" => {"component" => "vcap.some_component"}, # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(parsed_results.get("tags")).to include "vcap" } + + end + end + + describe "failed" do + when_parsing_log( + "@source" => {"component" => "some value"}, # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(parsed_results.get("tags")).to be_nil } + + it { expect(parsed_results.get("@type")).to be_nil } # keeps the same + it { expect(parsed_results.get("@source")["component"]).to eq "some value" } # keeps unchanged + it { expect(parsed_results.get("@message")).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when message is" do + + context "plain-text format" do + when_parsing_log( + "@source" => {"component" => "vcap.consul-agent"}, + "@level" => "Dummy level", + # plain-text format + "@message" => "2016/07/07 00:56:10 [WARN] agent: Check 'service:routing-api' is now critical" + ) do + + it { expect(parsed_results.get("tags")).to eq ["vcap"] } # vcap tag, no fail tag + + it { expect(parsed_results.get("@type")).to eq "vcap" } + + it { expect(parsed_results.get("@source")["component"]).to eq "consul-agent" } + + it { expect(parsed_results.get("@message")) + .to eq "2016/07/07 00:56:10 [WARN] agent: Check 'service:routing-api' is now critical" } # keeps the same value + it { expect(parsed_results.get("@level")).to eq "Dummy level" } # keeps the same + + it { expect(parsed_results.get("parsed_json_field")).to be_nil } # no json fields + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } # no json fields + + end + end + + context "JSON format" do + when_parsing_log( + "@source"=> { "component" => "vcap.nats" }, + # JSON format + "@message" => "{\"timestamp\":1467852972.554088,\"source\":\"NatsStreamForwarder\",\"log_level\":\"info\",\"message\":\"router.register\",\"data\":{\"nats_message\": \"{\\\"uris\\\":[\\\"redis-broker.64.78.234.207.xip.io\\\"],\\\"host\\\":\\\"192.168.111.201\\\",\\\"port\\\":80}\",\"reply_inbox\":\"_INBOX.7e93f2a1d5115844163cc930b5\"}}" + ) do + + # no parsing errors + it { expect(parsed_results.get("tags")).to eq ["vcap"] } # vcap tag, no fail tag + + it { expect(parsed_results.get("@type")).to eq "vcap" } + + it { expect(parsed_results.get("@source")["component"]).to eq "nats" } + + it "sets JSON fields" do + expect(parsed_results.get("parsed_json_field")).not_to be_nil + expect(parsed_results.get("parsed_json_field")["timestamp"].to_f).to eq 1467852972.554088 + expect(parsed_results.get("parsed_json_field")["source"]).to eq "NatsStreamForwarder" + expect(parsed_results.get("parsed_json_field")["data"]["nats_message"]).to eq "{\"uris\":[\"redis-broker.64.78.234.207.xip.io\"],\"host\":\"192.168.111.201\",\"port\":80}" + expect(parsed_results.get("parsed_json_field")["data"]["reply_inbox"]).to eq "_INBOX.7e93f2a1d5115844163cc930b5" + expect(parsed_results.get("parsed_json_field_name")).to eq "nats" # set from @source.component + end + + it "sets @message from JSON" do + expect(parsed_results.get("@message")).to eq "router.register" + expect(parsed_results.get("parsed_json_field")["message"]).to be_nil + end + + it "sets @level from JSON" do + expect(parsed_results.get("@level")).to eq "info" + expect(parsed_results.get("parsed_json_field")["log_level"]).to be_nil + end + + end + end + + context "JSON format (invalid)" do + when_parsing_log( + "@source" => { "component" => "vcap.nats" }, + "@level" => "Dummy value", + # JSON format + "@message" => "{\"timestamp\":14678, abcd}}" # invalid JSON + ) do + + # parsing error + it { expect(parsed_results.get("tags")).to eq ["vcap", "fail/cloudfoundry/platform-vcap/json"] } + + it { expect(parsed_results.get("@type")).to eq "vcap" } + it { expect(parsed_results.get("@message")).to eq "{\"timestamp\":14678, abcd}}" } # keeps unchanged + it { expect(parsed_results.get("@source")["component"]).to eq "nats" } # keeps unchanged + it { expect(parsed_results.get("@level")).to eq "Dummy value" } # keeps unchanged + it { expect(parsed_results.get("parsed_json_field")).to be_nil } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + + end + end + + + end + + describe "#level translate numeric" do + + context "(DEBUG)" do + when_parsing_log( + "@source" => {"component" => "vcap.dummy"}, + "@message" => "{\"log_level\":0}" + ) do + + it { expect(parsed_results.get("@level")).to eq "DEBUG" } # translated + + end + end + + context "(INFO)" do + when_parsing_log( + "@source" => {"component" => "vcap.dummy"}, + "@message" => "{\"log_level\":1}" + ) do + + it { expect(parsed_results.get("@level")).to eq "INFO" } # translated + + end + end + + context "(ERROR)" do + when_parsing_log( + "@source" => {"component" => "vcap.dummy"}, + "@message" => "{\"log_level\":2}" + ) do + + it { expect(parsed_results.get("@level")).to eq "ERROR" } # translated + + end + end + + context "(FATAL)" do + when_parsing_log( + "@source" => {"component" => "vcap.dummy"}, + "@message" => "{\"log_level\":3}" + ) do + + it { expect(parsed_results.get("@level")).to eq "FATAL" } # translated + + end + end + + context "(fallback)" do + when_parsing_log( + "@source" => {"component" => "vcap.dummy"}, + "@message" => "{\"log_level\":8}" # unknown log level + ) do + + it { expect(parsed_results.get("@level")).to eq "8" } # just converted to string + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/setup_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/setup_spec.rb new file mode 100644 index 00000000..1d01995b --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/setup_spec.rb @@ -0,0 +1,98 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "setup.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/setup.conf")} + } + CONFIG + end + + describe "when message" do + context "is useless (empty)" do + when_parsing_log( + "@message" => "" # empty + ) do + + # event is dropped + it { expect(parsed_results).to be_nil} + + end + end + + context "is useless (blank)" do + when_parsing_log( + "@message" => " " # blank + ) do + + # event is dropped + it { expect(parsed_results).to be_nil} + + end + end + + context "contains unicode (\u0000)" do + when_parsing_log( + "@message" => "a\u0000bc" # unicode + ) do + + # unicode removed + it { expect(parsed_results.get("@message")).to eq "abc" } + + end + end + + context "is OK" do + when_parsing_log( + "@type" => "some-type", + "syslog_program" => "some-program", + "syslog_pri" => 5, + "@message" => "Some message" # OK + ) do + + # fields + it { expect(parsed_results.get("@index_type")).to eq "platform" } + it { expect(parsed_results.get("@metadata")["index"]).to eq "platform" } + it { expect(parsed_results.get("@input")).to eq "some-type" } + it { expect(parsed_results.get("@shipper")["priority"]).to eq 5 } + it { expect(parsed_results.get("@shipper")["name"]).to eq "some-program_some-type" } + it { expect(parsed_results.get("@source")["component"]).to eq "some-program" } + + end + end + + end + + describe "when index is " do + context "app" do + when_parsing_log( + "syslog_program" => "doppler", # app logs + "@message" => "Some message" + ) do + + # fields + it { expect(parsed_results.get("@index_type")).to eq "app" } + it { expect(parsed_results.get("@metadata")["index"]).to eq "app" } + + end + end + + context "platform" do + when_parsing_log( + "syslog_program" => "not doppler", # platform logs + "@message" => "Some message" + ) do + + # fields + it { expect(parsed_results.get("@index_type")).to eq "platform" } + it { expect(parsed_results.get("@metadata")["index"]).to eq "platform" } + + end + end + + end + +end diff --git a/src/logsearch-filters/spec/logstash-filters/snippets/teardown_spec.rb b/src/logsearch-filters/spec/logstash-filters/snippets/teardown_spec.rb new file mode 100644 index 00000000..19dc9596 --- /dev/null +++ b/src/logsearch-filters/spec/logstash-filters/snippets/teardown_spec.rb @@ -0,0 +1,204 @@ +# encoding: utf-8 +require 'spec_helper' + +describe "teardown.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/teardown.conf")} + } + CONFIG + end + + describe "sets @level based on [syslog_severity_code]" do + + context "([syslog_severity_code] = 2)" do + when_parsing_log( "syslog_severity_code" => 2 ) do + it { expect(parsed_results.get("@level]")).to eq "ERROR" } + end + end + + context "([syslog_severity_code] = 3)" do + when_parsing_log( "syslog_severity_code" => 3 ) do + it { expect(parsed_results.get("@level")).to eq "ERROR" } + end + end + + context "([syslog_severity_code] = 4)" do + when_parsing_log( "syslog_severity_code" => 4 ) do + it { expect(parsed_results.get("@level")).to eq "WARN" } + end + end + + context "([syslog_severity_code] = 5)" do + when_parsing_log( "syslog_severity_code" => 5 ) do + it { expect(parsed_results.get("@level")).to eq "WARN" } + end + end + + context "([syslog_severity_code] = 6)" do + when_parsing_log( "syslog_severity_code" => 6 ) do + it { expect(parsed_results.get("@level")).to eq "INFO" } + end + end + + context "([syslog_severity_code] = 7)" do + when_parsing_log( "syslog_severity_code" => 7 ) do + it { expect(parsed_results.get("@level")).to eq "DEBUG" } + end + end + + context "([syslog_severity_code] = 8)" do + when_parsing_log( "syslog_severity_code" => 8 ) do + it { expect(parsed_results.get("@level")).to be_nil } + end + end + + context "(no [syslog_severity_code])" do + when_parsing_log( "some_field" => "some_value" ) do + it { expect(parsed_results.get("@level")).to be_nil } + end + end + + end + + describe "sets [@source][vm]" do + + context "when [@source]* fields are set" do + when_parsing_log( + "@source" => {"job" => "Abc", "index" => 123} + ) do + it { expect(parsed_results.get("@source")["vm"]).to eq "Abc/123" } + end + end + + context "when [@source][job] is missing" do + when_parsing_log( + "@source" => {"instance" => 123} + ) do + it { expect(parsed_results.get("@source")["vm"]).to be_nil } + end + end + + context "when [@source][index] is missing" do + when_parsing_log( + "@source" => {"job" => "Abc"} + ) do + it { expect(parsed_results.get("@source")["vm"]).to be_nil } + end + end + + context "when [@source]* fields are missing" do + when_parsing_log( + "@source" => {"some useless field" => "Abc"} + ) do + it { expect(parsed_results.get("@source")["vm"]).to be_nil } + end + end + + end + + describe "parses [host]" do + + context "when [@source][host] is set" do + when_parsing_log( + "host" => "1.2.3.4", + "@source" => {"host" => "5.6.7.8"} + ) do + it { expect(parsed_results.get("@source")["host"]).to eq "5.6.7.8" } + it { expect(parsed_results.get("host")).to be_nil } + end + end + + context "when [@source][host] is NOT set" do + when_parsing_log( + "host" => "1.2.3.4" + ) do + it { expect(parsed_results.get("@source")["host"]).to eq "1.2.3.4" } + it { expect(parsed_results.get("host")).to be_nil } + end + end + + end + + describe "renames [parsed_json_field]" do + + describe "when [parsed_json_field] and [parsed_json_field_name] are set" do + when_parsing_log( + "parsed_json_field" => "dummy value", + "parsed_json_field_name" => "Abc-defg.hI?jk#lm NOPQ" + ) do + # [parsed_json_field] renamed + it { expect(parsed_results.get("parsed_json_field")).to be_nil } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + it { expect(parsed_results.get("abc_defg_hi_jk_lm_nopq")).to eq "dummy value" } # renamed + end + end + + context "when [parsed_json_field] is NOT set" do + when_parsing_log( + "parsed_json_field_name" => "Abc-defg.hI?jk#lm NOPQ" + ) do + # nothing is set + it { expect(parsed_results.get("parsed_json_field")).to be_nil } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + it { expect(parsed_results.get("abc_defg_hi_jk_lm_nopq")).to be_nil } + end + end + + context "when [parsed_json_field_name] is NOT set" do + when_parsing_log( + "parsed_json_field" => "dummy value" + ) do + # keep [parsed_json_field] + it { expect(parsed_results.get("parsed_json_field")).to eq "dummy value" } + it { expect(parsed_results.get("parsed_json_field_name")).to be_nil } + end + end + + end + + describe "cleanup" do + + when_parsing_log( + "syslog_pri" => "abc", + "syslog_facility" => "def", + "syslog_facility_code" => "ghi", + "syslog_message" => "jkl", + "syslog_severity" => "mno", + "syslog_severity_code" => "pqr", + "syslog_program" => "stu", + "syslog_timestamp" => "vw", + "syslog_hostname" => "xy", + "syslog_pid" => "z", + "@level" => "lowercase value", + "@version" => "some version", + "host" => "1.2.3.4", + "_logstash_input" => "abc" + ) do + + it "removes syslog_ fields" do + expect(parsed_results.get("syslog_pri")).to be_nil + expect(parsed_results.get("syslog_facility")).to be_nil + expect(parsed_results.get("syslog_facility_code")).to be_nil + expect(parsed_results.get("syslog_message")).to be_nil + expect(parsed_results.get("syslog_severity")).to be_nil + expect(parsed_results.get("syslog_severity_code")).to be_nil + expect(parsed_results.get("syslog_program")).to be_nil + expect(parsed_results.get("syslog_timestamp")).to be_nil + expect(parsed_results.get("syslog_hostname")).to be_nil + expect(parsed_results.get("syslog_pid")).to be_nil + end + + it { expect(parsed_results.get("@level")).to eq "LOWERCASE VALUE" } + + it { expect(parsed_results.get("@version")).to be_nil } + it { expect(parsed_results.get("host")).to be_nil } + it { expect(parsed_results.get("_logstash_input")).to be_nil } + + end + + end + +end diff --git a/src/logsearch-filters/spec/spec_helper.rb b/src/logsearch-filters/spec/spec_helper.rb new file mode 100644 index 00000000..d3fc0865 --- /dev/null +++ b/src/logsearch-filters/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) +require 'helpers/filters_helper' +require 'helpers/app_helper' +require 'helpers/platform_helper' + +RSpec.configure do |config| + config.extend Helpers::FilterHelper + config.include Helpers::FilterHelper + + config.extend Helpers::AppHelper + config.include Helpers::AppHelper + + config.extend Helpers::PlatformHelper + config.include Helpers::PlatformHelper +end diff --git a/src/logsearch-filters/src/logstash-filters/default.conf.erb b/src/logsearch-filters/src/logstash-filters/default.conf.erb new file mode 100644 index 00000000..c2bc8bd9 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/default.conf.erb @@ -0,0 +1,33 @@ +# NOTE: All parsed data should include @message, @level and @source.component. +# Otherwise these fields are set from syslog_ fields in teardown script afterwards. + +# NOTE: @timestamp for CF components logs is set in logsearch-boshrelease from syslog_timestamp (timestamp set by metron_agent). +# Timestamp set by metron_agent for Firehose logs is .timestamp field. (That's why app.conf snippet sets date with it). + +# Setup snippet (should precede all other snippets) +<%= File.read('src/logstash-filters/snippets/setup.conf') %> + +##-- App +# (App snippet should precede all other app snippets) +<%= File.read('src/logstash-filters/snippets/app.conf') %> +# special cases parsing +<%= File.read('src/logstash-filters/snippets/app-logmessage.conf') %> +<%= File.read('src/logstash-filters/snippets/app-logmessage-app.conf') %> +<%= File.read('src/logstash-filters/snippets/app-logmessage-rtr.conf') %> +<%= File.read('src/logstash-filters/snippets/app-error.conf') %> +<%= File.read('src/logstash-filters/snippets/app-containermetric.conf') %> +<%= File.read('src/logstash-filters/snippets/app-valuemetric.conf') %> +<%= File.read('src/logstash-filters/snippets/app-counterevent.conf') %> +<%= File.read('src/logstash-filters/snippets/app-http.conf') %> + +##-- Platform +# (Platform snippet should precede all other platform snippets) +<%= File.read('src/logstash-filters/snippets/platform.conf') %> +# special cases parsing +<%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> +<%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> +<%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> +<%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> + +# Teardown snippet (should follow all other snippets) +<%= File.read('src/logstash-filters/snippets/teardown.conf') %> diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-containermetric.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-containermetric.conf new file mode 100644 index 00000000..86934b1a --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-containermetric.conf @@ -0,0 +1,27 @@ +##------------------------------------------------------------------ +# Parses ContainerMetric message. | +# A ContainerMetric records resource usage of an app in a container.| +##------------------------------------------------------------------ +if( [@type] == "ContainerMetric" ) { + + mutate { + add_tag => [ "containermetric" ] + } + + # related application + if [@cf][app_id] and [@cf][app_id] != "" { + mutate { + rename => { "[parsed_json_field][instance_index]" => "[@cf][app_instance]" } + } + } else { + mutate { + remove_field => "[@cf][app_id]" + remove_field => "[parsed_json_field][instance_index]" + } + } + + # @message + mutate { + replace => {"@message" => "cpu=%{[parsed_json_field][cpu_percentage]}, memory=%{[parsed_json_field][memory_bytes]}, disk=%{[parsed_json_field][disk_bytes]}"} + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-counterevent.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-counterevent.conf new file mode 100644 index 00000000..d849aba9 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-counterevent.conf @@ -0,0 +1,12 @@ +##------------------------------------------------------ +# Parses CounterEvent message. | +# A CounterEvent represents the increment of a counter. | +##------------------------------------------------------ +if( [@type] == "CounterEvent" ) { + + mutate { + add_tag => [ "counterevent" ] + + replace => {"@message" => "%{[parsed_json_field][name]} (delta=%{[parsed_json_field][delta]}, total=%{[parsed_json_field][total]})"} + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-error.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-error.conf new file mode 100644 index 00000000..5f46eae8 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-error.conf @@ -0,0 +1,12 @@ +##--------------------------------------------------------------- +# Parses app Error events. | +# An Error event represents an error in the originating process. | +##--------------------------------------------------------------- +if( [@type] == "Error" ) { + + mutate { + add_tag => [ "error" ] + + rename => { "[parsed_json_field][message]" => "@message" } + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-http.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-http.conf new file mode 100644 index 00000000..7f6213d7 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-http.conf @@ -0,0 +1,24 @@ +##-------------------------------------------------------------------------- +# Parses HttpStartStop event. | +# An HttpStartStop event represents the whole lifecycle of an HTTP request. | +##-------------------------------------------------------------------------- +if( [@type] == "HttpStartStop" ) { + + mutate { + add_tag => [ "http" ] + } + + # Related application + if ![parsed_json_field][instance_id] or [parsed_json_field][instance_id] == "" { + mutate { + remove_field => "[parsed_json_field][instance_id]" + remove_field => "[parsed_json_field][instance_index]" + } + } + + # Set @message + mutate { + replace => {"@message" => "%{[parsed_json_field][status_code]} %{[parsed_json_field][method]} %{[parsed_json_field][uri]} (%{[parsed_json_field][duration_ms]} ms)"} + } + +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf new file mode 100644 index 00000000..a699c4c8 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf @@ -0,0 +1,82 @@ +##------------------------------ +## Parses LogMessage APP events.| +##------------------------------ + +if [@type] == "LogMessage" and [@source][type] =~ /APP(|\/.*)$/ { + + mutate { + add_tag => [ "logmessage-app" ] + } + + mutate { + # Firehose sets values like "APP/PROC/WEB". Rename just to "APP" for simplicity. + replace => { "[@source][type]" => "APP" } + } + + # Parse application logs based on msg format. + # Marks unknown format with [unknown_msg_format] tag. + + ## ---- Format 1: JSON + if [@message] =~ /^\s*{".*}\s*$/ { # if it looks like JSON + + json { + source => "@message" + target => "app" + id => "cloudfoundry/app-app/json" + } + + if !("_jsonparsefailure" in [tags]) { + + mutate { + rename => { "[app][message]" => "@message" } # @message + } + # concat message and exception + if [app][exception] { + mutate { + ## NOTE: keep line break and new line spacing (new line is inserted in logstash in such a way) + replace => { "@message" => "%{@message} +%{[app][exception]}" } + remove_field => [ "[app][exception]" ] + } + } + + mutate { + rename => { "[app][level]" => "@level" } # @level + } + + } else { + + mutate { + add_tag => [ "unknown_msg_format" ] + remove_tag => ["_jsonparsefailure"] + } + } + + ## ---- Format 2: "[CONTAINER] .." (Tomcat logs) + } else if [@message] =~ /^\s*\[CONTAINER\]/ { + + # Tomcat specific parsing (in accordance with https://github.com/cloudfoundry/java-buildpack-support/blob/master/tomcat-logging-support/src/main/java/com/gopivotal/cloudfoundry/tomcat/logging/CloudFoundryFormatter.java) + grok { + match => [ "@message", "(?\[CONTAINER\]%{SPACE}%{NOTSPACE})%{SPACE}%{LOGLEVEL:@level}%{SPACE}%{GREEDYDATA:@message}" ] + overwrite => [ "@message", "@level" ] + tag_on_failure => [ "unknown_msg_format" ] + id => "cloudfoundry/app-app/tomcat/grok" + } + mutate { + rename => { "app_logger" => "[app][logger]" } + } + + } else { + + ## ---- Format 3: Logback status logs + grok { + match => [ "@message", "%{TIME} \|\-%{LOGLEVEL:@level} in %{NOTSPACE:[app][logger]} - %{GREEDYDATA:@message}" ] + overwrite => [ "@message", "@level" ] + + ## ---- Unknown Format: if no formats succeeded set with 'unknown_msg_format' tag + tag_on_failure => [ "unknown_msg_format" ] + id => "cloudfoundry/app-app/logback/grok" + } + } + +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf new file mode 100644 index 00000000..d74ba7da --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf @@ -0,0 +1,112 @@ +##------------------------------ +## Parses LogMessage RTR events.| +##------------------------------ + +if ( [@type] == "LogMessage" and [@source][type] == "RTR" ) { + + mutate { + add_tag => [ "logmessage-rtr" ] + } + + grok { + match => [ "@message", [ + # cf-deployment v12.27.0+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", + # cf-deployment v12.17.0+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_time:%{NUMBER:[rtr][app_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int|-}\"", + # cf-release v252+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"", + # cf-release v250+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{HOSTPORT}|-)\" \"(%{HOSTPORT}|-)\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}", + # older + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" %{HOSTPORT} x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}" + ] + ] + id => "cloudfoundry/app-rtr/grok" + + tag_on_failure => [ "fail/cloudfoundry/app-rtr/grok" ] + } + + if !("fail/cloudfoundry/app-rtr/grok" in [tags]) { + + # Set [rtr][timestamp] + mutate { + rename => { "rtr_time" => "[rtr][timestamp]" } + } + + # Set [rtr][x_forwarded_for] + mutate { + gsub => ["[rtr][x_forwarded_for]","[\s\"]",""] # remove quotes and whitespace + split => ["[rtr][x_forwarded_for]", ","] # format is client, proxy1, proxy2 ... + } + + # Set [rtr][remote_addr] + mutate { + add_field => ["[rtr][remote_addr]", "%{[rtr][x_forwarded_for][0]}"] + } + if [rtr][remote_addr] =~ /([0-9]{1,3}\.){3}[0-9]{1,3}/ { + geoip { + source => "[rtr][remote_addr]" + } + } + + # Set [rtr][response_time_ms] + mutate { + add_field => { "[rtr][response_time_ms]" => "%{[rtr][response_time_sec]}000" } + } + mutate { + gsub => ["[rtr][response_time_ms]", "\.(\d)(\d)(\d)([\d]{0,3}).*","\1\2\3.\4"] + } + mutate { + convert => { "[rtr][response_time_ms]" => "float" } + } + + # Set [rtr][gorouter_time_ms] + if [rtr][gorouter_time_sec] =~ /.+/ { + mutate { + add_field => { "[rtr][gorouter_time_ms]" => "%{[rtr][gorouter_time_sec]}000" } + } + mutate { + gsub => ["[rtr][gorouter_time_ms]", "\.(\d)(\d)(\d)([\d]{0,3}).*","\1\2\3.\4"] + } + mutate { + convert => { "[rtr][gorouter_time_ms]" => "float" } + } + } + + # Process extra headers based on the following 2 properties available in CF-Deployment + # https://bosh.io/jobs/gorouter?source=github.com/cloudfoundry/routing-release&version=0.198.0#p%3drouter.extra_headers_to_log + # https://bosh.io/jobs/gorouter?source=github.com/cloudfoundry/routing-release&version=0.198.0#p%3drouter.tracing.enable_zipkin + if [kvpairs] =~ /.+/ { + kv { + source => "kvpairs" + remove_field => [ "kvpairs" ] + include_brackets => false + value_split => ":" + target => "[rtr][extra]" + tag_on_failure => "fail/cloudfoundry/app-rtr/kv" + } + } + + # Set @message + mutate { + replace => {"@message" => "%{[rtr][status]} %{[rtr][verb]} %{[rtr][path]} (%{[rtr][response_time_ms]} ms)"} + } + + # Set @level (based on HTTP status) + if [rtr][status] >= 500 { + mutate { + replace => { "@level" => "ERROR" } + } + } else if [rtr][status] >= 400 { + mutate { + replace => { "@level" => "WARN" } + } + } else { + mutate { + replace => { "@level" => "INFO" } + } + } + } + +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage.conf new file mode 100644 index 00000000..d5072f66 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage.conf @@ -0,0 +1,35 @@ +##------------------------------------------------------------ +# General parsing of LogMessage events. | +# A LogMessage contains a "log line" and associated metadata. | +##------------------------------------------------------------ + +if([@type] == "LogMessage") { + + mutate { + add_tag => [ "logmessage" ] + } + + # Drop useless message log. + if [@message] =~ /^\s*$/ or [@message] =~ /^#.*$/ { + drop { } + } + + # Override [@source][type] + mutate { + rename => { "[parsed_json_field][source_type]" => "[@source][type]" } + uppercase => [ "[@source][type]" ] # uppercase for consistency + } + + # Set [@cf][app_instance] + if [@cf][app_id] and [@cf][app_id] != "" { + mutate { + rename => { "[parsed_json_field][source_instance]" => "[@cf][app_instance]" } + convert => { "[@cf][app_instance]" => "integer" } + } + } else { + mutate { + remove_field => "[@cf][app_id]" + remove_field => "[parsed_json_field][source_instance]" + } + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-valuemetric.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-valuemetric.conf new file mode 100644 index 00000000..2180ff07 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-valuemetric.conf @@ -0,0 +1,12 @@ +##--------------------------------------------------------------------- +# Parses ValueMetric message. | +# A ValueMetric indicates the value of a metric at an instant in time. | +##--------------------------------------------------------------------- +if( [@type] == "ValueMetric" ) { + + mutate { + add_tag => [ "valuemetric" ] + + replace => {"@message" => "%{[parsed_json_field][name]} = %{[parsed_json_field][value]} (%{[parsed_json_field][unit]})"} + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app.conf b/src/logsearch-filters/src/logstash-filters/snippets/app.conf new file mode 100644 index 00000000..e7afc709 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/app.conf @@ -0,0 +1,140 @@ +##-------------------------- +# App conf. Parses app logs.| +##-------------------------- + +if [@index_type] == "app" { + + mutate { + add_tag => [ "app" ] + } + + # Parse Cloud Foundry logs from doppler firehose + # (for message format see https://github.com/cloudfoundry-community/firehose-to-syslog + # and https://github.com/cloudfoundry/dropsonde-protocol/tree/master/events) + json { + source => "@message" + target => "parsed_json_field" + id => "cloudfoundry/app/json" + } + + if "_jsonparsefailure" in [tags] { + + # Amend the failure tag to match our fail/${addon}/${filter}/${detail} standard + mutate { + add_tag => ["fail/cloudfoundry/app/json"] + remove_tag => ["_jsonparsefailure"] + } + + } else { + + # Set @timestamp + + if [parsed_json_field][timestamp] or [parsed_json_field][start_timestamp] { + + # set @timestamp from event's timestamp if it is passed + + mutate { + rename => { "[parsed_json_field][start_timestamp]" => "event_timestamp" } # HttpStartStop event case + rename => { "[parsed_json_field][timestamp]" => "event_timestamp" } + convert => { "event_timestamp" => "string" } + gsub => ["event_timestamp", "\d{6}$", ""] + } + date { + match => [ "event_timestamp", "UNIX_MS" ] + remove_field => [ "event_timestamp" ] + } + } else if [parsed_json_field][time] { + + # if event's timestamp is not passed then we set @timestamp = event shipping time from firehose-to-syslog + + date { + match => [ "[parsed_json_field][time]", "ISO8601"] + } + } + mutate { + remove_field => [ "[parsed_json_field][time]" ] + } + + + # Set @message and @level + + # 1) Replace the unicode \u2028 with \n, which Kibana will display as a new line. + # 2) Replace the unicode Null character \u0000 with "" + mutate { + gsub => [ "[parsed_json_field][msg]", '\u2028', " +" # Seems that passing a string with an actual newline in it is the only way to make gsub work. + ] + gsub => [ "[parsed_json_field][msg]", '\u0000', ""] + } + + mutate { + rename => { "[parsed_json_field][msg]" => "@message" } # @message + rename => { "[parsed_json_field][level]" => "@level" } # @level + } + + # Set @source fields + mutate { + rename => { "[parsed_json_field][ip]" => "[@source][host]" } + rename => { "[parsed_json_field][deployment]" => "[@source][deployment]" } + rename => { "[parsed_json_field][job]" => "[@source][job]" } + rename => { "[parsed_json_field][job_index]" => "[@source][job_index]" } + # override + rename => { "[parsed_json_field][origin]" => "[@source][component]" } + } + + if ![parsed_json_field][event_type] { + mutate { + add_field => { "[parsed_json_field][event_type]" => "UnknownEvent" } + } + } + + # Set @type (based on event_type) + mutate { + replace => { "@type" => "%{[parsed_json_field][event_type]}" } + } + + # Set [@source][type] (based on @type by default, + # for LogMessage we override it with source_type field that comes in event's json) + translate { + field => "@type" + dictionary => [ "LogMessage", "LOG", + "Error", "ERR", + "ContainerMetric", "CONTAINER", + "ValueMetric", "METRIC", + "CounterEvent", "COUNT", + "HttpStartStop", "HTTP" + ] + destination => "[@source][type]" + override => true + fallback => "NA" + } + + # Set @cf fields + mutate { + rename => { "[parsed_json_field][cf_org_id]" => "[@cf][org_id]" } + rename => { "[parsed_json_field][cf_org_name]" => "[@cf][org]" } + rename => { "[parsed_json_field][cf_space_id]" => "[@cf][space_id]" } + rename => { "[parsed_json_field][cf_space_name]" => "[@cf][space]" } + rename => { "[parsed_json_field][cf_app_id]" => "[@cf][app_id]" } + rename => { "[parsed_json_field][cf_app_name]" => "[@cf][app]" } + + remove_field => "[parsed_json_field][cf_origin]" # cf_origin = firehose all the time + remove_field => "[parsed_json_field][cf_ignored_app]" # cf_ignored_app = false all the time (https://github.com/cloudfoundry-community/firehose-to-syslog/issues/137) + } + + # Define [parsed_json_field_name] + mutate { + add_field => { "parsed_json_field_name" => "%{@type}"} + } + + # Override @metadata.index + if [@cf][org] { + mutate { replace => { "[@metadata][index]" => "%{[@metadata][index]}-%{[@cf][org]}" } } + if [@cf][space] { + mutate { replace => { "[@metadata][index]" => "%{[@metadata][index]}-%{[@cf][space]}" } } + } + mutate { lowercase => [ "[@metadata][index]" ] } + } + } + +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf new file mode 100644 index 00000000..19a19826 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf @@ -0,0 +1,17 @@ +##--------------------------------- +# Parses cloud_controller_ng logs.| +##--------------------------------- +if [@source][component] == "cloud_controller_ng" { + + mutate { + replace => { "@type" => "cloud_controller_ng" } + add_tag => "cloud_controller_ng" + } + + grok { + match => { "@message" => "%{URIHOST:Request_Host} %{NOTSPACE} \[%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{ISO8601_TIMEZONE}\] \"%{WORD:Request_Method} %{URIPATHPARAM:Request_URL} %{SYSLOGPROG:Request_Protocol}\" %{NUMBER:Status_Code:int} %{NUMBER:Bytes_Received:int} \"%{NOTSPACE:Referer}\" \"%{DATA:User_Agent}\" %{URIHOST:Backend_Address} vcap_request_id:%{DATA:X_Vcap_Request_ID} response_time:%{NUMBER:Response_Time}" } + tag_on_failure => "fail/cloudfoundry/platform-cloud_controller_ng/grok" + id => "cloudfoundry/platform-cloud_controller_ng/grok" + } + +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf new file mode 100644 index 00000000..90065830 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf @@ -0,0 +1,35 @@ +##------------------------------------+ +# Gorouter conf. Parses gorouter logs.| +##------------------------------------+ + +if [@index_type] == "platform" and [@source][component] == "gorouter" { + if [@message] =~ "\A\{.+\}\z" { + json { + source => "@message" + add_tag => [ "router/syslog" ] + tag_on_failure => [ "router/parsing_failed" ] + id => "router/accesslog/json" + } + } else { + grok { + match => [ "@message", [ + # cf-deployment v12.27.0+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", + # cf-deployment v12.17.0+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_time:%{NUMBER:[rtr][app_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int}\"", + # cf-release v252+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"", + # cf-release v250+ + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{HOSTPORT}|-)\" \"(%{HOSTPORT}|-)\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}", + # very old + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" %{HOSTPORT} x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}" + ] + ] + id => "router/accesslog/grok" + overwrite => [ "@message" ] + add_tag => "router/accesslog" + tag_on_failure => "router/parsing_failed" + } + + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf new file mode 100644 index 00000000..b130d041 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf @@ -0,0 +1,48 @@ +##---------------------------------- +# Haproxy conf. Parses haproxy logs.| +##---------------------------------- +if [@source][component] == "haproxy" { + + mutate { + replace => { "@type" => "haproxy" } + add_tag => "haproxy" + } + + # Grok patterns are based on http://www.haproxy.org/download/1.7/doc/configuration.txt + # Two formats are used accordingly: + # 8.2.3. HTTP log format + # 8.2.5. Error log format + + grok { + match => [ "@message", "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{INT:[haproxy][time_request]:int}/%{INT:[haproxy][time_queue]:int}/%{INT:[haproxy][time_backend_connect]:int}/%{INT:[haproxy][time_backend_response]:int}/%{INT:[haproxy][time_duration]:int} %{INT:[haproxy][http_status_code]:int} %{NOTSPACE:[haproxy][bytes_read]:int} %{DATA:[haproxy][captured_request_cookie]} %{DATA:[haproxy][captured_response_cookie]} %{NOTSPACE:[haproxy][termination_state]} %{INT:[haproxy][actconn]:int}/%{INT:[haproxy][feconn]:int}/%{INT:[haproxy][beconn]:int}/%{INT:[haproxy][srvconn]:int}/%{NOTSPACE:[haproxy][retries]:int} %{INT:[haproxy][srv_queue]:int}/%{INT:[haproxy][backend_queue]:int} (\{%{DATA:[haproxy][captured_request_headers]}\})?( )?(\{%{DATA:[haproxy][captured_response_headers]}\})?( )?\"(?(?(|((%{WORD:[haproxy][http_request_verb]})?( %{GREEDYDATA})?))))\"" ] + match => [ "@message", "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]}/%{NOTSPACE:[haproxy][bind_name]}:%{SPACE}%{GREEDYDATA:message}" ] + id => "cloudfoundry/platform-haproxy/grok" + tag_on_failure => "fail/cloudfoundry/platform-haproxy/grok" + } + + if !("fail/cloudfoundry/platform-haproxy/grok" in [tags]) { + + if [haproxy_http_request] { + mutate { + rename => {"haproxy_http_request" => "[haproxy][http_request]"} + } + } + + mutate { + rename => {"message" => "@message"} # @message + } + + # @level + if [haproxy][http_status_code] { + if [haproxy][http_status_code] >= 400 { + mutate { + add_field => { "@level" => "ERROR" } + } + } else { + mutate { + add_field => { "@level" => "INFO" } + } + } + } + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf new file mode 100644 index 00000000..eb3d52f2 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf @@ -0,0 +1,71 @@ +##-------------------------- +# Uaa conf. Parses uaa logs.| +##-------------------------- +if [@source][component] == "vcap.uaa" { + + # ---- Parse UAA events (general) + + mutate { + replace => { "[@source][component]" => "uaa" } # remove vcap. prefix + replace => { "@type" => "uaa" } + add_tag => "uaa" + } + + grok { + match => { "@message" => "\[%{TIMESTAMP_ISO8601:[uaa][timestamp]}\]%{SPACE}uaa%{SPACE}-%{SPACE}%{NUMBER:[uaa][pid]:int}%{SPACE}\[%{DATA:[uaa][thread]}\]%{SPACE}....%{SPACE}%{LOGLEVEL:@level}%{SPACE}---%{SPACE}%{DATA:[uaa][log_category]}:%{SPACE}%{GREEDYDATA:@message}"} + overwrite => ["@message", "@level"] # @message, @level + id => "cloudfoundry/platform-uaa/grok" + tag_on_failure => "fail/cloudfoundry/platform-uaa/grok" + } + + if [uaa][log_category] == "Audit" { + + # override + mutate { + replace => { "@type" => "uaa-audit" } + add_tag => "audit" + } + + # ---- Additional parsing: Audit events + + grok { + match => { "@message" => "(?(%{WORD:[uaa][audit][type]}%{SPACE}\('%{DATA:[uaa][audit][data]}'\))):%{SPACE}principal=%{DATA:[uaa][audit][principal]},%{SPACE}origin=\[%{DATA:[uaa][audit][origin]}\],%{SPACE}identityZoneId=\[%{DATA:[uaa][audit][identity_zone_id]}\]"} + id => "cloudfoundry/platform-uaa/audit/grok" + tag_on_failure => "fail/cloudfoundry/platform-uaa/audit/grok" + } + + if !("fail/cloudfoundry/platform-uaa/audit/grok" in [tags]) { + + # Audit @message + mutate { + rename => { "uaa_audit_message" => "@message" } + } + + # extract audit_event_remote_address and geoip it + if "PrincipalAuthenticationFailure" == [uaa][audit][type] { + mutate { + add_field => { "[uaa][audit][remote_address]" => "%{[uaa][audit][origin]}" } + } + } + if [uaa][audit][origin] =~ /remoteAddress=/ { + grok { + match => { "[uaa][audit][origin]" => "remoteAddress=%{IP:[uaa][audit][remote_address]}" } + id => "cloudfoundry/platform-uaa/audit/origin/grok" + } + } + if [uaa][audit][remote_address] { + geoip { + source => "[uaa][audit][remote_address]" + } + } + + # split origin + mutate { + split => { "[uaa][audit][origin]" => ", " } + } + + } + + } +} + diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf new file mode 100644 index 00000000..768dcb56 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf @@ -0,0 +1,53 @@ +##----------------------------- +# Vcap conf. Parses vcap* logs.| +##----------------------------- +if [@source][component] != "vcap.uaa" and [@source][component] =~ /vcap\..*/ { + + # minus vcap. prefix + mutate { + gsub => ["[@source][component]", "^vcap\.", ""] + } + + mutate { + replace => { "@type" => "vcap" } + add_tag => "vcap" + } + + # Parse Cloud Foundry logs + if [@message] =~ /^\s*{".*}\s*$/ { # looks like JSON + + # parse JSON message + json { + source => "@message" + target => "parsed_json_field" + remove_field => [ "@message" ] + add_field => { "parsed_json_field_name" => "%{[@source][component]}"} + id => "cloudfoundry/platform-vcap/json" + } + + if "_jsonparsefailure" in [tags] { + # Amend the failure tag to match our fail/${addon}/${filter}/${detail} standard + mutate { + add_tag => ["fail/cloudfoundry/platform-vcap/json"] + remove_tag => ["_jsonparsefailure"] + } + + } else { + + mutate { + rename => { "[parsed_json_field][message]" => "@message" } # @message + } + + # @level + translate { + field => "[parsed_json_field][log_level]" + dictionary => [ "0", "DEBUG", "1", "INFO", "2", "ERROR", "3", "FATAL" ] + destination => "@level" + override => true + fallback => "%{[parsed_json_field][log_level]}" + remove_field => "[parsed_json_field][log_level]" + } + } + + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform.conf new file mode 100644 index 00000000..c5c8d5a7 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/platform.conf @@ -0,0 +1,49 @@ +##------------------------------ +# Platform conf. Parses CF logs.| +##------------------------------ +if [@index_type] == "platform" { + + mutate { + replace => { "[@source][type]" => "system" } # default for platform logs + add_tag => "platform" + } + + # Syslog message with RFC 5424 and the enterprise number is CF + if [syslog_sd_id] == "instance@47450" { + mutate { + add_field => { + "[@source][az]" => "%{[syslog_sd_params][az]}" + "[@source][deployment]" => "%{[syslog_sd_params][deployment]}" + "[@source][director]" => "%{[syslog_sd_params][director]}" + "[@source][id]" => "%{[syslog_sd_params][id]}" + "[@source][job]" => "%{[syslog_sd_params][group]}" + } + replace => { + "[@source][type]" => "cf" + "@type" => "cf" + } + add_tag => "cf" + } + } else { + # Try parsing with possible CF formats + grok { + # Metron agent format (https://github.com/cloudfoundry/loggregator/blob/master/jobs/metron_agent/templates/syslog_forwarder.conf.erb#L53) + match => [ "@message", "\[job=%{NOTSPACE:[@source][job]} index=%{INT:[@source][index]:int}\]%{SPACE}%{GREEDYDATA:@message}" ] + + # Syslog release format (https://github.com/cloudfoundry/syslog-release/blob/master/jobs/syslog_forwarder/templates/rsyslog.conf.erb#L56) + match => [ "@message", "\[bosh instance=%{NOTSPACE:[@source][deployment]}/%{NOTSPACE:[@source][job]}/%{NOTSPACE:[@source][job_index]}\]%{SPACE}%{GREEDYDATA:@message}" ] + + overwrite => [ "@message" ] # @message + id => "cloudfoundry/platform/grok" + tag_on_failure => "fail/cloudfoundry/platform/grok" + } + + if !("fail/cloudfoundry/platform/grok" in [tags]) { + mutate { + replace => { "[@source][type]" => "cf" } + replace => { "@type" => "cf" } + add_tag => "cf" + } + } + } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/setup.conf b/src/logsearch-filters/src/logstash-filters/snippets/setup.conf new file mode 100644 index 00000000..fd052016 --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/setup.conf @@ -0,0 +1,38 @@ +##-------------------------------- +# Setup conf. Sets general fields.| +##-------------------------------- + +# Replace the unicode empty character \u0000 with "" +# Drop useless logs +mutate { + gsub => [ "@message", '\u0000', ""] +} +if [@message] =~ /^\s*$/ or [@message] =~ /^#.*$/ { + drop { } +} + +# Set index +# @index_type stores type of index: app/platform +# [@metadata][index] stores full index prefix (for app logs additionally includes org and space name) +mutate { + add_field => { "@index_type" => "platform" } # by default logs go to 'platform' +} +if [syslog_program] == "doppler" { + mutate { + update => { "@index_type" => "app" } + } +} +mutate { + add_field => { "[@metadata][index]" => "%{@index_type}" } +} + +# Initialize @input, @shipper and @source +mutate { + add_field => { "@input" => "%{@type}" } + + rename => { "syslog_pri" => "[@shipper][priority]" } + replace => { "[@shipper][name]" => "%{syslog_program}_%{[@type]}" } + + add_field => { "[@source][component]" => "%{syslog_program}" } + add_field => { "[@source][type]" => "syslog" } +} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/teardown.conf b/src/logsearch-filters/src/logstash-filters/snippets/teardown.conf new file mode 100644 index 00000000..a7f7db1a --- /dev/null +++ b/src/logsearch-filters/src/logstash-filters/snippets/teardown.conf @@ -0,0 +1,77 @@ +##---------------------------------------------------------- +# Teardown conf. Performs fields post-processing & clean up.| +##---------------------------------------------------------- + +# -- Apply default settings for mandatory fields (if not set) + +# set syslog @level (if @level is not set yet) +if ![@level] and [syslog_severity_code] { # @level + + if [syslog_severity_code] <= 3 { # 0-Emergency, 1-Alert, 2-Critical, 3-Error + mutate { + add_field => { "@level" => "ERROR" } + } + } else if [syslog_severity_code] <= 5 { # 4-Warning, 5-Notice + mutate { + add_field => { "@level" => "WARN" } + } + } else if [syslog_severity_code] == 6 { # 6-Informational + mutate { + add_field => { "@level" => "INFO" } + } + } else if [syslog_severity_code] == 7 { #7-Debug + mutate { + add_field => { "@level" => "DEBUG" } + } + } +} +mutate { + uppercase => [ "@level" ] +} + +# -- Rework fields + +if [@source][job] and [@source][index] { + mutate { add_field => { "[@source][vm]" => "%{[@source][job]}/%{[@source][index]}" } } +} + +if ![@source][host] { + mutate { rename => { "[host]" => "[@source][host]" } } +} + +# Downcase [parsed_json_field_name] and replace special characters with '_').. +# .. and rename dynamic [parsed_json_field] field to this calculated name. +if [parsed_json_field] and [parsed_json_field_name] { + mutate { + lowercase => [ "parsed_json_field_name" ] + gsub => [ "parsed_json_field_name", "[\s/\\?#-\.]", "_" ] + } + mutate { + rename => { "parsed_json_field" => "%{parsed_json_field_name}" } + } +} + +mutate { + remove_field => [ "parsed_json_field_name" ] +} + +# -- Cleanup unnecessary fields + +# Remove syslog_ fields +mutate { + remove_field => "syslog_pri" + remove_field => "syslog_facility" + remove_field => "syslog_facility_code" + remove_field => "syslog_message" + remove_field => "syslog_severity" + remove_field => "syslog_severity_code" + remove_field => "syslog_program" + remove_field => "syslog_timestamp" + remove_field => "syslog_hostname" + remove_field => "syslog_pid" +} + +# Cleanup +mutate { + remove_field => [ "@version", "host", "port", "_logstash_input" ] +} diff --git a/src/logsearch-filters/test/logstash-filters/it_app_helper.rb b/src/logsearch-filters/test/logstash-filters/it_app_helper.rb new file mode 100644 index 00000000..8787462e --- /dev/null +++ b/src/logsearch-filters/test/logstash-filters/it_app_helper.rb @@ -0,0 +1,156 @@ +# encoding: utf-8 + +## -- Setup methods +$app_event_dummy = { + "@type" => "syslog", + "syslog_program" => "doppler", + "syslog_pri" => 6, + "syslog_severity_code" => 3, # error + "host" => "bed08922-4734-4d62-9eba-3291aed1b8ce", + "@message" => "Dummy message"} + +$envelope_fields = { + "cf_origin" => "firehose", + "deployment" => "cf-full", + "ip" => "192.168.111.32", + "job" => "runner_z1", + "job_index" => "4abc5def", + "origin" => "MetronAgent", + "time" => "2016-08-16T22:46:24Z" +} + +$app_data_fields = { + "cf_app_id" => "31b928ee-4110-4e7b-996c-334c5d7ac2ac", + "cf_app_name" => "loggenerator", + "cf_org_id" => "9887ad0a-f9f7-449e-8982-76307bd17239", + "cf_org_name" => "admin", + "cf_space_id" => "59cf41f2-3a1d-42db-88e7-9540b02945e8", + "cf_space_name" => "demo" +} + +def append_fields_from_hash(hash) + result = "" + hash.each do |key, value| + result += append_field(key, value) + end + result +end + +def append_field(field_name, field_value) + '"' + field_name + '":' + (field_value.is_a?(String) ? + '"' + field_value + '"' : field_value.to_s) + ',' +end + +def construct_event(event_type, is_include_app_data, event_fields_hash) + + # envelope + result = '{' + + append_field("event_type", event_type) + + append_fields_from_hash($envelope_fields) + + # app data + if is_include_app_data + result += append_fields_from_hash($app_data_fields) + end + + # event fields + result += append_fields_from_hash(event_fields_hash) + + result = result[0...-1] + '}' # cut last comma (,) + +end + +## -- Verification methods +def verify_app_general_fields (metadata_index, type, source_type, message, level) + + + # no app parsing error + it "sets tags" do + expect(subject["tags"]).not_to include "fail/cloudfoundry/app/json" + expect(subject["tags"]).to include "app" + end + + it { expect(subject["@type"]).to eq type } + + it { expect(subject["@index_type"]).to eq "app" } + it { expect(subject["@metadata"]["index"]).to eq metadata_index } + + it { expect(subject["@input"]).to eq "syslog" } + + it { expect(subject["@shipper"]["priority"]).to eq 6 } + it { expect(subject["@shipper"]["name"]).to eq "doppler_syslog" } + + it "sets @source fields" do + expect(subject["@source"]["deployment"]).to eq "cf-full" + expect(subject["@source"]["host"]).to eq "192.168.111.32" + expect(subject["@source"]["job"]).to eq "runner_z1" + expect(subject["@source"]["job_index"]).to eq "4abc5def" + expect(subject["@source"]["component"]).to eq "MetronAgent" + expect(subject["@source"]["type"]).to eq source_type + end + + it { expect(subject["@message"]).to eq message } + it { expect(subject["@level"]).to eq level } + + # verify no (default) dynamic JSON fields + it { expect(subject["parsed_json_field"]).to be_nil } + it { expect(subject["parsed_json_field_name"]).to be_nil } + +end + +def verify_app_cf_fields (app_instance) + + it "sets @cf fields" do + expect(subject["@cf"]["app"]).to eq "loggenerator" + expect(subject["@cf"]["app_id"]).to eq "31b928ee-4110-4e7b-996c-334c5d7ac2ac" + expect(subject["@cf"]["app_instance"]).to eq app_instance + expect(subject["@cf"]["space"]).to eq "demo" + expect(subject["@cf"]["space_id"]).to eq "59cf41f2-3a1d-42db-88e7-9540b02945e8" + expect(subject["@cf"]["org"]).to eq "admin" + expect(subject["@cf"]["org_id"]).to eq "9887ad0a-f9f7-449e-8982-76307bd17239" + end + +end + +## -- Special cases +def verify_parsing_logmessage_app_CF_versions(level, msg, expected_level, expected_message, &block) + context "in (Diego CF)" do + verify_parsing_logmessage_app(true, # Diego + level, msg, expected_level, expected_message, &block) + end + + context "in (Dea CF)" do + verify_parsing_logmessage_app(false, # Dea + level, msg, expected_level, expected_message, &block) + end +end + +def verify_parsing_logmessage_app(isDiego, level, msg, expected_level, expected_message, &block) + sample_fields = {"source_type" => isDiego ? "APP" : "App", # Diego/Dea specific + "source_instance" => "99", + "message_type" => "OUT", + "timestamp" => 1471387745714800488, + "level" => level, + "msg" => "Dummy msg"} + + sample_fields["msg"] = msg + + sample_event = $app_event_dummy.clone + sample_event["@message"] = construct_event("LogMessage", true, sample_fields) + + when_parsing_log(sample_event) do + + verify_app_general_fields("app-admin-demo", "LogMessage", "APP", + expected_message, expected_level) + + verify_app_cf_fields(99) + + # verify event-specific fields + it { expect(subject["tags"]).to include("logmessage", "logmessage-app") } + it { expect(subject["logmessage"]["message_type"]).to eq "OUT" } + + # additional verifications + describe("", &block) + + end +end diff --git a/src/logsearch-filters/test/logstash-filters/it_platform_helper.rb b/src/logsearch-filters/test/logstash-filters/it_platform_helper.rb new file mode 100644 index 00000000..eaa0919a --- /dev/null +++ b/src/logsearch-filters/test/logstash-filters/it_platform_helper.rb @@ -0,0 +1,108 @@ +# encoding: utf-8 +class MessagePayload + attr_accessor :deployment, :job, :message_text +end + +class MessagePayloadBuilder + attr_accessor :message_payload + + def initialize + @message_payload = MessagePayload.new + end + + def build + @message_payload + end + + def job(job) + @message_payload.job = job + self + end + + def deployment(deployment) + @message_payload.deployment = deployment + self + end + + def message_text(message_text) + @message_payload.message_text = message_text + self + end + +end + +def construct_cf_message__metronagent_format (message_payload) + '[job='+ message_payload.job + ' index=555] ' + message_payload.message_text +end + +def construct_cf_message__syslogrelease_format (message_payload) + '[bosh instance='+ message_payload.deployment + '/' + message_payload.job + '/5abc6def7ghi] ' + message_payload.message_text +end + +# -- Verify methods -- +def verify_platform_cf_fields__metronagent_format (expected_shipper, expected_component, expected_job, + expected_type, expected_tags, + expected_message, expected_level) + + verify_platform_cf_fields(expected_shipper, expected_job, expected_component, expected_type, expected_tags, + expected_message, expected_level); + + it { expect(subject["@source"]["index"]).to eq 555 } + it { expect(subject["@source"]["vm"]).to eq expected_job + '/555' } + it { expect(subject["@source"]["job_index"]).to be_nil } + it { expect(subject["@source"]["deployment"]).to be_nil } + + +end + +def verify_platform_cf_fields__syslogrelease_format (expected_shipper, expected_deployment, expected_component, expected_job, + expected_type, expected_tags, + expected_message, expected_level) + + verify_platform_cf_fields(expected_shipper, expected_job, expected_component, expected_type, expected_tags, + expected_message, expected_level); + + it { expect(subject["@source"]["job_index"]).to eq "5abc6def7ghi" } + it { expect(subject["@source"]["index"]).to be_nil } + it { expect(subject["@source"]["vm"]).to be_nil } + it { expect(subject["@source"]["deployment"]).to eq expected_deployment } + + +end + +## helper methods + +def verify_platform_cf_fields (expected_shipper, expected_job, expected_component, + expected_type, expected_tags, + expected_message, expected_level) + + verify_platform_fields(expected_shipper, expected_component, expected_type, expected_tags, + expected_message, expected_level) + + # verify CF format parsing + it { expect(subject["tags"]).not_to include "fail/cloudfoundry/platform/grok" } + it { expect(subject["@source"]["type"]).to eq "cf" } + it { expect(subject["@source"]["job"]).to eq expected_job } +end + +def verify_platform_fields (expected_shipper, expected_component, expected_type, expected_tags, + expected_message, expected_level) + + # fields + it { expect(subject["@message"]).to eq expected_message } + it { expect(subject["@level"]).to eq expected_level } + + it { expect(subject["@index_type"]).to eq "platform" } + it { expect(subject["@metadata"]["index"]).to eq "platform" } + it { expect(subject["@input"]).to eq "relp" } + it { expect(subject["@shipper"]["priority"]).to eq 14 } + it { expect(subject["@shipper"]["name"]).to eq expected_shipper } + it { expect(subject["@source"]["host"]).to eq "192.168.111.24" } + it { expect(subject["@source"]["component"]).to eq expected_component } + it { expect(subject["@type"]).to eq expected_type } + it { expect(subject["tags"]).to eq expected_tags } + + # verify no (default) dynamic JSON fields + it { expect(subject["parsed_json_field"]).to be_nil } + it { expect(subject["parsed_json_field_name"]).to be_nil } +end diff --git a/src/logsearch-filters/test/logstash-filters/snippets/platform-spec.rb b/src/logsearch-filters/test/logstash-filters/snippets/platform-spec.rb new file mode 100644 index 00000000..7a9cdb28 --- /dev/null +++ b/src/logsearch-filters/test/logstash-filters/snippets/platform-spec.rb @@ -0,0 +1,109 @@ +# encoding: utf-8 +require 'test/logstash-filters/filter_test_helpers' + +describe "platform.conf" do + + before(:all) do + load_filters <<-CONFIG + filter { + #{File.read("src/logstash-filters/snippets/platform.conf")} + } + CONFIG + end + + describe "#if" do + + describe "passed" do + when_parsing_log( + "@index_type" => "platform", # good value + "@message" => "Some message" + ) do + + # tag set => 'if' succeeded + it { expect(subject["tags"]).to include "platform" } + + end + end + + describe "failed" do + when_parsing_log( + "@index_type" => "some value", # bad value + "@message" => "Some message" + ) do + + # no tags set => 'if' failed + it { expect(subject["tags"]).to be_nil } + + it { expect(subject["@index_type"]).to eq "some value" } # keeps unchanged + it { expect(subject["@message"]).to eq "Some message" } # keeps unchanged + + end + end + + end + + # -- general case + describe "#fields when message is" do + + context "CF format (metron agent)" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "[job=nfs_z1 index=0] Some message" # CF metron agent format + ) do + + it { expect(subject["tags"]).to eq ["platform", "cf"] } # no fail tag + + it { expect(subject["@source"]["type"]).to eq "cf" } + it { expect(subject["@type"]).to eq "cf" } + + it "sets grok fields" do + expect(subject["@message"]).to eq "Some message" + expect(subject["@source"]["job"]).to eq "nfs_z1" + expect(subject["@source"]["index"]).to eq 0 + end + + end + end + + context "CF format (syslog release)" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "[bosh instance=cf_full/nfs_z1/abcdefg123] Some message" # CF syslog release format + ) do + + it { expect(subject["tags"]).to eq ["platform", "cf"] } # no fail tag + + it { expect(subject["@source"]["type"]).to eq "cf" } + it { expect(subject["@type"]).to eq "cf" } + + it "sets grok fields" do + expect(subject["@message"]).to eq "Some message" + expect(subject["@source"]["deployment"]).to eq "cf_full" + expect(subject["@source"]["job"]).to eq "nfs_z1" + expect(subject["@source"]["job_index"]).to eq "abcdefg123" + end + + end + end + + context "not CF format" do + when_parsing_log( + "@index_type" => "platform", + "@message" => "Some message that fails grok" # bad format + ) do + + # get parsing error + it { expect(subject["tags"]).to eq ["platform", "fail/cloudfoundry/platform/grok"] } + + # no fields set + it { expect(subject["@message"]).to eq "Some message that fails grok" } # keeps the same + it { expect(subject["@index_type"]).to eq "platform" } # keeps the same + it { expect(subject["@source"]["type"]).to eq "system" } + it { expect(subject["@type"]).to be_nil } + + end + end + + end + +end From 14bcdc03d926879af66bcd449d644894af1ed0d8 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 8 Oct 2024 10:18:53 -0400 Subject: [PATCH 11/13] adding in the logsearch features --- packages/logsearch-filters/spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/logsearch-filters/spec b/packages/logsearch-filters/spec index 7fe00293..90244c13 100644 --- a/packages/logsearch-filters/spec +++ b/packages/logsearch-filters/spec @@ -2,10 +2,10 @@ name: logsearch-filters dependencies: - - ruby-2.4-r5 + - ruby-3.1 files: - logsearch-filters/* - logsearch-filters/bin/* - logsearch-filters/src/logstash-filters/**/* - - ruby/rake-12.3.2.gem + - ruby/rake-13.0.6.gem From 3efbdcf1b3a4d5ca6a3595b1c97fe8d0aee0387f Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 8 Oct 2024 10:22:39 -0400 Subject: [PATCH 12/13] fixing ruby for this version --- packages/logsearch-filters/packaging | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/logsearch-filters/packaging b/packages/logsearch-filters/packaging index cd023afb..84bec6f0 100644 --- a/packages/logsearch-filters/packaging +++ b/packages/logsearch-filters/packaging @@ -7,7 +7,7 @@ RAKE_VERSION=$(find ruby -maxdepth 1 -name 'rake-*' | sed 's/ruby\/rake-\(.*\)\. # shellcheck disable=1090 source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/ruby-3.1/bosh/compile.env" -gem install "ruby/rake-${RAKE_VERSION}.gem" --local --no-ri --no-rdoc +gem install "ruby/rake-${RAKE_VERSION}.gem" --local logsearch-filters/bin/build From 747000106af6d4f6701c70125f376a0751e8d86c Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 8 Oct 2024 10:41:13 -0400 Subject: [PATCH 13/13] removing the cop --- jobs/ingestor_logsearch/templates/bin/ingestor_logsearch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/ingestor_logsearch/templates/bin/ingestor_logsearch b/jobs/ingestor_logsearch/templates/bin/ingestor_logsearch index 1795dd70..f390f0be 100644 --- a/jobs/ingestor_logsearch/templates/bin/ingestor_logsearch +++ b/jobs/ingestor_logsearch/templates/bin/ingestor_logsearch @@ -45,8 +45,8 @@ export LOGSTASH_WORKERS=`grep -c ^processor /proc/cpuinfo` <% else %> export LOGSTASH_WORKERS=<%= p('logstash_parser.workers') %> <% end %> -export TIMECOP_REJECT_GREATER_THAN_HOURS=<%= p('logstash_parser.timecop.reject_greater_than_hours') %> -export TIMECOP_REJECT_LESS_THAN_HOURS=<%= p('logstash_parser.timecop.reject_less_than_hours') %> +#export TIMECOP_REJECT_GREATER_THAN_HOURS=<%= p('logstash_parser.timecop.reject_greater_than_hours') %> +#export TIMECOP_REJECT_LESS_THAN_HOURS=<%= p('logstash_parser.timecop.reject_less_than_hours') %> export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * <%= p("logstash.heap_percentage") %> ) / 100 ))K <% if_p('logstash.heap_size') do |heap_size| %> HEAP_SIZE=<%= heap_size %> @@ -90,7 +90,7 @@ cat ${JOB_DIR}/config/filters_post.conf >> ${JOB_DIR}/config/logstash.conf <% if p('logstash_parser.enable_json_filter') %> cat /var/vcap/packages/logsearch-config/if_it_looks_like_json.conf >> ${JOB_DIR}/config/logstash.conf <% end %> -cat /var/vcap/packages/logsearch-config/timecop.conf >> ${JOB_DIR}/config/logstash.conf +#cat /var/vcap/packages/logsearch-config/timecop.conf >> ${JOB_DIR}/config/logstash.conf cat /var/vcap/packages/logsearch-config/deployment.conf >> ${JOB_DIR}/config/logstash.conf echo "} #close filters" >> ${JOB_DIR}/config/logstash.conf