diff --git a/README.md b/README.md index 1ae8e5e46..dabd12eaf 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ See also dockerhub tags page: https://hub.docker.com/r/fluent/fluentd-kubernetes - `v1.7.1-debian-logzio-1.0,v1.7-debian-logzio-1` [docker-image/v1.7/debian-logzio/Dockerfile](docker-image/v1.7/debian-logzio/Dockerfile) - `v1.7.1-debian-kafka-1.0,v1.7-debian-kafka-1` [docker-image/v1.7/debian-kafka/Dockerfile](docker-image/v1.7/debian-kafka/Dockerfile) - `v1.7.1-debian-kinesis-1.0,v1.7-debian-kinesis-1` [docker-image/v1.7/debian-kinesis/Dockerfile](docker-image/v1.7/debian-kinesis/Dockerfile) +- `v1.7.1-debian-logsense-1.0,v1.7-debian-logsense-1` [docker-image/v1.7/debian-logsense/Dockerfile](docker-image/v1.7/debian-logsense/Dockerfile) You can also use `v1-debian-PLUGIN` tag to refer latest v1 image, e.g. `v1-debian-elasticsearch`. On production, strict tag is better to avoid unexpected update. @@ -154,6 +155,7 @@ Some images are contributed by users. If you have a problem/question for followi - Amazon Kinesis : @shiftky - logz.io : @SaMnCo / @jamielennox - splunkhec: @FutureSharks +- logsense: @perk / @pmaciolek ### References diff --git a/docker-image/v1.7/debian-logsense/.dockerignore b/docker-image/v1.7/debian-logsense/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.7/debian-logsense/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.7/debian-logsense/Dockerfile b/docker-image/v1.7/debian-logsense/Dockerfile new file mode 100644 index 000000000..a8feb964d --- /dev/null +++ b/docker-image/v1.7/debian-logsense/Dockerfile @@ -0,0 +1,49 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.7.1-debian-1.0 + +LABEL maintainer="Eduardo Silva " +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/2.6.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.6.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.6.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ + RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps net-tools \ + && gem install bundler --version 1.16.2 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.7/debian-logsense/Gemfile b/docker-image/v1.7/debian-logsense/Gemfile new file mode 100644 index 000000000..fcc97d2aa --- /dev/null +++ b/docker-image/v1.7/debian-logsense/Gemfile @@ -0,0 +1,17 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.7.1" +gem "oj", "3.8.1" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.3.0" +gem "fluent-plugin-grok-parser", "~> 2.5.0" +gem "fluent-plugin-prometheus", "~> 1.5.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.0.0" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.2.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.3.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.0.1" diff --git a/docker-image/v1.7/debian-logsense/Gemfile.lock b/docker-image/v1.7/debian-logsense/Gemfile.lock new file mode 100644 index 000000000..c22fa5e7b --- /dev/null +++ b/docker-image/v1.7/debian-logsense/Gemfile.lock @@ -0,0 +1,113 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + concurrent-ruby (1.1.5) + cool.io (1.5.4) + dig_rb (1.0.1) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + ffi (1.11.1) + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.3.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-grok-parser (2.5.1) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (2.3.0) + fluentd (>= 0.14.0, < 2) + kubeclient (< 5) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-prometheus (1.5.0) + fluentd (>= 0.14.20, < 2) + prometheus-client + fluent-plugin-record-modifier (2.0.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.2.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.0.2) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 1.3.2) + fluentd (1.7.1) + cool.io (>= 1.4.5, < 2.0.0) + dig_rb (~> 1.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.2.0, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (~> 2.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (3.3.0) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.0) + http_parser.rb (~> 0.6.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.1.1) + http_parser.rb (0.6.0) + kubeclient (4.4.0) + http (~> 3.0) + recursive-open-struct (~> 1.0, >= 1.0.4) + rest-client (~> 2.0) + lru_redux (1.1.0) + mime-types (3.3) + mime-types-data (~> 3.2015) + mime-types-data (3.2019.0904) + msgpack (1.3.1) + netrc (0.11.0) + oj (3.8.1) + prometheus-client (0.9.0) + quantile (~> 0.2.1) + public_suffix (4.0.1) + quantile (0.2.1) + recursive-open-struct (1.1.0) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + serverengine (2.1.1) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.3) + systemd-journal (1.3.3) + ffi (~> 1.9) + tzinfo (2.0.0) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2019.3) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.6) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.3.0) + fluent-plugin-grok-parser (~> 2.5.0) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 2.3.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-prometheus (~> 1.5.0) + fluent-plugin-record-modifier (~> 2.0.0) + fluent-plugin-rewrite-tag-filter (~> 2.2.0) + fluent-plugin-systemd (~> 1.0.1) + fluentd (= 1.7.1) + oj (= 3.8.1) + +BUNDLED WITH + 1.17.2 diff --git a/docker-image/v1.7/debian-logsense/conf/fluent.conf b/docker-image/v1.7/debian-logsense/conf/fluent.conf new file mode 100644 index 000000000..2769031c9 --- /dev/null +++ b/docker-image/v1.7/debian-logsense/conf/fluent.conf @@ -0,0 +1,29 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + + + @type record_transformer + + logsense_token "#{ENV['LOGSENSE_TOKEN']}" + + + + + @type forward + send_timeout 60s + recover_wait 10s + hard_timeout 60s + transport tls + tls_verify_hostname true + + host "logs.logsense.com" + port 32714 + + + diff --git a/docker-image/v1.7/debian-logsense/conf/kubernetes.conf b/docker-image/v1.7/debian-logsense/conf/kubernetes.conf new file mode 100644 index 000000000..f44eed803 --- /dev/null +++ b/docker-image/v1.7/debian-logsense/conf/kubernetes.conf @@ -0,0 +1,192 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + + + @type null + + + + @type tail + @id in_tail_container_logs + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + tag kubernetes.* + read_from_head true + + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file /var/log/fluentd-salt.pos + tag salt + + @type regexp + expression /^(? + + + + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file /var/log/fluentd-startupscript.log.pos + tag startupscript + + @type syslog + + + + + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file /var/log/fluentd-docker.log.pos + tag docker + + @type regexp + expression /^time="(? + + + + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file /var/log/fluentd-etcd.log.pos + tag etcd + + @type none + + + + + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file /var/log/fluentd-kubelet.log.pos + tag kubelet + + @type kubernetes + + + + + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file /var/log/fluentd-kube-proxy.log.pos + tag kube-proxy + + @type kubernetes + + + + + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file /var/log/fluentd-kube-apiserver.log.pos + tag kube-apiserver + + @type kubernetes + + + + + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file /var/log/fluentd-kube-controller-manager.log.pos + tag kube-controller-manager + + @type kubernetes + + + + + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file /var/log/fluentd-kube-scheduler.log.pos + tag kube-scheduler + + @type kubernetes + + + + + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file /var/log/fluentd-rescheduler.log.pos + tag rescheduler + + @type kubernetes + + + + + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file /var/log/fluentd-glbc.log.pos + tag glbc + + @type kubernetes + + + + + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file /var/log/fluentd-cluster-autoscaler.log.pos + tag cluster-autoscaler + + @type kubernetes + + + +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" + + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file /var/log/kube-apiserver-audit.log.pos + tag kube-apiserver-audit + + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(? + + + + @type kubernetes_metadata + @id filter_kube_metadata + + diff --git a/docker-image/v1.7/debian-logsense/conf/prometheus.conf b/docker-image/v1.7/debian-logsense/conf/prometheus.conf new file mode 100644 index 000000000..384908515 --- /dev/null +++ b/docker-image/v1.7/debian-logsense/conf/prometheus.conf @@ -0,0 +1,14 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics + + @type prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" + + + + @type prometheus_output_monitor + diff --git a/docker-image/v1.7/debian-logsense/conf/systemd.conf b/docker-image/v1.7/debian-logsense/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.7/debian-logsense/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + + read_from_head true + tag kubelet + + +# Logs from docker-systemd + + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + + read_from_head true + tag docker.systemd + + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + + read_from_head true + tag bootkube + + diff --git a/docker-image/v1.7/debian-logsense/entrypoint.sh b/docker-image/v1.7/debian-logsense/entrypoint.sh new file mode 100755 index 000000000..bf0bbd75b --- /dev/null +++ b/docker-image/v1.7/debian-logsense/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.7/debian-logsense/hooks/post_push b/docker-image/v1.7/debian-logsense/hooks/post_push new file mode 100644 index 000000000..961f9184e --- /dev/null +++ b/docker-image/v1.7/debian-logsense/hooks/post_push @@ -0,0 +1,15 @@ +#!/bin/bash +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/post_push.erb + +set -e + +# Parse image name for repo name +tagStart=$(expr index "$IMAGE_NAME" :) +repoName=${IMAGE_NAME:0:tagStart-1} + +# Tag and push image for each additional tag +for tag in {v1.7.1-debian-logsense-1.0,v1.7-debian-logsense-1}; do + docker tag $IMAGE_NAME ${repoName}:${tag} + docker push ${repoName}:${tag} +done diff --git a/docker-image/v1.7/debian-logsense/plugins/.gitkeep b/docker-image/v1.7/debian-logsense/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.7/debian-logsense/plugins/parser_kubernetes.rb b/docker-image/v1.7/debian-logsense/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.7/debian-logsense/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# +# @type kubernetes +# +# +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?\w)(?