diff --git a/Makefile b/Makefile index 0fd97d9e02..05795700b8 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,8 @@ ALL_IMAGES := \ v1.3/debian-papertrail:v1.3.3-debian-papertrail-1.1,v1.3-debian-papertrail-1 \ v1.3/debian-logzio:v1.3.3-debian-logzio-1.1,v1.3-debian-logzio-1 \ v1.3/debian-kafka:v1.3.3-debian-kafka-1.1,v1.3-debian-kafka-1 \ - v1.3/debian-kinesis:v1.3.3-debian-kinesis-1.1,v1.3-debian-kinesis-1 + v1.3/debian-kinesis:v1.3.3-debian-kinesis-1.1,v1.3-debian-kinesis-1 \ + v1.3/debian-logsense:v1.3.3-debian-logsense-1.1,v1.3-debian-logsense-1 # :,,,... diff --git a/README.md b/README.md index afe332f845..0d2ed9b432 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ See also dockerhub tags page: https://hub.docker.com/r/fluent/fluentd-kubernetes - `v1.3.3-debian-logzio-1.1,v1.3-debian-logzio-1` [docker-image/v1.3/debian-logzio/Dockerfile](docker-image/v1.3/debian-logzio/Dockerfile) - `v1.3.3-debian-kafka-1.1,v1.3-debian-kafka-1` [docker-image/v1.3/debian-kafka/Dockerfile](docker-image/v1.3/debian-kafka/Dockerfile) - `v1.3.3-debian-kinesis-1.1,v1.3-debian-kinesis-1` [docker-image/v1.3/debian-kinesis/Dockerfile](docker-image/v1.3/debian-kinesis/Dockerfile) +- `v1.3.3-debian-logsense-1.1,v1.3-debian-logsense-1` [docker-image/v1.3/debian-logsense/Dockerfile](docker-image/v1.3/debian-logsense/Dockerfile) - `v0.12-debian-elasticsearch` [docker-image/v0.12/debian-elasticsearch/Dockerfile](docker-image/v0.12/debian-elasticsearch/Dockerfile) - `v0.12-debian-loggly` [docker-image/v0.12/debian-loggly/Dockerfile](docker-image/v0.12/debian-loggly/Dockerfile) - `v0.12-debian-logentries` [docker-image/v0.12/debian-logentries/Dockerfile](docker-image/v0.12/debian-logentries/Dockerfile) @@ -114,6 +115,7 @@ Some images are contributed by users. If you have a problem/question for followi - Amazon Kinesis : @shiftky - logz.io : @SaMnCo / @jamielennox - splunkhec: @FutureSharks +- logsense: @perk / @pmaciolek ### References diff --git a/docker-image/v1.3/debian-logsense/.dockerignore b/docker-image/v1.3/debian-logsense/.dockerignore new file mode 100644 index 0000000000..921ed3fe9e --- /dev/null +++ b/docker-image/v1.3/debian-logsense/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.3/debian-logsense/Dockerfile b/docker-image/v1.3/debian-logsense/Dockerfile new file mode 100644 index 0000000000..5917816a26 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/Dockerfile @@ -0,0 +1,51 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.3.3-debian-1.0 + +LABEL maintainer="Eduardo Silva " +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/2.3.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.3.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.3.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ + RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps net-tools libjemalloc1 \ + && gem install bundler --version 1.16.2 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" + +# See https://packages.debian.org/stretch/amd64/libjemalloc1/filelist +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.1" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.3/debian-logsense/Gemfile b/docker-image/v1.3/debian-logsense/Gemfile new file mode 100644 index 0000000000..975d09b361 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/Gemfile @@ -0,0 +1,13 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.3.3" +gem "oj", "3.5.1" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.3.0" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.1.4" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.0.1" diff --git a/docker-image/v1.3/debian-logsense/Gemfile.lock b/docker-image/v1.3/debian-logsense/Gemfile.lock new file mode 100644 index 0000000000..68da82ced0 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/Gemfile.lock @@ -0,0 +1,104 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (5.2.2) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + addressable (2.6.0) + public_suffix (>= 2.0.2, < 4.0) + concurrent-ruby (1.1.4) + cool.io (1.5.3) + dig_rb (1.0.1) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + ffi (1.10.0) + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.3.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-kubernetes_metadata_filter (2.1.6) + fluentd (>= 0.14.0, < 2) + kubeclient (~> 1.1.4) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-rewrite-tag-filter (2.1.1) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.0.2) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 1.3.2) + fluentd (1.3.3) + cool.io (>= 1.4.5, < 2.0.0) + dig_rb (~> 1.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 0.7.0, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (~> 1.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (0.9.8) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 1.0.1) + http_parser.rb (~> 0.6.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (1.0.3) + http_parser.rb (0.6.0) + i18n (1.6.0) + concurrent-ruby (~> 1.0) + kubeclient (1.1.4) + activesupport + http (= 0.9.8) + recursive-open-struct (= 1.0.0) + rest-client + lru_redux (1.1.0) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) + minitest (5.11.3) + msgpack (1.2.7) + netrc (0.11.0) + oj (3.5.1) + public_suffix (3.0.3) + recursive-open-struct (1.0.0) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + serverengine (2.1.0) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.3) + systemd-journal (1.3.3) + ffi (~> 1.9) + thread_safe (0.3.6) + tzinfo (1.2.5) + thread_safe (~> 0.1) + tzinfo-data (1.2018.9) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.3.0) + fluent-plugin-kubernetes_metadata_filter (~> 2.1.4) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-rewrite-tag-filter (~> 2.1.0) + fluent-plugin-systemd (~> 1.0.1) + fluentd (= 1.3.3) + oj (= 3.5.1) + +BUNDLED WITH + 1.17.2 diff --git a/docker-image/v1.3/debian-logsense/conf/fluent.conf b/docker-image/v1.3/debian-logsense/conf/fluent.conf new file mode 100644 index 0000000000..9212ad0da8 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/conf/fluent.conf @@ -0,0 +1,28 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + + + @type record_transformer + + cs_customer_token "#{ENV['LOGSENSE_TOKEN']}" + + + + + @type forward + send_timeout 60s + recover_wait 10s + hard_timeout 60s + transport tls + tls_verify_hostname true + + host logs.logsense.com + port 32714 + + + diff --git a/docker-image/v1.3/debian-logsense/conf/kubernetes.conf b/docker-image/v1.3/debian-logsense/conf/kubernetes.conf new file mode 100644 index 0000000000..b1b6cb5797 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/conf/kubernetes.conf @@ -0,0 +1,192 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + + + @type null + + + + @type tail + @id in_tail_container_logs + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + tag kubernetes.* + read_from_head true + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file /var/log/fluentd-salt.pos + tag salt + + @type regexp + expression /^(? + + + + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file /var/log/fluentd-startupscript.log.pos + tag startupscript + + @type syslog + + + + + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file /var/log/fluentd-docker.log.pos + tag docker + + @type regexp + expression /^time="(? + + + + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file /var/log/fluentd-etcd.log.pos + tag etcd + + @type none + + + + + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file /var/log/fluentd-kubelet.log.pos + tag kubelet + + @type kubernetes + + + + + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file /var/log/fluentd-kube-proxy.log.pos + tag kube-proxy + + @type kubernetes + + + + + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file /var/log/fluentd-kube-apiserver.log.pos + tag kube-apiserver + + @type kubernetes + + + + + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file /var/log/fluentd-kube-controller-manager.log.pos + tag kube-controller-manager + + @type kubernetes + + + + + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file /var/log/fluentd-kube-scheduler.log.pos + tag kube-scheduler + + @type kubernetes + + + + + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file /var/log/fluentd-rescheduler.log.pos + tag rescheduler + + @type kubernetes + + + + + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file /var/log/fluentd-glbc.log.pos + tag glbc + + @type kubernetes + + + + + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file /var/log/fluentd-cluster-autoscaler.log.pos + tag cluster-autoscaler + + @type kubernetes + + + +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" + + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file /var/log/kube-apiserver-audit.log.pos + tag kube-apiserver-audit + + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(? + + + + @type kubernetes_metadata + @id filter_kube_metadata + + diff --git a/docker-image/v1.3/debian-logsense/conf/systemd.conf b/docker-image/v1.3/debian-logsense/conf/systemd.conf new file mode 100644 index 0000000000..cfc73eba80 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + + read_from_head true + tag kubelet + + +# Logs from docker-systemd + + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + + read_from_head true + tag docker.systemd + + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + + read_from_head true + tag bootkube + + diff --git a/docker-image/v1.3/debian-logsense/entrypoint.sh b/docker-image/v1.3/debian-logsense/entrypoint.sh new file mode 100755 index 0000000000..e2ce270c08 --- /dev/null +++ b/docker-image/v1.3/debian-logsense/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.3/debian-logsense/hooks/post_push b/docker-image/v1.3/debian-logsense/hooks/post_push new file mode 100644 index 0000000000..1d9d00496d --- /dev/null +++ b/docker-image/v1.3/debian-logsense/hooks/post_push @@ -0,0 +1,15 @@ +#!/bin/bash +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/post_push.erb + +set -e + +# Parse image name for repo name +tagStart=$(expr index "$IMAGE_NAME" :) +repoName=${IMAGE_NAME:0:tagStart-1} + +# Tag and push image for each additional tag +for tag in {v1.3.3-debian-logsense-1.1,v1.3-debian-logsense-1}; do + docker tag $IMAGE_NAME ${repoName}:${tag} + docker push ${repoName}:${tag} +done diff --git a/docker-image/v1.3/debian-logsense/plugins/.gitkeep b/docker-image/v1.3/debian-logsense/plugins/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker-image/v1.3/debian-logsense/plugins/parser_kubernetes.rb b/docker-image/v1.3/debian-logsense/plugins/parser_kubernetes.rb new file mode 100644 index 0000000000..0f593caafa --- /dev/null +++ b/docker-image/v1.3/debian-logsense/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# +# @type kubernetes +# +# +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?\w)(?