From 6d1d870d30150cee94da506d5efc4375f1d8bd31 Mon Sep 17 00:00:00 2001 From: Dmitry Logunov Date: Wed, 21 Nov 2018 13:42:01 +0300 Subject: [PATCH 1/3] fixing values chart and adding helm-update --- charts/loghouse/values.yaml | 9 ++++++--- helm-update | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100755 helm-update diff --git a/charts/loghouse/values.yaml b/charts/loghouse/values.yaml index 749c068..0cf47bb 100644 --- a/charts/loghouse/values.yaml +++ b/charts/loghouse/values.yaml @@ -5,6 +5,9 @@ # result: YWRtaW46JGFwcjEkMzdxSEwvTVIkcEFvdzEzZDUwMkd5VFc2VDNlQmJiMAoK auth: YWRtaW46JGFwcjEkelhESkU5YTkkRkU0OFdnZlBMZlJJQjk0bVhXZVprMAoK +app: + version: 0.2.2 + # Password for default user in clickhouse clickhouse_pass_original: password @@ -24,9 +27,9 @@ ingress: host: tabix.domain.com path: "/" tls_secret_name: loghouse -# annotations: -# - 'kubernetes.io/ingress.class: traefik' -# - 'traefik.frontend.passHostHeader: "true"' + annotations: + - 'kubernetes.io/ingress.class: traefik' + - 'traefik.frontend.passHostHeader: "true"' # Enable tabix enable_tabix: false diff --git a/helm-update b/helm-update new file mode 100755 index 0000000..e5cf0c0 --- /dev/null +++ b/helm-update @@ -0,0 +1,33 @@ +#! /bin/bash + +NAMESPACE=$1 + +if [[ "${NAMESPACE}" = "" ]]; then + echo " It's nessary to define NAMESPACE environment variable!" + echo " Format: ./helm-update []" + exit +fi + +KUBE_CONFIGURATION_DIR="${HOME}/arcadia-bedrock/gpt-core" + +if [[ "$2" != "" ]]; then +KUBE_CONFIGURATION_DIR="$2" +fi + +if [[ ! -f ${KUBE_CONFIGURATION_DIR}/config || ! -f ${KUBE_CONFIGURATION_DIR}/.helm ]]; then + echo " Check files ${KUBE_CONFIGURATION_DIR}/config and ${KUBE_CONFIGURATION_DIR}/.helm exist!" + echo " " + echo " It's nessasry to save Kubernetes and Helm configs in folder '~/arcadia-bedrock/gpt-core' (default config folder)" + echo " Also you can define another config folder in the second parameter of command" + echo " Format: ./helm-update []" + exit +fi + +export KUBECONFIG=/home/lunx/projects/arcadia/dlp-bedrock-new/assets/dev1-config +export HELM_HOME=~/.helm + +echo " Deleting ${NAMESPACE} ... " +( cd ./charts && helm delete ${NAMESPACE} --purge ) + +echo " Installing ${NAMESPACE} ... " +( cd ./charts && helm install --namespace ${NAMESPACE} -n ${NAMESPACE} loghouse --tls ) From 701e4c6e24f2a69a20b8931135579a6605dab40f Mon Sep 17 00:00:00 2001 From: Dmitry Logunov Date: Fri, 23 Nov 2018 17:25:08 +0300 Subject: [PATCH 2/3] parsing params --- .gitignore | 1 + .../templates/fluentd/fluentd-configmap.yaml | 16 +++++---- .../loghouse/templates/fluentd/fluentd.yaml | 7 ++-- charts/loghouse/values.yaml | 8 ++--- helm-update | 33 ------------------- 5 files changed, 20 insertions(+), 45 deletions(-) delete mode 100755 helm-update diff --git a/.gitignore b/.gitignore index 7112d2f..1591b93 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ ## Environment normalization: /.bundle /vendor/bundle +helm diff --git a/charts/loghouse/templates/fluentd/fluentd-configmap.yaml b/charts/loghouse/templates/fluentd/fluentd-configmap.yaml index 48bde7d..fca53b8 100644 --- a/charts/loghouse/templates/fluentd/fluentd-configmap.yaml +++ b/charts/loghouse/templates/fluentd/fluentd-configmap.yaml @@ -83,20 +83,24 @@ data: @type record_modifier _json_log_ ${ log = record["log"].strip; if log[0].eql?('{') && log[-1].eql?('}'); begin; JSON.parse(log); rescue JSON::ParserError; end; end } + _params_select_ ${ record["_json_log_"] ? record["_json_log_"].select{|k, v| k == 'params' && v.is_a?(Hash)} : nil } + _params_hash_ ${ !record["_params_select_"].nil? && !record["_params_select_"]["params"].nil? ? record["_params_select_"]["params"] : nil } + _params_keys_ ${ !record["_params_hash_"].nil? ? record["_params_hash_"].keys.map{ |key| 'params_' + key } : []} + _params_values_ ${ !record["_params_hash_"].nil? ? record["_params_hash_"].values.map(&:to_s) : []} timestamp ${time} nsec ${record["time"].split('.').last.to_i} # static fields source "kubernetes" namespace ${record["kubernetes"]["namespace_name"]} - host ${record["kubernetes"]["host"] || ENV["K8S_NODE_NAME"]} + jsonhost ${record["kubernetes"]["host"] || ENV["K8S_NODE_NAME"]} pod_name ${record["kubernetes"]["pod_name"]} container_name ${record["kubernetes"]["container_name"]} stream ${record["stream"]} # dynamic fields labels.names ${record["kubernetes"]["labels"].keys} labels.values ${record["kubernetes"]["labels"].values} - string_fields.names ${record["_json_log_"] ? record["_json_log_"].select{|_, v| !v.nil? && !v.is_a?(Numeric) && !v.is_a?(TrueClass) && !v.is_a?(FalseClass)}.keys : ["log"]} - string_fields.values ${record["_json_log_"] ? record["_json_log_"].select{|_, v| !v.nil? && !v.is_a?(Numeric) && !v.is_a?(TrueClass) && !v.is_a?(FalseClass)}.values.map(&:to_s) : [record["log"]]} + string_fields.names ${record["_json_log_"] ? record["_json_log_"].select{|k, v| k != 'params' && !v.nil? && !v.is_a?(Numeric) && !v.is_a?(TrueClass) && !v.is_a?(FalseClass)}.keys + record["_params_keys_"] : ["log"]} + string_fields.values ${record["_json_log_"] ? record["_json_log_"].select{|k, v| k != 'params' && !v.nil? && !v.is_a?(Numeric) && !v.is_a?(TrueClass) && !v.is_a?(FalseClass)}.values.map(&:to_s) + record["_params_values_"] : [record["log"]]} number_fields.names ${record["_json_log_"] ? record["_json_log_"].select{|_, v| v.is_a?(Numeric)}.keys : []} number_fields.values ${record["_json_log_"] ? record["_json_log_"].select{|_, v| v.is_a?(Numeric)}.values : []} @@ -106,7 +110,7 @@ data: null_fields.names ${record["_json_log_"] ? record["_json_log_"].select{|_, v| v.nil?}.keys : []} - remove_keys kubernetes, docker, master_url, time, log, _json_log_ + remove_keys kubernetes, docker, master_url, time, log, _json_log_, _params_select_, _params_hash_, _params_keys_, _params_values_ @@ -137,9 +141,9 @@ data: @type record_modifier - whitelist_keys timestamp, nsec, source, namespace, host, pod_name, container_name, stream, labels.names, labels.values, string_fields.names, string_fields.values, number_fields.names, number_fields.values, boolean_fields.names, boolean_fields.values, null_fields.names + whitelist_keys timestamp, nsec, source, namespace, host, pod_name, container_name, stream, labels.names, labels.values, string_fields.names, string_fields.values, number_fields.names, number_fields.values, boolean_fields.names, boolean_fields.values, null_fields.names - + @type exec command bash /usr/local/bin/insert_ch.sh diff --git a/charts/loghouse/templates/fluentd/fluentd.yaml b/charts/loghouse/templates/fluentd/fluentd.yaml index cdb193d..007336d 100644 --- a/charts/loghouse/templates/fluentd/fluentd.yaml +++ b/charts/loghouse/templates/fluentd/fluentd.yaml @@ -12,9 +12,10 @@ spec: metadata: labels: k8s-app: fluentd - kubernetes.io/cluster-service: "true" + # kubernetes.io/cluster-service: "true" annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' + # scheduler.alpha.kubernetes.io/critical-pod: '' + checksum/config: {{ include (print $.Template.BasePath "/fluentd/fluentd-configmap.yaml") . | sha256sum }} spec: serviceAccountName: fluentd containers: @@ -22,6 +23,8 @@ spec: image: flant/loghouse-fluentd:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} env: + - name: TD_AGENT_OPTIONS + value: -vv - name: FLUENTD_ARGS value: --no-supervisor -q - name: CLICKHOUSE_SERVER diff --git a/charts/loghouse/values.yaml b/charts/loghouse/values.yaml index 0cf47bb..76e90ca 100644 --- a/charts/loghouse/values.yaml +++ b/charts/loghouse/values.yaml @@ -14,13 +14,13 @@ clickhouse_pass_original: password # Settings for ingress ingress: enable: true - enable_https: true + enable_https: false clickhouse: host: clickhouse.domain.com path: "/" tls_secret_name: clickhouse loghouse: - host: loghouse.domain.com + host: loghouse.dev1.cgdlp.io path: "/" tls_secret_name: loghouse tabix: @@ -66,7 +66,7 @@ partition_period: 1 # effect: 'NoSchedule' # If you not want install fluentd on master node -install_master: false +install_master: true imagePullPolicy: Always @@ -93,7 +93,7 @@ fluentd: cpu: 1 memory: 512Mi requests: - cpu: 0.5 + cpu: 0.1 memory: 256Mi loghouse: resources: diff --git a/helm-update b/helm-update deleted file mode 100755 index e5cf0c0..0000000 --- a/helm-update +++ /dev/null @@ -1,33 +0,0 @@ -#! /bin/bash - -NAMESPACE=$1 - -if [[ "${NAMESPACE}" = "" ]]; then - echo " It's nessary to define NAMESPACE environment variable!" - echo " Format: ./helm-update []" - exit -fi - -KUBE_CONFIGURATION_DIR="${HOME}/arcadia-bedrock/gpt-core" - -if [[ "$2" != "" ]]; then -KUBE_CONFIGURATION_DIR="$2" -fi - -if [[ ! -f ${KUBE_CONFIGURATION_DIR}/config || ! -f ${KUBE_CONFIGURATION_DIR}/.helm ]]; then - echo " Check files ${KUBE_CONFIGURATION_DIR}/config and ${KUBE_CONFIGURATION_DIR}/.helm exist!" - echo " " - echo " It's nessasry to save Kubernetes and Helm configs in folder '~/arcadia-bedrock/gpt-core' (default config folder)" - echo " Also you can define another config folder in the second parameter of command" - echo " Format: ./helm-update []" - exit -fi - -export KUBECONFIG=/home/lunx/projects/arcadia/dlp-bedrock-new/assets/dev1-config -export HELM_HOME=~/.helm - -echo " Deleting ${NAMESPACE} ... " -( cd ./charts && helm delete ${NAMESPACE} --purge ) - -echo " Installing ${NAMESPACE} ... " -( cd ./charts && helm install --namespace ${NAMESPACE} -n ${NAMESPACE} loghouse --tls ) From a29173749c5ecc053004184c8820482c0057be54 Mon Sep 17 00:00:00 2001 From: DmitryLogunov Date: Fri, 23 Nov 2018 17:28:05 +0300 Subject: [PATCH 3/3] Update fluentd-configmap.yaml --- charts/loghouse/templates/fluentd/fluentd-configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loghouse/templates/fluentd/fluentd-configmap.yaml b/charts/loghouse/templates/fluentd/fluentd-configmap.yaml index fca53b8..6e7ee06 100644 --- a/charts/loghouse/templates/fluentd/fluentd-configmap.yaml +++ b/charts/loghouse/templates/fluentd/fluentd-configmap.yaml @@ -92,7 +92,7 @@ data: # static fields source "kubernetes" namespace ${record["kubernetes"]["namespace_name"]} - jsonhost ${record["kubernetes"]["host"] || ENV["K8S_NODE_NAME"]} + host ${record["kubernetes"]["host"] || ENV["K8S_NODE_NAME"]} pod_name ${record["kubernetes"]["pod_name"]} container_name ${record["kubernetes"]["container_name"]} stream ${record["stream"]}