delete extra logs, fix ch for cozy

aenix-io · Oct 17, 2024 · bb3c429 · bb3c429
1 parent 2e16f78
commit bb3c429
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 14 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,10 +7,19 @@ repos:
     - id: mixed-line-ending
       args: [--fix=lf]
     - id: check-yaml
-      exclude: packages/apps/postgres/templates/init-script.yaml
+      exclude: '^.*templates/.*\.yaml$'
       args: [--unsafe]
 - repo: https://github.com/igorshubovych/markdownlint-cli
   rev: v0.41.0
   hooks:
   - id: markdownlint
     args: [--fix, --disable, MD013, MD041, --]
+-   repo: local
+    hooks:
+    - id: gen-versions-map
+      name: Generate versions map and check for changes
+      entry: bash -c 'cd packages/apps && make check-version-map'
+      language: system
+      types: [file]
+      pass_filenames: false
+      description: Run the script and fail if it generates changes
diff --git a/packages/apps/clickhouse/Chart.yaml b/packages/apps/clickhouse/Chart.yaml
@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.0
+version: 0.6.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml
@@ -32,11 +32,12 @@ kind: "ClickHouseInstallation"
 metadata:
   name: "{{ .Release.Name }}"
 spec:
-  {{- with .Values.size }}
+  namespaceDomainPattern:  "%s.svc.cozy.local"
   defaults:
     templates:
       dataVolumeClaimTemplate: data-volume-template
-  {{- end }}
+      podTemplate: clickhouse-per-host
+      serviceTemplate: svc-template
   configuration:
     {{- with $users }}
     users:
@@ -46,24 +47,91 @@ spec:
       {{ $name }}/networks/ip: ["::/0"]
       {{- end }}
     {{- end }}
+    files:
+      config.d/z_log_disable.xml: |
+        <clickhouse>
+            <asynchronous_metric_log remove="1"/>
+            <metric_log remove="1"/>
+            <query_views_log remove="1" />
+            <part_log remove="1"/>
+            <session_log remove="1"/>
+            <text_log remove="1" />
+            <trace_log remove="1"/>
+            <crash_log remove="1"/>
+            <opentelemetry_span_log remove="1"/>
+        </clickhouse>
+      config.d/query_log_ttl.xml: |
+        <clickhouse>
+            <query_log replace="1">
+                <database>system</database>
+                <table>query_log</table>
+                <engine>ENGINE = MergeTree PARTITION BY (event_date)
+                        ORDER BY (event_time)
+                        TTL event_date + INTERVAL 14 DAY DELETE
+                </engine>
+                <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+            </query_log>
+            <query_thread_log replace="1">
+                <database>system</database>
+                <table>query_thread_log</table>
+                <engine>ENGINE = MergeTree PARTITION BY (event_date)
+                        ORDER BY (event_time)
+                        TTL event_date + INTERVAL 14 DAY DELETE
+                </engine>
+                <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+            </query_thread_log>
+        </clickhouse>
     profiles:
       readonly/readonly: "1"
     clusters:
       - name: "clickhouse"
         layout:
           shardsCount: {{ .Values.shards }}
           replicasCount: {{ .Values.replicas }}
-  {{- with .Values.size }}
   templates:
     volumeClaimTemplates:
       - name: data-volume-template
         spec:
           accessModes:
             - ReadWriteOnce
-          {{- with $.Values.storageClass }}
-          storageClassName: {{ . }}
-          {{- end }}
           resources:
             requests:
-              storage: {{ . }}
-  {{- end }}
+              storage: {{ .Values.size }}
+      - name: log-volume-template
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: {{ .Values.logStorageSize }}
+    podTemplates:
+      - name: clickhouse-per-host
+        spec:
+          affinity:
+            podAntiAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchExpressions:
+                      - key: "clickhouse.altinity.com/chi"
+                        operator: In
+                        values:
+                          - "{{ .Release.Name }}"
+                  topologyKey: "kubernetes.io/hostname"
+          containers:
+            - name: clickhouse
+              imagePullPolicy: Always
+              image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+              volumeMounts:
+                - name: data-volume-template
+                  mountPath: /var/lib/clickhouse
+                - name: log-volume-template
+                  mountPath: /var/log/clickhouse-server
+    serviceTemplates:
+      - name: svc-template
+        generateName: chendpoint-{chi}
+        spec:
+          ports:
+            - name: http
+              port: 8123
+            - name: tcp
+              port: 9000
diff --git a/packages/apps/clickhouse/values.schema.json b/packages/apps/clickhouse/values.schema.json
@@ -7,6 +7,11 @@
             "description": "Persistent Volume size",
             "default": "10Gi"
         },
+        "logStorageSize": {
+            "type": "string",
+            "description": "logStorageSize Persistent Volume for logs size",
+            "default": "2Gi"
+        },
         "shards": {
             "type": "number",
             "description": "Number of Clickhouse replicas",
@@ -20,7 +25,7 @@
         "storageClass": {
             "type": "string",
             "description": "StorageClass used to store the data",
-            "default": ""
+            "default": "local"
         },
         "backup": {
             "type": "object",
@@ -68,4 +73,4 @@
             }
         }
     }
-}
+}
diff --git a/packages/apps/clickhouse/values.yaml b/packages/apps/clickhouse/values.yaml
@@ -1,14 +1,16 @@
 ## @section Common parameters
 
 ## @param size Persistent Volume size
+## @param logStorageSize Persistent Volume for logs size
 ## @param shards Number of Clickhouse replicas
 ## @param replicas Number of Clickhouse shards
 ## @param storageClass StorageClass used to store the data
 ##
 size: 10Gi
+logStorageSize: 2Gi
 shards: 1
 replicas: 2
-storageClass: ""
+storageClass: "local"
 
 ## @section Configuration parameters
 
@@ -42,3 +44,7 @@ backup:
   s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
   s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
   resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
+
+image:
+  repository: clickhouse/clickhouse-server
+  tag: "24.2.2.71"
diff --git a/packages/apps/versions_map b/packages/apps/versions_map
@@ -4,7 +4,8 @@ clickhouse 0.2.0 7cd7de73
 clickhouse 0.2.1 5ca8823
 clickhouse 0.3.0 b00621e
 clickhouse 0.4.0 320fc32
-clickhouse 0.5.0 HEAD
+clickhouse 0.5.0 2a4768a5
+clickhouse 0.6.0 HEAD
 ferretdb 0.1.0 4ffa8615
 ferretdb 0.1.1 5ca8823
 ferretdb 0.2.0 adaf603

diff --git a/packages/system/clickhouse-operator/values.yaml b/packages/system/clickhouse-operator/values.yaml
@@ -4,3 +4,7 @@ altinity-clickhouse-operator:
       config.yaml:
         watch:
           namespaces: [".*"]
+        clickhouse:
+          configuration:
+            network:
+              hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cozy\\.local$"