diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md index f198cefc..9850aa50 100644 --- a/build/charts/theia/README.md +++ b/build/charts/theia/README.md @@ -38,6 +38,14 @@ Kubernetes: `>= 1.16.0-0` | clickhouse.monitor.skipRoundsNum | int | `3` | The number of rounds for the monitor to stop after a deletion to wait for the ClickHouse MergeTree Engine to release memory. | | clickhouse.monitor.threshold | float | `0.5` | The storage percentage at which the monitor starts to delete old records. Vary from 0 to 1. | | clickhouse.service.httpPort | int | `8123` | HTTP port number for the ClickHouse service. | +| clickhouse.service.secureConnection.commonName | string | `"clickhouse-clickhouse.flow-visibility.svc"` | Common name to use in the certificate. This is ignored if selfSignedCert is false. | +| clickhouse.service.secureConnection.daysValid | int | `365` | Number of days for which the certificate will be valid. There is no automatic rotation with this method. This is ignored if selfSignedCert is false. | +| clickhouse.service.secureConnection.dnsNames | list | `[]` | DNS names to use in the certificate. This is ignored if selfSignedCert is false. | +| clickhouse.service.secureConnection.enable | bool | `false` | Determine whether to enable secure connection to ClickHouse | +| clickhouse.service.secureConnection.httpsPort | int | `8443` | HTTP port number for the ClickHouse service. | +| clickhouse.service.secureConnection.ipAddresses | list | `[]` | IP addresses to use in the certificate. This is ignored if selfSignedCert is false. | +| clickhouse.service.secureConnection.secureTcpPort | int | `9440` | TCP port number for the ClickHouse service. | +| clickhouse.service.secureConnection.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "clickhouse-tls" must be provided with the following keys: tls.crt and tls.key. | | clickhouse.service.tcpPort | int | `9000` | TCP port number for the ClickHouse service. | | clickhouse.service.type | string | `"ClusterIP"` | The type of Service exposing ClickHouse. It can be one of ClusterIP, NodePort or LoadBalancer. | | clickhouse.storage.createPersistentVolume.local.affinity | object | `{}` | Affinity for the Local PersistentVolume. By default it requires to label the Node used to store the ClickHouse data with "antrea.io/clickhouse-data-node=". | diff --git a/build/charts/theia/provisioning/tls/settings.xml b/build/charts/theia/provisioning/tls/settings.xml new file mode 100644 index 00000000..bc0542c6 --- /dev/null +++ b/build/charts/theia/provisioning/tls/settings.xml @@ -0,0 +1,15 @@ + + {{ .Values.clickhouse.service.secureConnection.httpsPort }} + {{ .Values.clickhouse.service.secureConnection.secureTcpPort }} + + + /opt/certs/tls.crt + /opt/certs/tls.key + none + true + true + sslv2,sslv3 + true + + + diff --git a/build/charts/theia/templates/_helpers.tpl b/build/charts/theia/templates/_helpers.tpl index 3609eea8..a8b4c775 100644 --- a/build/charts/theia/templates/_helpers.tpl +++ b/build/charts/theia/templates/_helpers.tpl @@ -42,12 +42,21 @@ {{- $clickhouse := .clickhouse }} {{- $enablePV := .enablePV }} {{- $Chart := .Chart }} +{{- $tls := .clickhouse.service.secureConnection }} - name: clickhouse image: {{ include "clickHouseServerImage" . | quote }} imagePullPolicy: {{ $clickhouse.image.pullPolicy }} volumeMounts: - name: clickhouse-configmap-volume mountPath: /docker-entrypoint-initdb.d + {{- if $tls.enable }} + - name: clickhouse-tls + mountPath: /opt/certs/tls.crt + subPath: tls.crt + - name: clickhouse-tls + mountPath: /opt/certs/tls.key + subPath: tls.key + {{- end }} {{- if not $enablePV }} - name: clickhouse-storage-volume mountPath: /var/lib/clickhouse @@ -73,6 +82,7 @@ {{- define "clickhouse.volume" }} {{- $clickhouse := .clickhouse }} +{{- $tls := .clickhouse.service.secureConnection }} {{- $enablePV := .enablePV }} {{- $Files := .Files }} - name: clickhouse-configmap-volume @@ -87,6 +97,12 @@ - key: {{ regexReplaceAll "(.*)/" $path "" }} path: migrators/{{ regexReplaceAll "(.*)/" $path "" }} {{- end }} +{{- if $tls.enable }} +- name: clickhouse-tls + secret: + secretName: clickhouse-tls + optional: true +{{- end }} {{- if not $enablePV }} - name: clickhouse-storage-volume emptyDir: @@ -99,6 +115,14 @@ name: clickhouse-monitor-coverage {{- end }} +{{- define "clickhouse.tlsConfig" -}} +{{- $Files := .Files }} +{{- $Global := .Global }} +{{- range $path, $_ := .Files.Glob "provisioning/tls/*" }} +{{ regexReplaceAll "(.*)/" $path "" }}: | +{{ tpl ($.Files.Get $path) $Global | indent 2 }} +{{- end }} +{{- end -}} {{- define "theiaImageTag" -}} {{- $tag := .tag -}} diff --git a/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml b/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml index 38781ea8..fa22aae8 100644 --- a/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml +++ b/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml @@ -16,8 +16,20 @@ spec: {{ .Values.clickhouse.connectionSecret.readOnlyUsername }}/networks/ip: "::/0" profiles: readonly/readonly: 1 + {{- if .Values.clickhouse.service.secureConnection.enable }} + files: + {{- include "clickhouse.tlsConfig" (dict "Files" .Files "Global" .) | indent 6 }} + {{- end }} clusters: - name: "clickhouse" + settings: + tcp_port: 9000 # keep for localhost + http_port: {{ .Values.clickhouse.service.httpPort }} + {{- if .Values.clickhouse.service.secureConnection.enable }} + tcp_port_secure: {{ .Values.clickhouse.service.secureConnection.secureTcpPort }} + https_port: {{ .Values.clickhouse.service.secureConnection.httpsPort }} + secure: "yes" + {{- end }} layout: {{- if gt (int .Values.clickhouse.cluster.replicas) 1 }} shards: @@ -68,6 +80,13 @@ spec: port: {{ .Values.clickhouse.service.httpPort }} - name: tcp port: {{ .Values.clickhouse.service.tcpPort }} + targetPort: 9000 + {{- if .Values.clickhouse.service.secureConnection.enable }} + - name: https + port: {{ .Values.clickhouse.service.secureConnection.httpsPort }} + - name: secureclient + port: {{ .Values.clickhouse.service.secureConnection.secureTcpPort }} + {{- end }} podTemplates: {{- if gt (int .Values.clickhouse.cluster.replicas) 1 }} - name: pod-template-without-monitor diff --git a/build/charts/theia/templates/clickhouse/secret.yaml b/build/charts/theia/templates/clickhouse/secret.yaml index ba1ec108..13671e01 100644 --- a/build/charts/theia/templates/clickhouse/secret.yaml +++ b/build/charts/theia/templates/clickhouse/secret.yaml @@ -1,3 +1,28 @@ +{{- $tls := .Values.clickhouse.service.secureConnection }} +{{- if and $tls.enable $tls.selfSignedCert }} +{{- $cert := genSelfSignedCert $tls.commonName $tls.ipAddresses (uniq (append $tls.dnsNames $tls.commonName)) ($tls.daysValid | int) }} +{{- $certPEM := $cert.Cert | b64enc }} +{{- $keyPEM := $cert.Key | b64enc }} +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: clickhouse-tls + namespace: {{ .Release.Namespace }} +data: + tls.crt: {{ $certPEM | quote }} + tls.key: {{ $keyPEM | quote }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: clickhouse-ca + namespace: {{ .Release.Namespace }} +data: + ca.crt: {{ $certPEM | quote }} +{{- end }} +--- apiVersion: v1 kind: Secret metadata: diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml index 279f1f83..41ff2058 100644 --- a/build/charts/theia/values.yaml +++ b/build/charts/theia/values.yaml @@ -47,6 +47,33 @@ clickhouse: tcpPort: 9000 # -- HTTP port number for the ClickHouse service. httpPort: 8123 + # As ClickHouse Operator currently does not support the cert management, + # we recommend to use Ingress or Load Balancer to expose ClickHouse securely. + # But we also provide this option to provide the secure connection from + # ClickHouse server end. To use this option, please rotate the certs manually. + secureConnection: + # -- Determine whether to enable secure connection to ClickHouse + enable: false + # -- TCP port number for the ClickHouse service. + secureTcpPort: 9440 + # -- HTTP port number for the ClickHouse service. + httpsPort: 8443 + # -- Indicates whether to use auto-generated self-signed TLS certificates. + # If false, a Secret named "clickhouse-tls" must be provided with the + # following keys: tls.crt and tls.key. + selfSignedCert: true + # -- Common name to use in the certificate. This is ignored if selfSignedCert + # is false. + commonName: "clickhouse-clickhouse.flow-visibility.svc" + # -- IP addresses to use in the certificate. This is ignored if selfSignedCert + # is false. + ipAddresses: [] + # -- DNS names to use in the certificate. This is ignored if selfSignedCert + # is false. + dnsNames: [] + # -- Number of days for which the certificate will be valid. There is no automatic + # rotation with this method. This is ignored if selfSignedCert is false. + daysValid: 365 # -- Time to live for data in the ClickHouse. Can be a plain integer using # one of these unit suffixes SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, # YEAR. diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 6ebf0a4a..d5d78741 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -7210,6 +7210,9 @@ spec: replicasCount: 1 shardsCount: 1 name: clickhouse + settings: + http_port: 8123 + tcp_port: 9000 profiles: readonly/readonly: 1 settings: @@ -7343,4 +7346,5 @@ spec: port: 8123 - name: tcp port: 9000 + targetPort: 9000 type: ClusterIP diff --git a/test/e2e/framework.go b/test/e2e/framework.go index c9dcee81..d795a1b0 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1301,9 +1301,11 @@ func (data *TestData) deployFlowVisibilityCommon(chOperatorYML, flowVisibilityYM // after applying a new Flow Visibility manifest. func (data *TestData) waitForClickHousePod() error { startUpdating := false - // ClickHouse Operator takes around 2 minute to restart the ClickHouse Pod + // After ClickHouse Operator is applied, ClickHouse pod will restart once + // before doing upgrading in the new flow visibility manifest. + // ClickHouse Operator takes around 2 minute for each restart of the ClickHouse Pod // which requires more time to make sure that ClickHouse Pod is ready. - err := wait.Poll(defaultInterval, defaultTimeout*4, func() (bool, error) { + err := wait.Poll(defaultInterval, defaultTimeout*6, func() (bool, error) { clickHouseStatefulSetName := fmt.Sprintf("%s-0-0", clickHousePodNamePrefix) ss, err := data.clientset.AppsV1().StatefulSets(flowVisibilityNamespace).Get(context.TODO(), clickHouseStatefulSetName, metav1.GetOptions{}) if err != nil { @@ -1326,6 +1328,12 @@ func (data *TestData) waitForClickHousePod() error { } else if err != nil { return err } + // The statefulset updating complete implies the new ClickHouse pod is starting + // Check for ClickHouse Pod ready + clickHousePodName := fmt.Sprintf("%s-0-0-0", clickHousePodNamePrefix) + if err = data.podWaitForReady(2*defaultTimeout, clickHousePodName, flowVisibilityNamespace); err != nil { + return err + } return nil }