Change ingress to ClickHouse server with tls enabled

Signed-off-by: Yun-Tang Hsu <[email protected]>
antrea-io · Jul 19, 2023 · 161c062 · 161c062
1 parent 861e52b
commit 161c062
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 171 deletions.
diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -48,8 +48,6 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 TESTBED_CMD="$THIS_DIR/kind-setup.sh"
 YML_CMD="$THIS_DIR/../../hack/generate-manifest.sh"
 FLOWAGGREGATOR_YML_CMD="$THIS_DIR/../../hack/generate-manifest-flow-aggregator.sh"
-CLICKHOUSE_TLS_YML_CMD="$THIS_DIR/../../hack/generate-manifest-clickhouse-tls.sh"
-INGRESS_NGINX_CONTROLLER="$THIS_DIR/../../build/yamls/ingress-nginx-controller.yml"
 FLOW_VISIBILITY_HELM_VALUES="$THIS_DIR/values-flow-exporter.yml"
 CH_OPERATOR_YML="$THIS_DIR/../../build/yamls/clickhouse-operator-install-bundle.yml"
 FLOW_VISIBILITY_CHART="$THIS_DIR/../../test/e2e/charts/flow-visibility"
@@ -258,15 +256,13 @@ function run_test {
           $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
       fi
       $HELM template "$FLOW_VISIBILITY_CHART"  | docker exec -i kind-control-plane dd of=/root/flow-visibility.yml
-      $HELM template "$FLOW_VISIBILITY_CHART" --set "ingress.enable=true" | docker exec -i kind-control-plane dd of=/root/flow-visibility-tls.yml
+      $HELM template "$FLOW_VISIBILITY_CHART" --set "secureConnection.enable=true" | docker exec -i kind-control-plane dd of=/root/flow-visibility-tls.yml
 
       curl -o $CH_OPERATOR_YML https://raw.githubusercontent.com/Altinity/clickhouse-operator/release-0.21.0/deploy/operator/clickhouse-operator-install-bundle.yaml
-      curl -o $INGRESS_NGINX_CONTROLLER https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.8.1/deploy/static/provider/cloud/deploy.yaml
       sed -i -e "s|\"image\": \"clickhouse/clickhouse-server:22.3\"|\"image\": \"projects.registry.vmware.com/antrea/clickhouse-server:23.4\"|g" $CH_OPERATOR_YML
       sed -i -e "s|image: altinity/clickhouse-operator:0.21.0|image: projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0|g" $CH_OPERATOR_YML
       sed -i -e "s|image: altinity/metrics-exporter:0.21.0|image: projects.registry.vmware.com/antrea/metrics-exporter:0.21.0|g" $CH_OPERATOR_YML
       cat $CH_OPERATOR_YML | docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yml
-      cat $INGRESS_NGINX_CONTROLLER | docker exec -i kind-control-plane dd of=/root/ingress-nginx-controller.yml
   fi
 
   if $proxy_all; then

diff --git a/test/e2e/charts/flow-visibility/provisioning/tls/settings.xml b/test/e2e/charts/flow-visibility/provisioning/tls/settings.xml
@@ -0,0 +1,15 @@
+<yandex>
+    <https_port>8443</https_port>
+    <tcp_port_secure>9440</tcp_port_secure>
+    <openSSL>
+        <server>
+            <certificateFile>/opt/certs/tls.crt</certificateFile>
+            <privateKeyFile>/opt/certs/tls.key</privateKeyFile>
+            <verificationMode>none</verificationMode>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+        </server>
+    </openSSL>
+</yandex>
diff --git a/test/e2e/charts/flow-visibility/templates/clickhouseinstallation.yaml b/test/e2e/charts/flow-visibility/templates/clickhouseinstallation.yaml
@@ -1,3 +1,12 @@
+{{- define "clickhouse.tlsConfig" -}}
+{{- $Files := .Files }}
+{{- $Global := .Global }}
+{{- range $path, $_ :=  .Files.Glob  "provisioning/tls/*" }}
+{{ regexReplaceAll "(.*)/" $path "" }}: |
+{{ tpl ($.Files.Get $path) $Global | indent 2 }}
+{{- end }}
+{{- end -}}
+
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"
 metadata:
@@ -10,8 +19,22 @@ spec:
     users:
       clickhouse_operator/k8s_secret_password: flow-visibility/clickhouse-secret/password
       clickhouse_operator/networks/ip: "::/0"
+    profiles:
+      readonly/readonly: 1
+    {{- if .Values.secureConnection.enable }}
+    files:
+      {{- include "clickhouse.tlsConfig" (dict "Files" .Files "Global" .) | indent 6 }}
+    {{- end }}
     clusters:
       - name: "clickhouse"
+        {{- if .Values.secureConnection.enable }}
+        secure: "yes"
+        settings:
+          tcp_port: 9000
+          tcp_port_secure: 9440
+          https_port: 8443
+          http_port: 8123
+        {{- end }}
         layout:
           shardsCount: 1
           replicasCount: 1
@@ -29,6 +52,12 @@ spec:
               port: 8123
             - name: tcp
               port: 9000
+            {{- if .Values.secureConnection.enable }}
+            - name: https
+              port: 8443
+            - name: secureclient
+              port: 9440
+            {{- end }}
     podTemplates:
       - name: pod-template
         spec:
@@ -41,6 +70,14 @@ spec:
                   mountPath: /docker-entrypoint-initdb.d
                 - name: clickhouse-storage-volume
                   mountPath: /var/lib/clickhouse
+                {{- if .Values.secureConnection.enable }}
+                - name: clickhouse-tls
+                  mountPath: /opt/certs/tls.crt
+                  subPath: tls.crt
+                - name: clickhouse-tls
+                  mountPath: /opt/certs/tls.key
+                  subPath: tls.key
+                {{- end }}
           volumes:
             - name: clickhouse-configmap-volume
               configMap:
@@ -49,3 +86,9 @@ spec:
               emptyDir:
                 medium: Memory
                 sizeLimit: 8Gi
+            {{- if .Values.secureConnection.enable }}
+            - name: clickhouse-tls
+              secret:
+                secretName: clickhouse-tls
+                optional: true
+            {{- end }}
diff --git a/test/e2e/charts/flow-visibility/templates/ingress.yaml b/test/e2e/charts/flow-visibility/templates/ingress.yaml
diff --git a/test/e2e/charts/flow-visibility/templates/namespace.yaml b/test/e2e/charts/flow-visibility/templates/namespace.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enable }}
+{{- if .Values.secureConnection.enable }}
 apiVersion: v1
 kind: Namespace
 metadata:

diff --git a/test/e2e/charts/flow-visibility/templates/secret.yaml b/test/e2e/charts/flow-visibility/templates/secret.yaml
@@ -1,5 +1,5 @@
-{{- if .Values.ingress.enable }}
-{{- $cert := genSelfSignedCert .Values.ingress.commonName .Values.ingress.ipAddresses (uniq (append .Values.ingress.dnsNames .Values.ingress.commonName)) (.Values.ingress.daysValid | int) }}
+{{- if .Values.secureConnection.enable }}
+{{- $cert := genSelfSignedCert .Values.secureConnection.commonName .Values.secureConnection.ipAddresses (uniq (append .Values.secureConnection.dnsNames .Values.secureConnection.commonName)) (.Values.secureConnection.daysValid | int) }}
 {{- $certPEM := $cert.Cert | b64enc }}
 {{- $keyPEM := $cert.Key | b64enc }}
 ---

diff --git a/test/e2e/charts/flow-visibility/values.yaml b/test/e2e/charts/flow-visibility/values.yaml
@@ -1,8 +1,8 @@
-# Settings for ingress
-ingress:
+# Settings for ClickHouse
+secureConnection:
   enable: false
   # -- Common name to use in the certificate.
-  commonName: "clickhouse.localdev.me"
+  commonName: "clickhouse-clickhouse.flow-visibility.svc"
   # -- IP addresses to use in the certificate.
   ipAddresses: []
   # -- DNS names to use in the certificate.

diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go
@@ -478,13 +478,6 @@ func teardownFlowAggregator(tb testing.TB, data *TestData) {
 	}
 }
 
-func teardownIngressNginxController(tb testing.TB, data *TestData) {
-	tb.Logf("Deleting ingress-nginx-controller")
-	if err := data.deleteIngressNginxController(); err != nil {
-		tb.Logf("Error when removing ingress-nginx-controller: %v", err)
-	}
-}
-
 func teardownTest(tb testing.TB, data *TestData) {
 	exportLogs(tb, data, "beforeTeardown", true)
 	if empty, _ := IsDirEmpty(data.logsDirForTestCase); empty {

diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go
@@ -165,20 +165,41 @@ type testFlow struct {
 func TestFlowAggregatorSecureConnection(t *testing.T) {
 	skipIfNotFlowVisibilityTest(t)
 	skipIfHasWindowsNodes(t)
-	options := []flowVisibilityTestOptions{
+	testCases := []struct {
+		testOption flowVisibilityTestOptions
+		name       string
+	}{
 		{
-			databaseURL: "http://clickhouse-clickhouse.flow-visibility.svc:8123",
-			ingress:     false,
-			name:        "http",
+			testOption: flowVisibilityTestOptions{
+				databaseURL:      "tcp://clickhouse-clickhouse.flow-visibility.svc:9000",
+				secureConnection: false,
+			},
+			name: "tcp",
 		},
 		{
-			databaseURL: "https://clickhouse.localdev.me",
-			ingress:     true,
-			name:        "https",
+			testOption: flowVisibilityTestOptions{
+				databaseURL:      "http://clickhouse-clickhouse.flow-visibility.svc:8123",
+				secureConnection: false,
+			},
+			name: "http",
+		},
+		{
+			testOption: flowVisibilityTestOptions{
+				databaseURL:      "tls://clickhouse-clickhouse.flow-visibility.svc:9440",
+				secureConnection: true,
+			},
+			name: "tls",
+		},
+		{
+			testOption: flowVisibilityTestOptions{
+				databaseURL:      "https://clickhouse-clickhouse.flow-visibility.svc:8443",
+				secureConnection: true,
+			},
+			name: "https",
 		},
 	}
-	for _, o := range options {
-		data, v4Enabled, v6Enabled, err := setupTestForFlowAggregator(t, o)
+	for _, o := range testCases {
+		data, v4Enabled, v6Enabled, err := setupTestForFlowAggregator(t, o.testOption)
 		if err != nil {
 			t.Fatalf("Error when setting up test: %v", err)
 		}
@@ -188,9 +209,6 @@ func TestFlowAggregatorSecureConnection(t *testing.T) {
 				// Execute teardownFlowAggregator later than teardownTest to ensure that the log
 				// of Flow Aggregator has been exported.
 				teardownFlowAggregator(t, data)
-				if o.ingress {
-					teardownIngressNginxController(t, data)
-				}
 			}()
 			podAIPs, podBIPs, _, _, _, err := createPerftestPods(data)
 			if err != nil {
@@ -211,7 +229,7 @@ func TestFlowAggregator(t *testing.T) {
 	skipIfHasWindowsNodes(t)
 
 	data, v4Enabled, v6Enabled, err := setupTestForFlowAggregator(t, flowVisibilityTestOptions{
-		databaseURL: defaultDatabaseURL,
+		databaseURL: defaultCHDatabaseURL,
 	})
 	if err != nil {
 		t.Fatalf("Error when setting up test: %v", err)