From 253a6e5ee863043929e7ad872d4e209fd5d02d5d Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Wed, 27 Nov 2024 17:54:45 -0700 Subject: [PATCH] Use shorter DNS name for pod-to-pod networking (#764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Proposed changes Eliminates the use of the hardcoded DNS suffix "svc.cluster.local", e.g. to support ["VPC scope DNS"](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns#vpc_scope_dns). ### Workspace RPC For workspace RPC connectivity, the operator now uses the "namespace-scoped" short name of the workspace pod, e.g. `guestbook-workspace.default` rather than `guestbook-workspace.default.svc.cluster.local.`. ``` ❯ kubectl get workspace NAME IMAGE READY ADDRESS kubernetes-py-guestbook pulumi/pulumi:3.134.1-nonroot True kubernetes-py-guestbook-workspace.default:50051 ``` This approach is an improvement but isn't ideal for cross-cluster communication. A possible future enhancement would be to fully resolve the address to be persisted in the status block (e.g. to `guestbook-workspace.default.svc.cluster1.`). See [DNS for Services and Pods](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#namespaces-of-services) for more information, which says: > In summary, a Pod in the `test` namespace can successfully resolve either `data.prod` or `data.prod.svc.cluster.local`. ### Program Artifact The program artifact is fetched by workspace pods from the controller itself, based on the configured advertised address. This PR shortens the advertised address used by the default installation manifest, and updates the Helm chart to support a configurable override. ``` ❯ kubectl get program NAME AGE URL random-yaml 28m http://pulumi-kubernetes-operator.pulumi-kubernetes-operator:80/programs/default/random-yaml ``` ### Related issues (optional) Closes #759 --- CHANGELOG.md | 1 + .../pulumi-operator/templates/_helpers.tpl | 11 +++++ .../pulumi-operator/templates/deployment.yaml | 2 +- deploy/helm/pulumi-operator/values.yaml | 2 + deploy/yaml/install.yaml | 2 +- operator/cmd/main_test.go | 4 ++ operator/config/manager/manager.yaml | 2 +- .../{random-yaml => auto}/workspace.yaml | 11 ++--- .../{random-yaml => program}/program.yaml | 0 operator/examples/program/stack.yaml | 41 +++++++++++++++++++ operator/examples/random-yaml/update-.yaml | 7 ---- .../controller/auto/workspace_controller.go | 2 +- .../pulumi/program_controller_test.go | 2 +- 13 files changed, 67 insertions(+), 20 deletions(-) rename operator/examples/{random-yaml => auto}/workspace.yaml (86%) rename operator/examples/{random-yaml => program}/program.yaml (100%) create mode 100644 operator/examples/program/stack.yaml delete mode 100644 operator/examples/random-yaml/update-.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ea2b712e..aa1b9a96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ CHANGELOG - Stack Controller: fix an issue where new commits weren't detected when using git sources. https://github.com/pulumi/pulumi-kubernetes-operator/issues/762 - Ensure cleanup of Stack in foreground deletion. [#760](https://github.com/pulumi/pulumi-kubernetes-operator/pull/760) - Register API resources into the "pulumi" category [#765](https://github.com/pulumi/pulumi-kubernetes-operator/pull/765) +- Use shorter DNS name for pod-to-pod networking. [#764](https://github.com/pulumi/pulumi-kubernetes-operator/pull/764) ## 2.0.0-beta.2 (2024-11-11) diff --git a/deploy/helm/pulumi-operator/templates/_helpers.tpl b/deploy/helm/pulumi-operator/templates/_helpers.tpl index 95312fe5..de26ed9d 100755 --- a/deploy/helm/pulumi-operator/templates/_helpers.tpl +++ b/deploy/helm/pulumi-operator/templates/_helpers.tpl @@ -60,3 +60,14 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Create the advertised address for the controller +*/}} +{{- define "pulumi-kubernetes-operator.advertisedAddress" -}} +{{- if .Values.controller.advertisedAddress }} +{{- .Values.controller.advertisedAddress }} +{{- else }} +{{- include "pulumi-kubernetes-operator.fullname" . }}.$(POD_NAMESPACE) +{{- end }} +{{- end }} diff --git a/deploy/helm/pulumi-operator/templates/deployment.yaml b/deploy/helm/pulumi-operator/templates/deployment.yaml index c055fd95..dea02247 100644 --- a/deploy/helm/pulumi-operator/templates/deployment.yaml +++ b/deploy/helm/pulumi-operator/templates/deployment.yaml @@ -38,7 +38,7 @@ spec: - --leader-elect - --health-probe-bind-address=:8081 - --metrics-bind-address=:8383 - - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80 + - --program-fs-adv-addr={{ include "pulumi-kubernetes-operator.advertisedAddress" . }}:80 - --zap-log-level={{ .Values.controller.logLevel }} - --zap-time-encoding=iso8601 env: diff --git a/deploy/helm/pulumi-operator/values.yaml b/deploy/helm/pulumi-operator/values.yaml index 298de84c..50096c29 100644 --- a/deploy/helm/pulumi-operator/values.yaml +++ b/deploy/helm/pulumi-operator/values.yaml @@ -23,6 +23,8 @@ imagePullSecrets: "" controller: # -- Log Level ('debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity) logLevel: error + # -- the advertised address for the controller's service + # advertisedAddress: "pulumi-kubernetes-operator.pulumi-kubernetes-operator.svc.cluster.local" # -- Extra Environments to be passed to the operator extraEnv: [] diff --git a/deploy/yaml/install.yaml b/deploy/yaml/install.yaml index 4d71958f..39914828 100644 --- a/deploy/yaml/install.yaml +++ b/deploy/yaml/install.yaml @@ -28587,7 +28587,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 - - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80 + - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE):80 - --zap-log-level=info - --zap-time-encoding=iso8601 env: diff --git a/operator/cmd/main_test.go b/operator/cmd/main_test.go index 814345e6..929755a4 100644 --- a/operator/cmd/main_test.go +++ b/operator/cmd/main_test.go @@ -42,6 +42,10 @@ func TestDetermineAdvAddr(t *testing.T) { addr: "0.0.0.0:9090", want: fakehostname + ":9090", }, + { + addr: "fake.default:9090", + want: "fake.default:9090", + }, { addr: "fake.default.svc.cluster.local:9090", want: "fake.default.svc.cluster.local:9090", diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml index dee09b36..f08f880b 100644 --- a/operator/config/manager/manager.yaml +++ b/operator/config/manager/manager.yaml @@ -43,7 +43,7 @@ spec: - /manager - --leader-elect - --health-probe-bind-address=:8081 - - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80 + - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE):80 - --zap-log-level=info - --zap-time-encoding=iso8601 ports: diff --git a/operator/examples/random-yaml/workspace.yaml b/operator/examples/auto/workspace.yaml similarity index 86% rename from operator/examples/random-yaml/workspace.yaml rename to operator/examples/auto/workspace.yaml index a54bdbdf..540d09c6 100644 --- a/operator/examples/random-yaml/workspace.yaml +++ b/operator/examples/auto/workspace.yaml @@ -14,14 +14,9 @@ spec: image: pulumi/pulumi:3.134.1-nonroot securityProfile: restricted serviceAccountName: default - - # git: - # url: https://github.com/pulumi/examples.git - # revision: 1e2fc471709448f3c9f7a250f28f1eafcde7017b - # dir: random-yaml - flux: - url: http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1e2fc471709448f3c9f7a250f28f1eafcde7017b.tar.gz - digest: sha256:6560311e95689086aa195a82c0310080adc31bea2457936ce528a014d811407a + git: + url: https://github.com/pulumi/examples.git + revision: 1e2fc471709448f3c9f7a250f28f1eafcde7017b dir: random-yaml env: - name: PULUMI_ACCESS_TOKEN diff --git a/operator/examples/random-yaml/program.yaml b/operator/examples/program/program.yaml similarity index 100% rename from operator/examples/random-yaml/program.yaml rename to operator/examples/program/program.yaml diff --git a/operator/examples/program/stack.yaml b/operator/examples/program/stack.yaml new file mode 100644 index 00000000..2a5dff74 --- /dev/null +++ b/operator/examples/program/stack.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: random-yaml-program + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: random-yaml-program:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: random-yaml-program + namespace: default +--- +apiVersion: pulumi.com/v1 +kind: Stack +metadata: + name: random-yaml-program + namespace: default +spec: + serviceAccountName: random-yaml-program + programRef: + name: random-yaml + stack: random-yaml-program + refresh: true + destroyOnFinalize: true + envRefs: + PULUMI_ACCESS_TOKEN: + type: Secret + secret: + name: pulumi-api-secret + key: accessToken + workspaceTemplate: + spec: + image: pulumi/pulumi:3.134.1-nonroot + diff --git a/operator/examples/random-yaml/update-.yaml b/operator/examples/random-yaml/update-.yaml deleted file mode 100644 index 96226ff3..00000000 --- a/operator/examples/random-yaml/update-.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: auto.pulumi.com/v1alpha1 -kind: Update -metadata: - generateName: random-yaml-1e2fc47- -spec: - workspaceName: random-yaml-1e2fc47 - stackName: dev \ No newline at end of file diff --git a/operator/internal/controller/auto/workspace_controller.go b/operator/internal/controller/auto/workspace_controller.go index ac2487bb..535131ae 100644 --- a/operator/internal/controller/auto/workspace_controller.go +++ b/operator/internal/controller/auto/workspace_controller.go @@ -395,7 +395,7 @@ func nameForService(w *autov1alpha1.Workspace) string { } func fqdnForService(w *autov1alpha1.Workspace) string { - return fmt.Sprintf("%s.%s.svc.cluster.local", nameForService(w), w.Namespace) + return fmt.Sprintf("%s.%s", nameForService(w), w.Namespace) } func labelsForStatefulSet(w *autov1alpha1.Workspace) map[string]string { diff --git a/operator/internal/controller/pulumi/program_controller_test.go b/operator/internal/controller/pulumi/program_controller_test.go index e89b73de..330e4037 100644 --- a/operator/internal/controller/pulumi/program_controller_test.go +++ b/operator/internal/controller/pulumi/program_controller_test.go @@ -42,7 +42,7 @@ var _ = Describe("Program Controller", func() { BeforeEach(func() { ctx = context.Background() - advertisedAddress = "http://fake-svc.fake-namespace.svc.cluster.local" + advertisedAddress = "http://fake-svc.fake-namespace" r = &ProgramReconciler{ Client: k8sClient,