Set some default resource requests on the workspace pod (#707)

<!--Thanks for your contribution. See [CONTRIBUTING](CONTRIBUTING.md)
    for Pulumi's contribution guidelines.

    Help us merge your changes more quickly by adding more details such
    as labels, milestones, and reviewers.-->

### Proposed changes

<!--Give us a brief description of what you've done and what it solves.
-->
Implements good defaults for the workspace resource, using a
["burstable"](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#burstable)
approach.
Since a workspace pod's utilization is bursty - with low resource usage
during idle times and with high resource usage during deployment ops -
the pod requests a small amount of resources (64mb, 100m) to be able to
idle. A deployment op is able to use much more memory - all available
memory on the host.

Users may customize the resources (e.g. to apply different requests
and/or limits). For large/complex Pulumi apps, it might make sense to
reserve more memory and/or use
#694.

The agent takes some pains to stay within the requested amount, using a
programmatic form of the
[GOMEMLIMIT](https://weaviate.io/blog/gomemlimit-a-game-changer-for-high-memory-applications)
environment variable. The agent detects the requested amount via the
Downward API. We don't use `GOMEMLIMIT` to avoid propagating it to
sub-processes, and because the format is a Kubernetes 'quantity'.

It was observed that zombies weren't being cleaned up, and this was
leading to resource exhaustion. Fixed by using
[tini](https://github.com/krallin/tini/) as the entrypoint process (PID
1).

### Related issues (optional)

<!--Refer to related PRs or issues: #1234, or 'Fixes #1234' or 'Closes
#1234'.
Or link to full URLs to issues or pull requests in other GitHub
repositories. -->
Closes #698

Dockerfile

@@ -1,5 +1,10 @@
 # Build a base image with modules cached.
 FROM --platform=${BUILDPLATFORM} golang:1.23 AS base
+ARG TARGETARCH
+
+# Install tini to reap zombie processes.
+ENV TINI_VERSION=v0.19.0
+ADD --chmod=755 https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-${TARGETARCH} /tini
 
 COPY /go.mod go.mod
 COPY /go.sum go.sum
@@ -37,8 +42,11 @@ RUN --mount=type=cache,target=${GOCACHE} \
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static-debian12:debug-nonroot
+
+COPY --from=base /tini /tini
 COPY --from=op-builder /manager /manager
 COPY --from=agent-builder /agent /agent
 USER 65532:65532
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/tini", "--"]
+CMD ["/manager"]

agent/cmd/serve.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime/debug"
 	"syscall"
 
 	"github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/server"
@@ -30,6 +31,7 @@ import (
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapio"
+	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 var (
@@ -52,6 +54,15 @@ var serveCmd = &cobra.Command{
 		log.Infow("Pulumi Kubernetes Agent", "version", version.Version)
 		log.Debugw("executing serve command", "WorkDir", _workDir)
 
+		// limit the agent's memory usage to the configured quantity (e.g. 64Mi)
+		if limit, ok := os.LookupEnv("AGENT_MEMLIMIT"); ok {
+			val := resource.MustParse(limit)
+			if !val.IsZero() {
+				log.Debugf("setting memory limit to %s", limit)
+				debug.SetMemoryLimit(val.Value())
+			}
+		}
+
 		// open the workspace using auto api
 		workspaceOpts := []auto.LocalWorkspaceOption{}
 		workDir, err := filepath.EvalSymlinks(_workDir) // resolve the true location of the workspace

deploy/crds/auto.pulumi.com_workspaces.yaml

@@ -8413,8 +8413,12 @@ spec:
                 type: object
               securityProfile:
                 default: restricted
-                description: SecurityProfile applies a security profile to the workspace,
-                  'restricted' by default.
+                description: |-
+                  SecurityProfile applies a security profile to the workspace.
+                  The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                  the Restricted policy of the Pod Security Standards.
+                  The baseline profile runs the pod as the root user and with a security context that conforms with
+                  the Baseline policy of the Pod Security Standards.
                 type: string
               serviceAccountName:
                 default: default

deploy/crds/pulumi.com_stacks.yaml

@@ -9334,8 +9334,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default
@@ -18871,8 +18875,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default

deploy/helm/pulumi-operator/crds/auto.pulumi.com_workspaces.yaml

@@ -8413,8 +8413,12 @@ spec:
                 type: object
               securityProfile:
                 default: restricted
-                description: SecurityProfile applies a security profile to the workspace,
-                  'restricted' by default.
+                description: |-
+                  SecurityProfile applies a security profile to the workspace.
+                  The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                  the Restricted policy of the Pod Security Standards.
+                  The baseline profile runs the pod as the root user and with a security context that conforms with
+                  the Baseline policy of the Pod Security Standards.
                 type: string
               serviceAccountName:
                 default: default

deploy/helm/pulumi-operator/crds/pulumi.com_stacks.yaml

@@ -9334,8 +9334,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default
@@ -18871,8 +18875,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default

deploy/helm/pulumi-operator/templates/deployment.yaml

@@ -33,13 +33,12 @@ spec:
       {{- toYaml .Values.extraSidecars  | nindent  8 }}
       {{- end}}
       - name: manager
-        command:
-        - /manager
         args:
+        - /manager
         - --leader-elect
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=:8383
-        - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local
+        - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80
         - --zap-log-level={{ .Values.controller.logLevel }}
         - --zap-time-encoding=iso8601
         env:

deploy/yaml/install.yaml

@@ -9569,8 +9569,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default
@@ -19106,8 +19110,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default
@@ -27868,8 +27876,12 @@ spec:
                 type: object
               securityProfile:
                 default: restricted
-                description: SecurityProfile applies a security profile to the workspace,
-                  'restricted' by default.
+                description: |-
+                  SecurityProfile applies a security profile to the workspace.
+                  The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                  the Restricted policy of the Pod Security Standards.
+                  The baseline profile runs the pod as the root user and with a security context that conforms with
+                  the Baseline policy of the Pod Security Standards.
                 type: string
               serviceAccountName:
                 default: default
@@ -28388,14 +28400,13 @@ spec:
     spec:
       containers:
       - args:
+        - /manager
         - --leader-elect
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=:8383
-        - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local
+        - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80
         - --zap-log-level=error
         - --zap-time-encoding=iso8601
-        command:
-        - /manager
         env:
         - name: POD_NAMESPACE
           valueFrom:

operator/Makefile

@@ -145,7 +145,7 @@ build: manifests generate fmt vet ## Build manager binary.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/main.go --program-fs-adv-addr=localhost:9090
+	WORKSPACE_LOCALHOST=localhost:50051 SOURCE_CONTROLLER_LOCALHOST=localhost:9090 go run ./cmd/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
@@ -183,7 +183,7 @@ endif
 
 .PHONY: install
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply --server-side=true -f -
+	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply --server-side=true --force-conflicts -f -
 
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.

operator/api/auto/v1alpha1/workspace_types.go

@@ -34,11 +34,6 @@ const (
 	SecurityProfileBaseline SecurityProfile = "baseline"
 	// SecurityProfileRestricted applies the restricted security profile.
 	SecurityProfileRestricted SecurityProfile = "restricted"
-
-	// SecurityProfileBaselineDefaultImage is the default image used when the security profile is 'baseline'.
-	SecurityProfileBaselineDefaultImage = "pulumi/pulumi:latest"
-	// SecurityProfileRestrictedDefaultImage is the default image used when the security profile is 'restricted'.
-	SecurityProfileRestrictedDefaultImage = "pulumi/pulumi:latest-nonroot"
 )
 
 // WorkspaceSpec defines the desired state of Workspace
@@ -47,7 +42,11 @@ type WorkspaceSpec struct {
 	// +kubebuilder:default="default"
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 
-	// SecurityProfile applies a security profile to the workspace, 'restricted' by default.
+	// SecurityProfile applies a security profile to the workspace.
+	// The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+	// the Restricted policy of the Pod Security Standards.
+	// The baseline profile runs the pod as the root user and with a security context that conforms with
+	// the Baseline policy of the Pod Security Standards.
 	// +kubebuilder:default="restricted"
 	// +optional
 	SecurityProfile SecurityProfile `json:"securityProfile,omitempty"`

operator/cmd/main.go

@@ -27,7 +27,6 @@ import (
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2"
@@ -150,7 +149,10 @@ func main() {
 
 	// Create a new ProgramHandler to handle Program objects. Both the ProgramReconciler and the file server need to
 	// access the ProgramHandler, so it is created here and passed to both.
-	pHandler := newProgramHandler(mgr.GetClient(), programFSAdvAddr)
+	if programFSAdvAddr == "" {
+		programFSAdvAddr = determineAdvAddr(programFSAddr)
+	}
+	pHandler := pulumicontroller.NewProgramHandler(mgr.GetClient(), programFSAdvAddr)
 
 	if err = (&autocontroller.WorkspaceReconciler{
 		Client:   mgr.GetClient(),
@@ -246,14 +248,6 @@ func (fs pFileserver) Start(ctx context.Context) error {
 	}
 }
 
-func newProgramHandler(k8sClient client.Client, advAddr string) *pulumicontroller.ProgramHandler {
-	if advAddr == "" {
-		advAddr = determineAdvAddr(advAddr)
-	}
-
-	return pulumicontroller.NewProgramHandler(k8sClient, advAddr)
-}
-
 func determineAdvAddr(addr string) string {
 	host, port, err := net.SplitHostPort(addr)
 	if err != nil {

operator/config/crd/bases/auto.pulumi.com_workspaces.yaml

@@ -8413,8 +8413,12 @@ spec:
                 type: object
               securityProfile:
                 default: restricted
-                description: SecurityProfile applies a security profile to the workspace,
-                  'restricted' by default.
+                description: |-
+                  SecurityProfile applies a security profile to the workspace.
+                  The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                  the Restricted policy of the Pod Security Standards.
+                  The baseline profile runs the pod as the root user and with a security context that conforms with
+                  the Baseline policy of the Pod Security Standards.
                 type: string
               serviceAccountName:
                 default: default

operator/config/crd/bases/pulumi.com_stacks.yaml

@@ -9334,8 +9334,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default
@@ -18871,8 +18875,12 @@ spec:
                         type: object
                       securityProfile:
                         default: restricted
-                        description: SecurityProfile applies a security profile to
-                          the workspace, 'restricted' by default.
+                        description: |-
+                          SecurityProfile applies a security profile to the workspace.
+                          The restricted profile (default) runs the pod as a non-root user and with a security context that conforms with
+                          the Restricted policy of the Pod Security Standards.
+                          The baseline profile runs the pod as the root user and with a security context that conforms with
+                          the Baseline policy of the Pod Security Standards.
                         type: string
                       serviceAccountName:
                         default: default

operator/config/manager/manager.yaml

@@ -33,18 +33,18 @@ spec:
         runAsUser: 65532
         runAsGroup: 65532
       containers:
-        - command:
-            - /manager
+        - name: manager
           env:
             - name: POD_NAMESPACE
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
           args:
+            - /manager
             - --leader-elect
             - --health-probe-bind-address=:8081
             - --metrics-bind-address=:8383
-            - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local
+            - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80
             - --zap-log-level=error
             - --zap-time-encoding=iso8601
           ports:
@@ -56,7 +56,6 @@ spec:
             protocol: TCP
           image: controller:latest
           imagePullPolicy: IfNotPresent
-          name: manager
           securityContext:
             allowPrivilegeEscalation: false
             capabilities: