Use a per-stack PULUMI_HOME directory (#490)

<!--Thanks for your contribution. See [CONTRIBUTING](CONTRIBUTING.md)
    for Pulumi's contribution guidelines.

    Help us merge your changes more quickly by adding more details such
    as labels, milestones, and reviewers.-->

### Proposed changes
Closes #483 

This PR seeks to isolate the credentials associated with a given
`Stack`, to solve the problem of credentials leaking across stacks. Some
underlying details here:
1. Pulumi CLI stores login credentials in PULUMI_HOME (e.g.
`~/.pulumi/credentials.json`).
2. A side-effect of using `PULUMI_ACCESS_TOKEN` is that the CLI login
credentials are set.
4. Pulumi CLI prefers the persisted login credentials to
`PULUMI_ACCESS_TOKEN`.

This PR takes the conservative approach of encapsulating the PULUMI_HOME
into a per-stack working directory, as opposed to reusing `~/.pulumi`
across stacks. The working directory is retained across reconciliation
passes, and cleaned up during stack finalization. Note that the
_workspace_ directory is erased at the end of each reconciliation pass,
as is the current behavior.

This PR does NOT solve the (lack of) mutability of `PULUMI_ACCESS_TOKEN`
across stack updates.

_Note that this PR contains some commits (related to hacking on the
operator) that will be moved to a separate PR._

### Technical Details
Relevant terminology used within the controller codebase:
- **root directory** - a temporary directory for each stack, retained
until finalization
- **home directory** - the `PULUMI_HOME` directory, located within the
stack's root directory
- **workspace directory** - the Pulumi workspace directory containing
the program and stack configuration.

The current behavior of the operator is to erase the workspace directory
on each reconciliation pass, e.g. to ensure a clean git checkout. This
PR retains this behavior while keeping the home directory across passes,
e.g. to reuse the providers.

### Related issues (optional)
- pulumi/pulumi#13919

.gitignore

@@ -88,3 +88,4 @@ tags
 ### IntelliJ ###
 .idea
 config/
+.envrc

CHANGELOG.md

@@ -3,6 +3,7 @@ CHANGELOG
 
 ## HEAD (unreleased)
 - Changed indentation in deploy/helm/pulumi-operator/templates/deployment.yaml for volumes and volumeMounts.
+- Use a separate PULUMI_HOME for each stack. [#490](https://github.com/pulumi/pulumi-kubernetes-operator/pull/490)
 
 ## 1.13.0 (2023-08-04)
 - Use digest field for Flux source artifact if present [#459](https://github.com/pulumi/pulumi-kubernetes-operator/pull/459)

pkg/controller/stack/flux.go

@@ -21,6 +21,9 @@ const maxArtifactDownloadSize = 50 * 1024 * 1024
 func (sess *reconcileStackSession) SetupWorkdirFromFluxSource(ctx context.Context, source unstructured.Unstructured, fluxSource *shared.FluxSource) (string, error) {
 	// this source artifact fetching code is based closely on
 	// https://github.com/fluxcd/kustomize-controller/blob/db3c321163522259595894ca6c19ed44a876976d/controllers/kustomization_controller.go#L529
+	homeDir := sess.getPulumiHome()
+	workspaceDir := sess.getWorkspaceDir()
+	sess.logger.Debug("Setting up pulumi workspace for stack", "stack", sess.stack, "workspace", workspaceDir)
 
 	artifactURL, err := getArtifactField(source, "url")
 	if err != nil {
@@ -43,14 +46,16 @@ func (sess *reconcileStackSession) SetupWorkdirFromFluxSource(ctx context.Contex
 	}
 
 	fetcher := fetch.NewArchiveFetcher(1, maxArtifactDownloadSize, maxArtifactDownloadSize*10, "")
-	if err = fetcher.Fetch(artifactURL, digest, sess.rootDir); err != nil {
+	if err = fetcher.Fetch(artifactURL, digest, workspaceDir); err != nil {
 		return "", fmt.Errorf("failed to get artifact from source: %w", err)
 	}
 
-	// woo! now there's a directory with source in `rootdir`. Construct a workspace.
-
 	secretsProvider := auto.SecretsProvider(sess.stack.SecretsProvider)
-	w, err := auto.NewLocalWorkspace(ctx, auto.WorkDir(filepath.Join(sess.rootDir, fluxSource.Dir)), secretsProvider)
+	w, err := auto.NewLocalWorkspace(
+		ctx,
+		auto.PulumiHome(homeDir),
+		auto.WorkDir(filepath.Join(workspaceDir, fluxSource.Dir)),
+		secretsProvider)
 	if err != nil {
 		return "", fmt.Errorf("failed to create local workspace: %w", err)
 	}

pkg/controller/stack/stack_controller.go

@@ -461,16 +461,20 @@ func (r *ReconcileStack) Reconcile(ctx context.Context, request reconcile.Reques
 	stack := instance.Spec
 	sess := newReconcileStackSession(reqLogger, stack, r.client, request.Namespace)
 
+	// Create a long-term working directory containing the home and workspace directories.
+	// The working directory is deleted during stack finalization.
+	// Any problem here is unexpected, and treated as a controller error.
+	_, err = sess.MakeRootDir(instance.GetNamespace(), instance.GetName())
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("unable to create root directory for stack: %w", err)
+	}
+
 	// We can exit early if there is no clean-up to do.
 	if isStackMarkedToBeDeleted && !stack.DestroyOnFinalize {
 		// We know `!(isStackMarkedToBeDeleted && !contains(finalizer))` from above, and now
 		// `isStackMarkedToBeDeleted`, implying `contains(finalizer)`; but this would be correct
 		// even if it's a no-op.
-		err := sess.removeFinalizerAndUpdate(ctx, instance)
-		if err != nil {
-			sess.logger.Error(err, "Failed to delete Pulumi finalizer", "Stack.Name", instance.Spec.Stack)
-		}
-		return reconcile.Result{}, err
+		return reconcile.Result{}, sess.finalize(ctx, instance)
 	}
 
 	// This makes sure the status reflects the outcome of reconcilation. Any non-error return means
@@ -579,18 +583,15 @@ func (r *ReconcileStack) Reconcile(ctx context.Context, request reconcile.Reques
 		return found
 	}
 
-	// Create the build working directory. Any problem here is unexpected, and treated as a
+	// Create the workspace directory. Any problem here is unexpected, and treated as a
 	// controller error.
-	workingDir, err := makeWorkingDir(instance)
+	_, err = sess.MakeWorkspaceDir()
 	if err != nil {
 		return reconcile.Result{}, fmt.Errorf("unable to create tmp directory for workspace: %w", err)
 	}
-	sess.rootDir = workingDir
-	defer func() {
-		if workingDir != "" {
-			os.RemoveAll(workingDir)
-		}
-	}()
+
+	// Delete the workspace directory after the reconciliation is completed (regardless of success or failure).
+	defer sess.CleanupWorkspaceDir()
 
 	// Check which kind of source we have.
 
@@ -633,7 +634,7 @@ func (r *ReconcileStack) Reconcile(ctx context.Context, request reconcile.Reques
 
 		if currentCommit, err = sess.SetupWorkdirFromGitSource(ctx, gitAuth, gitSource); err != nil {
 			r.emitEvent(instance, pulumiv1.StackInitializationFailureEvent(), "Failed to initialize stack: %v", err.Error())
-			reqLogger.Error(err, "Failed to setup Pulumi workdir", "Stack.Name", stack.Stack)
+			reqLogger.Error(err, "Failed to setup Pulumi workspace", "Stack.Name", stack.Stack)
 			r.markStackFailed(sess, instance, err, "", "")
 			if isStalledError(err) {
 				instance.Status.MarkStalledCondition(pulumiv1.StalledCrossNamespaceRefForbiddenReason, err.Error())
@@ -683,7 +684,7 @@ func (r *ReconcileStack) Reconcile(ctx context.Context, request reconcile.Reques
 		currentCommit, err = sess.SetupWorkdirFromFluxSource(ctx, sourceObject, fluxSource)
 		if err != nil {
 			r.emitEvent(instance, pulumiv1.StackInitializationFailureEvent(), "Failed to initialize stack: %v", err.Error())
-			reqLogger.Error(err, "Failed to setup Pulumi workdir", "Stack.Name", stack.Stack)
+			reqLogger.Error(err, "Failed to setup Pulumi workspace", "Stack.Name", stack.Stack)
 			r.markStackFailed(sess, instance, err, "", "")
 			if isStalledError(err) {
 				instance.Status.MarkStalledCondition(pulumiv1.StalledCrossNamespaceRefForbiddenReason, err.Error())
@@ -698,7 +699,7 @@ func (r *ReconcileStack) Reconcile(ctx context.Context, request reconcile.Reques
 		programRef := stack.ProgramRef
 		if currentCommit, err = sess.SetupWorkdirFromYAML(ctx, *programRef); err != nil {
 			r.emitEvent(instance, pulumiv1.StackInitializationFailureEvent(), "Failed to initialize stack: %v", err.Error())
-			reqLogger.Error(err, "Failed to setup Pulumi workdir", "Stack.Name", stack.Stack)
+			reqLogger.Error(err, "Failed to setup Pulumi workspace", "Stack.Name", stack.Stack)
 			r.markStackFailed(sess, instance, err, "", "")
 			if errors.Is(err, errProgramNotFound) {
 				instance.Status.MarkStalledCondition(pulumiv1.StalledSourceUnavailableReason, err.Error())
@@ -714,9 +715,6 @@ func (r *ReconcileStack) Reconcile(ctx context.Context, request reconcile.Reques
 		}
 	}
 
-	// Delete the temporary directory after the reconciliation is completed (regardless of success or failure).
-	defer sess.CleanupPulumiDir()
-
 	// Step 2. If there are extra environment variables, read them in now and use them for subsequent commands.
 	if err = sess.SetEnvs(ctx, stack.Envs, request.Namespace); err != nil {
 		err := fmt.Errorf("could not find ConfigMap for Envs: %w", err)
@@ -959,8 +957,11 @@ func (sess *reconcileStackSession) finalize(ctx context.Context, stack *pulumiv1
 		sess.logger.Error(err, "Failed to run Pulumi finalizer", "Stack.Name", stack.Spec.Stack)
 		return err
 	}
-
-	return sess.removeFinalizerAndUpdate(ctx, stack)
+	if err := sess.removeFinalizerAndUpdate(ctx, stack); err != nil {
+		sess.logger.Error(err, "Failed to delete Pulumi finalizer", "Stack.Name", stack.Spec.Stack)
+		return err
+	}
+	return nil
 }
 
 // removeFinalizerAndUpdate makes sure this controller's finalizer is not present in the instance
@@ -988,6 +989,10 @@ func (sess *reconcileStackSession) finalizeStack(ctx context.Context) error {
 			return err
 		}
 	}
+
+	// Delete the root directory for this stack.
+	sess.cleanupRootDir()
+
 	sess.logger.Info("Successfully finalized stack")
 	return nil
 }
@@ -1220,27 +1225,76 @@ func (sess *reconcileStackSession) lookupPulumiAccessToken(ctx context.Context)
 	return "", false
 }
 
-// Make a working directory for building the given stack. These are stable paths, so that (for one
+// Make a root directory for the given stack, containing the home and workspace directories.
+func (sess *reconcileStackSession) MakeRootDir(ns, name string) (string, error) {
+	rootDir := filepath.Join(os.TempDir(), buildDirectoryPrefix, ns, name)
+	sess.logger.Debug("Creating root dir for stack", "stack", sess.stack, "root", rootDir)
+	if err := os.MkdirAll(rootDir, 0700); err != nil {
+		return "", fmt.Errorf("error creating working dir: %w", err)
+	}
+	sess.rootDir = rootDir
+
+	homeDir := sess.getPulumiHome()
+	if err := os.MkdirAll(homeDir, 0700); err != nil {
+		return "", fmt.Errorf("error creating .pulumi dir: %w", err)
+	}
+	return rootDir, nil
+}
+
+// cleanupRootDir cleans the root directory that contains the Pulumi home and workspace directories.
+func (sess *reconcileStackSession) cleanupRootDir() {
+	if sess.rootDir == "" {
+		return
+	}
+	sess.logger.Debug("Cleaning up root dir for stack", "stack", sess.stack, "root", sess.rootDir)
+	if err := os.RemoveAll(sess.rootDir); err != nil {
+		sess.logger.Error(err, "Failed to delete temporary root dir: %s", sess.rootDir)
+	}
+	sess.rootDir = ""
+}
+
+// getPulumiHome returns the home directory (containing CLI artifacts such as plugins and credentials).
+func (sess *reconcileStackSession) getPulumiHome() string {
+	return filepath.Join(sess.rootDir, ".pulumi")
+}
+
+// Make a workspace directory for building the given stack. These are stable paths, so that (for one
 // thing) the go build cache does not treat new clones of the same repo as distinct files. Since a
 // stack is processed by at most one thread at a time, and stacks have unique qualified names, and
-// the directory is expected to be removed after processing, this won't cause collisions; but, we
-// check anyway, treating the existence of the build directory as a crude lock.
-func makeWorkingDir(s *pulumiv1.Stack) (_path string, _err error) {
-	path := filepath.Join(os.TempDir(), buildDirectoryPrefix, s.GetNamespace(), s.GetName())
-	_, err := os.Stat(path)
+// the workspace directory is expected to be removed after processing, this won't cause collisions; but, we
+// check anyway, treating the existence of the workspace directory as a crude lock.
+func (sess *reconcileStackSession) MakeWorkspaceDir() (string, error) {
+	workspaceDir := filepath.Join(sess.rootDir, "workspace")
+	_, err := os.Stat(workspaceDir)
 	switch {
 	case os.IsNotExist(err):
 		break
 	case err == nil:
-		return "", fmt.Errorf("expected build directory %q for stack not to exist already, but it does", path)
+		return "", fmt.Errorf("expected workspace directory %q for stack not to exist already, but it does", workspaceDir)
 	case err != nil:
-		return "", fmt.Errorf("error while checking for build directory: %w", err)
+		return "", fmt.Errorf("error while checking for workspace directory: %w", err)
 	}
+	if err := os.MkdirAll(workspaceDir, 0700); err != nil {
+		return "", fmt.Errorf("error creating workspace dir: %w", err)
+	}
+	return workspaceDir, nil
+}
 
-	if err = os.MkdirAll(path, 0700); err != nil {
-		return "", fmt.Errorf("error creating working dir: %w", err)
+// CleanupWorkspace cleans the Pulumi workspace directory, located within the root directory.
+func (sess *reconcileStackSession) CleanupWorkspaceDir() {
+	if sess.rootDir == "" {
+		return
+	}
+	workspaceDir := sess.getWorkspaceDir()
+	sess.logger.Debug("Cleaning up pulumi workspace for stack", "stack", sess.stack, "workspace", workspaceDir)
+	if err := os.RemoveAll(workspaceDir); err != nil {
+		sess.logger.Error(err, "Failed to delete workspace dir: %s", workspaceDir)
 	}
-	return path, nil
+}
+
+// getWorkspaceDir returns the workspace directory (containing the Pulumi project).
+func (sess *reconcileStackSession) getWorkspaceDir() string {
+	return filepath.Join(sess.rootDir, "workspace")
 }
 
 func (sess *reconcileStackSession) SetupWorkdirFromGitSource(ctx context.Context, gitAuth *auto.GitAuth, source *shared.GitSource) (string, error) {
@@ -1251,12 +1305,20 @@ func (sess *reconcileStackSession) SetupWorkdirFromGitSource(ctx context.Context
 		Branch:      source.Branch,
 		Auth:        gitAuth,
 	}
+	homeDir := sess.getPulumiHome()
+	workspaceDir := sess.getWorkspaceDir()
 
-	sess.logger.Debug("Setting up pulumi workdir for stack", "stack", sess.stack)
+	sess.logger.Debug("Setting up pulumi workspace for stack", "stack", sess.stack, "workspace", workspaceDir)
 	// Create a new workspace.
+
 	secretsProvider := auto.SecretsProvider(sess.stack.SecretsProvider)
 
-	w, err := auto.NewLocalWorkspace(ctx, auto.WorkDir(sess.rootDir), auto.Repo(repo), secretsProvider)
+	w, err := auto.NewLocalWorkspace(
+		ctx,
+		auto.PulumiHome(homeDir),
+		auto.WorkDir(workspaceDir),
+		auto.Repo(repo),
+		secretsProvider)
 	if err != nil {
 		return "", fmt.Errorf("failed to create local workspace: %w", err)
 	}
@@ -1277,10 +1339,11 @@ type ProjectFile struct {
 }
 
 func (sess *reconcileStackSession) SetupWorkdirFromYAML(ctx context.Context, programRef shared.ProgramReference) (string, error) {
-	sess.logger.Debug("Setting up pulumi workdir for stack", "stack", sess.stack)
+	homeDir := sess.getPulumiHome()
+	workspaceDir := sess.getWorkspaceDir()
+	sess.logger.Debug("Setting up pulumi workspace for stack", "stack", sess.stack, "workspace", workspaceDir)
 
 	// Create a new workspace.
-
 	secretsProvider := auto.SecretsProvider(sess.stack.SecretsProvider)
 
 	program := pulumiv1.Program{}
@@ -1304,13 +1367,17 @@ func (sess *reconcileStackSession) SetupWorkdirFromYAML(ctx context.Context, pro
 		return "", fmt.Errorf("failed to marshal program object to YAML: %w", err)
 	}
 
-	err = os.WriteFile(filepath.Join(sess.rootDir, "Pulumi.yaml"), out, 0600)
+	err = os.WriteFile(filepath.Join(workspaceDir, "Pulumi.yaml"), out, 0600)
 	if err != nil {
 		return "", fmt.Errorf("failed to write YAML to file: %w", err)
 	}
 
 	var w auto.Workspace
-	w, err = auto.NewLocalWorkspace(ctx, auto.WorkDir(sess.rootDir), secretsProvider)
+	w, err = auto.NewLocalWorkspace(
+		ctx,
+		auto.PulumiHome(homeDir),
+		auto.WorkDir(workspaceDir),
+		secretsProvider)
 	if err != nil {
 		return "", fmt.Errorf("failed to create local workspace: %w", err)
 	}
@@ -1407,14 +1474,6 @@ func (sess *reconcileStackSession) ensureStackSettings(ctx context.Context, w au
 	return nil
 }
 
-func (sess *reconcileStackSession) CleanupPulumiDir() {
-	if sess.rootDir != "" {
-		if err := os.RemoveAll(sess.rootDir); err != nil {
-			sess.logger.Error(err, "Failed to delete temporary root dir: %s", sess.rootDir)
-		}
-	}
-}
-
 // Determine the actual commit information from the working directory (Spec commit etc. is optional).
 func revisionAtWorkingDir(workingDir string) (string, error) {
 	gitRepo, err := git.PlainOpenWithOptions(workingDir, &git.PlainOpenOptions{DetectDotGit: true})

test/s3backend/.gitignore

@@ -0,0 +1,2 @@
+/bin/
+/node_modules/