dotnet remote transforms

integration_tests/transformations_remote/.gitignore

@@ -0,0 +1,3 @@
+/.pulumi/
+[Bb]in/
+[Oo]bj/

integration_tests/transformations_remote/Program.cs

@@ -0,0 +1,129 @@
+// Copyright 2016-2024, Pulumi Corporation.  All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using Pulumi;
+using Pulumi.Random;
+
+class MyComponent : ComponentResource
+{
+    public RandomString Child { get; }
+
+    public MyComponent(string name, ComponentResourceOptions? options = null)
+        : base("my:component:MyComponent", name, options)
+    {
+        this.Child = new RandomString($"{name}-child",
+            new RandomStringArgs { Length = 5 },
+            new CustomResourceOptions {Parent = this, AdditionalSecretOutputs = {"special"} });
+    }
+}
+
+// Scenario #5 - cross-resource transformations that inject the output of one resource to the input
+// of the other one.
+class MyOtherComponent : ComponentResource
+{
+    public RandomString Child1 { get; }
+    public RandomString Child2 { get; }
+
+    public MyOtherComponent(string name, ComponentResourceOptions? options = null)
+        : base("my:component:MyComponent", name, options)
+    {
+        this.Child1 = new RandomString($"{name}-child1",
+            new RandomStringArgs { Length = 5 },
+            new CustomResourceOptions { Parent = this });
+
+        this.Child2 = new RandomString($"{name}-child2",
+            new RandomStringArgs { Length = 6 },
+            new CustomResourceOptions { Parent = this });
+    }
+}
+
+class TransformationsStack : Stack
+{   
+    public TransformationsStack() : base(new StackOptions { ResourceTransforms = {Scenario3} })
+    {
+        // Scenario #1 - apply a transformation to a CustomResource
+        var res1 = new RandomString("res1", new RandomStringArgs { Length = 5 }, new CustomResourceOptions
+        {
+            ResourceTransforms =
+            { 
+                args =>
+                {
+                    var options = CustomResourceOptions.Merge(
+                        (CustomResourceOptions)args.Options,
+                        new CustomResourceOptions {AdditionalSecretOutputs = {"length"}});
+                    return new ResourceTransformResult(args.Args, options);
+                }
+            }
+        });
+
+        // Scenario #2 - apply a transformation to a Component to transform its children
+        var res2 = new MyComponent("res2", new ComponentResourceOptions
+        {
+            ResourceTransforms =
+            {
+                args =>
+                {
+                    if (args.Resource.GetResourceType() == RandomStringType && args.Args is RandomStringArgs oldArgs)
+                    {
+                        var resultArgs = new RandomStringArgs {Length = oldArgs.Length, MinUpper = 2};
+                        var resultOpts = CustomResourceOptions.Merge((CustomResourceOptions)args.Options,
+                            new CustomResourceOptions {AdditionalSecretOutputs = {"length"}});
+                        return new ResourceTransformResult(resultArgs, resultOpts);
+                    }
+
+                    return null;
+                }
+            }
+        });
+
+        // Scenario #3 - apply a transformation to the Stack to transform all resources in the stack.
+        var res3 = new RandomString("res3", new RandomStringArgs { Length = 5 });
+
+        // Scenario #4 - transformations are applied in order of decreasing specificity
+        // 1. (not in this example) Child transformation
+        // 2. First parent transformation
+        // 3. Second parent transformation
+        // 4. Stack transformation
+        var res4 = new MyComponent("res4", new ComponentResourceOptions
+        {
+            ResourceTransforms = { args => scenario4(args, "value1"), args => scenario4(args, "value2") }
+        });
+
+        ResourceTransformResult? scenario4(ResourceTransformArgs args, string v)
+        {
+            if (args.Resource.GetResourceType() == RandomStringType && args.Args is RandomStringArgs oldArgs)
+            {
+                var resultArgs = new RandomStringArgs
+                    {Length = oldArgs.Length, OverrideSpecial = Output.Format($"{oldArgs.OverrideSpecial}{v}")};
+                return new ResourceTransformResult(resultArgs, args.Options);
+            }
+
+            return null;
+        }
+    }
+
+    // Scenario #3 - apply a transformation to the Stack to transform all (future) resources in the stack
+    private static ResourceTransformResult? Scenario3(ResourceTransformArgs args)
+    {
+        if (args.Resource.GetResourceType() == RandomStringType && args.Args is RandomStringArgs oldArgs)
+        {
+            var resultArgs = new RandomStringArgs
+            {
+                Length = oldArgs.Length,
+                MinUpper = oldArgs.MinUpper,
+                OverrideSpecial = Output.Format($"{oldArgs.OverrideSpecial}stackvalue")
+            };
+            return new ResourceTransformResult(resultArgs, args.Options);
+        }
+
+        return null;
+    }   
+
+    private const string RandomStringType = "random:index/randomString:RandomString";
+}
+
+class Program
+{
+    static Task<int> Main(string[] args) => Deployment.RunAsync<TransformationsStack>();
+}

integration_tests/transformations_remote/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: transformations_dotnet
+description: A simple .NET program that uses transformations.
+runtime: dotnet

integration_tests/transformations_remote/Transformations.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Pulumi.Random" Version="4.8.2" />
+  </ItemGroup>
+
+</Project>

integration_tests/transformations_simple_test.go

@@ -85,3 +85,84 @@ func dotNetValidator() func(t *testing.T, stack integration.RuntimeValidationSta
 		assert.True(t, foundRes5Child)
 	}
 }
+
+func TestDotNetTransforms(t *testing.T) {
+	testDotnetProgram(t, &integration.ProgramTestOptions{
+		Dir:                    "transformations_remote",
+		Quick:                  true,
+		ExtraRuntimeValidation: Validator,
+		LocalProviders: []integration.LocalDependency{
+			{
+				Package: "testprovider",
+				Path:    "testprovider",
+			},
+		},
+	})
+}
+
+func Validator(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+	randomResName := "testprovider:index:Random"
+	foundRes1 := false
+	foundRes2Child := false
+	foundRes3 := false
+	foundRes4Child := false
+	foundRes5 := false
+	for _, res := range stack.Deployment.Resources {
+		// "res1" has a transformation which adds additionalSecretOutputs
+		if res.URN.Name() == "res1" {
+			foundRes1 = true
+			assert.Equal(t, res.Type, tokens.Type(randomResName))
+			assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("result"))
+		}
+		// "res2" has a transformation which adds additionalSecretOutputs to it's
+		// "child"
+		if res.URN.Name() == "res2-child" {
+			foundRes2Child = true
+			assert.Equal(t, res.Type, tokens.Type(randomResName))
+			assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
+			assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("result"))
+			assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("length"))
+		}
+		// "res3" is impacted by a global stack transformation which sets
+		// optionalDefault to "stackDefault"
+		if res.URN.Name() == "res3" {
+			foundRes3 = true
+			assert.Equal(t, res.Type, tokens.Type(randomResName))
+			optionalPrefix := res.Inputs["prefix"]
+			assert.NotNil(t, optionalPrefix)
+			assert.Equal(t, "stackDefault", optionalPrefix.(string))
+			length := res.Inputs["length"]
+			assert.NotNil(t, length)
+			// length should be secret
+			secret, ok := length.(map[string]interface{})
+			assert.True(t, ok, "length should be a secret")
+			assert.Equal(t, resource.SecretSig, secret[resource.SigKey])
+			assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("result"))
+		}
+		// "res4" is impacted by two component parent transformations which set
+		// optionalDefault to "default1" and then "default2" and also a global stack
+		// transformation which sets optionalDefault to "stackDefault".  The end
+		// result should be "stackDefault".
+		if res.URN.Name() == "res4-child" {
+			foundRes4Child = true
+			assert.Equal(t, res.Type, tokens.Type(randomResName))
+			assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
+			optionalPrefix := res.Inputs["prefix"]
+			assert.NotNil(t, optionalPrefix)
+			assert.Equal(t, "stackDefault", optionalPrefix.(string))
+		}
+		// "res5" should have mutated the length
+		if res.URN.Name() == "res5" {
+			foundRes5 = true
+			assert.Equal(t, res.Type, tokens.Type(randomResName))
+			length := res.Inputs["length"]
+			assert.NotNil(t, length)
+			assert.Equal(t, 20.0, length.(float64))
+		}
+	}
+	assert.True(t, foundRes1)
+	assert.True(t, foundRes2Child)
+	assert.True(t, foundRes3)
+	assert.True(t, foundRes4Child)
+	assert.True(t, foundRes5)
+}

sdk/Pulumi/Deployment/Callbacks.cs

@@ -0,0 +1,153 @@
+using Grpc.Core;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Pulumi
+{
+    internal delegate Task<Google.Protobuf.ByteString> Callback(Google.Protobuf.ByteString message, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// This class implements the callbacks server used by the engine to invoke remote functions in the dotnet process.
+    /// </summary>
+    internal sealed class Callbacks : Pulumirpc.Callbacks.CallbacksBase
+    {
+        private readonly ConcurrentDictionary<string, Callback> _callbacks = new ConcurrentDictionary<string, Callback>();
+        private readonly Task<string> _target;
+
+        public Callbacks(Task<string> target)
+        {
+            _target = target;
+        }
+
+        public async Task<Pulumirpc.Callback> AllocateCallback(Callback callback)
+        {
+            // Find a unique token for this callback, this will generally succed on first attempt.
+            var token = Guid.NewGuid().ToString();
+            while (!_callbacks.TryAdd(token, callback))
+            {
+                token = Guid.NewGuid().ToString();
+            }
+
+            var result = new Pulumirpc.Callback();
+            result.Token = token;
+            result.Target = await _target;
+            return result;
+        }
+
+        public override async Task<Pulumirpc.CallbackInvokeResponse> Invoke(Pulumirpc.CallbackInvokeRequest request, ServerCallContext context)
+        {
+            if (!_callbacks.TryGetValue(request.Token, out var callback))
+            {
+                throw new Exception(string.Format("Callback not found: {}", request.Token));
+            }
+
+            var response = new Pulumirpc.CallbackInvokeResponse();
+            response.Response = await callback(request.Request, context.CancellationToken);
+            return response;
+        }
+    }
+
+    internal sealed class CallbacksHost : IAsyncDisposable
+    {
+        private readonly TaskCompletionSource<string> _targetTcs = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
+        private readonly IHost _host;
+        private readonly CancellationTokenRegistration _portRegistration;
+
+        public CallbacksHost()
+        {
+            this._host = Host.CreateDefaultBuilder()
+                .ConfigureWebHostDefaults(webBuilder =>
+                {
+                    webBuilder
+                        .ConfigureKestrel(kestrelOptions =>
+                        {
+                            kestrelOptions.Listen(IPAddress.Loopback, 0, listenOptions =>
+                            {
+                                listenOptions.Protocols = HttpProtocols.Http2;
+                            });
+                        })
+                        .ConfigureAppConfiguration((context, config) =>
+                        {
+                            // clear so we don't read appsettings.json
+                            // note that we also won't read environment variables for config
+                            config.Sources.Clear();
+                        })
+                        .ConfigureLogging(loggingBuilder =>
+                        {
+                            // disable default logging
+                            loggingBuilder.ClearProviders();
+                        })
+                        .ConfigureServices(services =>
+                        {
+                            // Injected into Callbacks constructor
+                            services.AddSingleton(_targetTcs);
+
+                            services.AddGrpc(grpcOptions =>
+                            {
+                                // MaxRpcMesageSize raises the gRPC Max Message size from `4194304` (4mb) to `419430400` (400mb)
+                                var maxRpcMesageSize = 1024 * 1024 * 400;
+
+                                grpcOptions.MaxReceiveMessageSize = maxRpcMesageSize;
+                                grpcOptions.MaxSendMessageSize = maxRpcMesageSize;
+                            });
+                        })
+                        .Configure(app =>
+                        {
+                            app.UseRouting();
+                            app.UseEndpoints(endpoints =>
+                            {
+                                endpoints.MapGrpcService<Callbacks>();
+                            });
+                        });
+                })
+                .Build();
+
+            // before starting the host, set up this callback to tell us what port was selected
+            this._portRegistration = this._host.Services.GetRequiredService<IHostApplicationLifetime>().ApplicationStarted.Register(() =>
+            {
+                try
+                {
+                    var serverFeatures = this._host.Services.GetRequiredService<IServer>().Features;
+                    var addresses = serverFeatures.Get<IServerAddressesFeature>()!.Addresses.ToList();
+                    Debug.Assert(addresses.Count == 1, "Server should only be listening on one address");
+                    this._targetTcs.TrySetResult(addresses[0]);
+                }
+                catch (Exception ex)
+                {
+                    this._targetTcs.TrySetException(ex);
+                }
+            });
+        }
+
+        public Callbacks Callbacks
+        {
+            get
+            {
+                return this._host.Services.GetRequiredService<Callbacks>();
+            }
+        }
+
+        public Task StartAsync(CancellationToken cancellationToken)
+            => this._host.StartAsync(cancellationToken);
+
+        public async ValueTask DisposeAsync()
+        {
+            this._portRegistration.Unregister();
+            await this._host.StopAsync().ConfigureAwait(false);
+            this._host.Dispose();
+        }
+    }
+}