From 585e6d023cfaf9e85553ed60d0ff06c1c53eef79 Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Mon, 4 Mar 2024 11:45:00 +0000 Subject: [PATCH] dotnet remote transforms --- .changes/unreleased/Improvements-234.yaml | 6 + .github/workflows/pr.yml | 2 + .../transformations_remote/.gitignore | 3 + .../transformations_remote/Program.cs | 132 ++++ .../transformations_remote/Pulumi.yaml | 3 + .../transformations_remote/Random.cs | 29 + .../Transformations.csproj | 9 + .../transformations_simple/Program.cs | 28 +- .../transformations_simple_test.go | 81 ++ sdk/Pulumi.Automation/Pulumi.Automation.xml | 53 ++ .../Resources/CustomTimeoutsTests.cs | 26 + sdk/Pulumi/Core/Alias.cs | 84 ++ sdk/Pulumi/Deployment/Callbacks.cs | 169 ++++ sdk/Pulumi/Deployment/Deployment.cs | 32 + sdk/Pulumi/Deployment/Deployment_Prepare.cs | 250 ++++-- .../Deployment/Deployment_RegisterResource.cs | 27 +- .../Deployment/Deployment_Serialization.cs | 2 +- sdk/Pulumi/Pulumi.xml | 748 ++++++++++++++++-- sdk/Pulumi/Resources/CustomTimeouts.cs | 124 +++ sdk/Pulumi/Resources/ResourceOptions.cs | 17 +- sdk/Pulumi/Resources/ResourceOptions_Copy.cs | 3 +- sdk/Pulumi/Resources/ResourceTransform.cs | 66 ++ sdk/Pulumi/Resources/StackOptions.cs | 17 +- sdk/Pulumi/Serialization/Serializer.cs | 6 +- sdk/Pulumi/Stack.cs | 3 +- sdk/Pulumi/Testing/MockMonitor.cs | 6 +- 26 files changed, 1782 insertions(+), 144 deletions(-) create mode 100644 .changes/unreleased/Improvements-234.yaml create mode 100644 integration_tests/transformations_remote/.gitignore create mode 100644 integration_tests/transformations_remote/Program.cs create mode 100644 integration_tests/transformations_remote/Pulumi.yaml create mode 100644 integration_tests/transformations_remote/Random.cs create mode 100644 integration_tests/transformations_remote/Transformations.csproj create mode 100644 sdk/Pulumi.Tests/Resources/CustomTimeoutsTests.cs create mode 100644 sdk/Pulumi/Deployment/Callbacks.cs create mode 100644 sdk/Pulumi/Resources/ResourceTransform.cs diff --git a/.changes/unreleased/Improvements-234.yaml b/.changes/unreleased/Improvements-234.yaml new file mode 100644 index 00000000..83878502 --- /dev/null +++ b/.changes/unreleased/Improvements-234.yaml @@ -0,0 +1,6 @@ +component: sdk +kind: Improvements +body: Add experimental support for the new transforms system +time: 2024-03-04T13:12:07.2773112Z +custom: + PR: "234" diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a3d47370..6a52f441 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -142,6 +142,8 @@ jobs: run: dotnet run integration test TestProvider - name: TestDeletedWith run: dotnet run integration test TestDeletedWith + - name: TestDotNetTransforms + run: dotnet run integration test TestDotNetTransforms info: name: gather diff --git a/integration_tests/transformations_remote/.gitignore b/integration_tests/transformations_remote/.gitignore new file mode 100644 index 00000000..e9519ab9 --- /dev/null +++ b/integration_tests/transformations_remote/.gitignore @@ -0,0 +1,3 @@ +/.pulumi/ +[Bb]in/ +[Oo]bj/ diff --git a/integration_tests/transformations_remote/Program.cs b/integration_tests/transformations_remote/Program.cs new file mode 100644 index 00000000..4df09d50 --- /dev/null +++ b/integration_tests/transformations_remote/Program.cs @@ -0,0 +1,132 @@ +// Copyright 2016-2024, Pulumi Corporation. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Pulumi; + +class MyComponent : ComponentResource +{ + public Random Child { get; } + + public MyComponent(string name, ComponentResourceOptions? options = null) + : base("my:component:MyComponent", name, options) + { + this.Child = new Random($"{name}-child", + new RandomArgs { Length = 5 }, + new CustomResourceOptions {Parent = this, AdditionalSecretOutputs = {"length"} }); + } +} + +class TransformsStack : Stack +{ + public TransformsStack() : base(new StackOptions { XResourceTransforms = {Scenario3} }) + { + // Scenario #1 - apply a transformation to a CustomResource + var res1 = new Random("res1", new RandomArgs { Length = 5 }, new CustomResourceOptions + { + XResourceTransforms = + { + async (args, _) => + { + var options = CustomResourceOptions.Merge( + (CustomResourceOptions)args.Options, + new CustomResourceOptions {AdditionalSecretOutputs = {"result"}}); + return new ResourceTransformResult(args.Args, options); + } + } + }); + + // Scenario #2 - apply a transformation to a Component to transform its children + var res2 = new MyComponent("res2", new ComponentResourceOptions + { + XResourceTransforms = + { + async (args, _) => + { + if (args.Type == "testprovider:index:Random") + { + var resultArgs = args.Args; + resultArgs = resultArgs.SetItem("prefix", "newDefault"); + + var resultOpts = CustomResourceOptions.Merge( + (CustomResourceOptions)args.Options, + new CustomResourceOptions {AdditionalSecretOutputs = {"result"}}); + + return new ResourceTransformResult(resultArgs, resultOpts); + } + + return null; + } + } + }); + + // Scenario #3 - apply a transformation to the Stack to transform all resources in the stack. + var res3 = new Random("res3", new RandomArgs { Length = Output.CreateSecret(5) }); + + // Scenario #4 - Transforms are applied in order of decreasing specificity + // 1. (not in this example) Child transformation + // 2. First parent transformation + // 3. Second parent transformation + // 4. Stack transformation + var res4 = new MyComponent("res4", new ComponentResourceOptions + { + XResourceTransforms = { (args, _) => scenario4(args, "default1"), (args, _) => scenario4(args, "default2") } + }); + + async Task scenario4(ResourceTransformArgs args, string v) + { + if (args.Type == "testprovider:index:Random") + { + var resultArgs = args.Args; + resultArgs = resultArgs.SetItem("prefix", v); + return new ResourceTransformResult(resultArgs, args.Options); + } + + return null; + } + + // Scenario #5 - mutate the properties of a resource + var res5 = new Random("res5", new RandomArgs { Length = 10 }, new CustomResourceOptions + { + XResourceTransforms = + { + async (args, _) => + { + if (args.Type == "testprovider:index:Random") + { + var resultArgs = args.Args; + var length = (double)resultArgs["length"] * 2; + resultArgs = resultArgs.SetItem("length", length); + return new ResourceTransformResult(resultArgs, args.Options); + } + + return null; + } + } + }); + } + + // Scenario #3 - apply a transformation to the Stack to transform all (future) resources in the stack + private static async Task Scenario3(ResourceTransformArgs args, CancellationToken ct) + { + if (args.Type == "testprovider:index:Random") + { + var resultArgs = args.Args; + resultArgs = resultArgs.SetItem("prefix", "stackDefault"); + + var resultOpts = CustomResourceOptions.Merge( + (CustomResourceOptions)args.Options, + new CustomResourceOptions {AdditionalSecretOutputs = {"result"}}); + + return new ResourceTransformResult(resultArgs, resultOpts); + } + + return null; + } +} + +class Program +{ + static Task Main(string[] args) => Deployment.RunAsync(); +} diff --git a/integration_tests/transformations_remote/Pulumi.yaml b/integration_tests/transformations_remote/Pulumi.yaml new file mode 100644 index 00000000..49b78794 --- /dev/null +++ b/integration_tests/transformations_remote/Pulumi.yaml @@ -0,0 +1,3 @@ +name: transformations_dotnet +description: A simple .NET program that uses transformations. +runtime: dotnet diff --git a/integration_tests/transformations_remote/Random.cs b/integration_tests/transformations_remote/Random.cs new file mode 100644 index 00000000..77ded72f --- /dev/null +++ b/integration_tests/transformations_remote/Random.cs @@ -0,0 +1,29 @@ +// Copyright 2016-2021, Pulumi Corporation. All rights reserved. + +// Exposes the Random resource from the testprovider. + +using Pulumi; + +public partial class Random : Pulumi.CustomResource +{ + [Output("length")] + public Output Length { get; private set; } = null!; + + [Output("result")] + public Output Result { get; private set; } = null!; + + public Random(string name, RandomArgs args, CustomResourceOptions? options = null) + : base("testprovider:index:Random", name, args ?? new RandomArgs(), options) + { + } +} + +public sealed class RandomArgs : Pulumi.ResourceArgs +{ + [Input("length", required: true)] + public Input Length { get; set; } = null!; + + public RandomArgs() + { + } +} diff --git a/integration_tests/transformations_remote/Transformations.csproj b/integration_tests/transformations_remote/Transformations.csproj new file mode 100644 index 00000000..1d22a369 --- /dev/null +++ b/integration_tests/transformations_remote/Transformations.csproj @@ -0,0 +1,9 @@ + + + + Exe + net6.0 + enable + + + diff --git a/integration_tests/transformations_simple/Program.cs b/integration_tests/transformations_simple/Program.cs index ccd56de6..3c19f72a 100644 --- a/integration_tests/transformations_simple/Program.cs +++ b/integration_tests/transformations_simple/Program.cs @@ -8,7 +8,7 @@ class MyComponent : ComponentResource { public RandomString Child { get; } - + public MyComponent(string name, ComponentResourceOptions? options = null) : base("my:component:MyComponent", name, options) { @@ -24,14 +24,14 @@ class MyOtherComponent : ComponentResource { public RandomString Child1 { get; } public RandomString Child2 { get; } - + public MyOtherComponent(string name, ComponentResourceOptions? options = null) : base("my:component:MyComponent", name, options) { this.Child1 = new RandomString($"{name}-child1", new RandomStringArgs { Length = 5 }, new CustomResourceOptions { Parent = this }); - + this.Child2 = new RandomString($"{name}-child2", new RandomStringArgs { Length = 6 }, new CustomResourceOptions { Parent = this }); @@ -39,14 +39,14 @@ public MyOtherComponent(string name, ComponentResourceOptions? options = null) } class TransformationsStack : Stack -{ +{ public TransformationsStack() : base(new StackOptions { ResourceTransformations = {Scenario3} }) { // Scenario #1 - apply a transformation to a CustomResource var res1 = new RandomString("res1", new RandomStringArgs { Length = 5 }, new CustomResourceOptions { ResourceTransformations = - { + { args => { var options = CustomResourceOptions.Merge( @@ -56,7 +56,7 @@ public TransformationsStack() : base(new StackOptions { ResourceTransformations } } }); - + // Scenario #2 - apply a transformation to a Component to transform its children var res2 = new MyComponent("res2", new ComponentResourceOptions { @@ -76,10 +76,10 @@ public TransformationsStack() : base(new StackOptions { ResourceTransformations } } }); - + // Scenario #3 - apply a transformation to the Stack to transform all resources in the stack. var res3 = new RandomString("res3", new RandomStringArgs { Length = 5 }); - + // Scenario #4 - transformations are applied in order of decreasing specificity // 1. (not in this example) Child transformation // 2. First parent transformation @@ -89,7 +89,7 @@ public TransformationsStack() : base(new StackOptions { ResourceTransformations { ResourceTransformations = { args => scenario4(args, "value1"), args => scenario4(args, "value2") } }); - + ResourceTransformationResult? scenario4(ResourceTransformationArgs args, string v) { if (args.Resource.GetResourceType() == RandomStringType && args.Args is RandomStringArgs oldArgs) @@ -145,18 +145,18 @@ ResourceTransformation transformChild1DependsOnChild2() return child2Args.Length; }) .Apply(output => output); - + var newArgs = new RandomStringArgs {Length = child2Length}; - + return new ResourceTransformationResult(newArgs, args.Options); } } - + return null; } } } - + // Scenario #3 - apply a transformation to the Stack to transform all (future) resources in the stack private static ResourceTransformationResult? Scenario3(ResourceTransformationArgs args) { @@ -172,7 +172,7 @@ ResourceTransformation transformChild1DependsOnChild2() } return null; - } + } private const string RandomStringType = "random:index/randomString:RandomString"; } diff --git a/integration_tests/transformations_simple_test.go b/integration_tests/transformations_simple_test.go index c70e1d2e..24b8012c 100644 --- a/integration_tests/transformations_simple_test.go +++ b/integration_tests/transformations_simple_test.go @@ -85,3 +85,84 @@ func dotNetValidator() func(t *testing.T, stack integration.RuntimeValidationSta assert.True(t, foundRes5Child) } } + +func TestDotNetTransforms(t *testing.T) { + testDotnetProgram(t, &integration.ProgramTestOptions{ + Dir: "transformations_remote", + Quick: true, + ExtraRuntimeValidation: Validator, + LocalProviders: []integration.LocalDependency{ + { + Package: "testprovider", + Path: "testprovider", + }, + }, + }) +} + +func Validator(t *testing.T, stack integration.RuntimeValidationStackInfo) { + randomResName := "testprovider:index:Random" + foundRes1 := false + foundRes2Child := false + foundRes3 := false + foundRes4Child := false + foundRes5 := false + for _, res := range stack.Deployment.Resources { + // "res1" has a transformation which adds additionalSecretOutputs + if res.URN.Name() == "res1" { + foundRes1 = true + assert.Equal(t, res.Type, tokens.Type(randomResName)) + assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("result")) + } + // "res2" has a transformation which adds additionalSecretOutputs to it's + // "child" + if res.URN.Name() == "res2-child" { + foundRes2Child = true + assert.Equal(t, res.Type, tokens.Type(randomResName)) + assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent")) + assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("result")) + assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("length")) + } + // "res3" is impacted by a global stack transformation which sets + // optionalDefault to "stackDefault" + if res.URN.Name() == "res3" { + foundRes3 = true + assert.Equal(t, res.Type, tokens.Type(randomResName)) + optionalPrefix := res.Inputs["prefix"] + assert.NotNil(t, optionalPrefix) + assert.Equal(t, "stackDefault", optionalPrefix.(string)) + length := res.Inputs["length"] + assert.NotNil(t, length) + // length should be secret + secret, ok := length.(map[string]interface{}) + assert.True(t, ok, "length should be a secret") + assert.Equal(t, resource.SecretSig, secret[resource.SigKey]) + assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("result")) + } + // "res4" is impacted by two component parent transformations which set + // optionalDefault to "default1" and then "default2" and also a global stack + // transformation which sets optionalDefault to "stackDefault". The end + // result should be "stackDefault". + if res.URN.Name() == "res4-child" { + foundRes4Child = true + assert.Equal(t, res.Type, tokens.Type(randomResName)) + assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent")) + optionalPrefix := res.Inputs["prefix"] + assert.NotNil(t, optionalPrefix) + assert.Equal(t, "stackDefault", optionalPrefix.(string)) + } + // "res5" should have mutated the length + if res.URN.Name() == "res5" { + foundRes5 = true + assert.Equal(t, res.Type, tokens.Type(randomResName)) + length := res.Inputs["length"] + assert.NotNil(t, length) + assert.Equal(t, 20.0, length.(float64)) + } + } + assert.True(t, foundRes1) + assert.True(t, foundRes2Child) + assert.True(t, foundRes3) + assert.True(t, foundRes4Child) + assert.True(t, foundRes5) +} diff --git a/sdk/Pulumi.Automation/Pulumi.Automation.xml b/sdk/Pulumi.Automation/Pulumi.Automation.xml index 9bdee7c5..8e076662 100644 --- a/sdk/Pulumi.Automation/Pulumi.Automation.xml +++ b/sdk/Pulumi.Automation/Pulumi.Automation.xml @@ -7,6 +7,54 @@ Compares two dictionaries for equality by content, as F# maps would. + + + Options to configure a instance. + + + + + The version of the Pulumi CLI to install or the minimum version requirement for an existing installation. + + + + + The directory where to install the Pulumi CLI to or where to find an existing installation. + + + + + If true, skips the version validation that checks if an existing Pulumi CLI installation is compatible with the SDK. + + + + + A implementation that uses a locally installed Pulumi CLI. + + + + + + + + Creates a new LocalPulumiCommand instance. + + Options to configure the LocalPulumiCommand. + A cancellation token. + + + + + Installs the Pulumi CLI if it is not already installed and returns a new LocalPulumiCommand instance. + + Options to configure the LocalPulumiCommand. + A cancellation token. + + + + The version of the Pulumi CLI that is being used. + + Options controlling the behavior of an operation. @@ -669,6 +717,11 @@ The directory to override for CLI metadata. + + + The Pulumi CLI installation to use. + + The secrets provider to user for encryption and decryption of stack secrets. diff --git a/sdk/Pulumi.Tests/Resources/CustomTimeoutsTests.cs b/sdk/Pulumi.Tests/Resources/CustomTimeoutsTests.cs new file mode 100644 index 00000000..deb25a11 --- /dev/null +++ b/sdk/Pulumi.Tests/Resources/CustomTimeoutsTests.cs @@ -0,0 +1,26 @@ +// Copyright 2016-2024, Pulumi Corporation + +using System; +using Xunit; + +namespace Pulumi.Tests.Resources +{ + public class CustomTimeoutsTests + { + [Fact] + public void TestDeserialize() + { + var rpc = new Pulumirpc.RegisterResourceRequest.Types.CustomTimeouts + { + Create = "1m", + Update = "1h3m12.99ms100us", + Delete = "" + }; + + var timeouts = CustomTimeouts.Deserialize(rpc); + Assert.Equal(TimeSpan.FromMinutes(1), timeouts.Create); + Assert.Equal(TimeSpan.FromHours(1) + TimeSpan.FromMinutes(3) + TimeSpan.FromMilliseconds(12.99) + TimeSpan.FromTicks(1), timeouts.Update); + Assert.Null(timeouts.Delete); + } + } +} diff --git a/sdk/Pulumi/Core/Alias.cs b/sdk/Pulumi/Core/Alias.cs index 57833672..9af07a2c 100644 --- a/sdk/Pulumi/Core/Alias.cs +++ b/sdk/Pulumi/Core/Alias.cs @@ -1,5 +1,7 @@ // Copyright 2016-2019, Pulumi Corporation +using System.Threading.Tasks; + namespace Pulumi { /// @@ -81,5 +83,87 @@ public sealed class Alias /// Only specify one of or or . /// public bool NoParent { get; set; } + + /// + /// Deserialize a wire protocol alias to an alias object. + /// + internal static Alias Deserialize(Pulumirpc.Alias alias) + { + if (alias.AliasCase == Pulumirpc.Alias.AliasOneofCase.Urn) + { + return new Alias + { + Urn = alias.Urn, + }; + } + + var spec = alias.Spec; + return new Alias + { + Name = spec.Name == "" ? null : (Input)spec.Name, + Type = spec.Type == "" ? null : (Input)spec.Type, + Stack = spec.Stack == "" ? null : (Input)spec.Stack, + Project = spec.Project == "" ? null : (Input)spec.Project, + Parent = spec.ParentUrn == "" ? null : new DependencyResource(spec.ParentUrn), + ParentUrn = spec.ParentUrn == "" ? null : (Input)spec.ParentUrn, + NoParent = spec.NoParent, + }; + } + + static async Task Resolve(Input? input, T whenUnknown) + { + return input == null + ? whenUnknown + : await input.ToOutput().GetValueAsync(whenUnknown).ConfigureAwait(false); + } + + internal async Task SerializeAsync() + { + if (Urn != null) + { + // Alias URN fully provided, use it as is + return new Pulumirpc.Alias + { + Urn = Urn, + }; + } + + var aliasSpec = new Pulumirpc.Alias.Types.Spec + { + Name = await Resolve(Name, ""), + Type = await Resolve(Type, ""), + Stack = await Resolve(Stack, ""), + Project = await Resolve(Project, ""), + }; + + // Here we specify whether the alias has a parent or not. + // aliasSpec must only specify one of NoParent or ParentUrn, not both! + // this determines the wire format of the alias which is used by the engine. + if (Parent == null && ParentUrn == null) + { + aliasSpec.NoParent = NoParent; + } + else if (Parent != null) + { + var aliasParentUrn = await Resolve(Parent.Urn, ""); + if (!string.IsNullOrEmpty(aliasParentUrn)) + { + aliasSpec.ParentUrn = aliasParentUrn; + } + } + else + { + var aliasParentUrn = await Resolve(ParentUrn, ""); + if (!string.IsNullOrEmpty(aliasParentUrn)) + { + aliasSpec.ParentUrn = aliasParentUrn; + } + } + + return new Pulumirpc.Alias + { + Spec = aliasSpec, + }; + } } } diff --git a/sdk/Pulumi/Deployment/Callbacks.cs b/sdk/Pulumi/Deployment/Callbacks.cs new file mode 100644 index 00000000..17120122 --- /dev/null +++ b/sdk/Pulumi/Deployment/Callbacks.cs @@ -0,0 +1,169 @@ +// Copyright 2016-2024, Pulumi Corporation + +using Grpc.Core; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Google.Protobuf; + +namespace Pulumi +{ + /// + /// A callback function that can be invoked by the engine to perform some operation. The input message will be a + /// byte serialized protobuf message, which the callback function should deserialize and process. The return value + /// is a protobuf message that the SDK will serialize and return to the engine. + /// + /// A byte serialized protobuf message. + /// The async cancellation token. + /// A protobuf message to be returned to the engine. + internal delegate Task Callback(ByteString message, CancellationToken cancellationToken = default); + + /// + /// This class implements the callbacks server used by the engine to invoke remote functions in the dotnet process. + /// + internal sealed class Callbacks : Pulumirpc.Callbacks.CallbacksBase + { + private readonly ConcurrentDictionary _callbacks = new ConcurrentDictionary(); + private readonly Task _target; + + public Callbacks(Task target) + { + _target = target; + } + + public async Task AllocateCallback(Callback callback) + { + // Find a unique token for this callback, this will generally succed on first attempt. + var token = Guid.NewGuid().ToString(); + while (!_callbacks.TryAdd(token, callback)) + { + token = Guid.NewGuid().ToString(); + } + + var result = new Pulumirpc.Callback(); + result.Token = token; + result.Target = await _target.ConfigureAwait(false); + return result; + } + + public override async Task Invoke(Pulumirpc.CallbackInvokeRequest request, ServerCallContext context) + { + if (!_callbacks.TryGetValue(request.Token, out var callback)) + { + throw new RpcException(new Status(StatusCode.InvalidArgument, string.Format("Callback not found: {}", request.Token))); + } + + try + { + var result = await callback(request.Request, context.CancellationToken).ConfigureAwait(false); + var response = new Pulumirpc.CallbackInvokeResponse(); + response.Response = result.ToByteString(); + return response; + } + catch (Exception ex) + { + throw new RpcException(new Status(StatusCode.Unknown, ex.Message)); + } + } + } + + internal sealed class CallbacksHost : IAsyncDisposable + { + private readonly TaskCompletionSource _targetTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly IHost _host; + private readonly CancellationTokenRegistration _portRegistration; + + public CallbacksHost() + { + this._host = Host.CreateDefaultBuilder() + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder + .ConfigureKestrel(kestrelOptions => + { + kestrelOptions.Listen(IPAddress.Loopback, 0, listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http2; + }); + }) + .ConfigureAppConfiguration((context, config) => + { + // clear so we don't read appsettings.json + // note that we also won't read environment variables for config + config.Sources.Clear(); + }) + .ConfigureLogging(loggingBuilder => + { + // disable default logging + loggingBuilder.ClearProviders(); + }) + .ConfigureServices(services => + { + // Injected into Callbacks constructor + services.AddSingleton(new Callbacks(_targetTcs.Task)); + + services.AddGrpc(grpcOptions => + { + // MaxRpcMesageSize raises the gRPC Max Message size from `4194304` (4mb) to `419430400` (400mb) + var maxRpcMesageSize = 1024 * 1024 * 400; + + grpcOptions.MaxReceiveMessageSize = maxRpcMesageSize; + grpcOptions.MaxSendMessageSize = maxRpcMesageSize; + }); + }) + .Configure(app => + { + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapGrpcService(); + }); + }); + }) + .Build(); + + // before starting the host, set up this callback to tell us what port was selected + this._portRegistration = this._host.Services.GetRequiredService().ApplicationStarted.Register(() => + { + try + { + var serverFeatures = this._host.Services.GetRequiredService().Features; + // Server should only be listening on one address + var address = serverFeatures.Get()!.Addresses.Single(); + var uri = new Uri(address); + // grpc expects just hostname:port, not http://hostname:port + var target = $"{uri.Host}:{uri.Port}"; + this._targetTcs.TrySetResult(target); + } + catch (Exception ex) + { + this._targetTcs.TrySetException(ex); + } + }); + } + + public Callbacks Callbacks => this._host.Services.GetRequiredService(); + + public Task StartAsync(CancellationToken cancellationToken) + => this._host.StartAsync(cancellationToken); + + public async ValueTask DisposeAsync() + { + this._portRegistration.Unregister(); + await this._host.StopAsync().ConfigureAwait(false); + this._host.Dispose(); + } + } +} diff --git a/sdk/Pulumi/Deployment/Deployment.cs b/sdk/Pulumi/Deployment/Deployment.cs index ef1a1f6a..843e2f8e 100644 --- a/sdk/Pulumi/Deployment/Deployment.cs +++ b/sdk/Pulumi/Deployment/Deployment.cs @@ -175,6 +175,30 @@ internal Deployment(IEngine engine, IMonitor monitor, TestOptions? options) IEngineLogger IDeploymentInternal.Logger => _logger; IRunner IDeploymentInternal.Runner => _runner; + CallbacksHost? _callbacks; + internal async Task GetCallbacksAsync(CancellationToken cancellationToken) + { + if (_callbacks != null) + { + return _callbacks; + } + + // Atomically allocate a callbacks instance to use + var callbacks = new CallbacksHost(); + var current = Interlocked.CompareExchange(ref _callbacks, callbacks, null); + if (current == null) + { + // We swapped in the new host so start it up + await callbacks.StartAsync(cancellationToken); + return callbacks; + } + + // Someone beat us to it, just return the existing one and dispose of the new one we made. + await callbacks.DisposeAsync(); + return current; + } + + Stack IDeploymentInternal.Stack { get => Stack; @@ -231,6 +255,14 @@ internal Task MonitorSupportsAliasSpecs() return MonitorSupportsFeature("aliasSpecs"); } + /// + /// Returns whether the resource monitor we are connected to supports the "transforms" feature across the RPC interface. + /// + internal Task MonitorSupportsTransforms() + { + return MonitorSupportsFeature("transforms"); + } + // Because the secrets feature predates the Pulumi .NET SDK, we assume // that the monitor supports secrets. } diff --git a/sdk/Pulumi/Deployment/Deployment_Prepare.cs b/sdk/Pulumi/Deployment/Deployment_Prepare.cs index 7651f926..9fc4ae61 100644 --- a/sdk/Pulumi/Deployment/Deployment_Prepare.cs +++ b/sdk/Pulumi/Deployment/Deployment_Prepare.cs @@ -1,9 +1,11 @@ // Copyright 2016-2021, Pulumi Corporation using System; -using System.Linq; using System.Collections.Generic; using System.Collections.Immutable; +using System.Linq; +using System.Reflection; +using System.Threading; using System.Threading.Tasks; using Google.Protobuf.WellKnownTypes; @@ -119,6 +121,23 @@ await _logger.WarnAsync( var resourceMonitorSupportsAliasSpecs = await MonitorSupportsAliasSpecs().ConfigureAwait(false); var aliases = await PrepareAliases(res, options, resourceMonitorSupportsAliasSpecs).ConfigureAwait(false); + var transforms = new List(); + if (options.XResourceTransforms.Count > 0) + { + var resourceMonitorSupportsTransforms = await MonitorSupportsTransforms().ConfigureAwait(false); + if (!resourceMonitorSupportsTransforms) + { + throw new Exception("The Pulumi CLI does not support resource transforms. Please update the Pulumi CLI."); + } + + var callbacks = await this.GetCallbacksAsync(CancellationToken.None).ConfigureAwait(false); + + foreach (var t in options.XResourceTransforms) + { + transforms.Add(await AllocateTransform(callbacks.Callbacks, t).ConfigureAwait(false)); + } + } + return new PrepareResult( serializedProps, parentUrn ?? "", @@ -127,7 +146,8 @@ await _logger.WarnAsync( allDirectDependencyUrns, propertyToDirectDependencyUrns, aliases, - resourceMonitorSupportsAliasSpecs); + resourceMonitorSupportsAliasSpecs, + transforms); void LogExcessive(string message) { @@ -136,6 +156,177 @@ void LogExcessive(string message) } } + private static async Task AllocateTransform(Callbacks callbacks, ResourceTransform transform) + { + var wrapper = new Callback(async (message, token) => + { + var request = Pulumirpc.TransformRequest.Parser.ParseFrom(message); + + var props = ImmutableDictionary.CreateBuilder(); + foreach (var kv in request.Properties.Fields) + { + var outputData = Serialization.Deserializer.Deserialize(kv.Value); + // If it's plain just use the value directly + if (outputData.IsKnown && !outputData.IsSecret && outputData.Resources.IsEmpty) + { + props.Add(kv.Key, outputData.Value!); + } + else + { + // Else need to wrap it in an Output + var elementType = typeof(object); + if (outputData.Value != null) + { + elementType = outputData.Value.GetType(); + } + var outputDataType = typeof(Pulumi.Serialization.OutputData<>).MakeGenericType(elementType); + var createOutputData = outputDataType.GetConstructor(new[] + { + typeof(ImmutableHashSet), + elementType, + typeof(bool), + typeof(bool) + }); + if (createOutputData == null) + { + throw new InvalidOperationException( + $"Could not find constructor for type OutputData with parameters " + + $"{nameof(ImmutableHashSet)}, {elementType.Name}, bool, bool"); + } + + var typedOutputData = createOutputData.Invoke(new object?[] { + ImmutableHashSet.Empty, + outputData.Value, + outputData.IsKnown, + outputData.IsSecret }); + + var createOutputMethod = + typeof(Output<>) + .MakeGenericType(elementType) + .GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance) + .First(ctor => + { + // expected parameter type == Task> + var parameters = ctor.GetParameters(); + return parameters.Length == 1 && + parameters[0].ParameterType == typeof(Task<>).MakeGenericType(outputDataType); + })!; + + var fromResultMethod = + typeof(Task) + .GetMethod("FromResult")! + .MakeGenericMethod(outputDataType)!; + + var outputObject = createOutputMethod.Invoke(new[] { fromResultMethod.Invoke(null, new[] { typedOutputData }) }); + props.Add(kv.Key, outputObject); + } + } + + ResourceOptions opts; + // Copy all the options from the request + if (request.Custom) + { + var copts = new CustomResourceOptions(); + copts.DeleteBeforeReplace = request.Options.DeleteBeforeReplace; + copts.AdditionalSecretOutputs = request.Options.AdditionalSecretOutputs.ToList(); + opts = copts; + } + else + { + var copts = new ComponentResourceOptions(); + copts.Providers = request.Options.Providers.Select(p => (ProviderResource)new DependencyProviderResource(p.Value)).ToList(); + opts = copts; + } + opts.Aliases = request.Options.Aliases.Select(a => (Input)Alias.Deserialize(a)).ToList(); + opts.CustomTimeouts = request.Options.CustomTimeouts != null + ? CustomTimeouts.Deserialize(request.Options.CustomTimeouts) + : null; + opts.DeletedWith = request.Options.DeletedWith == "" ? null : new DependencyResource(request.Options.DeletedWith); + opts.DependsOn = request.Options.DependsOn.Select(u => (Resource)new DependencyResource(u)).ToList(); + opts.IgnoreChanges = request.Options.IgnoreChanges.ToList(); + opts.Parent = request.Parent == "" ? null : new DependencyResource(request.Parent); + opts.PluginDownloadURL = request.Options.PluginDownloadUrl == "" ? null : request.Options.PluginDownloadUrl; + opts.Protect = request.Options.Protect; + opts.Provider = request.Options.Provider == "" ? null : new DependencyProviderResource(request.Options.Provider); + opts.ReplaceOnChanges = request.Options.ReplaceOnChanges.ToList(); + opts.RetainOnDelete = request.Options.RetainOnDelete; + opts.Version = request.Options.Version; + + var args = new ResourceTransformArgs( + request.Name, + request.Type, + request.Custom, + props.ToImmutable(), + opts); + + var result = await transform(args, token).ConfigureAwait(false); + + var response = new Pulumirpc.TransformResponse(); + if (result != null) + { + response.Properties = Serialization.Serializer.CreateStruct(result.Value.Args); + + // Copy the options back + var aliases = new List(); + foreach (var alias in result.Value.Options.Aliases) + { + var defaultAliasWhenUnknown = new Alias(); + var resolvedAlias = await alias.ToOutput().GetValueAsync(whenUnknown: defaultAliasWhenUnknown).ConfigureAwait(false); + if (ReferenceEquals(resolvedAlias, defaultAliasWhenUnknown)) + { + // alias contains unknowns, skip it. + continue; + } + aliases.Add(await resolvedAlias.SerializeAsync().ConfigureAwait(false)); + } + + response.Options = new Pulumirpc.TransformResourceOptions(); + response.Options.Aliases.AddRange(aliases); + response.Options.CustomTimeouts = result.Value.Options.CustomTimeouts == null ? null : result.Value.Options.CustomTimeouts.Serialize(); + response.Options.DeletedWith = result.Value.Options.DeletedWith == null ? "" : await result.Value.Options.DeletedWith.Urn.GetValueAsync("").ConfigureAwait(false); + await foreach (var dep in result.Value.Options.DependsOn) + { + if (dep == null) + { + continue; + } + + var res = await dep.ToOutput().GetValueAsync(null!).ConfigureAwait(false); + if (res == null) + { + continue; + } + + var urn = await res.Urn.GetValueAsync("").ConfigureAwait(false); + response.Options.DependsOn.Add(urn); + } + response.Options.IgnoreChanges.AddRange(result.Value.Options.IgnoreChanges); + response.Options.PluginDownloadUrl = result.Value.Options.PluginDownloadURL ?? ""; + response.Options.Protect = result.Value.Options.Protect ?? false; + response.Options.Provider = result.Value.Options.Provider == null ? "" : await result.Value.Options.Provider.Urn.GetValueAsync("").ConfigureAwait(false); + response.Options.ReplaceOnChanges.AddRange(result.Value.Options.ReplaceOnChanges); + response.Options.RetainOnDelete = result.Value.Options.RetainOnDelete ?? false; + response.Options.Version = result.Value.Options.Version; + + if (result.Value.Options is CustomResourceOptions customOptions) + { + response.Options.DeleteBeforeReplace = customOptions.DeleteBeforeReplace ?? false; + response.Options.AdditionalSecretOutputs.AddRange(customOptions.AdditionalSecretOutputs); + } + if (result.Value.Options is ComponentResourceOptions componentOptions) + { + foreach (var provider in componentOptions.Providers) + { + var urn = await provider.Urn.GetValueAsync("").ConfigureAwait(false); + response.Options.Providers.Add(provider.Package, urn); + } + } + } + return response; + }); + return await callbacks.AllocateCallback(wrapper); + } + static async Task Resolve(Input? input, T whenUnknown) { return input == null @@ -162,53 +353,7 @@ static async Task Resolve(Input? input, T whenUnknown) continue; } - if (resolvedAlias.Urn != null) - { - // Alias URN fully provided, use it as is - aliases.Add(new Pulumirpc.Alias - { - Urn = resolvedAlias.Urn - }); - - continue; - } - - var aliasSpec = new Pulumirpc.Alias.Types.Spec - { - Name = await Resolve(resolvedAlias.Name, ""), - Type = await Resolve(resolvedAlias.Type, ""), - Stack = await Resolve(resolvedAlias.Stack, ""), - Project = await Resolve(resolvedAlias.Project, ""), - }; - - // Here we specify whether the alias has a parent or not. - // aliasSpec must only specify one of NoParent or ParentUrn, not both! - // this determines the wire format of the alias which is used by the engine. - if (resolvedAlias.Parent == null && resolvedAlias.ParentUrn == null) - { - aliasSpec.NoParent = resolvedAlias.NoParent; - } - else if (resolvedAlias.Parent != null) - { - var aliasParentUrn = await Resolve(resolvedAlias.Parent.Urn, ""); - if (!string.IsNullOrEmpty(aliasParentUrn)) - { - aliasSpec.ParentUrn = aliasParentUrn; - } - } - else - { - var aliasParentUrn = await Resolve(resolvedAlias.ParentUrn, ""); - if (!string.IsNullOrEmpty(aliasParentUrn)) - { - aliasSpec.ParentUrn = aliasParentUrn; - } - } - - aliases.Add(new Pulumirpc.Alias - { - Spec = aliasSpec - }); + aliases.Add(await resolvedAlias.SerializeAsync().ConfigureAwait(false)); } } else @@ -223,7 +368,7 @@ static async Task Resolve(Input? input, T whenUnknown) options.Parent); foreach (var aliasUrn in allAliases) { - var aliasValue = await Resolve(aliasUrn, ""); + var aliasValue = await Resolve(aliasUrn, "").ConfigureAwait(false); if (!string.IsNullOrEmpty(aliasValue) && uniqueAliases.Add(aliasValue)) { aliases.Add(new Pulumirpc.Alias @@ -434,6 +579,7 @@ private readonly struct PrepareResult public readonly HashSet AllDirectDependencyUrns; public readonly Dictionary> PropertyToDirectDependencyUrns; public readonly List Aliases; + public readonly List Transforms; /// /// Returns whether the resource monitor we are connected to supports the "aliasSpec" feature across the RPC interface. /// When that is not the case, use only use the URNs of the aliases to populate the AliasURNs field of RegisterResourceRequest, @@ -450,7 +596,8 @@ public PrepareResult( Dictionary> propertyToDirectDependencyUrns, List aliases, - bool supportsAliasSpec) + bool supportsAliasSpec, + List transforms) { SerializedProps = serializedProps; ParentUrn = parentUrn; @@ -460,6 +607,7 @@ public PrepareResult( PropertyToDirectDependencyUrns = propertyToDirectDependencyUrns; SupportsAliasSpec = supportsAliasSpec; Aliases = aliases; + Transforms = transforms; } } } diff --git a/sdk/Pulumi/Deployment/Deployment_RegisterResource.cs b/sdk/Pulumi/Deployment/Deployment_RegisterResource.cs index 74cb72c3..0f794185 100644 --- a/sdk/Pulumi/Deployment/Deployment_RegisterResource.cs +++ b/sdk/Pulumi/Deployment/Deployment_RegisterResource.cs @@ -1,4 +1,4 @@ -// Copyright 2016-2021, Pulumi Corporation +// Copyright 2016-2024, Pulumi Corporation using System; using System.Collections.Immutable; @@ -69,6 +69,7 @@ private static void PopulateRequest(RegisterResourceRequest request, PrepareResu request.AliasURNs.AddRange(aliasUrns); } + request.Transforms.AddRange(prepareResult.Transforms); request.Dependencies.AddRange(prepareResult.AllDirectDependencyUrns); foreach (var (key, resourceUrns) in prepareResult.PropertyToDirectDependencyUrns) @@ -103,12 +104,7 @@ private static async Task CreateRegisterResourceRequest AcceptResources = !_disableResourceReferences, DeleteBeforeReplace = deleteBeforeReplace ?? false, DeleteBeforeReplaceDefined = deleteBeforeReplace != null, - CustomTimeouts = new RegisterResourceRequest.Types.CustomTimeouts - { - Create = TimeoutString(options.CustomTimeouts?.Create), - Delete = TimeoutString(options.CustomTimeouts?.Delete), - Update = TimeoutString(options.CustomTimeouts?.Update), - }, + CustomTimeouts = options.CustomTimeouts?.Serialize(), Remote = remote, RetainOnDelete = options.RetainOnDelete ?? false, DeletedWith = deletedWith, @@ -124,22 +120,5 @@ private static async Task CreateRegisterResourceRequest return request; } - - private static string TimeoutString(TimeSpan? timeSpan) - { - if (timeSpan == null) - return ""; - - // This will eventually be parsed by go's ParseDuration function here: - // https://github.com/pulumi/pulumi/blob/06d4dde8898b2a0de2c3c7ff8e45f97495b89d82/pkg/resource/deploy/source_eval.go#L967 - // - // So we generate a legal duration as allowed by - // https://golang.org/pkg/time/#ParseDuration. - // - // Simply put, we simply convert our ticks to the integral number of nanoseconds - // corresponding to it. Since each tick is 100ns, this can trivially be done just by - // appending "00" to it. - return timeSpan.Value.Ticks + "00ns"; - } } } diff --git a/sdk/Pulumi/Deployment/Deployment_Serialization.cs b/sdk/Pulumi/Deployment/Deployment_Serialization.cs index 272ebbf2..186e92fa 100644 --- a/sdk/Pulumi/Deployment/Deployment_Serialization.cs +++ b/sdk/Pulumi/Deployment/Deployment_Serialization.cs @@ -126,7 +126,7 @@ public RawSerializationResult( public SerializationResult ToSerializationResult() => new SerializationResult( - Serializer.CreateStruct(PropertyValues), + Serializer.CreateStruct(PropertyValues!), PropertyToDependentResources); } } diff --git a/sdk/Pulumi/Pulumi.xml b/sdk/Pulumi/Pulumi.xml index ca8fe505..849b2ce6 100644 --- a/sdk/Pulumi/Pulumi.xml +++ b/sdk/Pulumi/Pulumi.xml @@ -327,6 +327,11 @@ Only specify one of or or . + + + Deserialize a wire protocol alias to an alias object. + + An Archive represents a collection of named assets. @@ -890,6 +895,21 @@ and the parent name changed. + + + A callback function that can be invoked by the engine to perform some operation. The input message will be a + byte serialized protobuf message, which the callback function should deserialize and process. The return value + is a protobuf message that the SDK will serialize and return to the engine. + + A byte serialized protobuf message. + The async cancellation token. + A protobuf message to be returned to the engine. + + + + This class implements the callbacks server used by the engine to invoke remote functions in the dotnet process. + + Options to help control the behavior of . @@ -971,6 +991,11 @@ In which case we no longer compute alias combinations ourselves but instead delegate the work to the engine. + + + Returns whether the resource monitor we are connected to supports the "transforms" feature across the RPC interface. + + Logs a debug-level message that is generally hidden from end-users. @@ -2137,6 +2162,14 @@ parents walking from the resource up to the stack. + + + Optional list of transforms to apply to this resource during construction.The transforms are applied in + order, and are applied prior to transform applied to parents walking from the resource up to the stack. + + This property is experimental. + + An optional list of aliases to treat this resource as matching. @@ -2173,6 +2206,43 @@ if specified resource is being deleted as well. + + + ResourceTransform is the callback signature for . A + transform is passed the same set of inputs provided to the constructor, and can + optionally return back alternate values for the properties and/or options prior to the resource + actually being created. The effect will be as though those properties and/or options were passed + in place of the original call to the constructor. If the transform returns , this indicates that the resource will not be transformed. + + The new values to use for the args and options of the in place of + the originally provided values. + + + + The name of the resource being transformed. + + + + + The type of the resource being transformed. + + + + + If this is a custom resource. + + + + + The original properties passed to the Resource constructor. + + + + + The original resource options passed to the Resource constructor. + + ResourceTransformation is the callback signature for + + + Optional list of transforms to apply to this stack's resources during construction. The transforms are + applied in order, and are applied after all the transforms of custom and component resources in the stack. + + This property is experimental. + + Manages a reference to a Pulumi stack and provides access to the referenced stack's outputs. @@ -2362,6 +2440,12 @@ cref="P:Google.Protobuf.WellKnownTypes.Struct.Fields"/> returned by the engine. + + + Attribute used by a Pulumi Cloud Provider Package to mark + constructor parameters with a name override. + + Attribute used by a Pulumi Cloud Provider Package to mark enum types. @@ -2377,12 +2461,6 @@ but is recommended and is what our codegen does. - - - Attribute used by a Pulumi Cloud Provider Package to mark - constructor parameters with a name override. - - Unknown values are encoded as a distinguished string value. @@ -2875,6 +2953,11 @@ Disabled policies do not run during a deployment. + + + Remediated policies actually fixes problems instead of issuing diagnostics. + + Field number for the "type" field. @@ -3220,6 +3303,72 @@ URN of the resource that violates the policy. + + + Remediation is a single resource remediation result. + + + + Field number for the "policyName" field. + + + + Name of the policy that performed the remediation. + + + + Field number for the "policyPackName" field. + + + + Name of the policy pack the transform is in. + + + + Field number for the "policyPackVersion" field. + + + + Version of the policy pack. + + + + Field number for the "description" field. + + + + Description of transform rule. e.g., "auto-tag resources." + + + + Field number for the "properties" field. + + + + the transformed properties to use. + + + + Field number for the "diagnostic" field. + + + + an optional warning diagnostic to emit, if a transform failed. + + + + + RemediateResponse contains a sequence of remediations applied, in order. + + + + Field number for the "remediations" field. + + + + the list of remediations that were applied. + + AnalyzerInfo provides metadata about a PolicyPack inside an analyzer. @@ -3414,6 +3563,15 @@ The context of the server-side call handler being invoked. The response to send back to the client (wrapped by a task). + + + Remediate optionally transforms a single resource object. This effectively rewrites + a single resource object's properties instead of using what was generated by the program. + + The request received from the client. + The context of the server-side call handler being invoked. + The response to send back to the client (wrapped by a task). + GetAnalyzerInfo returns metadata about the analyzer (e.g., list of policies contained). @@ -3540,6 +3698,46 @@ The options for the call. The call object. + + + Remediate optionally transforms a single resource object. This effectively rewrites + a single resource object's properties instead of using what was generated by the program. + + The request to send to the server. + The initial metadata to send with the call. This parameter is optional. + An optional deadline for the call. The call will be cancelled if deadline is hit. + An optional token for canceling the call. + The response received from the server. + + + + Remediate optionally transforms a single resource object. This effectively rewrites + a single resource object's properties instead of using what was generated by the program. + + The request to send to the server. + The options for the call. + The response received from the server. + + + + Remediate optionally transforms a single resource object. This effectively rewrites + a single resource object's properties instead of using what was generated by the program. + + The request to send to the server. + The initial metadata to send with the call. This parameter is optional. + An optional deadline for the call. The call will be cancelled if deadline is hit. + An optional token for canceling the call. + The call object. + + + + Remediate optionally transforms a single resource object. This effectively rewrites + a single resource object's properties instead of using what was generated by the program. + + The request to send to the server. + The options for the call. + The call object. + GetAnalyzerInfo returns metadata about the analyzer (e.g., list of policies contained). @@ -3661,6 +3859,138 @@ Service methods will be bound by calling AddMethod on this object. An object implementing the server-side handling logic. + + Holder for reflection information generated from pulumi/callback.proto + + + File descriptor for pulumi/callback.proto + + + Field number for the "target" field. + + + + the gRPC target of the callback service. + + + + Field number for the "token" field. + + + + the service specific unique token for this callback. + + + + Field number for the "token" field. + + + + the token for the callback. + + + + Field number for the "request" field. + + + + the serialized protobuf message of the arguments for this callback. + + + + Field number for the "response" field. + + + + the serialized protobuf message of the response for this callback. + + + + + Callbacks is a service for invoking functions in one runtime from other processes. + + + + Service descriptor + + + Base class for server-side implementations of Callbacks + + + + Invoke invokes a given callback, identified by its token. + + The request received from the client. + The context of the server-side call handler being invoked. + The response to send back to the client (wrapped by a task). + + + Client for Callbacks + + + Creates a new client for Callbacks + The channel to use to make remote calls. + + + Creates a new client for Callbacks that uses a custom CallInvoker. + The callInvoker to use to make remote calls. + + + Protected parameterless constructor to allow creation of test doubles. + + + Protected constructor to allow creation of configured clients. + The client configuration. + + + + Invoke invokes a given callback, identified by its token. + + The request to send to the server. + The initial metadata to send with the call. This parameter is optional. + An optional deadline for the call. The call will be cancelled if deadline is hit. + An optional token for canceling the call. + The response received from the server. + + + + Invoke invokes a given callback, identified by its token. + + The request to send to the server. + The options for the call. + The response received from the server. + + + + Invoke invokes a given callback, identified by its token. + + The request to send to the server. + The initial metadata to send with the call. This parameter is optional. + An optional deadline for the call. The call will be cancelled if deadline is hit. + An optional token for canceling the call. + The call object. + + + + Invoke invokes a given callback, identified by its token. + + The request to send to the server. + The options for the call. + The call object. + + + Creates a new instance of client from given ClientBaseConfiguration. + + + Creates service definition that can be registered with a server + An object implementing the server-side handling logic. + + + Register service method with a service binder with or without implementation. Useful when customizing the service binding logic. + Note: this method is part of an experimental API that can change or be removed without any prior notice. + Service methods will be bound by calling AddMethod on this object. + An object implementing the server-side handling logic. + Holder for reflection information generated from pulumi/codegen/hcl.proto @@ -3863,6 +4193,30 @@ the provider PluginDownloadURL to use for the resource, if any. + + Field number for the "logical_name" field. + + + + the logical name of the resource. + + + + Field number for the "is_component" field. + + + + true if this is a component resource. + + + + Field number for the "is_remote" field. + + + + true if this is a remote resource. Ignored if is_component is false. + + Field number for the "resources" field. @@ -3871,6 +4225,14 @@ a list of resources to import. + + Field number for the "diagnostics" field. + + + + any diagnostics from state conversion. + + Field number for the "source_directory" field. @@ -4352,6 +4714,48 @@ File descriptor for pulumi/language.proto + + + ProgramInfo are the common set of options that a language runtime needs to execute or query a program. This + is filled in by the engine based on where the `Pulumi.yaml` file was, the `runtime.options` property, and + the `main` property. + + + + Field number for the "root_directory" field. + + + + the root of the project, where the `Pulumi.yaml` file is located. + + + + Field number for the "program_directory" field. + + + + the absolute path to the directory of the program to execute. Generally, but not required to be, + underneath the root directory. + + + + Field number for the "entry_point" field. + + + + the entry point of the program, normally '.' meaning to just use the program directory, but can also be + a filename inside the program directory. How that filename is interpreted, if at all, is language + specific. + + + + Field number for the "options" field. + + + + JSON style options from the `Pulumi.yaml` runtime options section. + + AboutResponse returns runtime information about the language. @@ -4386,7 +4790,7 @@ - the project name. + the project name, the engine always sets this to "deprecated" now. @@ -4413,6 +4817,14 @@ if transitive dependencies should be included in the result. + + Field number for the "info" field. + + + + the program info to use to calculate dependencies. + + Field number for the "name" field. @@ -4442,7 +4854,7 @@ - the project name. + the project name, the engine always sets this to "deprecated" now. @@ -4461,6 +4873,14 @@ the path to the program. + + Field number for the "info" field. + + + + the program info to use to calculate plugins. + + Field number for the "plugins" field. @@ -4570,6 +4990,22 @@ the organization of the stack being deployed into. + + Field number for the "configPropertyMap" field. + + + + the configuration variables to apply before running. + + + + Field number for the "info" field. + + + + the program info to use to execute the program. + + RunResponse is the response back from the interpreter/source back to the monitor. @@ -4609,6 +5045,14 @@ if we are running in a terminal and should use ANSI codes + + Field number for the "info" field. + + + + the program info to use to execute the plugin. + + Field number for the "stdout" field. @@ -4657,6 +5101,14 @@ any environment variables to set as part of the program. + + Field number for the "info" field. + + + + the program info to use to execute the plugin. + + Field number for the "stdout" field. @@ -4805,6 +5257,14 @@ The target of a codegen.LoaderServer to use for loading schemas. + + Field number for the "diagnostics" field. + + + + any diagnostics from code generation. + + Field number for the "package_directory" field. @@ -5474,6 +5934,14 @@ when true, diff and update will be called with the old outputs and the old inputs. + + Field number for the "sends_old_inputs_to_delete" field. + + + + when true, delete will be called with the old outputs and the old inputs. + + Field number for the "acceptSecrets" field. @@ -5594,38 +6062,6 @@ a map from argument keys to the dependencies of the argument. - - Field number for the "provider" field. - - - - an optional reference to the provider to use for this invoke. - - - - Field number for the "version" field. - - - - the version of the provider to use when servicing this request. - - - - Field number for the "pluginDownloadURL" field. - - - - the pluginDownloadURL of the provider to use when servicing this request. - - - - Field number for the "pluginChecksums" field. - - - - a map of checksums of the provider to use when servicing this request. - - Field number for the "project" field. @@ -5690,12 +6126,12 @@ the organization of the stack being deployed into. - - Field number for the "sourcePosition" field. + + Field number for the "accepts_output_values" field. - + - the optional source position of the user code that initiated the call. + the engine can be passed output values back, returnDependencies can be left blank if returning output values. @@ -6236,6 +6672,14 @@ the delete request timeout represented in seconds. + + Field number for the "old_inputs" field. + + + + the old input values of the resource to delete. + + Field number for the "project" field. @@ -6429,6 +6873,14 @@ whether to retain the resource in the cloud provider when it is deleted + + Field number for the "accepts_output_values" field. + + + + the engine can be passed output values back, stateDependencies can be left blank if returning output values. + + Container for nested types declared in the ConstructRequest message type. @@ -7976,6 +8428,14 @@ the optional source position of the user code that initiated the register. + + Field number for the "transforms" field. + + + + a list of transforms to apply to the resource before registering it. + + Container for nested types declared in the RegisterResourceRequest message type. @@ -8176,6 +8636,206 @@ the optional source position of the user code that initiated the invoke. + + Field number for the "tok" field. + + + + the function token to invoke. + + + + Field number for the "args" field. + + + + the arguments for the function invocation. + + + + Field number for the "argDependencies" field. + + + + a map from argument keys to the dependencies of the argument. + + + + Field number for the "provider" field. + + + + an optional reference to the provider to use for this invoke. + + + + Field number for the "version" field. + + + + the version of the provider to use when servicing this request. + + + + Field number for the "pluginDownloadURL" field. + + + + the pluginDownloadURL of the provider to use when servicing this request. + + + + Field number for the "pluginChecksums" field. + + + + a map of checksums of the provider to use when servicing this request. + + + + Field number for the "sourcePosition" field. + + + + the optional source position of the user code that initiated the call. + + + + Container for nested types declared in the ResourceCallRequest message type. + + + + ArgumentDependencies describes the resources that a particular argument depends on. + + + + Field number for the "urns" field. + + + + A list of URNs this argument depends on. + + + + + TransformResourceOptions is a subset of all resource options that are relevant to transforms. + + + + Field number for the "depends_on" field. + + + Field number for the "protect" field. + + + Field number for the "ignore_changes" field. + + + Field number for the "replace_on_changes" field. + + + Field number for the "version" field. + + + Field number for the "aliases" field. + + + Field number for the "provider" field. + + + Field number for the "custom_timeouts" field. + + + Field number for the "plugin_download_url" field. + + + Field number for the "retain_on_delete" field. + + + Field number for the "deleted_with" field. + + + Field number for the "delete_before_replace" field. + + + Gets whether the "delete_before_replace" field is set + + + Clears the value of the "delete_before_replace" field + + + Field number for the "additional_secret_outputs" field. + + + Field number for the "providers" field. + + + Field number for the "plugin_checksums" field. + + + Field number for the "type" field. + + + + the type of the resource. + + + + Field number for the "name" field. + + + + the name of the resource. + + + + Field number for the "custom" field. + + + + true if the resource is a custom resource, else it's a component resource. + + + + Field number for the "parent" field. + + + + the parent of the resource, this can't be changed by the transform. + + + + Field number for the "properties" field. + + + + the input properties of the resource. + + + + Field number for the "options" field. + + + + the options for the resource. + + + + Field number for the "properties" field. + + + + the transformed input properties. + + + + Field number for the "options" field. + + + + the options for the resource. + + ResourceMonitor is the interface a source uses to talk back to the planning monitor orchestrating the execution. diff --git a/sdk/Pulumi/Resources/CustomTimeouts.cs b/sdk/Pulumi/Resources/CustomTimeouts.cs index f710db33..bb8c2e8c 100644 --- a/sdk/Pulumi/Resources/CustomTimeouts.cs +++ b/sdk/Pulumi/Resources/CustomTimeouts.cs @@ -31,5 +31,129 @@ public sealed class CustomTimeouts Delete = timeouts.Delete, Update = timeouts.Update, }; + + + private static string TimeoutString(TimeSpan? timeSpan) + { + if (timeSpan == null) + return ""; + + // This will eventually be parsed by go's ParseDuration function here: + // https://github.com/pulumi/pulumi/blob/06d4dde8898b2a0de2c3c7ff8e45f97495b89d82/pkg/resource/deploy/source_eval.go#L967 + // + // So we generate a legal duration as allowed by + // https://golang.org/pkg/time/#ParseDuration. + // + // Simply put, we simply convert our ticks to the integral number of nanoseconds + // corresponding to it. Since each tick is 100ns, this can trivially be done just by + // appending "00" to it. + return timeSpan.Value.Ticks + "00ns"; + } + + internal Pulumirpc.RegisterResourceRequest.Types.CustomTimeouts Serialize() + { + return new Pulumirpc.RegisterResourceRequest.Types.CustomTimeouts + { + Create = TimeoutString(Create), + Update = TimeoutString(Update), + Delete = TimeoutString(Delete), + }; + } + + internal static CustomTimeouts Deserialize(Pulumirpc.RegisterResourceRequest.Types.CustomTimeouts customTimeouts) + { + static TimeSpan? parse(string s) + { + if (s == null || s == "") + { + return null; + } + + // A duration string is a possibly signed sequence of decimal numbers, each with optional + // fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", + // "us" (or "µs"), "ms", "s", "m", "h". + + var span = s.AsSpan(); + + var neg = false; + if (span[0] == '-' || span[0] == '+') + { + neg = span[0] == '-'; + span = span[1..]; + } + if (span == "0") + { + return TimeSpan.Zero; + } + if (span.IsEmpty) + { + throw new ArgumentException("invalid duration " + s); + } + var duration = TimeSpan.Zero; + while (!span.IsEmpty) + { + // find the next timeunit + var i = 0; + while (i < span.Length && (('0' <= span[i] && span[i] <= '9') || span[i] == '.')) + { + i++; + } + // parse the number + var v = double.Parse(span[0..i]); + // parse the unit + span = span[i..]; + if (span.IsEmpty) + { + throw new ArgumentException("missing unit in duration " + s); + } + if (span.StartsWith("µs") || span.StartsWith("us")) + { + duration += TimeSpan.FromTicks((long)(v / 100)); + span = span[2..]; + } + else if (span.StartsWith("ms")) + { + duration += TimeSpan.FromMilliseconds(v); + span = span[2..]; + } + else if (span.StartsWith("s")) + { + duration += TimeSpan.FromSeconds(v); + span = span[1..]; + } + else if (span.StartsWith("m")) + { + duration += TimeSpan.FromMinutes(v); + span = span[1..]; + } + else if (span.StartsWith("h")) + { + duration += TimeSpan.FromHours(v); + span = span[1..]; + } + else if (span.StartsWith("d")) + { + duration += TimeSpan.FromDays(v); + span = span[1..]; + } + else + { + throw new ArgumentException("invalid unit in duration " + s); + } + } + if (neg) + { + duration = -duration; + } + return duration; + }; + + return new CustomTimeouts + { + Create = parse(customTimeouts.Create), + Update = parse(customTimeouts.Update), + Delete = parse(customTimeouts.Delete), + }; + } } } diff --git a/sdk/Pulumi/Resources/ResourceOptions.cs b/sdk/Pulumi/Resources/ResourceOptions.cs index 916a7cd8..d42756f4 100644 --- a/sdk/Pulumi/Resources/ResourceOptions.cs +++ b/sdk/Pulumi/Resources/ResourceOptions.cs @@ -1,5 +1,6 @@ -// Copyright 2016-2021, Pulumi Corporation +// Copyright 2016-2024, Pulumi Corporation +using System; using System.Collections.Generic; namespace Pulumi @@ -86,6 +87,20 @@ public List ResourceTransformations set => _resourceTransformations = value; } + private List? _resourceTransforms; + + /// + /// Optional list of transforms to apply to this resource during construction.The transforms are applied in + /// order, and are applied prior to transform applied to parents walking from the resource up to the stack. + /// + /// This property is experimental. + /// + public List XResourceTransforms + { + get => _resourceTransforms ??= new List(); + set => _resourceTransforms = value; + } + /// /// An optional list of aliases to treat this resource as matching. /// diff --git a/sdk/Pulumi/Resources/ResourceOptions_Copy.cs b/sdk/Pulumi/Resources/ResourceOptions_Copy.cs index 1a5342e8..5811bf50 100644 --- a/sdk/Pulumi/Resources/ResourceOptions_Copy.cs +++ b/sdk/Pulumi/Resources/ResourceOptions_Copy.cs @@ -24,7 +24,8 @@ public partial class ResourceOptions Version = options.Version, PluginDownloadURL = options.PluginDownloadURL, RetainOnDelete = options.RetainOnDelete, - DeletedWith = options.DeletedWith + DeletedWith = options.DeletedWith, + XResourceTransforms = options.XResourceTransforms.ToList(), }; internal static CustomResourceOptions CreateCustomResourceOptionsCopy(ResourceOptions? options) diff --git a/sdk/Pulumi/Resources/ResourceTransform.cs b/sdk/Pulumi/Resources/ResourceTransform.cs new file mode 100644 index 00000000..2ac67d71 --- /dev/null +++ b/sdk/Pulumi/Resources/ResourceTransform.cs @@ -0,0 +1,66 @@ +// Copyright 2016-2019, Pulumi Corporation + +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; + +namespace Pulumi +{ + /// + /// ResourceTransform is the callback signature for . A + /// transform is passed the same set of inputs provided to the constructor, and can + /// optionally return back alternate values for the properties and/or options prior to the resource + /// actually being created. The effect will be as though those properties and/or options were passed + /// in place of the original call to the constructor. If the transform returns , this indicates that the resource will not be transformed. + /// + /// The new values to use for the args and options of the in place of + /// the originally provided values. + public delegate Task ResourceTransform(ResourceTransformArgs args, CancellationToken cancellationToken = default); + + public readonly struct ResourceTransformArgs + { + /// + /// The name of the resource being transformed. + /// + public string Name { get; } + /// + /// The type of the resource being transformed. + /// + public string Type { get; } + /// + /// If this is a custom resource. + /// + public bool Custom { get; } + /// + /// The original properties passed to the Resource constructor. + /// + public ImmutableDictionary Args { get; } + /// + /// The original resource options passed to the Resource constructor. + /// + public ResourceOptions Options { get; } + + public ResourceTransformArgs( + string name, string type, bool custom, ImmutableDictionary args, ResourceOptions options) + { + Name = name; + Type = type; + Custom = custom; + Args = args; + Options = options; + } + } + + public readonly struct ResourceTransformResult + { + public ImmutableDictionary Args { get; } + public ResourceOptions Options { get; } + + public ResourceTransformResult(ImmutableDictionary args, ResourceOptions options) + { + Args = args; + Options = options; + } + } +} diff --git a/sdk/Pulumi/Resources/StackOptions.cs b/sdk/Pulumi/Resources/StackOptions.cs index 11b2b684..c154308f 100644 --- a/sdk/Pulumi/Resources/StackOptions.cs +++ b/sdk/Pulumi/Resources/StackOptions.cs @@ -1,5 +1,6 @@ -// Copyright 2016-2020, Pulumi Corporation +// Copyright 2016-2024, Pulumi Corporation +using System; using System.Collections.Generic; namespace Pulumi @@ -21,5 +22,19 @@ public List ResourceTransformations get => _resourceTransformations ??= new List(); set => _resourceTransformations = value; } + + private List? _resourceTransforms; + + /// + /// Optional list of transforms to apply to this stack's resources during construction. The transforms are + /// applied in order, and are applied after all the transforms of custom and component resources in the stack. + /// + /// This property is experimental. + /// + public List XResourceTransforms + { + get => _resourceTransforms ??= new List(); + set => _resourceTransforms = value; + } } } diff --git a/sdk/Pulumi/Serialization/Serializer.cs b/sdk/Pulumi/Serialization/Serializer.cs index 9e626aac..2aa7b4c1 100644 --- a/sdk/Pulumi/Serialization/Serializer.cs +++ b/sdk/Pulumi/Serialization/Serializer.cs @@ -483,8 +483,8 @@ internal static Value CreateValue(object? value) double d => Value.ForNumber(d), bool b => Value.ForBool(b), string s => Value.ForString(s), - ImmutableArray list => Value.ForList(list.Select(CreateValue).ToArray()), - ImmutableDictionary dict => Value.ForStruct(CreateStruct(dict)), + ImmutableArray list => Value.ForList(list.Select(CreateValue).ToArray()), + ImmutableDictionary dict => Value.ForStruct(CreateStruct(dict)), _ => throw new InvalidOperationException("Unsupported value when converting to protobuf: " + value.GetType().FullName), }; @@ -513,7 +513,7 @@ internal static bool ContainsUnknowns(object? value) /// Given a produced by , /// produces the equivalent that can be passed to the Pulumi engine. /// - public static Struct CreateStruct(ImmutableDictionary serializedDictionary) + public static Struct CreateStruct(ImmutableDictionary serializedDictionary) { var result = new Struct(); foreach (var key in serializedDictionary.Keys.OrderBy(k => k)) diff --git a/sdk/Pulumi/Stack.cs b/sdk/Pulumi/Stack.cs index b3b06d74..b0d0fec0 100644 --- a/sdk/Pulumi/Stack.cs +++ b/sdk/Pulumi/Stack.cs @@ -124,7 +124,8 @@ internal void RegisterPropertyOutputs() return new ComponentResourceOptions { - ResourceTransformations = options.ResourceTransformations + ResourceTransformations = options.ResourceTransformations, + XResourceTransforms = options.XResourceTransforms, }; } } diff --git a/sdk/Pulumi/Testing/MockMonitor.cs b/sdk/Pulumi/Testing/MockMonitor.cs index 63a30b82..8fd2e919 100644 --- a/sdk/Pulumi/Testing/MockMonitor.cs +++ b/sdk/Pulumi/Testing/MockMonitor.cs @@ -111,7 +111,7 @@ public async Task ReadResourceAsync(Resource resource, Rea return new ReadResourceResponse { Urn = urn, - Properties = Serializer.CreateStruct(serializedState), + Properties = Serializer.CreateStruct(serializedState!), }; } @@ -156,7 +156,7 @@ public async Task RegisterResourceAsync(Resource resou { Id = id ?? request.ImportId, Urn = urn, - Object = Serializer.CreateStruct(serializedState), + Object = Serializer.CreateStruct(serializedState!), }; } @@ -214,7 +214,7 @@ private async Task> SerializeToDictionary(ob private async Task SerializeAsync(object o) { var dict = await SerializeToDictionary(o).ConfigureAwait(false); - return Serializer.CreateStruct(dict); + return Serializer.CreateStruct(dict!); } } }