From 392fde47d1269ef3e7d3330b776e4012f3f31f6f Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Mon, 13 Nov 2023 10:35:39 -0800 Subject: [PATCH] HTTP APIs for .NET Isolated Parity (#2653) This PR adds HTTP API parity (CallHttpAsync()) for .NET Isolated. Changes include: New DurableHttpRequest and DurableHttpResponse types in the Worker project New TaskOrchestrationContext extension method for CallHttpAsync() Additional changes in OutOfProcMiddleware to execute the BuiltIn::HttpActivity activity function by using the existing TaskHttpActivityShim (BuiltIn::HttpActivity is the reserved name to know when a TaskActivity should be an HTTP activity). --- .../OutOfProcMiddleware.cs | 6 ++ .../Constants.cs | 2 + .../HTTP/DurableHttpRequest.cs | 72 ++++++++++++++++ .../HTTP/DurableHttpResponse.cs | 43 ++++++++++ .../HTTP/HttpHeadersConverter.cs | 84 +++++++++++++++++++ .../HTTP/HttpMethodConverter.cs | 33 ++++++++ .../HTTP/HttpRetryOptions.cs | 81 ++++++++++++++++++ ...askOrchestrationContextExtensionMethods.cs | 52 ++++++++++++ .../Worker.Extensions.DurableTask.csproj | 1 - 9 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 src/Worker.Extensions.DurableTask/HTTP/DurableHttpRequest.cs create mode 100644 src/Worker.Extensions.DurableTask/HTTP/DurableHttpResponse.cs create mode 100644 src/Worker.Extensions.DurableTask/HTTP/HttpHeadersConverter.cs create mode 100644 src/Worker.Extensions.DurableTask/HTTP/HttpMethodConverter.cs create mode 100644 src/Worker.Extensions.DurableTask/HTTP/HttpRetryOptions.cs create mode 100644 src/Worker.Extensions.DurableTask/TaskOrchestrationContextExtensionMethods.cs diff --git a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs index ac7040b2f..cf45c2521 100644 --- a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs +++ b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs @@ -419,6 +419,12 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F throw new InvalidOperationException($"An activity was scheduled but no {nameof(TaskScheduledEvent)} was found!"); } + if (scheduledEvent.Name?.StartsWith("BuiltIn::", StringComparison.OrdinalIgnoreCase) ?? false) + { + await next(); + return; + } + FunctionName functionName = new FunctionName(scheduledEvent.Name); OrchestrationInstance? instance = dispatchContext.GetProperty(); diff --git a/src/Worker.Extensions.DurableTask/Constants.cs b/src/Worker.Extensions.DurableTask/Constants.cs index 6f3c3cd41..e94200bef 100644 --- a/src/Worker.Extensions.DurableTask/Constants.cs +++ b/src/Worker.Extensions.DurableTask/Constants.cs @@ -9,4 +9,6 @@ internal static class Constants public const string IllegalAwaitErrorMessage = "An invalid asynchronous invocation was detected. This can be caused by awaiting non-durable tasks " + "in an orchestrator function's implementation or by middleware that invokes asynchronous code."; + + public const string HttpTaskActivityReservedName = "BuiltIn::HttpActivity"; } \ No newline at end of file diff --git a/src/Worker.Extensions.DurableTask/HTTP/DurableHttpRequest.cs b/src/Worker.Extensions.DurableTask/HTTP/DurableHttpRequest.cs new file mode 100644 index 000000000..333c74cfd --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HTTP/DurableHttpRequest.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Primitives; + +namespace Microsoft.Azure.Functions.Worker.Extensions.DurableTask.Http; + +/// +/// Request used to make an HTTP call through Durable Functions. +/// +public class DurableHttpRequest +{ + /// + /// Initializes a new instance of the class. + /// + public DurableHttpRequest(HttpMethod method, Uri uri) + { + this.Method = method; + this.Uri = uri; + } + + /// + /// HttpMethod used in the HTTP request made by the Durable Function. + /// + [JsonPropertyName("method")] + [JsonConverter(typeof(HttpMethodConverter))] + public HttpMethod Method { get; } + + /// + /// Uri used in the HTTP request made by the Durable Function. + /// + [JsonPropertyName("uri")] + public Uri Uri { get; } + + /// + /// Headers passed with the HTTP request made by the Durable Function. + /// + [JsonPropertyName("headers")] + [JsonConverter(typeof(HttpHeadersConverter))] + public IDictionary? Headers { get; set; } + + /// + /// Content passed with the HTTP request made by the Durable Function. + /// + [JsonPropertyName("content")] + public string? Content { get; set; } + + /// + /// Specifies whether the Durable HTTP APIs should automatically + /// handle the asynchronous HTTP pattern. + /// + [JsonPropertyName("asynchronousPatternEnabled")] + public bool AsynchronousPatternEnabled { get; set; } + + /// + /// Defines retry policy for handling of failures in making the HTTP Request. These could be non-successful HTTP status codes + /// in the response, a timeout in making the HTTP call, or an exception raised from the HTTP Client library. + /// + [JsonPropertyName("retryOptions")] + public HttpRetryOptions? HttpRetryOptions { get; set; } + + /// + /// The total timeout for the original HTTP request and any + /// asynchronous polling. + /// + [JsonPropertyName("timeout")] + public TimeSpan? Timeout { get; set; } +} \ No newline at end of file diff --git a/src/Worker.Extensions.DurableTask/HTTP/DurableHttpResponse.cs b/src/Worker.Extensions.DurableTask/HTTP/DurableHttpResponse.cs new file mode 100644 index 000000000..06875d9e4 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HTTP/DurableHttpResponse.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Net; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Primitives; + +namespace Microsoft.Azure.Functions.Worker.Extensions.DurableTask.Http; + +/// +/// Response received from the HTTP request made by the Durable Function. +/// +public class DurableHttpResponse +{ + /// + /// Initializes a new instance of the class. + /// + /// HTTP Status code returned from the HTTP call. + public DurableHttpResponse(HttpStatusCode statusCode) + { + this.StatusCode = statusCode; + } + + /// + /// Status code returned from an HTTP request. + /// + [JsonPropertyName("statusCode")] + public HttpStatusCode StatusCode { get; } + + /// + /// Headers in the response from an HTTP request. + /// + [JsonPropertyName("headers")] + [JsonConverter(typeof(HttpHeadersConverter))] + public IDictionary? Headers { get; init; } + + /// + /// Content returned from an HTTP request. + /// + [JsonPropertyName("content")] + public string? Content { get; init; } +} \ No newline at end of file diff --git a/src/Worker.Extensions.DurableTask/HTTP/HttpHeadersConverter.cs b/src/Worker.Extensions.DurableTask/HTTP/HttpHeadersConverter.cs new file mode 100644 index 000000000..d41acef5e --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HTTP/HttpHeadersConverter.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Primitives; + +namespace Microsoft.Azure.Functions.Worker.Extensions.DurableTask.Http; + +// StringValues does not deserialize as you would expect, so we need a custom mechanism +// for serializing HTTP header collections +internal class HttpHeadersConverter : JsonConverter> +{ + public override IDictionary Read( + ref Utf8JsonReader reader, + Type objectType, + JsonSerializerOptions options) + { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (reader.TokenType != JsonTokenType.StartObject) + { + return headers; + } + + var valueList = new List(); + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + string? propertyName = reader.GetString(); + + reader.Read(); + + // Header values can be either individual strings or string arrays + StringValues values = default(StringValues); + if (reader.TokenType == JsonTokenType.String) + { + values = new StringValues(reader.GetString()); + } + else if (reader.TokenType == JsonTokenType.StartArray) + { + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + valueList.Add(reader.GetString()); + } + + values = new StringValues(valueList.ToArray()); + valueList.Clear(); + } + + headers[propertyName] = values; + } + + return headers; + } + + public override void Write( + Utf8JsonWriter writer, + IDictionary value, + JsonSerializerOptions options) + { + writer.WriteStartObject(); + + var headers = (IDictionary)value; + foreach (var pair in headers) + { + if (pair.Value.Count == 1) + { + // serialize as a single string value + writer.WriteString(pair.Key, pair.Value[0]); + } + else + { + // serializes as an array + writer.WriteStartArray(pair.Key); + writer.WriteStringValue(pair.Value); + writer.WriteEndArray(); + } + } + + writer.WriteEndObject(); + } +} \ No newline at end of file diff --git a/src/Worker.Extensions.DurableTask/HTTP/HttpMethodConverter.cs b/src/Worker.Extensions.DurableTask/HTTP/HttpMethodConverter.cs new file mode 100644 index 000000000..540f1f981 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HTTP/HttpMethodConverter.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Azure.Functions.Worker.Extensions.DurableTask.Http; + +internal class HttpMethodConverter : JsonConverter +{ + public override bool CanConvert(Type objectType) + { + return typeof(HttpMethod).IsAssignableFrom(objectType); + } + + public override HttpMethod Read( + ref Utf8JsonReader reader, + Type objectType, + JsonSerializerOptions options) + { + return new HttpMethod(reader.GetString()); + } + + public override void Write( + Utf8JsonWriter writer, + HttpMethod value, + JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } +} diff --git a/src/Worker.Extensions.DurableTask/HTTP/HttpRetryOptions.cs b/src/Worker.Extensions.DurableTask/HTTP/HttpRetryOptions.cs new file mode 100644 index 000000000..959f7e047 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HTTP/HttpRetryOptions.cs @@ -0,0 +1,81 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Text.Json.Serialization; + +namespace Microsoft.Azure.Functions.Worker.Extensions.DurableTask.Http; + +/// +/// Defines retry policies that can be passed as parameters to various operations. +/// +public class HttpRetryOptions +{ + // Would like to make this durability provider specific, but since this is a developer + // facing type, that is difficult. + private static readonly TimeSpan DefaultMaxRetryinterval = TimeSpan.FromDays(6); + + /// + /// Creates a new instance SerializableRetryOptions with the supplied first retry and max attempts. + /// + public HttpRetryOptions(IList? statusCodesToRetry = null) + { + this.StatusCodesToRetry = statusCodesToRetry ?? new List(); + } + + /// + /// Gets or sets the first retry interval. + /// + /// + /// The TimeSpan to wait for the first retries. + /// + [JsonPropertyName("FirstRetryInterval")] + public TimeSpan FirstRetryInterval { get; set; } + + /// + /// Gets or sets the max retry interval. + /// + /// + /// The TimeSpan of the max retry interval, defaults to 6 days. + /// + [JsonPropertyName("MaxRetryInterval")] + public TimeSpan MaxRetryInterval { get; set; } = DefaultMaxRetryinterval; + + /// + /// Gets or sets the backoff coefficient. + /// + /// + /// The backoff coefficient used to determine rate of increase of backoff. Defaults to 1. + /// + [JsonPropertyName("BackoffCoefficient")] + public double BackoffCoefficient { get; set; } = 1; + + /// + /// Gets or sets the timeout for retries. + /// + /// + /// The TimeSpan timeout for retries, defaults to . + /// + [JsonPropertyName("RetryTimeout")] + public TimeSpan RetryTimeout { get; set; } = TimeSpan.MaxValue; + + /// + /// Gets or sets the max number of attempts. + /// + /// + /// The maximum number of retry attempts. + /// + [JsonPropertyName("MaxNumberOfAttempts")] + public int MaxNumberOfAttempts { get; set; } + + /// + /// Gets or sets the list of status codes upon which the + /// retry logic specified by this object shall be triggered. + /// If none are provided, all 4xx and 5xx status codes + /// will be retried. + /// + [JsonPropertyName("StatusCodesToRetry")] + public IList StatusCodesToRetry { get; } +} diff --git a/src/Worker.Extensions.DurableTask/TaskOrchestrationContextExtensionMethods.cs b/src/Worker.Extensions.DurableTask/TaskOrchestrationContextExtensionMethods.cs new file mode 100644 index 000000000..c5523d002 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/TaskOrchestrationContextExtensionMethods.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Extensions.DurableTask; +using Microsoft.Azure.Functions.Worker.Extensions.DurableTask.Http; + +namespace Microsoft.DurableTask; + +/// +/// Extensions for . +/// +public static class TaskOrchestrationContextExtensionMethods +{ + /// + /// Makes an HTTP call using the information in the DurableHttpRequest. + /// + /// The task orchestration context. + /// The DurableHttpRequest used to make the HTTP call. + /// DurableHttpResponse + public static Task CallHttpAsync(this TaskOrchestrationContext context, DurableHttpRequest request) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + return context.CallActivityAsync(Constants.HttpTaskActivityReservedName, request); + } + + /// + /// Makes an HTTP call to the specified uri. + /// + /// The task orchestration context. + /// HttpMethod used for api call. + /// uri used to make the HTTP call. + /// Content passed in the HTTP request. + /// The retry option for the HTTP task. + /// A Result of the HTTP call. + public static Task CallHttpAsync(this TaskOrchestrationContext context, HttpMethod method, Uri uri, string? content = null, HttpRetryOptions? retryOptions = null) + { + DurableHttpRequest request = new DurableHttpRequest(method, uri) + { + Content = content, + HttpRetryOptions = retryOptions, + }; + + return context.CallHttpAsync(request); + } +} diff --git a/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj b/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj index 3f63320c5..c7b7d1918 100644 --- a/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj +++ b/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj @@ -54,5 +54,4 @@ content/SBOM -