-
-
Notifications
You must be signed in to change notification settings - Fork 25
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for parallel execution (#16)
* Make sure that that SNS & SQS Handler are using scoped. So after usage scoped objects are disposed. * Async handling & Concurrent handling in for each loop * Added unit tests and full support to concurrently handle multiple records at the same time. Co-authored-by: Wouter Crooy <[email protected]> Co-authored-by: Renato Golia <[email protected]>
- Loading branch information
3 people
authored
May 19, 2020
1 parent
662bb12
commit 8a14c67
Showing
16 changed files
with
988 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| using System; | ||
| using System.Threading.Tasks; | ||
|
|
||
| using Amazon.Lambda.Core; | ||
| using Amazon.Lambda.SNSEvents; | ||
| using Kralizek.Lambda; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| [assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))] | ||
|
|
||
| namespace SnsEventFunction | ||
| { | ||
| public class Function : EventFunction<SNSEvent> | ||
| { | ||
| protected override void Configure(IConfigurationBuilder builder) | ||
| { | ||
| builder.AddEnvironmentVariables(); | ||
| } | ||
|
|
||
| protected override void ConfigureLogging(ILoggingBuilder logging, IExecutionEnvironment executionEnvironment) | ||
| { | ||
| logging.AddConfiguration(Configuration.GetSection("Logging")); | ||
|
|
||
| logging.AddLambdaLogger(new LambdaLoggerOptions | ||
| { | ||
| IncludeCategory = true, | ||
| IncludeLogLevel = true, | ||
| IncludeNewline = true | ||
| }); | ||
| } | ||
|
|
||
| protected override void ConfigureServices(IServiceCollection services, IExecutionEnvironment executionEnvironment) | ||
| { | ||
| services.ConfigureSnsParallelExecution(4); | ||
|
|
||
| services.UseNotificationHandler<CustomNotification, CustomNotificationHandler>(enableParallelExecution: true); | ||
| } | ||
| } | ||
|
|
||
| public class CustomNotification | ||
| { | ||
| public string Message { get; set; } | ||
| } | ||
|
|
||
| public class CustomNotificationHandler : INotificationHandler<CustomNotification> | ||
| { | ||
| private readonly ILogger<CustomNotificationHandler> _logger; | ||
|
|
||
| public CustomNotificationHandler(ILogger<CustomNotificationHandler> logger) | ||
| { | ||
| _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||
| } | ||
|
|
||
| public Task HandleAsync(CustomNotification notification, ILambdaContext context) | ||
| { | ||
| _logger.LogInformation($"Handling notification: {notification.Message}"); | ||
|
|
||
| return Task.CompletedTask; | ||
| } | ||
| } | ||
| } |
19 changes: 19 additions & 0 deletions
19
samples/SnsEventFunctionWithParallelism/SnsEventFunctionWithParallelism.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>netcoreapp3.1</TargetFramework> | ||
| <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles> | ||
| <AWSProjectType>Lambda</AWSProjectType> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Amazon.Lambda.Logging.AspNetCore" Version="2.1.0" /> | ||
| <PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="2.1.1" /> | ||
| <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.1.1" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\src\Kralizek.Lambda.Template.Sns\Kralizek.Lambda.Template.Sns.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
20 changes: 20 additions & 0 deletions
20
samples/SnsEventFunctionWithParallelism/aws-lambda-tools-defaults.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| { | ||
| "Information" : [ | ||
| "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", | ||
| "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", | ||
|
|
||
| "dotnet lambda help", | ||
|
|
||
| "All the command line options for the Lambda command can be specified in this file." | ||
| ], | ||
|
|
||
| "profile":"RG", | ||
| "region" : "eu-west-1", | ||
| "configuration" : "Release", | ||
| "framework": "netcoreapp2.1", | ||
| "function-runtime": "dotnetcore3.1", | ||
| "function-name": "test-lambda-template", | ||
| "function-memory-size" : 256, | ||
| "function-timeout" : 30, | ||
| "function-handler" : "SnsEventFunction::SnsEventFunction.Function::FunctionHandler" | ||
| } |
56 changes: 56 additions & 0 deletions
56
src/Kralizek.Lambda.Template.Sns/ParallelSnsEventHandler.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| using System; | ||
| using System.Linq; | ||
| using System.Text.Json; | ||
| using System.Threading.Tasks; | ||
| using Amazon.Lambda.Core; | ||
| using Amazon.Lambda.SNSEvents; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace Kralizek.Lambda | ||
| { | ||
| public class ParallelSnsExecutionOptions | ||
| { | ||
| public int MaxDegreeOfParallelism { get; set; } = System.Environment.ProcessorCount; | ||
| } | ||
|
|
||
| public class ParallelSnsEventHandler<TNotification>: IEventHandler<SNSEvent> where TNotification : class | ||
| { | ||
| private readonly ILogger _logger; | ||
| private readonly IServiceProvider _serviceProvider; | ||
| private readonly ParallelSnsExecutionOptions _options; | ||
|
|
||
| public ParallelSnsEventHandler(IServiceProvider serviceProvider, ILoggerFactory loggerFactory, IOptions<ParallelSnsExecutionOptions> options) | ||
| { | ||
| _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); | ||
| _logger = loggerFactory?.CreateLogger("SnsForEachAsyncEventHandler") ?? throw new ArgumentNullException(nameof(loggerFactory)); | ||
| _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); | ||
| } | ||
|
|
||
| public async Task HandleAsync(SNSEvent input, ILambdaContext context) | ||
| { | ||
| if (input.Records.Any()) | ||
| { | ||
| await input.Records.ForEachAsync(_options.MaxDegreeOfParallelism, async record => | ||
| { | ||
| using (var scope = _serviceProvider.CreateScope()) | ||
| { | ||
| var message = record.Sns.Message; | ||
| var notification = JsonSerializer.Deserialize<TNotification>(message); | ||
| _logger.LogDebug($"Message received: {message}"); | ||
|
|
||
| var messageHandler = scope.ServiceProvider.GetService<INotificationHandler<TNotification>>(); | ||
| if (messageHandler == null) | ||
| { | ||
| _logger.LogCritical($"No INotificationHandler<{typeof(TNotification).Name}> could be found."); | ||
| throw new InvalidOperationException($"No INotificationHandler<{typeof(TNotification).Name}> could be found."); | ||
| } | ||
|
|
||
| await messageHandler.HandleAsync(notification, context); | ||
| } | ||
| }); | ||
| } | ||
| } | ||
| } | ||
| } |
23 changes: 20 additions & 3 deletions
23
src/Kralizek.Lambda.Template.Sns/ServiceCollectionExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,19 +1,36 @@ | ||
| using System; | ||
| using Amazon.Lambda.SNSEvents; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
|
|
||
| namespace Kralizek.Lambda | ||
| { | ||
| public static class ServiceCollectionExtensions | ||
| { | ||
| public static IServiceCollection UseNotificationHandler<TNotification, THandler>(this IServiceCollection services) | ||
| public static IServiceCollection ConfigureSnsParallelExecution(this IServiceCollection services, int maxDegreeOfParallelism) | ||
| { | ||
| services.Configure<ParallelSnsExecutionOptions>(option => option.MaxDegreeOfParallelism = maxDegreeOfParallelism); | ||
|
|
||
| return services; | ||
| } | ||
|
|
||
| public static IServiceCollection UseNotificationHandler<TNotification, THandler>(this IServiceCollection services, bool enableParallelExecution = false) | ||
| where TNotification : class | ||
| where THandler : class, INotificationHandler<TNotification> | ||
| { | ||
| services.AddTransient<IEventHandler<SNSEvent>, SnsEventHandler<TNotification>>(); | ||
| services.AddOptions(); | ||
|
|
||
| if (enableParallelExecution) | ||
| { | ||
| services.AddTransient<IEventHandler<SNSEvent>, ParallelSnsEventHandler<TNotification>>(); | ||
| } | ||
| else | ||
| { | ||
| services.AddTransient<IEventHandler<SNSEvent>, SnsEventHandler<TNotification>>(); | ||
| } | ||
|
|
||
| services.AddTransient<INotificationHandler<TNotification>, THandler>(); | ||
|
|
||
| return services; | ||
| return services; | ||
| } | ||
| } | ||
| } |
58 changes: 58 additions & 0 deletions
58
src/Kralizek.Lambda.Template.Sqs/ParallelSqsEventHandler.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| using System; | ||
| using System.Linq; | ||
| using System.Text.Json; | ||
| using System.Threading.Tasks; | ||
| using Amazon.Lambda.Core; | ||
| using Amazon.Lambda.SQSEvents; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace Kralizek.Lambda | ||
| { | ||
| public class ParallelSqsExecutionOptions | ||
| { | ||
| public int MaxDegreeOfParallelism { get; set; } = System.Environment.ProcessorCount; | ||
| } | ||
|
|
||
| public class ParallelSqsEventHandler<TMessage>: IEventHandler<SQSEvent> where TMessage : class | ||
| { | ||
| private readonly ILogger _logger; | ||
| private readonly IServiceProvider _serviceProvider; | ||
| private readonly ParallelSqsExecutionOptions _options; | ||
|
|
||
| public ParallelSqsEventHandler(IServiceProvider serviceProvider, ILoggerFactory loggerFactory, IOptions<ParallelSqsExecutionOptions> options) | ||
| { | ||
| _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); | ||
| _logger = loggerFactory?.CreateLogger("SqsForEachAsyncEventHandler") ?? throw new ArgumentNullException(nameof(loggerFactory)); | ||
| _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); | ||
| } | ||
|
|
||
| public async Task HandleAsync(SQSEvent input, ILambdaContext context) | ||
| { | ||
| if (input.Records.Any()) | ||
| { | ||
|
|
||
| await input.Records.ForEachAsync(_options.MaxDegreeOfParallelism, async singleSqsMessage => | ||
| { | ||
| using (var scope = _serviceProvider.CreateScope()) | ||
| { | ||
| var sqsMessage = singleSqsMessage.Body; | ||
| _logger.LogDebug($"Message received: {sqsMessage}"); | ||
|
|
||
| var message = JsonSerializer.Deserialize<TMessage>(sqsMessage); | ||
|
|
||
| var messageHandler = scope.ServiceProvider.GetService<IMessageHandler<TMessage>>(); | ||
| if (messageHandler == null) | ||
| { | ||
| _logger.LogError($"No IMessageHandler<{typeof(TMessage).Name}> could be found."); | ||
| throw new InvalidOperationException($"No IMessageHandler<{typeof(TMessage).Name}> could be found."); | ||
| } | ||
|
|
||
| await messageHandler.HandleAsync(message, context); | ||
| } | ||
| }); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| using System; | ||
| using System.Collections.Concurrent; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Kralizek.Lambda | ||
| { | ||
| public static class AsyncExtensions | ||
| { | ||
| /// <summary> | ||
| /// Extensions on collection | ||
| /// Lambda style extensions to cater a foreach with concurrency. | ||
| /// </summary> | ||
| /// <typeparam name="T"></typeparam> | ||
| /// <param name="source">The collection please make sure the collection can handle the concurrency. If writing back to the objects in the collection</param> | ||
| /// <param name="maxDegreeOfParallelism">Concurrent threads doing the async</param> | ||
| /// <param name="body">The work that needs to be done.</param> | ||
| /// <returns></returns> | ||
| public static Task ForEachAsync<T>(this IEnumerable<T> source, int maxDegreeOfParallelism, Func<T, Task> body) | ||
| { | ||
| return Task.WhenAll( | ||
| from partition in Partitioner.Create(source).GetPartitions(maxDegreeOfParallelism) | ||
| select Task.Run(async delegate | ||
| { | ||
| using (partition) | ||
| while (partition.MoveNext()) | ||
| await body(partition.Current); | ||
| })); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.