From 3c0dd260d0a13cbc8aca9c55f5e923fed9014acb Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 30 Jan 2025 12:12:50 +0000 Subject: [PATCH] CDMS-238 skeleton code to redact and replicate data to SND data lake --- Btms.Analytics/MovementExceptions.cs | 3 +- Btms.Backend/Endpoints/SyncEndpoints.cs | 11 +++- Btms.Business/BusinessOptions.cs | 14 ++++- Btms.Business/Commands/DownloadCommand.cs | 28 +++++----- Btms.Business/Commands/ReplicateCommand.cs | 54 +++++++++++++++++++ Btms.Business/Commands/SyncHandler.cs | 1 + .../Commands/SyncPeriodExtensions.cs | 4 ++ .../Extensions/ServiceCollectionExtensions.cs | 3 +- 8 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 Btms.Business/Commands/ReplicateCommand.cs diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index 8b690d80..e597e31d 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -20,7 +20,8 @@ private static string FormatUnmatched(DecisionStatusEnum decisionStatus, Movemen return matched.ToString(); } - //Returns a summary of the exceptions or a list + + // Returns a summary of the exceptions or a list // Means we can share the same anonymous / query code without needing to create loads // of classes public (SingleSeriesDataset summary, List) GetAllExceptions(DateTime from, DateTime to, bool finalisedOnly, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) diff --git a/Btms.Backend/Endpoints/SyncEndpoints.cs b/Btms.Backend/Endpoints/SyncEndpoints.cs index e50ce141..58383d55 100644 --- a/Btms.Backend/Endpoints/SyncEndpoints.cs +++ b/Btms.Backend/Endpoints/SyncEndpoints.cs @@ -38,7 +38,7 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions GenerateDownload([FromServices] IBtmsMediator return Results.Ok(command.JobId); } + private static async Task ReplicateGet( + [FromServices] IBtmsMediator mediator, + SyncPeriod syncPeriod) + { + ReplicateCommand command = new() { SyncPeriod = syncPeriod }; + await mediator.SendJob(command); + return Results.Ok(command.JobId); + } + private static Task GetAllSyncJobs([FromServices] ISyncJobStore store) { return Task.FromResult(Results.Ok(new { jobs = store.GetJobs() })); diff --git a/Btms.Business/BusinessOptions.cs b/Btms.Business/BusinessOptions.cs index a25cf1db..c734730f 100644 --- a/Btms.Business/BusinessOptions.cs +++ b/Btms.Business/BusinessOptions.cs @@ -1,8 +1,20 @@ using System.ComponentModel.DataAnnotations; +using Btms.Azure; using Btms.Business.Commands; namespace Btms.Business; +public class ReplicationOptions : IAzureConfig +{ + public const string SectionName = nameof(ReplicationOptions); + + public string? AzureClientId { get; set; } + public string? AzureTenantId { get; set; } + public string? AzureClientSecret { get; set; } + + public bool Enabled { get; set; } = false; + +} public class BusinessOptions { public const string SectionName = nameof(BusinessOptions); @@ -10,7 +22,7 @@ public class BusinessOptions private readonly int defaultDegreeOfParallelism = Math.Max(Environment.ProcessorCount / 4, 1); [Required] public string DmpBlobRootFolder { get; set; } = "RAW"; - + public Dictionary> ConcurrencyConfiguration { get; set; } public enum Feature diff --git a/Btms.Business/Commands/DownloadCommand.cs b/Btms.Business/Commands/DownloadCommand.cs index 4fbdeffd..889d5611 100644 --- a/Btms.Business/Commands/DownloadCommand.cs +++ b/Btms.Business/Commands/DownloadCommand.cs @@ -26,6 +26,18 @@ public class DownloadCommand : IRequest, ISyncJob public string? RootFolder { get; set; } + public static List<(string path, Type dataType)> BlobFolders = new List<(string path, Type dataType)>() + { + ("IPAFFS/CHEDA", typeof(ImportNotification)), + ("IPAFFS/CHEDD", typeof(ImportNotification)), + ("IPAFFS/CHEDP", typeof(ImportNotification)), + ("IPAFFS/CHEDPP", typeof(ImportNotification)), + ("IPAFFS/ALVS", typeof(AlvsClearanceRequest)), + ("IPAFFS/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse)), + ("IPAFFS/DECISIONS", typeof(AlvsClearanceRequest)), + ("IPAFFS/FINALISATION", typeof(Finalisation)) + }; + internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IHostEnvironment env, IOptions businessOptions) : IRequestHandler { @@ -55,18 +67,10 @@ public async Task Handle(DownloadCommand request, CancellationToken cancellation } else { - await Download(request, rootFolder, $"{blobContainer}/IPAFFS/CHEDA", typeof(ImportNotification), null, cancellationToken); - await Download(request, rootFolder, $"{blobContainer}/IPAFFS/CHEDD", typeof(ImportNotification), null, cancellationToken); - await Download(request, rootFolder, $"{blobContainer}/IPAFFS/CHEDP", typeof(ImportNotification), null, cancellationToken); - await Download(request, rootFolder, $"{blobContainer}/IPAFFS/CHEDPP", typeof(ImportNotification), null, cancellationToken); - - await Download(request, rootFolder, $"{blobContainer}/ALVS", typeof(AlvsClearanceRequest), null, cancellationToken); - - await Download(request, rootFolder, $"{blobContainer}/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse), null, cancellationToken); - - await Download(request, rootFolder, $"{blobContainer}/DECISIONS", typeof(AlvsClearanceRequest), null, cancellationToken); - - await Download(request, rootFolder, $"{blobContainer}/FINALISATION", typeof(Finalisation), null, cancellationToken); + BlobFolders.ForEach(async f => + { + await Download(request, rootFolder, $"{blobContainer}/{f.path}", f.dataType, null, cancellationToken); + }); } diff --git a/Btms.Business/Commands/ReplicateCommand.cs b/Btms.Business/Commands/ReplicateCommand.cs new file mode 100644 index 00000000..1d76918c --- /dev/null +++ b/Btms.Business/Commands/ReplicateCommand.cs @@ -0,0 +1,54 @@ +using System.IO.Compression; +using System.Text.Json.Serialization; +using Btms.BlobService; +using Btms.Common.Extensions; +using MediatR; +using Btms.SensitiveData; +using Btms.Types.Ipaffs; +using Btms.SyncJob; +using Btms.Types.Alvs; +using Btms.Types.Gvms; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Btms.Business.Commands; + +public class ReplicateCommand() : IRequest, ISyncJob +{ + [JsonConverter(typeof(JsonStringEnumConverter))] + public SyncPeriod SyncPeriod { get; set; } + + public Guid JobId { get; } = Guid.NewGuid(); + public string Timespan { get; } = null!; + public string Resource { get; } = null!; + + public string? RootFolder { get; set; } + // IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IHostEnvironment env, + internal class Handler(ILogger logger, IOptions businessOptions, IOptions replicationOptions) : IRequestHandler + { + public async Task Handle(ReplicateCommand request, CancellationToken cancellationToken) + { + var blobContainer = string.IsNullOrEmpty(request.RootFolder) + ? businessOptions.Value.DmpBlobRootFolder + : request.RootFolder; + + if (replicationOptions.Value.Enabled) + { + logger.LogInformation("Replicating from {BlobContainer} to {DestinationContainer}", blobContainer, blobContainer); + + DownloadCommand.BlobFolders.ForEach(f => + { + logger.LogInformation("Replicate {Path}", f.path); + }); + } + else + { + logger.LogWarning("ReplicateCommand called but not replication not enabled in config."); + } + + await Task.CompletedTask; + } + + } +} \ No newline at end of file diff --git a/Btms.Business/Commands/SyncHandler.cs b/Btms.Business/Commands/SyncHandler.cs index deba95cf..222e7e71 100644 --- a/Btms.Business/Commands/SyncHandler.cs +++ b/Btms.Business/Commands/SyncHandler.cs @@ -15,6 +15,7 @@ namespace Btms.Business.Commands; public enum SyncPeriod { Today, + Yesterday, LastMonth, ThisMonth, From202411, diff --git a/Btms.Business/Commands/SyncPeriodExtensions.cs b/Btms.Business/Commands/SyncPeriodExtensions.cs index a082347e..69a0c982 100644 --- a/Btms.Business/Commands/SyncPeriodExtensions.cs +++ b/Btms.Business/Commands/SyncPeriodExtensions.cs @@ -20,6 +20,10 @@ public static string[] GetPeriodPaths(this SyncPeriod period) { return [DateTime.Today.ToString("/yyyy/MM/dd/")]; } + else if (period == SyncPeriod.Yesterday) + { + return [DateTime.Today.AddDays(-1).ToString("/yyyy/MM/dd/")]; + } else if (period == SyncPeriod.From202411) { return DateTime.Today diff --git a/Btms.Business/Extensions/ServiceCollectionExtensions.cs b/Btms.Business/Extensions/ServiceCollectionExtensions.cs index eccf417c..90192b59 100644 --- a/Btms.Business/Extensions/ServiceCollectionExtensions.cs +++ b/Btms.Business/Extensions/ServiceCollectionExtensions.cs @@ -30,7 +30,8 @@ public static IServiceCollection AddBusinessServices(this IServiceCollection ser services.AddBtmsMetrics(); services.BtmsAddOptions(configuration, SensitiveDataOptions.SectionName); services.BtmsAddOptions(configuration, BusinessOptions.SectionName); - + services.BtmsAddOptions(configuration, ReplicationOptions.SectionName); + services.AddMongoDbContext(configuration); services.AddBlobStorage(configuration); services.AddSingleton();