Appt 352/booking daily extract

azure-pipelines-booking-data-extract.yml

@@ -0,0 +1,57 @@
+trigger: none
+
+pool:
+  vmImage: "ubuntu-latest"
+
+schedules:
+  - cron: "0 2 * * *"
+    displayName: "Booking data extract schedule"
+    branches:
+      include:
+      - main
+    batch: false
+    always: true
+
+parameters:
+  - name: env
+    displayName: "Environment to import data into"
+    type: string
+    default: "prod"
+    values: ["dev", "int", "stag", "prod"]
+
+variables:
+  - group: covid19bookingkv${{parameters.env}}uks
+  - group: mya-booking-extract-${{parameters.env}}
+
+stages:
+  - stage: "CreateDailyExtract"
+    displayName: "Create booking data extract"
+    jobs:
+      - job: "GenerateBookingDailyExtract"
+        displayName: "Generate and send booking extract"
+        steps:
+          - task: AzureCLI@2
+            displayName: "Get Cosmos DB account connection string"
+            inputs:
+              azureSubscription: "nbs-mya-rg-dev"
+              scriptType: pscore
+              scriptLocation: scriptPath
+              scriptPath: "$(Build.SourcesDirectory)/scripts/cosmos-build/get_cosmos_connection_string.ps1"
+              arguments: "-resourceGroup $(resourceGroupName) -cosmosAccountName $(cosmosAccountName)"
+          - task: DotNetCoreCLI@2
+            displayName: "Run tool to create booking data extract"
+            inputs:
+              command: "run"
+              projects: "data/BookingsDataExtracts/BookingsDataExtracts.csproj"
+            env:
+              COSMOS_ENDPOINT: $(COSMOS_ENDPOINT)
+              COSMOS_TOKEN: $(COSMOS_TOKEN)
+              MESH_MAILBOX_DESTINATION: $(toMeshMailboxId)
+              MESH_WORKFLOW: $(meshWorkflowId)
+              KeyVault__KeyVaultName: covid19bookingkv${{parameters.env}}uks
+              KeyVault__TenantId: $(tenantId)
+              KeyVault__ClientId: $(ClientId)
+              KeyVault__ClientSecret: $(ClientSecret)
+              MeshClientOptions__BaseUrl: $(meshApiBaseUri)
+              MeshAuthorizationOptions__MailboxId: $(fromMeshMailboxId)
+              MeshAuthorizationOptions__CertificateName: $(meshCertificateName)          

azure-pipelines.yml

@@ -23,6 +23,7 @@ stages:
           coreUnitTestProjectPath: "tests/Nhs.Appointments.Core.UnitTests/Nhs.Appointments.Core.UnitTests.csproj"
           persistanceUnitTestProjectPath: "tests/Nhs.Appointments.Persistance.UnitTests/Nhs.Appointments.Persistance.UnitTests.csproj"
           csvDataToolTestProjectPath: "tests/CsvDataTool.UnitTests/CsvDataTool.UnitTests.csproj"
+          bookingExtractsTestProjectPath: "tests/BookingDataExtracts.UnitTests/BookingDataExtracts.UnitTests.csproj"
         steps:
           - task: SonarCloudPrepare@3
             displayName: "Prepare analysis on SonarCloud"
@@ -70,6 +71,12 @@ stages:
               command: "test"
               projects: $(csvDataToolTestProjectPath)
               arguments: '--collect "XPlat Code Coverage;Format=opencover"'
+          - task: DotNetCoreCLI@2
+            displayName: "Run Booking Extracts unit tests"
+            inputs:
+              command: "test"
+              projects: $(bookingExtractsTestProjectPath)
+              arguments: '--collect "XPlat Code Coverage;Format=opencover"'
           - task: SonarCloudAnalyze@3
             displayName: "Run Code Analysis"
           - task: SonarCloudPublish@3
@@ -140,6 +147,7 @@ stages:
         variables:
           PFX_CERT_PATH: "$(Build.SourcesDirectory)/.aspnet/https"
           integrationTestProjectPath: "tests/Nhs.Appointments.Api.Integration/Nhs.Appointments.Api.Integration.csproj"
+          bookingDataExtractTestProjectPath: "tests/BookingDataExtractTests/BookingDataExtracts.Integration.csproj"
           cosmosDbSeederProjectPath: "data/CosmosDbSeeder/CosmosDbSeeder.csproj"
           prBuildResourceGroup: "nbs-myacicd-rg-int-uks"
           subscriptionId: "2cf44e0d-817d-4596-b471-0788f8a14ab2"
@@ -169,9 +177,39 @@ stages:
               mkdir -p $(PFX_CERT_PATH)
               dotnet dev-certs https -ep $(PFX_CERT_PATH)/aspnetapp.pfx -p password
             displayName: "Create aspnet developer certificate"
+          - script: |
+                cd data/BookingsDataExtracts && docker compose up --build --detach
+            displayName: "Start MESH docker container"
           - script: |
               docker compose up --build --detach
-            displayName: "Start docker containers"
+            displayName: "Start application docker containers"
+            env:
+              COSMOS_ENDPOINT: $(COSMOS_ENDPOINT)
+              COSMOS_TOKEN: $(COSMOS_TOKEN)
+          - task: DotNetCoreCLI@2
+            displayName: "Run Booking Data Extract Integration Tests"
+            inputs:
+              command: "test"
+              projects: "$(bookingDataExtractTestProjectPath)"
+              arguments: '--logger:"trx;logfilename=myaDataExtractTests.trx";verbosity=detailed'
+            env:
+              COSMOS_ENDPOINT: $(COSMOS_ENDPOINT)
+              COSMOS_TOKEN: $(COSMOS_TOKEN)
+          - task: PublishTestResults@2
+            displayName: "Publish data extract integration test results"
+            inputs:
+              searchFolder: $(Agent.TempDirectory)
+              testResultsFormat: "VSTest"
+              testResultsFiles: "myaDataExtractTests.trx"
+              testRunTitle: "MYA Data Extract Integration Tests"
+              publishRunAttachments: true
+            condition: succeededOrFailed()
+          - task: DotNetCoreCLI@2
+            displayName: "Reset and seed Cosmos DB containers"
+            inputs:
+              command: "run"
+              projects: $(cosmosDbSeederProjectPath)
+              arguments: "delete-containers"
             env:
               COSMOS_ENDPOINT: $(COSMOS_ENDPOINT)
               COSMOS_TOKEN: $(COSMOS_TOKEN)

data/BookingsDataExtracts/BookingDataConverter.cs

@@ -0,0 +1,46 @@
+using BookingsDataExtracts.Documents;
+using Nhs.Appointments.Core;
+using Nhs.Appointments.Persistance.Models;
+
+namespace BookingsDataExtracts;
+
+public class BookingDataConverter(IEnumerable<SiteDocument> sites)
+{
+    public string ExtractICB(BookingDocument bookingDocument) => sites.Single(s => s.Id == bookingDocument.Site).IntegratedCareBoard;
+
+    public string ExtractRegion(BookingDocument bookingDocument) => sites.Single(s => s.Id == bookingDocument.Site).Region;
+
+    public string ExtractSiteName(BookingDocument bookingDocument) => sites.Single(s => s.Id == bookingDocument.Site).Name;
+
+    public double ExtractLongitude(BookingDocument bookingDocument) => sites.Single(s => s.Id == bookingDocument.Site).Location.Coordinates[0];
+
+    public double ExtractLatitude(BookingDocument bookingDocument) => sites.Single(s => s.Id == bookingDocument.Site).Location.Coordinates[1];
+
+    public string ExtractOdsCode(BookingDocument booking) => sites.Single(s => s.Id == booking.Site).OdsCode;
+
+    public static string ExtractNhsNumber(BookingDocument booking) => booking.AttendeeDetails.NhsNumber;
+
+    public static string ExtractAppointmentDateTime(BookingDocument booking) => booking.From.ToString("yyyy-MM-dd HH:mm:ss");
+
+    public static string ExtractCreatedDateTime(BookingDocument booking) => booking.Created.ToString("yyyy-MM-dd HH:mm:ss");
+
+    public static string ExtractAppointmentStatus(BookingDocument booking) => booking.Status switch
+    {        
+        AppointmentStatus.Orphaned => "B",
+        AppointmentStatus.Booked => "B",
+        AppointmentStatus.Cancelled => "C",
+        _ => throw new ArgumentOutOfRangeException(nameof(booking.Status)),
+    };
+
+    public static bool ExtractSelfReferral(NbsBookingDocument booking) => booking.AdditionalData?.ReferralType == "SelfReferred";
+
+    public static string ExtractSource(NbsBookingDocument booking) => booking.AdditionalData != null ? booking.AdditionalData.Source : "Unknown";
+
+    public static string ExtractDateOfBirth(BookingDocument booking) => booking.AttendeeDetails.DateOfBirth.ToString("yyyy-MM-dd");
+
+    public static string ExtractBookingReference(BookingDocument booking) => booking.Reference;
+
+    public static string ExtractService(BookingDocument booking) => booking.Service;
+
+    public static string ExtractCancelledDateTime(BookingDocument booking) => booking.Status == AppointmentStatus.Cancelled ? booking.StatusUpdated.ToString("yyyy-MM-dd HH:mm:ss") : string.Empty;
+}

data/BookingsDataExtracts/BookingDataExtractFields.cs

@@ -0,0 +1,24 @@
+namespace BookingsDataExtracts;
+
+public static class BookingDataExtractFields
+{
+    public static string OdsCode => "ODS_CODE";
+    public static string NhsNumber => "NHS_NUMBER";
+    public static string AppointmentDateTime => "APPOINTMENT_DATE_TIME";
+    public static string AppointmentStatus => "APPOINTMENT_STATUS";
+    public static string SelfRerral => "SELF_REFERRAL";
+    public static string Source => "SOURCE";
+    public static string DateOfBirth => "DATE_OF_BIRTH";
+    public static string BookingReferenceNumber => "BOOKING_REFERENCE_NUMBER";
+    public static string Service => "SERVICE";
+    public static string CreatedDateTime => "CREATED_DATE_TIME";
+    public static string Latitude => "LATITUDE";
+    public static string Longitude => "LONGITUDE";
+    public static string SiteName => "SITE_NAME";
+    public static string Region => "REGION";
+    public static string IntegratedCareBoard => "ICB";
+    public static string BookingSystem => "BOOKING_SYSTEM";
+    public static string CancelledDateTime => "CANCELLED_DATE_TIME";
+    public static string CancellationReason => "CANCELLATION_REASON";
+
+}

data/BookingsDataExtracts/BookingsDataExtract.cs

@@ -0,0 +1,64 @@
+using BookingsDataExtracts.Documents;
+using Nhs.Appointments.Core;
+using Nhs.Appointments.Persistance.Models;
+using Parquet;
+using Parquet.Schema;
+
+namespace BookingsDataExtracts;
+
+public class BookingDataExtract(
+    CosmosStore<NbsBookingDocument> bookingsStore,
+    CosmosStore<SiteDocument> sitesStore,
+    TimeProvider timeProvider)
+{
+    public async Task RunAsync(FileInfo outputFile)
+    {
+        Console.WriteLine("Loading bookings");
+
+        var allBookings = await bookingsStore.RunQueryAsync(b => b.DocumentType == "booking" && b.StatusUpdated > timeProvider.GetUtcNow().Date.AddDays(-1) && b.StatusUpdated < timeProvider.GetUtcNow().Date, b => b);
+        var bookings = allBookings.Where(b => b.Status != AppointmentStatus.Provisional).ToList();
+
+        Console.WriteLine("Loading sites");
+        var sites = await sitesStore.RunQueryAsync(s => s.DocumentType == "site", s => s);
+        var dataConverter = new BookingDataConverter(sites);
+
+        var dataFactories = new List<DataFactory>
+        {
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.OdsCode, dataConverter.ExtractOdsCode),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.NhsNumber, BookingDataConverter.ExtractNhsNumber),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.AppointmentDateTime, BookingDataConverter.ExtractAppointmentDateTime),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.AppointmentStatus, BookingDataConverter.ExtractAppointmentStatus),
+            new DataFactory<NbsBookingDocument, bool>(BookingDataExtractFields.SelfRerral, BookingDataConverter.ExtractSelfReferral),
+            new DataFactory<NbsBookingDocument, string>(BookingDataExtractFields.Source, BookingDataConverter.ExtractSource),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.DateOfBirth, BookingDataConverter.ExtractDateOfBirth),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.BookingReferenceNumber, BookingDataConverter.ExtractBookingReference),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.Service, BookingDataConverter.ExtractService),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.CreatedDateTime, BookingDataConverter.ExtractCreatedDateTime),
+            new DataFactory<BookingDocument, double>(BookingDataExtractFields.Latitude, dataConverter.ExtractLatitude),
+            new DataFactory<BookingDocument, double>(BookingDataExtractFields.Longitude, dataConverter.ExtractLongitude),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.SiteName, dataConverter.ExtractSiteName),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.Region, dataConverter.ExtractRegion),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.IntegratedCareBoard, dataConverter.ExtractICB),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.BookingSystem, doc => "MYA"),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.CancelledDateTime, BookingDataConverter.ExtractCancelledDateTime),
+            new DataFactory<BookingDocument, string>(BookingDataExtractFields.CancellationReason, doc => null),
+        };
+
+        Console.WriteLine("Preparing to write");
+
+        var schema = new ParquetSchema(dataFactories.Select(df => df.Field).ToArray());
+        using (Stream fs = outputFile.OpenWrite())
+        {
+            using (var writer = await ParquetWriter.CreateAsync(schema, fs))
+            {
+                using (var groupWriter = writer.CreateRowGroup())
+                {
+                    foreach (var dataFactory in dataFactories)
+                        await groupWriter.WriteColumnAsync(dataFactory.CreateColumn(bookings));                    
+                }
+            }
+        }
+
+        Console.WriteLine("done");
+    }    
+}

data/BookingsDataExtracts/BookingsDataExtracts.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk.Worker">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>disable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.46.0" />
+    <PackageReference Include="Microsoft.Azure.AppConfiguration.AspNetCore" Version="7.0.0" />
+    <PackageReference Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Parquet.Net" Version="5.0.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\api\Nhs.Appointments.Api\Nhs.Appointments.Api.csproj" />
+    <ProjectReference Include="..\..\src\api\Nhs.Appointments.Persistance\Nhs.Appointments.Persistance.csproj" />
+    <ProjectReference Include="..\..\src\libraries\Nbs.MeshClient\Nbs.MeshClient.csproj" />
+  </ItemGroup>
+
+</Project>

data/BookingsDataExtracts/CosmosStore.cs

@@ -0,0 +1,55 @@
+using System.Linq.Expressions;
+using BookingsDataExtracts.Documents;
+using Microsoft.Azure.Cosmos;
+using Microsoft.Azure.Cosmos.Linq;
+using Microsoft.Extensions.Options;
+using Nhs.Appointments.Persistance.Models;
+
+namespace BookingsDataExtracts;
+
+public class CosmosStore<TDocument>(CosmosClient cosmosClient, IOptions<CosmosStoreOptions> options)
+{
+    public Task<IEnumerable<TModel>> RunQueryAsync<TModel>(Expression<Func<TDocument, bool>> predicate, Expression<Func<TDocument, TModel>> projection)
+    {
+        var queryFeed = GetContainer().GetItemLinqQueryable<TDocument>().Where(predicate).ToFeedIterator();
+        return IterateResults(queryFeed, projection.Compile());
+    }
+
+    public Task<IEnumerable<TModel>> RunSqlQueryAsync<TModel>(QueryDefinition query)
+    {
+        var queryFeed = GetContainer().GetItemQueryIterator<TModel>(
+            queryDefinition: query);
+
+        return IterateResults(queryFeed, item => item);
+    }
+
+    private async Task<IEnumerable<TOutput>> IterateResults<TSource, TOutput>(FeedIterator<TSource> queryFeed, Func<TSource, TOutput> map)
+    {
+        var requestCharge = 0.0;
+        var results = new List<TOutput>();
+        using (queryFeed)
+        {
+            while (queryFeed.HasMoreResults)
+            {
+                var resultSet = await queryFeed.ReadNextAsync();
+                results.AddRange(resultSet.Select(map));
+                requestCharge += resultSet.RequestCharge;
+            }
+        }
+        return results;
+    }
+
+    private string GetContainerName() => typeof(TDocument).Name switch
+    {
+        nameof(NbsBookingDocument) => "booking_data",
+        nameof(SiteDocument) => "core_data",
+        _ => throw new NotSupportedException()
+    };
+
+    protected Container GetContainer() => cosmosClient.GetContainer(options.Value.DatabaseName, GetContainerName());
+}
+
+public class CosmosStoreOptions
+{
+    public string DatabaseName { get; set; }
+}