Merge pull request #4526 from dfe-analytical-services/dev

Merge Dev into Master

.eslintignore

@@ -1,6 +1,7 @@
 .idea/
 .next/
 build/
+dist/
 coverage/
 node_modules/
 *.min.js

.prettierignore

@@ -1,6 +1,7 @@
 .idea/
 .next/
 build/
+dist/
 coverage/
 node_modules/
 *.min.css

.stylelintignore

@@ -1,6 +1,7 @@
 .next
 build/
 coverage/
+dist/
 node_modules/
 *.min.css
 # Artifacts can leak between jobs on self-hosted runners

src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/TopicServicePermissionTests.cs

@@ -1,3 +1,4 @@
+#nullable enable
 using System;
 using AutoMapper;
 using GovUk.Education.ExploreEducationStatistics.Admin.Security;
@@ -100,16 +101,17 @@ public void DeleteTopic()
         }
 
         private TopicService SetupTopicService(
-            ContentDbContext contentContext = null,
-            StatisticsDbContext statisticsContext = null,
-            PersistenceHelper<ContentDbContext> persistenceHelper = null,
-            IMapper mapper = null,
-            IUserService userService = null,
-            IReleaseSubjectRepository releaseSubjectRepository = null,
-            IReleaseDataFileService releaseDataFileService = null,
-            IReleaseFileService releaseFileService = null,
-            IPublishingService publishingService = null,
-            IMethodologyService methodologyService = null)
+            ContentDbContext? contentContext = null,
+            StatisticsDbContext? statisticsContext = null,
+            PersistenceHelper<ContentDbContext>? persistenceHelper = null,
+            IMapper? mapper = null,
+            IUserService? userService = null,
+            IReleaseSubjectRepository? releaseSubjectRepository = null,
+            IReleaseDataFileService? releaseDataFileService = null,
+            IReleaseFileService? releaseFileService = null,
+            IPublishingService? publishingService = null,
+            IMethodologyService? methodologyService = null,
+            IReleasePublishingStatusRepository? releasePublishingStatusRepository = null)
         {
             return new TopicService(
                 Mock.Of<IConfiguration>(),
@@ -123,7 +125,8 @@ private TopicService SetupTopicService(
                 releaseFileService ?? Mock.Of<IReleaseFileService>(),
                 publishingService ?? Mock.Of<IPublishingService>(),
                 methodologyService ?? Mock.Of<IMethodologyService>(),
-                Mock.Of<IBlobCacheService>()
+                Mock.Of<IBlobCacheService>(),
+                releasePublishingStatusRepository ?? Mock.Of<IReleasePublishingStatusRepository>()
             );
         }
     }

src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/TopicServiceTests.cs

@@ -413,6 +413,7 @@ public async Task DeleteTopic()
             var methodologyService = new Mock<IMethodologyService>(Strict);
             var publishingService = new Mock<IPublishingService>(Strict);
             var cacheService = new Mock<IBlobCacheService>(Strict);
+            var releasePublishingStatusRepository = new Mock<IReleasePublishingStatusRepository>(Strict);
 
             await using (var contentContext = DbUtils.InMemoryApplicationDbContext(contextId))
             await using (var statisticsContext = InMemoryStatisticsDbContext(contextId))
@@ -425,7 +426,8 @@ public async Task DeleteTopic()
                     releaseSubjectRepository: releaseSubjectRepository.Object,
                     methodologyService: methodologyService.Object,
                     publishingService: publishingService.Object,
-                    cacheService: cacheService.Object);
+                    cacheService: cacheService.Object,
+                    releasePublishingStatusRepository: releasePublishingStatusRepository.Object);
 
                 releaseDataFileService
                     .Setup(s => s.DeleteAll(releaseId, true))
@@ -452,13 +454,18 @@ public async Task DeleteTopic()
                             ItIs.DeepEqualTo(new PrivateReleaseContentFolderCacheKey(releaseId))))
                     .Returns(Task.CompletedTask);
 
+                releasePublishingStatusRepository.Setup(mock =>
+                        mock.RemovePublisherReleaseStatuses(new List<Guid>{ release.Id }))
+                    .Returns(Task.CompletedTask);
+
                 var result = await service.DeleteTopic(topic.Id);
                 VerifyAllMocks(releaseDataFileService,
                     releaseFileService,
                     releaseSubjectRepository,
                     methodologyService,
                     publishingService,
-                    cacheService);
+                    cacheService,
+                    releasePublishingStatusRepository);
 
                 result.AssertRight();
 
@@ -558,6 +565,7 @@ public async Task DeleteTopic_ReleaseVersionsDeletedInCorrectOrder()
             var releaseSubjectRepository = new Mock<IReleaseSubjectRepository>(Strict);
             var publishingService = new Mock<IPublishingService>(Strict);
             var cacheService = new Mock<IBlobCacheService>(Strict);
+            var releasePublishingStatusRepository = new Mock<IReleasePublishingStatusRepository>(Strict);
 
             await using (var contentContext = DbUtils.InMemoryApplicationDbContext(contextId))
             await using (var statisticsContext = InMemoryStatisticsDbContext(contextId))
@@ -569,7 +577,8 @@ public async Task DeleteTopic_ReleaseVersionsDeletedInCorrectOrder()
                     releaseFileService: releaseFileService.Object,
                     releaseSubjectRepository: releaseSubjectRepository.Object,
                     publishingService: publishingService.Object,
-                    cacheService: cacheService.Object);
+                    cacheService: cacheService.Object,
+                    releasePublishingStatusRepository: releasePublishingStatusRepository.Object);
 
                 var releaseDataFileDeleteSequence = new MockSequence();
 
@@ -608,13 +617,18 @@ public async Task DeleteTopic_ReleaseVersionsDeletedInCorrectOrder()
                 publishingService.Setup(s => s.TaxonomyChanged())
                     .ReturnsAsync(Unit.Instance);
 
+                releasePublishingStatusRepository.Setup(mock =>
+                        mock.RemovePublisherReleaseStatuses(releaseIdsInExpectedDeleteOrder))
+                    .Returns(Task.CompletedTask);
+
                 var result = await service.DeleteTopic(topicId);
                 VerifyAllMocks(
                     releaseDataFileService,
                     releaseFileService,
                     releaseSubjectRepository,
                     publishingService,
-                    cacheService);
+                    cacheService,
+                    releasePublishingStatusRepository);
 
                 result.AssertRight();
 
@@ -872,6 +886,7 @@ public async Task DeleteTopic_OtherTopicsUnaffected()
             var methodologyService = new Mock<IMethodologyService>(Strict);
             var publishingService = new Mock<IPublishingService>(Strict);
             var cacheService = new Mock<IBlobCacheService>(Strict);
+            var releasePublishingStatusRepository = new Mock<IReleasePublishingStatusRepository>(Strict);
 
             await using (var contentContext = DbUtils.InMemoryApplicationDbContext(contextId))
             await using (var statisticsContext = InMemoryStatisticsDbContext(contextId))
@@ -884,7 +899,8 @@ public async Task DeleteTopic_OtherTopicsUnaffected()
                     releaseSubjectRepository: releaseSubjectRepository.Object,
                     methodologyService: methodologyService.Object,
                     publishingService: publishingService.Object,
-                    cacheService: cacheService.Object);
+                    cacheService: cacheService.Object,
+                    releasePublishingStatusRepository: releasePublishingStatusRepository.Object);
 
                 releaseDataFileService
                     .Setup(s => s.DeleteAll(releaseId, true))
@@ -911,13 +927,18 @@ public async Task DeleteTopic_OtherTopicsUnaffected()
                             ItIs.DeepEqualTo(new PrivateReleaseContentFolderCacheKey(releaseId))))
                     .Returns(Task.CompletedTask);
 
+                releasePublishingStatusRepository.Setup(mock =>
+                        mock.RemovePublisherReleaseStatuses(new List<Guid>{ releaseId }))
+                    .Returns(Task.CompletedTask);
+
                 var result = await service.DeleteTopic(topic.Id);
                 VerifyAllMocks(releaseDataFileService,
                     releaseFileService,
                     releaseSubjectRepository,
                     methodologyService,
                     publishingService,
-                    cacheService);
+                    cacheService,
+                    releasePublishingStatusRepository);
 
                 result.AssertRight();
 
@@ -942,6 +963,7 @@ private static TopicService SetupTopicService(
             IPublishingService? publishingService = null,
             IMethodologyService? methodologyService = null,
             IBlobCacheService? cacheService = null,
+            IReleasePublishingStatusRepository? releasePublishingStatusRepository = null,
             bool enableThemeDeletion = true)
         {
             var configuration =
@@ -959,7 +981,8 @@ private static TopicService SetupTopicService(
                 releaseFileService ?? Mock.Of<IReleaseFileService>(Strict),
                 publishingService ?? Mock.Of<IPublishingService>(Strict),
                 methodologyService ?? Mock.Of<IMethodologyService>(Strict),
-                cacheService ?? Mock.Of<IBlobCacheService>(Strict)
+                cacheService ?? Mock.Of<IBlobCacheService>(Strict),
+                releasePublishingStatusRepository ?? Mock.Of<IReleasePublishingStatusRepository>(Strict)
             );
         }
     }

src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleasePublishingStatusRepository.cs

@@ -8,5 +8,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces
     public interface IReleasePublishingStatusRepository
     {
         Task<IEnumerable<ReleasePublishingStatus>> GetAllByOverallStage(Guid releaseId, params ReleasePublishingStatusOverallStage[] overallStages);
+
+        Task RemovePublisherReleaseStatuses(List<Guid> releaseIds);
     }
-}
+}

src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleasePublishingStatusRepository.cs

@@ -3,7 +3,9 @@
 using System.Linq;
 using System.Threading.Tasks;
 using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces;
+using GovUk.Education.ExploreEducationStatistics.Common.Extensions;
 using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces;
+using GovUk.Education.ExploreEducationStatistics.Data.Model;
 using GovUk.Education.ExploreEducationStatistics.Publisher.Model;
 using Microsoft.Azure.Cosmos.Table;
 using static GovUk.Education.ExploreEducationStatistics.Common.TableStorageTableNames;
@@ -47,5 +49,35 @@ public Task<IEnumerable<ReleasePublishingStatus>> GetAllByOverallStage(Guid rele
             var query = new TableQuery<ReleasePublishingStatus>().Where(filter);
             return _publisherTableStorageService.ExecuteQueryAsync(PublisherReleaseStatusTableName, query);
         }
+
+        public async Task RemovePublisherReleaseStatuses(List<Guid> releaseIds)
+        {
+            if (releaseIds.IsNullOrEmpty())
+            {
+                // Return early as we want to do nothing in this case - without this,
+                // `filter` will be string.Empty and the query returns all table entities
+                return;
+            }
+
+            var filter = string.Empty;
+            foreach (var releaseId in releaseIds)
+            {
+                var newFilter = TableQuery.GenerateFilterCondition(nameof(ReleasePublishingStatus.PartitionKey),
+                    QueryComparisons.Equal, releaseId.ToString());
+
+                filter = filter == string.Empty
+                    ? newFilter
+                    : TableQuery.CombineFilters(filter, TableOperators.Or, newFilter);
+            }
+
+            var cloudTable = _publisherTableStorageService.GetTable(PublisherReleaseStatusTableName);
+            var query = new TableQuery<ReleasePublishingStatus>().Where(filter);
+            var releaseStatusesToRemove = cloudTable.ExecuteQuery(query);
+
+            foreach (var releaseStatus in releaseStatusesToRemove)
+            {
+                await cloudTable.ExecuteAsync(TableOperation.Delete(releaseStatus));
+            }
+        }
     }
-}
+}

src/GovUk.Education.ExploreEducationStatistics.Admin/Services/TopicService.cs

@@ -11,7 +11,6 @@
 using GovUk.Education.ExploreEducationStatistics.Admin.Validators;
 using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels;
 using GovUk.Education.ExploreEducationStatistics.Common.Cache;
-using GovUk.Education.ExploreEducationStatistics.Common.Extensions;
 using GovUk.Education.ExploreEducationStatistics.Common.Model;
 using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces;
 using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security;
@@ -43,6 +42,7 @@ public class TopicService : ITopicService
         private readonly IPublishingService _publishingService;
         private readonly IMethodologyService _methodologyService;
         private readonly IBlobCacheService _cacheService;
+        private readonly IReleasePublishingStatusRepository _releasePublishingStatusRepository;
         private readonly bool _topicDeletionAllowed;
 
         public TopicService(
@@ -57,7 +57,8 @@ public TopicService(
             IReleaseFileService releaseFileService,
             IPublishingService publishingService,
             IMethodologyService methodologyService,
-            IBlobCacheService cacheService)
+            IBlobCacheService cacheService,
+            IReleasePublishingStatusRepository releasePublishingStatusRepository)
         {
             _contentContext = contentContext;
             _statisticsContext = statisticsContext;
@@ -70,6 +71,7 @@ public TopicService(
             _publishingService = publishingService;
             _methodologyService = methodologyService;
             _cacheService = cacheService;
+            _releasePublishingStatusRepository = releasePublishingStatusRepository;
             _topicDeletionAllowed = configuration.GetValue<bool>("enableThemeDeletion");
         }
 
@@ -229,12 +231,18 @@ private async Task<Either<ActionResult, Unit>> DeleteReleasesForPublications(IEn
 
             var releaseIdsInDeleteOrder = VersionedEntityDeletionOrderUtil
                 .Sort(releaseIdsToDelete)
-                .Select(ids => Guid.Parse(ids.Id));
+                .Select(ids => Guid.Parse(ids.Id))
+                .ToList();
+
+            // Delete release entries in the Azure Storage ReleaseStatus table - if not it will attempt to publish
+            // deleted releases that were left scheduled
+            await _releasePublishingStatusRepository.RemovePublisherReleaseStatuses(releaseIdsInDeleteOrder);
 
             return await releaseIdsInDeleteOrder
                 .Select(DeleteContentAndStatsRelease)
                 .OnSuccessAll()
                 .OnSuccessVoid();
+
         }
 
         private async Task<Either<ActionResult, Unit>> DeletePublications(IEnumerable<Guid> publicationIds)

src/GovUk.Education.ExploreEducationStatistics.Data.Api/Controllers/TableBuilderController.cs

@@ -104,16 +104,22 @@ await _tableBuilderService.QueryToCsvStream(
         // both in order to support legacy URLs in bookmarks and in content, but also to remain consistent with the equivalent
         // endpoint in the Admin API, which does require the Release Id in order to differentiate between different 
         // DataBlockVersions rather than simply picking the latest published one.
-        [ResponseCache(Duration = 300)]
         [HttpGet("tablebuilder/release/{releaseId:guid}/data-block/{dataBlockParentId:guid}")]
         public async Task<ActionResult<TableBuilderResultViewModel>> QueryForTableBuilderResult(
             Guid dataBlockParentId)
         {
-            return await GetLatestPublishedDataBlockVersion(dataBlockParentId)
+            var actionResult = await GetLatestPublishedDataBlockVersion(dataBlockParentId)
                 .OnSuccessDo(dataBlockVersion => this
                     .CacheWithLastModifiedAndETag(lastModified: dataBlockVersion.Published, ApiVersion))
                 .OnSuccess(GetDataBlockTableResult)
                 .HandleFailuresOrOk();
+
+            if (actionResult.Result is not NotFoundResult)
+            {
+                Response.Headers["Cache-Control"] = "public,max-age=300";
+            }
+
+            return actionResult;
         }
 
         [HttpGet("tablebuilder/fast-track/{dataBlockParentId:guid}")]

src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs

@@ -108,14 +108,11 @@ await releaseIdsToUpdate
                 }
             });
 
-        var publicationSlugs = prePublishingStagesComplete
-            .Select(status => status.PublicationSlug)
-            .Distinct();
-
         var directlyRelatedPublicationIds = await _contentDbContext
-            .Publications
-            .Where(p => publicationSlugs.Contains(p.Slug))
-            .Select(p => p.Id)
+            .Releases
+            .Where(r => releaseIdsToUpdate.Contains(r.Id))
+            .Select(r => r.PublicationId)
+            .Distinct()
             .ToListAsync();
 
         await directlyRelatedPublicationIds

tests/playwright-tests/.env.example

@@ -1 +1,4 @@
-PUBLIC_URL=http://localhost:3000  # URL of the public frontend
+PUBLIC_URL=http://localhost:3000  # URL of the public frontend
+ADMIN_URL=https://localhost:5021  # URL for the Admin frontend
+ADMIN_EMAIL=ADMIN_EMAIL  # BAU1 user email
+ADMIN_PASSWORD=ADMIN_PASSWORD  # BAU1 user password

tests/playwright-tests/admin/azpage/AzureLoginPage.ts

@@ -0,0 +1,33 @@
+import { Locator, Page } from '@playwright/test';
+import environment from '../../utils/env';
+
+export default class AzureLoginPage {
+  readonly page: Page;
+  readonly emailAddress: Locator;
+  readonly nextButton: Locator;
+  readonly password: Locator;
+  readonly signInButton: Locator;
+  readonly noButton: Locator;
+
+  constructor(page: Page) {
+    this.page = page;
+    // Locators
+    this.emailAddress = page.locator(
+      'input[placeholder="Email, phone, or Skype"]',
+    );
+    this.nextButton = page.locator('input[type="submit"]');
+    this.password = page.locator('input[type="password"]');
+    this.signInButton = page.locator('input[id="idSIButton9"]');
+    this.noButton = page.locator('input[id="idBtn_Back"]');
+  }
+
+  async doSignIn() {
+    await this.emailAddress.fill(environment.ADMIN_EMAIL);
+    await this.nextButton.click();
+    await this.password.waitFor({ state: 'visible' });
+    await this.password.fill(environment.ADMIN_PASSWORD);
+    await this.signInButton.click();
+    await this.noButton.waitFor({ state: 'visible' });
+    await this.noButton.click();
+  }
+}