From f4692f74699d4404ea945f3aba75d2d7ecbc27de Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 8 Jul 2024 14:14:40 -0700 Subject: [PATCH 1/5] unflake test --- test/Common/DurableTaskEndToEndTests.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 57eb8f532..364319377 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -5091,15 +5091,21 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) // run an orchestration B that queues behind A for the lock (and thus gets stuck) TestDurableClient clientB = await host.StartOrchestratorAsync(nameof(TestOrchestrations.LockThenFailReplay), (orphanedEntityId, false), this.output, orchestrationB); - // remove empty entity and release orphaned lock - var response = await client.InnerClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + // remove release orphaned lock to unblock orchestration B + // Note: do NOT remove empty entities ye: we want to keep the empty entity so it can unblock orchestration B + var response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: false, releaseOrphanedLocks: true, CancellationToken.None); + Assert.Equal(0, response.NumberOfEmptyEntitiesRemoved); Assert.Equal(1, response.NumberOfOrphanedLocksRemoved); - Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); // wait for orchestration B to complete, now that the lock has been released status = await clientB.WaitForCompletionAsync(this.output); Assert.True(status.RuntimeStatus == OrchestrationRuntimeStatus.Completed); + // delete empty entities + response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: true, releaseOrphanedLocks: false, CancellationToken.None); + Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); + Assert.Equal(0, response.NumberOfOrphanedLocksRemoved); + // check that the empty entity record has been removed from storage result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); Assert.DoesNotContain(result.Entities, s => s.EntityId.Equals(emptyEntityId)); From b5b4720d47b21951d41561272468204d9c86791a Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 8 Jul 2024 15:22:22 -0700 Subject: [PATCH 2/5] reorder steps --- test/Common/DurableTaskEndToEndTests.cs | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 364319377..22bb632c2 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -5065,6 +5065,8 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) var orchestrationA = $"{prefix}-A"; var orchestrationB = $"{prefix}-B"; + // PART 1: Test removal of empty entities + // create an empty entity var client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.CreateEmptyEntities), new EntityId[] { emptyEntityId }, this.output); var status = await client.WaitForCompletionAsync(this.output); @@ -5084,6 +5086,17 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) var result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); Assert.Contains(result.Entities, s => s.EntityId.Equals(emptyEntityId)); + // test removal of empty entity + var response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: true, releaseOrphanedLocks: false, CancellationToken.None); + Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); + Assert.Equal(0, response.NumberOfOrphanedLocksRemoved); + + // check that the empty entity record has been removed from storage + result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); + Assert.DoesNotContain(result.Entities, s => s.EntityId.Equals(emptyEntityId)); + + // PART 2: Test recovery from orphaned locks + // run an orchestration A that leaves an orphaned lock TestDurableClient clientA = await host.StartOrchestratorAsync(nameof(TestOrchestrations.LockThenFailReplay), (orphanedEntityId, true), this.output, orchestrationA); status = await clientA.WaitForCompletionAsync(this.output); @@ -5092,8 +5105,8 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) TestDurableClient clientB = await host.StartOrchestratorAsync(nameof(TestOrchestrations.LockThenFailReplay), (orphanedEntityId, false), this.output, orchestrationB); // remove release orphaned lock to unblock orchestration B - // Note: do NOT remove empty entities ye: we want to keep the empty entity so it can unblock orchestration B - var response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: false, releaseOrphanedLocks: true, CancellationToken.None); + // Note: do NOT remove empty entities yet: we want to keep the empty entity so it can unblock orchestration B + response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: false, releaseOrphanedLocks: true, CancellationToken.None); Assert.Equal(0, response.NumberOfEmptyEntitiesRemoved); Assert.Equal(1, response.NumberOfOrphanedLocksRemoved); @@ -5101,17 +5114,8 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) status = await clientB.WaitForCompletionAsync(this.output); Assert.True(status.RuntimeStatus == OrchestrationRuntimeStatus.Completed); - // delete empty entities - response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: true, releaseOrphanedLocks: false, CancellationToken.None); - Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); - Assert.Equal(0, response.NumberOfOrphanedLocksRemoved); - - // check that the empty entity record has been removed from storage - result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); - Assert.DoesNotContain(result.Entities, s => s.EntityId.Equals(emptyEntityId)); - // clean again to remove the orphaned entity which is now empty also - response = await client.InnerClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: true, releaseOrphanedLocks: true, CancellationToken.None); Assert.Equal(0, response.NumberOfOrphanedLocksRemoved); Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); From e7e833f1949b6446e7462d83dd7a70afe27f14bd Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 8 Jul 2024 16:04:53 -0700 Subject: [PATCH 3/5] add delay --- test/Common/DurableTaskEndToEndTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 22bb632c2..fe734a935 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -5104,6 +5104,8 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) // run an orchestration B that queues behind A for the lock (and thus gets stuck) TestDurableClient clientB = await host.StartOrchestratorAsync(nameof(TestOrchestrations.LockThenFailReplay), (orphanedEntityId, false), this.output, orchestrationB); + await Task.Delay(TimeSpan.FromMinutes(1)); // wait for a stable entity executionID, needed until https://github.com/Azure/durabletask/pull/1128 is merged + // remove release orphaned lock to unblock orchestration B // Note: do NOT remove empty entities yet: we want to keep the empty entity so it can unblock orchestration B response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: false, releaseOrphanedLocks: true, CancellationToken.None); From b6c17b8eeec5e4c619ad1b3bd2adc3ed61f0bd37 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 8 Jul 2024 16:28:46 -0700 Subject: [PATCH 4/5] make test slightly longer --- test/Common/HttpApiHandlerTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index 8556ae074..615d20874 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -411,14 +411,14 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_HTTP TaskHub = TestConstants.TaskHub, ConnectionName = TestConstants.ConnectionName, }, - TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(3)); stopwatch.Stop(); Assert.Equal(HttpStatusCode.OK, httpResponseMessage.StatusCode); var content = await httpResponseMessage.Content.ReadAsStringAsync(); var value = JsonConvert.DeserializeObject(content); Assert.Equal("Hello Tokyo!", value); - Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(10)); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(15)); } [Fact] From 9166608efe2f234f8820e24bd30c418f3fa9fcc6 Mon Sep 17 00:00:00 2001 From: David Justo Date: Tue, 9 Jul 2024 10:26:37 -0700 Subject: [PATCH 5/5] incorporate feedback --- test/Common/DurableTaskEndToEndTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index fe734a935..489379cdd 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -5109,8 +5109,8 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) // remove release orphaned lock to unblock orchestration B // Note: do NOT remove empty entities yet: we want to keep the empty entity so it can unblock orchestration B response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: false, releaseOrphanedLocks: true, CancellationToken.None); - Assert.Equal(0, response.NumberOfEmptyEntitiesRemoved); Assert.Equal(1, response.NumberOfOrphanedLocksRemoved); + Assert.Equal(0, response.NumberOfEmptyEntitiesRemoved); // wait for orchestration B to complete, now that the lock has been released status = await clientB.WaitForCompletionAsync(this.output);