From 424fdfe4c6107939be4731cd9dd9ce2162c5d59e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 27 Sep 2024 10:04:00 +0700 Subject: [PATCH] implement project reset cleanup (#1070) --- .../Controllers/TestingController.cs | 9 +++ .../LexBoxApi/Jobs/CleanupResetBackupJob.cs | 3 +- backend/LexBoxApi/Services/HgService.cs | 65 ++++++++++++++++++- backend/LexCore/Config/HgConfig.cs | 1 + .../LexCore/ServiceInterfaces/IHgService.cs | 2 + backend/LexCore/Utils/FileUtils.cs | 15 ++++- .../Services/CleanupResetProjectsTests.cs | 53 +++++++++++++++ 7 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 backend/Testing/Services/CleanupResetProjectsTests.cs diff --git a/backend/LexBoxApi/Controllers/TestingController.cs b/backend/LexBoxApi/Controllers/TestingController.cs index bccf075be..93d91bfa5 100644 --- a/backend/LexBoxApi/Controllers/TestingController.cs +++ b/backend/LexBoxApi/Controllers/TestingController.cs @@ -3,6 +3,7 @@ using LexBoxApi.Services; using LexCore.Auth; using LexCore.Exceptions; +using LexCore.ServiceInterfaces; using LexData; using LexData.Entities; using Microsoft.AspNetCore.Authorization; @@ -16,6 +17,7 @@ namespace LexBoxApi.Controllers; public class TestingController( LexAuthService lexAuthService, LexBoxDbContext lexBoxDbContext, + IHgService hgService, SeedingData seedingData) : ControllerBase { @@ -84,4 +86,11 @@ public ActionResult ThrowsException() [AllowAnonymous] [ProducesResponseType(StatusCodes.Status500InternalServerError)] public ActionResult Test500NoError() => StatusCode(500); + + [HttpGet("test-cleanup-reset-backups")] + [AdminRequired] + public async Task TestCleanupResetBackups(bool dryRun = true) + { + return await hgService.CleanupResetBackups(dryRun); + } } diff --git a/backend/LexBoxApi/Jobs/CleanupResetBackupJob.cs b/backend/LexBoxApi/Jobs/CleanupResetBackupJob.cs index 3244c3084..93e3968be 100644 --- a/backend/LexBoxApi/Jobs/CleanupResetBackupJob.cs +++ b/backend/LexBoxApi/Jobs/CleanupResetBackupJob.cs @@ -12,8 +12,7 @@ protected override async Task ExecuteJob(IJobExecutionContext context) { logger.LogInformation("Starting cleanup reset backup job"); - //todo implement job - await Task.Delay(TimeSpan.FromSeconds(1)); + await hgService.CleanupResetBackups(); logger.LogInformation("Finished cleanup reset backup job"); } diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index f41027e18..b3d3f956c 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -18,7 +18,7 @@ namespace LexBoxApi.Services; -public class HgService : IHgService, IHostedService +public partial class HgService : IHgService, IHostedService { private const string DELETED_REPO_FOLDER = ProjectCode.DELETED_REPO_FOLDER; private const string TEMP_REPO_FOLDER = ProjectCode.TEMP_REPO_FOLDER; @@ -99,13 +99,18 @@ public async Task ResetRepo(ProjectCode code) { var tmpRepo = new DirectoryInfo(GetTempRepoPath(code, "reset")); InitRepoAt(tmpRepo); - await SoftDeleteRepo(code, $"{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}__reset"); + await SoftDeleteRepo(code, ResetSoftDeleteSuffix(DateTimeOffset.UtcNow)); //we must init the repo as uploading a zip is optional tmpRepo.MoveTo(PrefixRepoFilePath(code)); await InvalidateDirCache(code); await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } + public static string ResetSoftDeleteSuffix(DateTimeOffset resetAt) + { + return $"{FileUtils.ToTimestamp(resetAt)}__reset"; + } + public async Task FinishReset(ProjectCode code, Stream zipFile) { var tempRepoPath = GetTempRepoPath(code, "upload"); @@ -144,6 +149,55 @@ await Task.Run(() => await WaitForRepoEmptyState(code, expectedState); } + + public async Task CleanupResetBackups(bool dryRun = false) + { + using var activity = LexBoxActivitySource.Get().StartActivity(); + List deletedRepos = []; + int deletedCount = 0; + int totalCount = 0; + foreach (var deletedRepo in Directory.EnumerateDirectories(Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER))) + { + totalCount++; + var deletedRepoName = Path.GetFileName(deletedRepo); + var resetDate = GetResetDate(deletedRepoName); + if (resetDate is null) + { + continue; + } + var resetAge = DateTimeOffset.UtcNow - resetDate.Value; + //enforce a minimum age threshold, just in case the threshold is set too low + var ageThreshold = TimeSpan.FromDays(Math.Max(_options.Value.ResetCleanupAgeDays, 5)); + if (resetAge <= ageThreshold) continue; + try + { + if (!dryRun) + await Task.Run(() => Directory.Delete(deletedRepo, true)); + deletedRepos.Add(deletedRepoName); + deletedCount++; + } + catch (Exception e) + { + activity?.AddTag("app.hg.cleanupresetbackups.error", e.Message); + } + } + activity?.AddTag("app.hg.cleanupresetbackups", totalCount); + activity?.AddTag("app.hg.cleanupresetbackups.deleted", deletedCount); + activity?.AddTag("app.hg.cleanupresetbackups.dryrun", dryRun); + return deletedRepos.ToArray(); + } + + public static DateTimeOffset? GetResetDate(string repoName) + { + var match = ResetProjectsRegex().Match(repoName); + if (!match.Success) return null; + return FileUtils.ToDateTimeOffset(match.Groups[1].Value); + } + + [GeneratedRegex(@"__(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})__reset$")] + public static partial Regex ResetProjectsRegex(); + + /// /// deletes all files and folders in the repo folder except for .hg /// @@ -227,7 +281,7 @@ public Task RevertRepo(ProjectCode code, string revHash) public async Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix) { - var deletedRepoName = $"{code}__{deletedRepoSuffix}"; + var deletedRepoName = DeletedRepoName(code, deletedRepoSuffix); await Task.Run(() => { var deletedRepoPath = Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER); @@ -240,6 +294,11 @@ await Task.Run(() => }); } + public static string DeletedRepoName(ProjectCode code, string deletedRepoSuffix) + { + return $"{code}__{deletedRepoSuffix}"; + } + private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup | UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | diff --git a/backend/LexCore/Config/HgConfig.cs b/backend/LexCore/Config/HgConfig.cs index 843517769..bca0e1bf6 100644 --- a/backend/LexCore/Config/HgConfig.cs +++ b/backend/LexCore/Config/HgConfig.cs @@ -17,4 +17,5 @@ public class HgConfig public required string HgResumableUrl { get; init; } public bool AutoUpdateLexEntryCountOnSendReceive { get; init; } = false; public bool RequireContainerVersionMatch { get; init; } = true; + public int ResetCleanupAgeDays { get; init; } = 31; } diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index 21b5b574d..df2426bde 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -24,4 +24,6 @@ public interface IHgService Task InvalidateDirCache(ProjectCode code, CancellationToken token = default); bool HasAbandonedTransactions(ProjectCode projectCode); Task HgCommandHealth(); + + Task CleanupResetBackups(bool dryRun = false); } diff --git a/backend/LexCore/Utils/FileUtils.cs b/backend/LexCore/Utils/FileUtils.cs index 08226086e..8f097b302 100644 --- a/backend/LexCore/Utils/FileUtils.cs +++ b/backend/LexCore/Utils/FileUtils.cs @@ -5,11 +5,22 @@ namespace LexCore.Utils; public static class FileUtils { + private static readonly string TimestampPattern = DateTimeFormatInfo.InvariantInfo.SortableDateTimePattern.Replace(':', '-'); public static string ToTimestamp(DateTimeOffset dateTime) { - var timestamp = dateTime.ToString(DateTimeFormatInfo.InvariantInfo.SortableDateTimePattern); + var timestamp = dateTime.ToUniversalTime().ToString(TimestampPattern); // make it file-system friendly - return timestamp.Replace(':', '-'); + return timestamp; + } + + public static DateTimeOffset? ToDateTimeOffset(string timestamp) + { + if (DateTimeOffset.TryParseExact(timestamp, TimestampPattern, null, DateTimeStyles.AssumeUniversal, out var dateTime)) + { + return dateTime; + } + + return null; } public static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo target, UnixFileMode? permissions = null) diff --git a/backend/Testing/Services/CleanupResetProjectsTests.cs b/backend/Testing/Services/CleanupResetProjectsTests.cs new file mode 100644 index 000000000..6946903d5 --- /dev/null +++ b/backend/Testing/Services/CleanupResetProjectsTests.cs @@ -0,0 +1,53 @@ +using LexBoxApi.Services; +using LexCore.Utils; +using Shouldly; + +namespace Testing.Services; + +public class CleanupResetProjectsTests +{ + [Fact] + public void ResetRegexCanFindTimestampFromResetRepoName() + { + var date = DateTimeOffset.UtcNow; + var repoName = HgService.DeletedRepoName("test", HgService.ResetSoftDeleteSuffix(date)); + var match = HgService.ResetProjectsRegex().Match(repoName); + match.Success.ShouldBeTrue(); + match.Groups[1].Value.ShouldBe(FileUtils.ToTimestamp(date)); + } + + [Fact] + public void CanGetDateFromResetRepoName() + { + var expected = DateTimeOffset.Now; + var repoName = HgService.DeletedRepoName("test", HgService.ResetSoftDeleteSuffix(expected)); + var actual = HgService.GetResetDate(repoName); + actual.ShouldNotBeNull(); + TruncateToMinutes(actual.Value).ShouldBe(TruncateToMinutes(expected)); + } + + private DateTimeOffset TruncateToMinutes(DateTimeOffset date) + { + return new DateTimeOffset(date.Year, date.Month, date.Day, date.Hour, date.Minute, 0, date.Offset); + } + + [Theory] + [InlineData("grobish-test-flex__2023-11-29T16-52-38__reset", "2023-11-29T16-52-38")] + public void ResetRegexCanFindTimestamp(string repoName, string timestamp) + { + var match = HgService.ResetProjectsRegex().Match(repoName); + match.Success.ShouldBeTrue(); + match.Groups[1].Value.ShouldBe(timestamp); + } + + [Theory] + [InlineData("grobish-test-flex__2023-11-29T16-52-38")] + //even if the code has a pattern that would match the reset with timestamp it mush be at the end + [InlineData("code-with-bad-name-2023-11-29T16-52-38__reset__2023-11-29T16-52-38")] + public void ResetRegexDoesNotMatchNonResets(string repoName) + { + var match = HgService.ResetProjectsRegex().Match(repoName); + match.Success.ShouldBeFalse(); + } + +}