Skip to content

Commit

Permalink
Stop doing file IO directly in web server
Browse files Browse the repository at this point in the history
  • Loading branch information
NoahStolk committed Apr 5, 2023
1 parent 18b07fc commit 2470b1a
Show file tree
Hide file tree
Showing 28 changed files with 329 additions and 106 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,87 @@ public LeaderboardHistory GetLeaderboardHistoryByFilePath(string filePath)
}

#pragma warning disable SA1201
public string Root => throw new NotImplementedException();
public string GetLeaderboardHistoryPathFromDate(DateTime dateTime) => throw new NotImplementedException();
public string GetPath(DataSubDirectory subDirectory) => throw new NotImplementedException();
public string GetToolDistributionPath(string name, ToolPublishMethod publishMethod, ToolBuildType buildType, string version) => throw new NotImplementedException();
public bool DeleteFileIfExists(string path) => throw new NotImplementedException();
public bool FileExists(string path)
{
throw new NotImplementedException();
}
public bool DeleteDirectoryIfExists(string path, bool recursive) => throw new NotImplementedException();
public bool DirectoryExists(string path)
{
throw new NotImplementedException();
}
public void MoveDirectory(string sourcePath, string destinationPath)
{
throw new NotImplementedException();
}
public void CreateDirectory(string path)
{
throw new NotImplementedException();
}
public string[] GetFiles(string path)
{
throw new NotImplementedException();
}
public string[] GetFiles(string path, string searchPattern)
{
throw new NotImplementedException();
}
public byte[] ReadAllBytes(string path)
{
throw new NotImplementedException();
}
public Task<byte[]> ReadAllBytesAsync(string path)
{
throw new NotImplementedException();
}
public Task<byte[]> ReadAllBytesAsync(string path, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public void WriteAllBytes(string path, byte[] bytes)
{
throw new NotImplementedException();
}
public Task WriteAllBytesAsync(string path, byte[] bytes)
{
throw new NotImplementedException();
}
public Task WriteAllBytesAsync(string path, byte[] bytes, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public void MoveFile(string sourcePath, string destinationPath)
{
throw new NotImplementedException();
}
public string ReadAllText(string path)
{
throw new NotImplementedException();
}
public Task<string> ReadAllTextAsync(string path)
{
throw new NotImplementedException();
}
public Task<string> ReadAllTextAsync(string path, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public void WriteAllText(string path, string text)
{
throw new NotImplementedException();
}
public Task WriteAllTextAsync(string path, string text)
{
throw new NotImplementedException();
}
public Task WriteAllTextAsync(string path, string text, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public int GetCount() => throw new NotImplementedException();
public void Clear() => throw new NotImplementedException();
#pragma warning restore SA1201
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ public CustomEntryProcessorTests()
fileSystemService.Setup(m => m.GetPath(DataSubDirectory.Spawnsets)).Returns(spawnsetsPath);
fileSystemService.Setup(m => m.GetPath(DataSubDirectory.CustomEntryReplays)).Returns(replaysPath);

Directory.CreateDirectory(replaysPath);

Mock<ILogger<SpawnsetHashCache>> spawnsetHashCacheLogger = new();
Mock<SpawnsetHashCache> spawnsetHashCache = new(fileSystemService.Object, spawnsetHashCacheLogger.Object);
Mock<ILogger<CustomEntryProcessor>> customEntryProcessorLogger = new();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,10 @@ protected ModArchiveProcessorTests()
string modsPath = Path.Combine(TestUtils.ResourcePath, "Mods");
string modArchiveCachePath = Path.Combine(TestUtils.ResourcePath, "ModArchiveCache");

if (Directory.Exists(modsPath))
Directory.Delete(modsPath, true);

if (Directory.Exists(modArchiveCachePath))
Directory.Delete(modArchiveCachePath, true);

Mock<IFileSystemService> fileSystemService = new();
fileSystemService.Setup(m => m.GetPath(DataSubDirectory.Mods)).Returns(modsPath);
fileSystemService.Setup(m => m.GetPath(DataSubDirectory.ModArchiveCache)).Returns(modArchiveCachePath);

Directory.CreateDirectory(modsPath);
Directory.CreateDirectory(modArchiveCachePath);

Cache = new(fileSystemService.Object);
Accessor = new(fileSystemService.Object, Cache);
Processor = new(fileSystemService.Object, Cache, Accessor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@ public async Task DeleteCustomEntryAsync(int id)
await _dbContext.SaveChangesAsync();

string path = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.CustomEntryReplays), $"{id}.ddreplay");
bool fileExists = File.Exists(path);
if (fileExists)
File.Delete(path);
_fileSystemService.DeleteFileIfExists(path);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -461,10 +461,10 @@ private void ValidateCustomLeaderboard(
throw new CustomLeaderboardValidationException($"Spawnset with ID '{spawnsetId}' does not exist.");

string spawnsetFilePath = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.Spawnsets), spawnset.Name);
if (!File.Exists(spawnsetFilePath))
if (!_fileSystemService.FileExists(spawnsetFilePath))
throw new InvalidOperationException($"Spawnset file '{spawnset.Name}' does not exist. Spawnset with ID '{spawnsetId}' does not have a file which should never happen.");

if (!SpawnsetBinary.TryParse(File.ReadAllBytes(spawnsetFilePath), out SpawnsetBinary? spawnsetBinary))
if (!SpawnsetBinary.TryParse(_fileSystemService.ReadAllBytes(spawnsetFilePath), out SpawnsetBinary? spawnsetBinary))
throw new InvalidOperationException($"Could not parse survival file '{spawnset.Name}'. Please review the file. Also review how this file ended up in the 'spawnsets' directory, as it should not be possible to upload non-survival files from the Admin API.");

GameMode requiredGameMode = category.RequiredGameModeForCategory();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public async Task AddSpawnsetAsync(AddSpawnset addSpawnset)

// Add file.
string path = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.Spawnsets), addSpawnset.Name);
await File.WriteAllBytesAsync(path, addSpawnset.FileContents);
await _fileSystemService.WriteAllBytesAsync(path, addSpawnset.FileContents);

// Add entity.
SpawnsetEntity spawnset = new()
Expand Down Expand Up @@ -80,7 +80,7 @@ public async Task EditSpawnsetAsync(int id, EditSpawnset editSpawnset)
string directory = _fileSystemService.GetPath(DataSubDirectory.Spawnsets);
string oldPath = Path.Combine(directory, spawnset.Name);
string newPath = Path.Combine(directory, editSpawnset.Name);
File.Move(oldPath, newPath);
_fileSystemService.MoveFile(oldPath, newPath);

_spawnsetHashCache.Clear();
}
Expand All @@ -104,12 +104,8 @@ public async Task DeleteSpawnsetAsync(int id)
throw new AdminDomainException("Spawnset with custom leaderboard cannot be deleted.");

string path = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.Spawnsets), spawnset.Name);
bool fileExists = File.Exists(path);
if (fileExists)
{
File.Delete(path);
if (_fileSystemService.DeleteFileIfExists(path))
_spawnsetHashCache.Clear();
}

_dbContext.Spawnsets.Remove(spawnset);
await _dbContext.SaveChangesAsync();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ public async Task AddDistribution(string name, ToolPublishMethod publishMethod,
throw new AdminDomainException("Distribution already exists.");

string path = _fileSystemService.GetToolDistributionPath(name, publishMethod, buildType, version);
if (File.Exists(path))
if (_fileSystemService.FileExists(path))
throw new AdminDomainException("File for distribution already exists, but does not exist in the database. Please review the database and the file system.");

await File.WriteAllBytesAsync(path, zipFileContents);
await _fileSystemService.WriteAllBytesAsync(path, zipFileContents);

if (updateVersion)
tool.CurrentVersionNumber = version;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,16 @@ public CustomEntryRepository(ApplicationDbContext dbContext, IFileSystemService
public byte[] GetCustomEntryReplayBufferById(int id)
{
string path = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.CustomEntryReplays), $"{id}.ddreplay");
if (!File.Exists(path))
if (!_fileSystemService.FileExists(path))
throw new NotFoundException($"Replay file with ID '{id}' could not be found.");

return File.ReadAllBytes(path);
return _fileSystemService.ReadAllBytes(path);
}

public (string FileName, byte[] Contents) GetCustomEntryReplayById(int id)
{
string path = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.CustomEntryReplays), $"{id}.ddreplay");
if (!File.Exists(path))
if (!_fileSystemService.FileExists(path))
throw new NotFoundException($"Replay file with ID '{id}' could not be found.");

// ! Navigation property.
Expand All @@ -48,13 +48,13 @@ public byte[] GetCustomEntryReplayBufferById(int id)
throw new NotFoundException($"Custom entry replay '{id}' could not be found.");

string fileName = $"{customEntry.SpawnsetId}-{customEntry.SpawnsetName}-{customEntry.PlayerId}-{customEntry.PlayerName}.ddreplay";
return (fileName, File.ReadAllBytes(path));
return (fileName, _fileSystemService.ReadAllBytes(path));
}

public List<int> GetExistingCustomEntryReplayIds(List<int> ids)
{
return ids
.Where(id => File.Exists(Path.Combine(_fileSystemService.GetPath(DataSubDirectory.CustomEntryReplays), $"{id}.ddreplay")))
.Where(id => _fileSystemService.FileExists(Path.Combine(_fileSystemService.GetPath(DataSubDirectory.CustomEntryReplays), $"{id}.ddreplay")))
.Select(id => id)
.ToList();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public ToolRepository(ApplicationDbContext dbContext, IFileSystemService fileSys

private async Task<string> GetJsonString(string name)
{
return await File.ReadAllTextAsync(Path.Combine(_fileSystemService.GetPath(DataSubDirectory.Tools), $"{name}.json"));
return await _fileSystemService.ReadAllTextAsync(Path.Combine(_fileSystemService.GetPath(DataSubDirectory.Tools), $"{name}.json"));
}

public async Task<Tool?> GetToolAsync(string name)
Expand Down Expand Up @@ -69,13 +69,13 @@ private async Task<string> GetJsonString(string name)
public byte[] GetToolDistributionFile(string name, ToolPublishMethod publishMethod, ToolBuildType buildType, string version)
{
string path = _fileSystemService.GetToolDistributionPath(name, publishMethod, buildType, version);
if (!File.Exists(path))
if (!_fileSystemService.FileExists(path))
{
_logger.LogError("Tool distribution file at '{path}' does not exist!", path);
throw new NotFoundException("Tool distribution file does not exist.");
}

return File.ReadAllBytes(path);
return _fileSystemService.ReadAllBytes(path);
}

public async Task<ToolDistribution?> GetLatestToolDistributionAsync(string name, ToolPublishMethod publishMethod, ToolBuildType buildType)
Expand Down Expand Up @@ -125,7 +125,7 @@ public async Task UpdateToolDistributionStatisticsAsync(string name, ToolPublish
private int GetToolDistributionFileSize(string name, ToolPublishMethod publishMethod, ToolBuildType buildType, string version)
{
string path = _fileSystemService.GetToolDistributionPath(name, publishMethod, buildType, version);
if (File.Exists(path))
if (_fileSystemService.FileExists(path))
return (int)new FileInfo(path).Length;

return 0;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using DevilDaggersInfo.Web.Server.Domain.Models.LeaderboardHistory;
using DevilDaggersInfo.Web.Server.Domain.Services.Inversion;
using System.Collections.Concurrent;

namespace DevilDaggersInfo.Web.Server.Domain.Services.Caching;
Expand All @@ -7,13 +8,20 @@ public class LeaderboardHistoryCache : ILeaderboardHistoryCache
{
private readonly ConcurrentDictionary<string, LeaderboardHistory> _cache = new();

private readonly IFileSystemService _fileSystemService;

public LeaderboardHistoryCache(IFileSystemService fileSystemService)
{
_fileSystemService = fileSystemService;
}

public LeaderboardHistory GetLeaderboardHistoryByFilePath(string filePath)
{
string name = Path.GetFileNameWithoutExtension(filePath);
if (_cache.TryGetValue(name, out LeaderboardHistory? value))
return value;

LeaderboardHistory lb = LeaderboardHistory.CreateFromFile(File.ReadAllBytes(filePath));
LeaderboardHistory lb = LeaderboardHistory.CreateFromFile(_fileSystemService.ReadAllBytes(filePath));
_cache.TryAdd(name, lb);
return lb;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public ModArchiveCache(IFileSystemService fileSystemService)
public ModArchiveCacheData GetArchiveDataByBytes(string name, byte[] bytes)
{
// Check memory cache.
if (_cache.ContainsKey(name))
return _cache[name];
if (_cache.TryGetValue(name, out ModArchiveCacheData? value))
return value;

// Check file cache.
ModArchiveCacheData? fileCache = LoadFromFileCache(name);
Expand All @@ -42,8 +42,8 @@ public ModArchiveCacheData GetArchiveDataByFilePath(string filePath)
{
// Check memory cache.
string name = Path.GetFileNameWithoutExtension(filePath);
if (_cache.ContainsKey(name))
return _cache[name];
if (_cache.TryGetValue(name, out ModArchiveCacheData? value))
return value;

// Check file cache.
ModArchiveCacheData? fileCache = LoadFromFileCache(name);
Expand All @@ -61,10 +61,10 @@ public ModArchiveCacheData GetArchiveDataByFilePath(string filePath)
private ModArchiveCacheData? LoadFromFileCache(string name)
{
string fileCachePath = Path.Combine(_fileSystemService.GetPath(DataSubDirectory.ModArchiveCache), $"{name}.json");
if (!File.Exists(fileCachePath))
if (!_fileSystemService.FileExists(fileCachePath))
return null;

ModArchiveCacheData? fileCacheArchiveData = JsonConvert.DeserializeObject<ModArchiveCacheData>(File.ReadAllText(fileCachePath));
ModArchiveCacheData? fileCacheArchiveData = JsonConvert.DeserializeObject<ModArchiveCacheData>(_fileSystemService.ReadAllText(fileCachePath));
if (fileCacheArchiveData == null)
return null;

Expand Down Expand Up @@ -114,17 +114,17 @@ private ModArchiveCacheData CreateModArchiveCacheDataFromStream(string name, Str
private void WriteToFileCache(string name, ModArchiveCacheData archiveData)
{
string fileCacheDirectory = _fileSystemService.GetPath(DataSubDirectory.ModArchiveCache);
Directory.CreateDirectory(fileCacheDirectory);
_fileSystemService.CreateDirectory(fileCacheDirectory);

File.WriteAllText(Path.Combine(fileCacheDirectory, $"{name}.json"), JsonConvert.SerializeObject(archiveData));
_fileSystemService.WriteAllText(Path.Combine(fileCacheDirectory, $"{name}.json"), JsonConvert.SerializeObject(archiveData));
}

public void LoadEntireFileCache()
{
string fileCacheDirectory = _fileSystemService.GetPath(DataSubDirectory.ModArchiveCache);
Directory.CreateDirectory(fileCacheDirectory);
_fileSystemService.CreateDirectory(fileCacheDirectory);

foreach (string path in Directory.GetFiles(fileCacheDirectory, "*.json"))
foreach (string path in _fileSystemService.GetFiles(fileCacheDirectory, "*.json"))
{
string name = Path.GetFileNameWithoutExtension(path);
LoadFromFileCache(name);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ public SpawnsetHashCache(IFileSystemService fileSystemService, ILogger<SpawnsetH
if (spawnsetCacheData != null)
return spawnsetCacheData;

foreach (string spawnsetPath in Directory.GetFiles(_fileSystemService.GetPath(DataSubDirectory.Spawnsets)))
foreach (string spawnsetPath in _fileSystemService.GetFiles(_fileSystemService.GetPath(DataSubDirectory.Spawnsets)))
{
byte[] spawnsetBytes = File.ReadAllBytes(spawnsetPath);
byte[] spawnsetBytes = _fileSystemService.ReadAllBytes(spawnsetPath);
if (!SpawnsetBinary.TryParse(spawnsetBytes, out _))
{
_logger.LogError("Could not parse file at `{path}` to a spawnset. Skipping file for cache.", spawnsetPath);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using DevilDaggersInfo.Core.Spawnset.Summary;
using DevilDaggersInfo.Web.Server.Domain.Services.Inversion;
using System.Collections.Concurrent;

namespace DevilDaggersInfo.Web.Server.Domain.Services.Caching;
Expand All @@ -7,13 +8,20 @@ public class SpawnsetSummaryCache
{
private readonly ConcurrentDictionary<string, SpawnsetSummary> _cache = new();

private readonly IFileSystemService _fileSystemService;

public SpawnsetSummaryCache(IFileSystemService fileSystemService)
{
_fileSystemService = fileSystemService;
}

public SpawnsetSummary GetSpawnsetSummaryByFilePath(string filePath)
{
string name = Path.GetFileNameWithoutExtension(filePath);
if (_cache.ContainsKey(name))
return _cache[name];
if (_cache.TryGetValue(name, out SpawnsetSummary? value))
return value;

if (!SpawnsetSummary.TryParse(File.ReadAllBytes(filePath), out SpawnsetSummary? spawnsetSummary))
if (!SpawnsetSummary.TryParse(_fileSystemService.ReadAllBytes(filePath), out SpawnsetSummary? spawnsetSummary))
throw new($"Failed to get spawnset summary from spawnset file: '{name}'.");

_cache.TryAdd(name, spawnsetSummary);
Expand Down
Loading

0 comments on commit 2470b1a

Please sign in to comment.