From e167c7c1dc88cfe838e9d36db1f74f76f90c2a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 19 May 2024 12:06:43 +0200 Subject: [PATCH 1/5] Add initial benchmarks --- .github/workflows/build.yml | 22 ++++++++++ .github/workflows/ci.yml | 22 ++++++++++ Directory.Packages.props | 2 + Testably.Abstractions.sln | 10 +++++ .../ConstructorBenchmarks.cs | 44 +++++++++++++++++++ .../CreateDirectoryBenchmarks.cs | 25 +++++++++++ .../CreateFileBenchmarks.cs | 26 +++++++++++ .../Program.cs | 11 +++++ .../Testably.Abstractions.Benchmarks.csproj | 17 +++++++ 9 files changed, 179 insertions(+) create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/Program.cs create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a15f74171..580ed8afb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -282,6 +282,28 @@ jobs: cd Tests ../../tools/dotnet-stryker -f ../.github/stryker/Stryker.Config.Compression.json -v "${GITHUB_REF#refs/heads/}" -r "Dashboard" -r "cleartext" + benchmark: + name: Run Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: 8.0.x + - name: Run benchmark + run: cd Tests/Benchmarks/Testably.Abstractions.Benchmarks && dotnet run -c Release --framework net8.0 --exporters json --filter '*' + - name: Store benchmark result + uses: rhysd/github-action-benchmark@v1 + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + alert-threshold: '150%' + comment-on-alert: false + fail-on-alert: false + deploy: name: Deploy if: startsWith(github.ref, 'refs/heads/release/') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97c5fb95b..d6c79aa9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,3 +295,25 @@ jobs: -H "Authorization: token $GITHUB_TOKEN" \ -d "$body" fi + + benchmark: + name: Run Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: 8.0.x + - name: Run benchmark + run: cd Tests/Benchmarks/Testably.Abstractions.Benchmarks && dotnet run -c Release --framework net8.0 --exporters json --filter '*' + - name: Store benchmark result + uses: rhysd/github-action-benchmark@v1 + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: false + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: true diff --git a/Directory.Packages.props b/Directory.Packages.props index 9fe602fca..b8b1aecd8 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,6 +7,7 @@ + @@ -32,6 +33,7 @@ + diff --git a/Testably.Abstractions.sln b/Testably.Abstractions.sln index 6a4d696e2..019ad34a7 100644 --- a/Testably.Abstractions.sln +++ b/Testably.Abstractions.sln @@ -100,6 +100,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Testably.Abstractions.TestS EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Settings", "Settings", "{D32170F4-1455-4839-8EF4-9530F3AA642A}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks", "{500E38FD-F587-4CC7-A7CE-10DA4AC76CE1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Testably.Abstractions.Benchmarks", "Tests\Benchmarks\Testably.Abstractions.Benchmarks\Testably.Abstractions.Benchmarks.csproj", "{459BB7E6-67C5-4C7B-99A8-09AD7D4CAE0C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -162,6 +166,10 @@ Global {241E6413-819C-4B06-A739-4D10C2A81365}.Debug|Any CPU.Build.0 = Debug|Any CPU {241E6413-819C-4B06-A739-4D10C2A81365}.Release|Any CPU.ActiveCfg = Release|Any CPU {241E6413-819C-4B06-A739-4D10C2A81365}.Release|Any CPU.Build.0 = Release|Any CPU + {459BB7E6-67C5-4C7B-99A8-09AD7D4CAE0C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {459BB7E6-67C5-4C7B-99A8-09AD7D4CAE0C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {459BB7E6-67C5-4C7B-99A8-09AD7D4CAE0C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {459BB7E6-67C5-4C7B-99A8-09AD7D4CAE0C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -186,6 +194,8 @@ Global {0AEBB1A6-07D6-46DC-BC78-9771D380EBC0} = {2FDB2AAE-E5CE-483E-A1A6-EC0B0A4AD67B} {241E6413-819C-4B06-A739-4D10C2A81365} = {D32170F4-1455-4839-8EF4-9530F3AA642A} {D32170F4-1455-4839-8EF4-9530F3AA642A} = {61F335A6-9CE0-4040-A34F-E70B1A55077D} + {500E38FD-F587-4CC7-A7CE-10DA4AC76CE1} = {61F335A6-9CE0-4040-A34F-E70B1A55077D} + {459BB7E6-67C5-4C7B-99A8-09AD7D4CAE0C} = {500E38FD-F587-4CC7-A7CE-10DA4AC76CE1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {EC4D8481-B9FD-41B5-832A-369210993DF4} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs new file mode 100644 index 000000000..0f857ad62 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs @@ -0,0 +1,44 @@ +using BenchmarkDotNet.Attributes; +using System.Collections.Generic; +using System.Globalization; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using Testably.Abstractions.Testing; +using Testably.Abstractions.Testing.Initializer; +using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; + +namespace Testably.Abstractions.Benchmarks; + +[RPlotExporter] +[MemoryDiagnoser] +public class ConstructorBenchmarks +{ + private readonly Dictionary _testData = CreateTestData(); + + public static Dictionary CreateTestData() + { + int filesCount = 1000; + int maxDirectoryDepth = 8; + return Enumerable.Range(0, filesCount) + .ToDictionary( + i => @$"C:\{string.Join(@"\", Enumerable.Range(0, (i % maxDirectoryDepth) + 1).Select(j => j.ToString(CultureInfo.InvariantCulture)))}\{i}.bin", + i => i.ToString(CultureInfo.InvariantCulture)); + } + + [Benchmark] + public IFileSystem TestableIO_1000() + => new System.IO.Abstractions.TestingHelpers.MockFileSystem( + _testData.ToDictionary(x => x.Key, x => new MockFileData(x.Value))); + + [Benchmark] + public IFileSystem TestableIO_Empty() + => new System.IO.Abstractions.TestingHelpers.MockFileSystem(); + + [Benchmark] + public IFileSystem Testably_1000() => new MockFileSystem().Initialize() + .With(_testData.Select(x => new FileDescription(x.Key, x.Value)).ToArray()).FileSystem; + + [Benchmark] + public IFileSystem Testably_Empty() => new MockFileSystem(); +} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs new file mode 100644 index 000000000..c6a9aaff1 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs @@ -0,0 +1,25 @@ +using BenchmarkDotNet.Attributes; +using System.IO.Abstractions; +using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; + +namespace Testably.Abstractions.Benchmarks; + +[RPlotExporter] +[MemoryDiagnoser] +public class CreateDirectoryBenchmarks +{ + private readonly string DirectoryPath = @"C:\l1\l2\l3\l4\l5\l6\l7\l8\l9\l10"; + + private readonly IFileSystem SystemIOFileSystem = + new System.IO.Abstractions.TestingHelpers.MockFileSystem(); + + private readonly IFileSystem TestablyFileSystem = new MockFileSystem(); + + [Benchmark] + public void TestableIO_CreateDirectory() + => SystemIOFileSystem.Directory.CreateDirectory(DirectoryPath); + + [Benchmark] + public void Testably_CreateDirectory() + => TestablyFileSystem.Directory.CreateDirectory(DirectoryPath); +} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs new file mode 100644 index 000000000..c6b422376 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs @@ -0,0 +1,26 @@ +using BenchmarkDotNet.Attributes; +using System.IO.Abstractions; +using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; + +namespace Testably.Abstractions.Benchmarks; + +[RPlotExporter] +[MemoryDiagnoser] +public class CreateFileBenchmarks +{ + private readonly string FileContent = "some file content"; + private readonly string FileName = "filename.txt"; + + private readonly IFileSystem SystemIOFileSystem = + new System.IO.Abstractions.TestingHelpers.MockFileSystem(); + + private readonly IFileSystem TestablyFileSystem = new MockFileSystem(); + + [Benchmark] + public void TestableIO_CreateDirectory() + => SystemIOFileSystem.File.WriteAllText(FileName, FileContent); + + [Benchmark] + public void Testably_CreateDirectory() + => TestablyFileSystem.File.WriteAllText(FileName, FileContent); +} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Program.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Program.cs new file mode 100644 index 000000000..94780f0c8 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Program.cs @@ -0,0 +1,11 @@ +using BenchmarkDotNet.Running; + +namespace Testably.Abstractions.Benchmarks; + +internal static class Program +{ + public static void Main(string[] args) + { + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + } +} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj new file mode 100644 index 000000000..22d29ae10 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj @@ -0,0 +1,17 @@ + + + false + false + Exe + false + ..\..\..\Build\Tests\$(MSBuildProjectName) + net8.0 + + + + + + + + + From 87bd2e6b6d6896004bac394bc7dfd33445028db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 19 May 2024 12:25:20 +0200 Subject: [PATCH 2/5] Adapt build pipeline --- .github/workflows/build.yml | 19 +++++++--- .github/workflows/ci.yml | 19 +++++++--- .../ConstructorBenchmarks.cs | 33 ++++++++--------- .../CreateDirectoryBenchmarks.cs | 25 ------------- .../CreateFileBenchmarks.cs | 26 -------------- .../UsageBenchmarks.cs | 35 +++++++++++++++++++ 6 files changed, 82 insertions(+), 75 deletions(-) delete mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs delete mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 580ed8afb..43199d9f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -283,21 +283,32 @@ jobs: ../../tools/dotnet-stryker -f ../.github/stryker/Stryker.Config.Compression.json -v "${GITHUB_REF#refs/heads/}" -r "Dashboard" -r "cleartext" benchmark: - name: Run Benchmarks + name: Benchmarks runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v3 with: dotnet-version: 8.0.x - - name: Run benchmark + - name: Run benchmark tests run: cd Tests/Benchmarks/Testably.Abstractions.Benchmarks && dotnet run -c Release --framework net8.0 --exporters json --filter '*' - - name: Store benchmark result + - name: Store ConstructorBenchmarks results uses: rhysd/github-action-benchmark@v1 with: name: Benchmark.Net Benchmark tool: 'benchmarkdotnet' - output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks-report-full-compressed.json + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + alert-threshold: '150%' + comment-on-alert: false + fail-on-alert: false + - name: Store UsageBenchmarks results + uses: rhysd/github-action-benchmark@v1 + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true alert-threshold: '150%' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6c79aa9e..2a52b4d18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -297,21 +297,32 @@ jobs: fi benchmark: - name: Run Benchmarks + name: Benchmarks runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v3 with: dotnet-version: 8.0.x - - name: Run benchmark + - name: Run benchmark tests run: cd Tests/Benchmarks/Testably.Abstractions.Benchmarks && dotnet run -c Release --framework net8.0 --exporters json --filter '*' - - name: Store benchmark result + - name: Store ConstructorBenchmarks results uses: rhysd/github-action-benchmark@v1 with: name: Benchmark.Net Benchmark tool: 'benchmarkdotnet' - output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks-report-full-compressed.json + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: false + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: true + - name: Store UsageBenchmarks results + uses: rhysd/github-action-benchmark@v1 + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: false alert-threshold: '150%' diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs index 0f857ad62..ac1b3fdfb 100644 --- a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs @@ -16,6 +16,23 @@ public class ConstructorBenchmarks { private readonly Dictionary _testData = CreateTestData(); + [Benchmark] + public IFileSystem ConstructorEmpty_TestableIO() + => new System.IO.Abstractions.TestingHelpers.MockFileSystem(); + + [Benchmark] + public IFileSystem ConstructorEmpty_Testably() => new MockFileSystem(); + + [Benchmark] + public IFileSystem ConstructorWithInitialization_1000_TestableIO() + => new System.IO.Abstractions.TestingHelpers.MockFileSystem( + _testData.ToDictionary(x => x.Key, x => new MockFileData(x.Value))); + + [Benchmark] + public IFileSystem ConstructorWithInitialization_1000_Testably() => new MockFileSystem() + .Initialize() + .With(_testData.Select(x => new FileDescription(x.Key, x.Value)).ToArray()).FileSystem; + public static Dictionary CreateTestData() { int filesCount = 1000; @@ -25,20 +42,4 @@ public static Dictionary CreateTestData() i => @$"C:\{string.Join(@"\", Enumerable.Range(0, (i % maxDirectoryDepth) + 1).Select(j => j.ToString(CultureInfo.InvariantCulture)))}\{i}.bin", i => i.ToString(CultureInfo.InvariantCulture)); } - - [Benchmark] - public IFileSystem TestableIO_1000() - => new System.IO.Abstractions.TestingHelpers.MockFileSystem( - _testData.ToDictionary(x => x.Key, x => new MockFileData(x.Value))); - - [Benchmark] - public IFileSystem TestableIO_Empty() - => new System.IO.Abstractions.TestingHelpers.MockFileSystem(); - - [Benchmark] - public IFileSystem Testably_1000() => new MockFileSystem().Initialize() - .With(_testData.Select(x => new FileDescription(x.Key, x.Value)).ToArray()).FileSystem; - - [Benchmark] - public IFileSystem Testably_Empty() => new MockFileSystem(); } diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs deleted file mode 100644 index c6a9aaff1..000000000 --- a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateDirectoryBenchmarks.cs +++ /dev/null @@ -1,25 +0,0 @@ -using BenchmarkDotNet.Attributes; -using System.IO.Abstractions; -using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; - -namespace Testably.Abstractions.Benchmarks; - -[RPlotExporter] -[MemoryDiagnoser] -public class CreateDirectoryBenchmarks -{ - private readonly string DirectoryPath = @"C:\l1\l2\l3\l4\l5\l6\l7\l8\l9\l10"; - - private readonly IFileSystem SystemIOFileSystem = - new System.IO.Abstractions.TestingHelpers.MockFileSystem(); - - private readonly IFileSystem TestablyFileSystem = new MockFileSystem(); - - [Benchmark] - public void TestableIO_CreateDirectory() - => SystemIOFileSystem.Directory.CreateDirectory(DirectoryPath); - - [Benchmark] - public void Testably_CreateDirectory() - => TestablyFileSystem.Directory.CreateDirectory(DirectoryPath); -} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs deleted file mode 100644 index c6b422376..000000000 --- a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/CreateFileBenchmarks.cs +++ /dev/null @@ -1,26 +0,0 @@ -using BenchmarkDotNet.Attributes; -using System.IO.Abstractions; -using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; - -namespace Testably.Abstractions.Benchmarks; - -[RPlotExporter] -[MemoryDiagnoser] -public class CreateFileBenchmarks -{ - private readonly string FileContent = "some file content"; - private readonly string FileName = "filename.txt"; - - private readonly IFileSystem SystemIOFileSystem = - new System.IO.Abstractions.TestingHelpers.MockFileSystem(); - - private readonly IFileSystem TestablyFileSystem = new MockFileSystem(); - - [Benchmark] - public void TestableIO_CreateDirectory() - => SystemIOFileSystem.File.WriteAllText(FileName, FileContent); - - [Benchmark] - public void Testably_CreateDirectory() - => TestablyFileSystem.File.WriteAllText(FileName, FileContent); -} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs new file mode 100644 index 000000000..1051b2b52 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs @@ -0,0 +1,35 @@ +using BenchmarkDotNet.Attributes; +using System.IO.Abstractions; +using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; + +namespace Testably.Abstractions.Benchmarks; + +[RPlotExporter] +[MemoryDiagnoser] +public class UsageBenchmarks +{ + private readonly string _directoryPath = @"C:\l1\l2\l3\l4\l5\l6\l7\l8\l9\l10"; + private readonly string _fileContent = "some file content"; + private readonly string _fileName = "filename.txt"; + + private readonly IFileSystem _systemIoFileSystem = + new System.IO.Abstractions.TestingHelpers.MockFileSystem(); + + private readonly IFileSystem _testablyFileSystem = new MockFileSystem(); + + [Benchmark] + public void CreateDirectory_TestableIO() + => _systemIoFileSystem.Directory.CreateDirectory(_directoryPath); + + [Benchmark] + public void CreateDirectory_Testably() + => _testablyFileSystem.Directory.CreateDirectory(_directoryPath); + + [Benchmark] + public void WriteFile_TestableIO() + => _systemIoFileSystem.File.WriteAllText(_fileName, _fileContent); + + [Benchmark] + public void WriteFile_Testably() + => _testablyFileSystem.File.WriteAllText(_fileName, _fileContent); +} From c4207e3e51ee56ac63a24ecf27226dddb7495f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 19 May 2024 12:42:42 +0200 Subject: [PATCH 3/5] Set `benchmark-data-dir-path` --- .github/workflows/build.yml | 6 ++++-- .github/workflows/ci.yml | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43199d9f9..1279a5f73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -295,10 +295,11 @@ jobs: - name: Store ConstructorBenchmarks results uses: rhysd/github-action-benchmark@v1 with: - name: Benchmark.Net Benchmark + name: Constructor Benchmarks tool: 'benchmarkdotnet' output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} + benchmark-data-dir-path: '/docs/benchmarks' auto-push: true alert-threshold: '150%' comment-on-alert: false @@ -306,10 +307,11 @@ jobs: - name: Store UsageBenchmarks results uses: rhysd/github-action-benchmark@v1 with: - name: Benchmark.Net Benchmark + name: Usage Benchmarks tool: 'benchmarkdotnet' output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} + benchmark-data-dir-path: '/docs/benchmarks' auto-push: true alert-threshold: '150%' comment-on-alert: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a52b4d18..b28a37afa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,7 +309,7 @@ jobs: - name: Store ConstructorBenchmarks results uses: rhysd/github-action-benchmark@v1 with: - name: Benchmark.Net Benchmark + name: Constructor Benchmarks tool: 'benchmarkdotnet' output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} @@ -320,7 +320,7 @@ jobs: - name: Store UsageBenchmarks results uses: rhysd/github-action-benchmark@v1 with: - name: Benchmark.Net Benchmark + name: Usage Benchmarks tool: 'benchmarkdotnet' output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} From bb162759c375b1b87b95a139428ebfa4b0b2cbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 19 May 2024 12:59:26 +0200 Subject: [PATCH 4/5] Test1 --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b28a37afa..889834cfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,22 +309,24 @@ jobs: - name: Store ConstructorBenchmarks results uses: rhysd/github-action-benchmark@v1 with: - name: Constructor Benchmarks + name: Benchmarks tool: 'benchmarkdotnet' output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: false alert-threshold: '150%' + summary-always: true comment-on-alert: true fail-on-alert: true - name: Store UsageBenchmarks results uses: rhysd/github-action-benchmark@v1 with: - name: Usage Benchmarks + name: Benchmarks tool: 'benchmarkdotnet' output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: false alert-threshold: '150%' + summary-always: true comment-on-alert: true fail-on-alert: true From f6602a69084bcc9aaf76e74aed90290f6f91b049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 19 May 2024 17:36:31 +0200 Subject: [PATCH 5/5] Merge tests into one benchmark --- .github/workflows/build.yml | 18 ++-------- .github/workflows/ci.yml | 16 ++------- ...enchmarks.cs => Benchmarks.Constructor.cs} | 16 ++------- .../Benchmarks.Usage.cs | 22 ++++++++++++ .../Benchmarks.cs | 9 +++++ .../Testably.Abstractions.Benchmarks.csproj | 2 +- .../UsageBenchmarks.cs | 35 ------------------- 7 files changed, 39 insertions(+), 79 deletions(-) rename Tests/Benchmarks/Testably.Abstractions.Benchmarks/{ConstructorBenchmarks.cs => Benchmarks.Constructor.cs} (66%) create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Usage.cs create mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.cs delete mode 100644 Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1279a5f73..466c2a9b5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -292,24 +292,12 @@ jobs: dotnet-version: 8.0.x - name: Run benchmark tests run: cd Tests/Benchmarks/Testably.Abstractions.Benchmarks && dotnet run -c Release --framework net8.0 --exporters json --filter '*' - - name: Store ConstructorBenchmarks results + - name: Store benchmark results uses: rhysd/github-action-benchmark@v1 with: - name: Constructor Benchmarks + name: Benchmarks tool: 'benchmarkdotnet' - output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - benchmark-data-dir-path: '/docs/benchmarks' - auto-push: true - alert-threshold: '150%' - comment-on-alert: false - fail-on-alert: false - - name: Store UsageBenchmarks results - uses: rhysd/github-action-benchmark@v1 - with: - name: Usage Benchmarks - tool: 'benchmarkdotnet' - output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.Benchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} benchmark-data-dir-path: '/docs/benchmarks' auto-push: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 889834cfd..f02ae397b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,24 +306,12 @@ jobs: dotnet-version: 8.0.x - name: Run benchmark tests run: cd Tests/Benchmarks/Testably.Abstractions.Benchmarks && dotnet run -c Release --framework net8.0 --exporters json --filter '*' - - name: Store ConstructorBenchmarks results + - name: Store benchmark results uses: rhysd/github-action-benchmark@v1 with: name: Benchmarks tool: 'benchmarkdotnet' - output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.ConstructorBenchmarks-report-full-compressed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: false - alert-threshold: '150%' - summary-always: true - comment-on-alert: true - fail-on-alert: true - - name: Store UsageBenchmarks results - uses: rhysd/github-action-benchmark@v1 - with: - name: Benchmarks - tool: 'benchmarkdotnet' - output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.UsageBenchmarks-report-full-compressed.json + output-file-path: Tests/Benchmarks/Testably.Abstractions.Benchmarks/BenchmarkDotNet.Artifacts/results/Testably.Abstractions.Benchmarks.Benchmarks-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: false alert-threshold: '150%' diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Constructor.cs similarity index 66% rename from Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs rename to Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Constructor.cs index ac1b3fdfb..181cd6ec2 100644 --- a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/ConstructorBenchmarks.cs +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Constructor.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Globalization; using System.IO.Abstractions; -using System.IO.Abstractions.TestingHelpers; using System.Linq; using Testably.Abstractions.Testing; using Testably.Abstractions.Testing.Initializer; @@ -10,30 +9,19 @@ namespace Testably.Abstractions.Benchmarks; -[RPlotExporter] -[MemoryDiagnoser] -public class ConstructorBenchmarks +public partial class Benchmarks { private readonly Dictionary _testData = CreateTestData(); - [Benchmark] - public IFileSystem ConstructorEmpty_TestableIO() - => new System.IO.Abstractions.TestingHelpers.MockFileSystem(); - [Benchmark] public IFileSystem ConstructorEmpty_Testably() => new MockFileSystem(); - [Benchmark] - public IFileSystem ConstructorWithInitialization_1000_TestableIO() - => new System.IO.Abstractions.TestingHelpers.MockFileSystem( - _testData.ToDictionary(x => x.Key, x => new MockFileData(x.Value))); - [Benchmark] public IFileSystem ConstructorWithInitialization_1000_Testably() => new MockFileSystem() .Initialize() .With(_testData.Select(x => new FileDescription(x.Key, x.Value)).ToArray()).FileSystem; - public static Dictionary CreateTestData() + private static Dictionary CreateTestData() { int filesCount = 1000; int maxDirectoryDepth = 8; diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Usage.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Usage.cs new file mode 100644 index 000000000..182105097 --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.Usage.cs @@ -0,0 +1,22 @@ +using BenchmarkDotNet.Attributes; +using System.IO.Abstractions; +using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; + +namespace Testably.Abstractions.Benchmarks; + +public partial class Benchmarks +{ + private readonly string _directoryPath = @"C:\l1\l2\l3\l4\l5\l6\l7\l8\l9\l10"; + private readonly string _fileContent = "some file content"; + private readonly string _fileName = "filename.txt"; + + private readonly IFileSystem _fileSystem = new MockFileSystem(); + + [Benchmark] + public void CreateDirectory_Testably() + => _fileSystem.Directory.CreateDirectory(_directoryPath); + + [Benchmark] + public void WriteFile_Testably() + => _fileSystem.File.WriteAllText(_fileName, _fileContent); +} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.cs new file mode 100644 index 000000000..7c63df51c --- /dev/null +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Benchmarks.cs @@ -0,0 +1,9 @@ +using BenchmarkDotNet.Attributes; + +namespace Testably.Abstractions.Benchmarks; + +[RPlotExporter] +[MemoryDiagnoser] +public partial class Benchmarks +{ +} diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj index 22d29ae10..8daef84b5 100644 --- a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj +++ b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/Testably.Abstractions.Benchmarks.csproj @@ -6,10 +6,10 @@ false ..\..\..\Build\Tests\$(MSBuildProjectName) net8.0 + $(NoWarn);MA0049 - diff --git a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs b/Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs deleted file mode 100644 index 1051b2b52..000000000 --- a/Tests/Benchmarks/Testably.Abstractions.Benchmarks/UsageBenchmarks.cs +++ /dev/null @@ -1,35 +0,0 @@ -using BenchmarkDotNet.Attributes; -using System.IO.Abstractions; -using MockFileSystem = Testably.Abstractions.Testing.MockFileSystem; - -namespace Testably.Abstractions.Benchmarks; - -[RPlotExporter] -[MemoryDiagnoser] -public class UsageBenchmarks -{ - private readonly string _directoryPath = @"C:\l1\l2\l3\l4\l5\l6\l7\l8\l9\l10"; - private readonly string _fileContent = "some file content"; - private readonly string _fileName = "filename.txt"; - - private readonly IFileSystem _systemIoFileSystem = - new System.IO.Abstractions.TestingHelpers.MockFileSystem(); - - private readonly IFileSystem _testablyFileSystem = new MockFileSystem(); - - [Benchmark] - public void CreateDirectory_TestableIO() - => _systemIoFileSystem.Directory.CreateDirectory(_directoryPath); - - [Benchmark] - public void CreateDirectory_Testably() - => _testablyFileSystem.Directory.CreateDirectory(_directoryPath); - - [Benchmark] - public void WriteFile_TestableIO() - => _systemIoFileSystem.File.WriteAllText(_fileName, _fileContent); - - [Benchmark] - public void WriteFile_Testably() - => _testablyFileSystem.File.WriteAllText(_fileName, _fileContent); -}