diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 0e58722..0293443 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -17,16 +17,16 @@ jobs: global-json-file: global.json - name: Install dependencies - run: dotnet restore ./NUnit.Middlewares.sln --verbosity minimal && dotnet tool restore + run: dotnet restore ./nunit-extensions.sln --verbosity minimal && dotnet tool restore - name: Build - run: dotnet build --configuration Release ./NUnit.Middlewares.sln + run: dotnet build --configuration Release ./nunit-extensions.sln - name: Check codestyle - run: dotnet jb cleanupcode NUnit.Middlewares.sln --profile=CatalogueCleanup --verbosity=WARN && git diff --exit-code + run: dotnet jb cleanupcode nunit-extensions.sln --profile=CatalogueCleanup --verbosity=WARN && git diff --exit-code - name: Run tests - run: dotnet test --no-build --configuration Release ./NUnit.Middlewares.Tests/NUnit.Middlewares.Tests.csproj + run: dotnet test --no-build --configuration Release ./NUnit.Extensions.Tests/NUnit.Extensions.Tests.csproj publish: runs-on: windows-2019 needs: test @@ -60,7 +60,7 @@ jobs: Write-Host "Will create $release for package $packageName ($version)" -ForegroundColor "Green" - echo "RELEASE_NOTE=https://github.com/skbkontur/nunit-middlewares/releases/tag/$tagName" >> $env:GITHUB_ENV + echo "RELEASE_NOTE=https://github.com/skbkontur/nunit-extensions/releases/tag/$tagName" >> $env:GITHUB_ENV echo "PACKAGE_NAME=$packageName" >> $env:GITHUB_ENV echo "VERSION=$version" >> $env:GITHUB_ENV echo "PRE=$pre" >> $env:GITHUB_ENV diff --git a/Directory.Build.props b/Directory.Build.props index f95d661..f16ca8d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,28 +1,28 @@ - - 9 - enable - 8618 - + + 9 + enable + 8618 + - - - full - true - + + + full + true + - - - git - https://github.com/skbkontur/nunit-middlewares - $(RepositoryUrl) - true - $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb - + + + git + https://github.com/skbkontur/nunit-extensions + $(RepositoryUrl) + true + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb + - - - + + + \ No newline at end of file diff --git a/NUnit.Middlewares.Tests/ParallelParametrizedTestFixtureTest.cs b/NUnit.Extensions.Tests/Middlewares/ParallelParametrizedTestFixtureTest.cs similarity index 93% rename from NUnit.Middlewares.Tests/ParallelParametrizedTestFixtureTest.cs rename to NUnit.Extensions.Tests/Middlewares/ParallelParametrizedTestFixtureTest.cs index 3329e0f..6e14759 100644 --- a/NUnit.Middlewares.Tests/ParallelParametrizedTestFixtureTest.cs +++ b/NUnit.Extensions.Tests/Middlewares/ParallelParametrizedTestFixtureTest.cs @@ -4,7 +4,9 @@ using NUnit.Framework; -namespace SkbKontur.NUnit.Middlewares.Tests +using SkbKontur.NUnit.Middlewares; + +namespace SkbKontur.NUnit.Extensions.Tests.Middlewares { [TestFixture(-1)] [TestFixture(-2)] diff --git a/NUnit.Middlewares.Tests/ParallelTestContextUsageTest.cs b/NUnit.Extensions.Tests/Middlewares/ParallelTestContextUsageTest.cs similarity index 95% rename from NUnit.Middlewares.Tests/ParallelTestContextUsageTest.cs rename to NUnit.Extensions.Tests/Middlewares/ParallelTestContextUsageTest.cs index 6258f06..33ada94 100644 --- a/NUnit.Middlewares.Tests/ParallelTestContextUsageTest.cs +++ b/NUnit.Extensions.Tests/Middlewares/ParallelTestContextUsageTest.cs @@ -6,7 +6,9 @@ using NUnit.Framework; using NUnit.Framework.Interfaces; -namespace SkbKontur.NUnit.Middlewares.Tests +using SkbKontur.NUnit.Middlewares; + +namespace SkbKontur.NUnit.Extensions.Tests.Middlewares { public class Counter { diff --git a/NUnit.Middlewares.Tests/ParallelTestFixtureSetUpTest.cs b/NUnit.Extensions.Tests/Middlewares/ParallelTestFixtureSetUpTest.cs similarity index 95% rename from NUnit.Middlewares.Tests/ParallelTestFixtureSetUpTest.cs rename to NUnit.Extensions.Tests/Middlewares/ParallelTestFixtureSetUpTest.cs index d941200..de4334f 100644 --- a/NUnit.Middlewares.Tests/ParallelTestFixtureSetUpTest.cs +++ b/NUnit.Extensions.Tests/Middlewares/ParallelTestFixtureSetUpTest.cs @@ -2,7 +2,9 @@ using NUnit.Framework; -namespace SkbKontur.NUnit.Middlewares.Tests +using SkbKontur.NUnit.Middlewares; + +namespace SkbKontur.NUnit.Extensions.Tests.Middlewares { [Parallelizable(ParallelScope.Self)] public class FirstTestFixtureSetUpTest : SimpleTestBase diff --git a/NUnit.Middlewares.Tests/TestWithRetriesHasItsOwnSetup.cs b/NUnit.Extensions.Tests/Middlewares/TestWithRetriesHasItsOwnSetup.cs similarity index 96% rename from NUnit.Middlewares.Tests/TestWithRetriesHasItsOwnSetup.cs rename to NUnit.Extensions.Tests/Middlewares/TestWithRetriesHasItsOwnSetup.cs index 8500daf..64f571f 100644 --- a/NUnit.Middlewares.Tests/TestWithRetriesHasItsOwnSetup.cs +++ b/NUnit.Extensions.Tests/Middlewares/TestWithRetriesHasItsOwnSetup.cs @@ -5,7 +5,9 @@ using NUnit.Framework; using NUnit.Framework.Internal; -namespace SkbKontur.NUnit.Middlewares.Tests +using SkbKontur.NUnit.Middlewares; + +namespace SkbKontur.NUnit.Extensions.Tests.Middlewares { public class DisposableCounter : IDisposable { diff --git a/NUnit.Middlewares.Tests/NUnit.Middlewares.Tests.csproj b/NUnit.Extensions.Tests/NUnit.Extensions.Tests.csproj similarity index 79% rename from NUnit.Middlewares.Tests/NUnit.Middlewares.Tests.csproj rename to NUnit.Extensions.Tests/NUnit.Extensions.Tests.csproj index da6960a..ee96a4f 100644 --- a/NUnit.Middlewares.Tests/NUnit.Middlewares.Tests.csproj +++ b/NUnit.Extensions.Tests/NUnit.Extensions.Tests.csproj @@ -2,8 +2,8 @@ net48;net8.0 - SkbKontur.NUnit.Middlewares.Tests - SkbKontur.NUnit.Middlewares.Tests + SkbKontur.NUnit.Extensions.Tests + SkbKontur.NUnit.Extensions.Tests false true @@ -20,6 +20,7 @@ + \ No newline at end of file diff --git a/NUnit.Extensions.Tests/Retries/FixtureWithRetries.cs b/NUnit.Extensions.Tests/Retries/FixtureWithRetries.cs new file mode 100644 index 0000000..88f6363 --- /dev/null +++ b/NUnit.Extensions.Tests/Retries/FixtureWithRetries.cs @@ -0,0 +1,39 @@ +using System; + +using NUnit.Framework; +using NUnit.Framework.Internal; + +using SkbKontur.NUnit.Retries; + +namespace SkbKontur.NUnit.Extensions.Tests.Retries +{ + [RetryOnError(3)] + public class FixtureWithRetries + { + [Test] + public void Test1() + { + if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0 or 1) + { + Assert.Fail("Third time's a Charm"); + } + } + + [Test] + [RetryOnError(4)] + public void Test2() + { + if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0 or 1 or 2) + { + throw new Exception("Forth time's a Charm, now with exception"); + } + } + + [Test] + [NoRetry] + public void Test3() + { + Assert.Pass(); + } + } +} \ No newline at end of file diff --git a/NUnit.Extensions.Tests/Retries/RetrySuite.cs b/NUnit.Extensions.Tests/Retries/RetrySuite.cs new file mode 100644 index 0000000..b0a3120 --- /dev/null +++ b/NUnit.Extensions.Tests/Retries/RetrySuite.cs @@ -0,0 +1,12 @@ +using NUnit.Framework; + +using SkbKontur.NUnit.Retries; + +namespace SkbKontur.NUnit.Extensions.Tests.Retries +{ + [SetUpFixture] + [RetryOnError(2)] + public class RetrySuite + { + } +} \ No newline at end of file diff --git a/NUnit.Extensions.Tests/Retries/TestWithRetries.cs b/NUnit.Extensions.Tests/Retries/TestWithRetries.cs new file mode 100644 index 0000000..49d35c0 --- /dev/null +++ b/NUnit.Extensions.Tests/Retries/TestWithRetries.cs @@ -0,0 +1,29 @@ +using NUnit.Framework; +using NUnit.Framework.Internal; + +using SkbKontur.NUnit.Retries; + +namespace SkbKontur.NUnit.Extensions.Tests.Retries +{ + public class TestWithRetries + { + [Test] + public void Test1() + { + if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0) + { + Assert.Fail("Second time's a Charm"); + } + } + + [Test] + [RetryOnError(3)] + public void Test2() + { + if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0 or 1) + { + Assert.Fail("Third time's a Charm"); + } + } + } +} \ No newline at end of file diff --git a/NUnit.Middlewares/README.md b/NUnit.Middlewares/README.md index 8cf0e2a..795b680 100644 --- a/NUnit.Middlewares/README.md +++ b/NUnit.Middlewares/README.md @@ -1,7 +1,7 @@ # NUnit.Middlewares [![NuGet Status](https://img.shields.io/nuget/v/SkbKontur.NUnit.Middlewares.svg)](https://www.nuget.org/packages/SkbKontur.NUnit.Middlewares/) -[![Build status](https://github.com/skbkontur/nunit-middlewares/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-middlewares/actions) +[![Build status](https://github.com/skbkontur/nunit-extensions/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-extensions/actions) Use middleware pattern to write tests in concise and comprehensive manner. And ditch test bases. @@ -9,13 +9,13 @@ Use middleware pattern to write tests in concise and comprehensive manner. And d Inspired by ASP.NET Core [middlewares](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/middleware), the main idea of test middlewares can be summarized by this image: -![nunit-middlewares](https://github.com/skbkontur/nunit-middlewares/assets/5417867/9707428f-11ec-4353-ac96-7fdf70200a47) +![nunit-middlewares](https://github.com/skbkontur/nunit-extensions/assets/5417867/9707428f-11ec-4353-ac96-7fdf70200a47) Here we focus on *behaviours* that we want to add to our test rather than focusing on implementing test lifecycle methods provided by NUnit. `suite`, `fixture` and `test` in the image above are just `ISetupBuilder` that can accept either raw setup functions or anything that implements simple `ISetup` interface: -![setup-builder](https://github.com/skbkontur/nunit-middlewares/assets/5417867/e4adb7c6-2078-401e-9bac-539f89ffec54) +![setup-builder](https://github.com/skbkontur/nunit-extensions/assets/5417867/e4adb7c6-2078-401e-9bac-539f89ffec54) ## Simple test base @@ -126,7 +126,7 @@ To ensure everything is working as intended, parent's *context item*s should be In our example from first image, test context will look something like this: -![test-context](https://github.com/skbkontur/nunit-middlewares/assets/5417867/c70b41d6-5f3f-485a-9e9d-7616b3797232) +![test-context](https://github.com/skbkontur/nunit-extensions/assets/5417867/c70b41d6-5f3f-485a-9e9d-7616b3797232) Both `SimpleTestContext` and `GetFromThisOrParentContext` are just `ITest` wrappers that search for context value in `ITest`'s `Properties` recursively diff --git a/NUnit.Retries/CustomAttributeMethodWrapper.cs b/NUnit.Retries/CustomAttributeMethodWrapper.cs new file mode 100644 index 0000000..ff1357b --- /dev/null +++ b/NUnit.Retries/CustomAttributeMethodWrapper.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +using NUnit.Framework.Interfaces; + +namespace SkbKontur.NUnit.Retries +{ + public sealed class CustomAttributeMethodWrapper : IMethodInfo + { + public CustomAttributeMethodWrapper(IMethodInfo baseInfo, params Attribute[] extraAttributes) + { + this.baseInfo = baseInfo; + this.extraAttributes = extraAttributes; + } + + public ITypeInfo TypeInfo => baseInfo.TypeInfo; + public MethodInfo MethodInfo => baseInfo.MethodInfo; + public string Name => baseInfo.Name; + public bool IsAbstract => baseInfo.IsAbstract; + public bool IsPublic => baseInfo.IsPublic; + public bool IsStatic => baseInfo.IsStatic; + public bool ContainsGenericParameters => baseInfo.ContainsGenericParameters; + public bool IsGenericMethod => baseInfo.IsGenericMethod; + public bool IsGenericMethodDefinition => baseInfo.IsGenericMethodDefinition; + public ITypeInfo ReturnType => baseInfo.ReturnType; + + public T[] GetCustomAttributes(bool inherit) where T : class + { + var bases = baseInfo.GetCustomAttributes(inherit); + var extras = extraAttributes.OfType().ToArray(); + + return !extras.Any() + ? bases + : !bases.Any() + ? extras + : MergeAttributes(bases, extras); + } + + public bool IsDefined(bool inherit) where T : class + => baseInfo.IsDefined(inherit) || extraAttributes.OfType().Any(); + + public Type[] GetGenericArguments() => baseInfo.GetGenericArguments(); + public IParameterInfo[] GetParameters() => baseInfo.GetParameters(); + public object? Invoke(object? fixture, params object?[]? args) => baseInfo.Invoke(fixture, args); + public IMethodInfo MakeGenericMethod(params Type[] typeArguments) => baseInfo.MakeGenericMethod(typeArguments); + + private static T[] MergeAttributes(T[] bases, T[] extras) where T : class + { + var baseTypes = new HashSet(bases.Select(x => x.GetType())); + return bases + .Concat(extras.Where(e => !baseTypes.Contains(e.GetType()))) + .ToArray(); + } + + private readonly IMethodInfo baseInfo; + private readonly Attribute[] extraAttributes; + } +} \ No newline at end of file diff --git a/NUnit.Retries/IRetryStrategy.cs b/NUnit.Retries/IRetryStrategy.cs new file mode 100644 index 0000000..2ea6930 --- /dev/null +++ b/NUnit.Retries/IRetryStrategy.cs @@ -0,0 +1,13 @@ +using System; + +using NUnit.Framework.Internal; + +namespace SkbKontur.NUnit.Retries +{ + public interface IRetryStrategy + { + public int TryCount { get; } + public bool ShouldRetry(TestResult result); + public void OnTestFailed(TestExecutionContext context, DateTimeOffset start); + } +} \ No newline at end of file diff --git a/NUnit.Retries/NUnit.Retries.csproj b/NUnit.Retries/NUnit.Retries.csproj new file mode 100644 index 0000000..0572498 --- /dev/null +++ b/NUnit.Retries/NUnit.Retries.csproj @@ -0,0 +1,22 @@ + + + + netstandard2.0 + SkbKontur.NUnit.Retries + SkbKontur.NUnit.Retries + SkbKontur.NUnit.Retries + Retries for NUnit + README.md + NUnit Retry + Pavel Vostretsov + + + + + + + + + + + \ No newline at end of file diff --git a/NUnit.Retries/NoRetryAttribute.cs b/NUnit.Retries/NoRetryAttribute.cs new file mode 100644 index 0000000..8ffd7a4 --- /dev/null +++ b/NUnit.Retries/NoRetryAttribute.cs @@ -0,0 +1,9 @@ +using System; + +namespace SkbKontur.NUnit.Retries +{ + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] + public class NoRetryAttribute : Attribute + { + } +} \ No newline at end of file diff --git a/NUnit.Retries/README.md b/NUnit.Retries/README.md new file mode 100644 index 0000000..6b81842 --- /dev/null +++ b/NUnit.Retries/README.md @@ -0,0 +1,19 @@ +# NUnit.Retries + +[![NuGet Status](https://img.shields.io/nuget/v/SkbKontur.NUnit.Retries.svg)](https://www.nuget.org/packages/SkbKontur.NUnit.Retries/) +[![Build status](https://github.com/skbkontur/nunit-extensions/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-extensions/actions) + +Couple of helpful attributes for test retries: +- `RetryOnErrorAttribute` is like NUnit's own `RetryAttribute`, but it can be applied to whole Fixture/Suite/Assembly, and supports retry after exceptions in test, not only assertion failures +- On top of that, `RetryOnTeamCityAttribute` also supports TeamCity's [test retry](https://www.jetbrains.com/help/teamcity/2022.10/build-failure-conditions.html#test-retry) feature +- `NoRetryAttribute` for disabling retries + +Attributes can be overriden on any level, e.g. +- MyAssembly.dll: `[RetryOnError(2)]` +- MySuite.cs: `[NoRetry]` +- MyTestFixture.cs: `[RetryOnTeamCity(3)]` +- MyTestMethod(): `[RetryOnError(4)]` + +This means we have two retries on assembly level in `MyAssembly.dll`, but no retries in `MySuite`, +if `MyTestFixture` is also in `MySuite`, previous attributes are overriden by `RetryOnTeamCity`, +and method `MyTestMethod` in `MyTestFixture` is retried 4 times. \ No newline at end of file diff --git a/NUnit.Retries/RetryAttribute.cs b/NUnit.Retries/RetryAttribute.cs new file mode 100644 index 0000000..5ee9bd1 --- /dev/null +++ b/NUnit.Retries/RetryAttribute.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; + +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; +using NUnit.Framework.Internal.Commands; + +namespace SkbKontur.NUnit.Retries +{ + [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method)] + public class RetryAttribute : Attribute, IRepeatTest, IApplyToContext + { + protected RetryAttribute(IRetryStrategy strategy) + { + this.strategy = strategy; + } + + public TestCommand Wrap(TestCommand command) + { + return new RetryCommand(command, strategy); + } + + public void ApplyToContext(TestExecutionContext context) + { + ApplyToTestRecursively(context.CurrentTest, overwrite : true); + } + + private void ApplyToTestRecursively(Test test, bool overwrite) + { + if (test.GetCustomAttributes(true).Any() + || (!overwrite && test.GetCustomAttributes(true).Any())) + { + return; + } + + if (test is TestMethod method) + { + method.Method = new CustomAttributeMethodWrapper(method.Method, this); + return; + } + + foreach (var child in test.Tests.OfType()) + { + ApplyToTestRecursively(child, overwrite : false); + } + } + + private readonly IRetryStrategy strategy; + } +} \ No newline at end of file diff --git a/NUnit.Retries/RetryCommand.cs b/NUnit.Retries/RetryCommand.cs new file mode 100644 index 0000000..a631b49 --- /dev/null +++ b/NUnit.Retries/RetryCommand.cs @@ -0,0 +1,49 @@ +using System; + +using NUnit.Framework.Internal; +using NUnit.Framework.Internal.Commands; + +namespace SkbKontur.NUnit.Retries +{ + public class RetryCommand : DelegatingTestCommand + { + public RetryCommand(TestCommand innerCommand, IRetryStrategy strategy) + : base(innerCommand) + { + this.strategy = strategy; + } + + public override TestResult Execute(TestExecutionContext context) + { + var count = strategy.TryCount; + + while (count-- > 0) + { + var start = DateTimeOffset.UtcNow; + try + { + context.CurrentResult = innerCommand.Execute(context); + } + catch (Exception ex) + { + context.CurrentResult ??= context.CurrentTest.MakeTestResult(); + context.CurrentResult.RecordException(ex); + } + + if (count <= 0 || !strategy.ShouldRetry(context.CurrentResult)) + { + break; + } + + strategy.OnTestFailed(context, start); + + context.CurrentResult = context.CurrentTest.MakeTestResult(); + context.CurrentRepeatCount++; + } + + return context.CurrentResult; + } + + private readonly IRetryStrategy strategy; + } +} \ No newline at end of file diff --git a/NUnit.Retries/RetryOnErrorAttribute.cs b/NUnit.Retries/RetryOnErrorAttribute.cs new file mode 100644 index 0000000..b7a98c0 --- /dev/null +++ b/NUnit.Retries/RetryOnErrorAttribute.cs @@ -0,0 +1,13 @@ +using System; + +namespace SkbKontur.NUnit.Retries +{ + [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method)] + public class RetryOnErrorAttribute : RetryAttribute + { + public RetryOnErrorAttribute(int tryCount) + : base(new RetryOnErrorStrategy(tryCount)) + { + } + } +} \ No newline at end of file diff --git a/NUnit.Retries/RetryOnErrorStrategy.cs b/NUnit.Retries/RetryOnErrorStrategy.cs new file mode 100644 index 0000000..3fbd6ec --- /dev/null +++ b/NUnit.Retries/RetryOnErrorStrategy.cs @@ -0,0 +1,32 @@ +using System; + +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; + +namespace SkbKontur.NUnit.Retries +{ + public class RetryOnErrorStrategy : IRetryStrategy + { + public RetryOnErrorStrategy(int tryCount) + { + TryCount = tryCount; + } + + public int TryCount { get; } + + public bool ShouldRetry(TestResult result) + { + return result.ResultState == ResultState.Failure || result.ResultState == ResultState.Error; + } + + public void OnTestFailed(TestExecutionContext context, DateTimeOffset start) + { + var attempt = context.CurrentRepeatCount + 1; + + TestContext.Progress.WriteLine($"Test failed on attempt {attempt}/{TryCount}"); + TestContext.Progress.WriteLine(context.CurrentResult.Message); + TestContext.Progress.WriteLine("Retrying..."); + } + } +} \ No newline at end of file diff --git a/NUnit.Retries/TeamCity/RetryOnTeamCityAttribute.cs b/NUnit.Retries/TeamCity/RetryOnTeamCityAttribute.cs new file mode 100644 index 0000000..3249c22 --- /dev/null +++ b/NUnit.Retries/TeamCity/RetryOnTeamCityAttribute.cs @@ -0,0 +1,13 @@ +using System; + +namespace SkbKontur.NUnit.Retries.TeamCity +{ + [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method)] + public class RetryOnTeamCityAttribute : RetryAttribute + { + public RetryOnTeamCityAttribute(int tryCount) + : base(new RetryOnTeamCityStrategy(tryCount)) + { + } + } +} \ No newline at end of file diff --git a/NUnit.Retries/TeamCity/RetryOnTeamCityStrategy.cs b/NUnit.Retries/TeamCity/RetryOnTeamCityStrategy.cs new file mode 100644 index 0000000..d08b797 --- /dev/null +++ b/NUnit.Retries/TeamCity/RetryOnTeamCityStrategy.cs @@ -0,0 +1,29 @@ +using System; + +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; + +namespace SkbKontur.NUnit.Retries.TeamCity +{ + public class RetryOnTeamCityStrategy : IRetryStrategy + { + public RetryOnTeamCityStrategy(int tryCount) + { + TryCount = tryCount; + } + + public int TryCount { get; } + + public bool ShouldRetry(TestResult result) + { + return TestContextExtensions.IsOnTeamCity() && + (result.ResultState == ResultState.Failure || + result.ResultState == ResultState.Error); + } + + public void OnTestFailed(TestExecutionContext context, DateTimeOffset start) + { + context.WriteFailureForTeamCity(start, TryCount); + } + } +} \ No newline at end of file diff --git a/NUnit.Retries/TeamCity/ServiceMessageConstants.cs b/NUnit.Retries/TeamCity/ServiceMessageConstants.cs new file mode 100644 index 0000000..e690a09 --- /dev/null +++ b/NUnit.Retries/TeamCity/ServiceMessageConstants.cs @@ -0,0 +1,8 @@ +namespace SkbKontur.NUnit.Retries.TeamCity +{ + public static class ServiceMessageConstants + { + public const string ServiceMessageOpen = "##teamcity["; + public const string ServiceMessageClose = "]"; + } +} \ No newline at end of file diff --git a/NUnit.Retries/TeamCity/ServiceMessageFormatter.cs b/NUnit.Retries/TeamCity/ServiceMessageFormatter.cs new file mode 100644 index 0000000..b220987 --- /dev/null +++ b/NUnit.Retries/TeamCity/ServiceMessageFormatter.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SkbKontur.NUnit.Retries.TeamCity +{ + public static class ServiceMessageFormatter + { + /// + /// Serializes single value service message + /// https://github.com/JetBrains/TeamCity.ServiceMessages/blob/41f56446298b984719bd98b476f5b29f8aec8011/TeamCity.ServiceMessages/Write/ServiceMessageFormatter.cs#L102 + /// + /// message name + /// params of service message properties + /// service message string + public static string FormatMessage(string messageName, Dictionary properties) + { + if (string.IsNullOrEmpty(messageName)) + { + throw new ArgumentException("The message name must not be empty", nameof(messageName)); + } + + if (ServiceMessageReplacements.Encode(messageName) != messageName) + { + throw new ArgumentException("Message name contains illegal characters", nameof(messageName)); + } + + var sb = new StringBuilder(); + sb.Append(ServiceMessageConstants.ServiceMessageOpen); + sb.Append(messageName); + + foreach (var property in properties) + { + if (string.IsNullOrEmpty(property.Key)) + { + throw new InvalidOperationException("The property name must not be empty"); + } + + if (ServiceMessageReplacements.Encode(property.Key) != property.Key) + { + throw new InvalidOperationException($"The property name {property.Key} contains illegal characters"); + } + + sb.AppendFormat(" {0}='{1}'", property.Key, ServiceMessageReplacements.Encode(property.Value)); + } + + sb.Append(ServiceMessageConstants.ServiceMessageClose); + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/NUnit.Retries/TeamCity/ServiceMessageReplacements.cs b/NUnit.Retries/TeamCity/ServiceMessageReplacements.cs new file mode 100644 index 0000000..0fadee8 --- /dev/null +++ b/NUnit.Retries/TeamCity/ServiceMessageReplacements.cs @@ -0,0 +1,62 @@ +using System.Text; + +namespace SkbKontur.NUnit.Retries.TeamCity +{ + public static class ServiceMessageReplacements + { + /// + /// Performs TeamCity-format escaping of a string. + /// https://github.com/JetBrains/TeamCity.ServiceMessages/blob/41f56446298b984719bd98b476f5b29f8aec8011/TeamCity.ServiceMessages/ServiceMessageReplacements.cs#L19 + /// + public static string Encode(string value) + { + var sb = new StringBuilder(value.Length * 2); + foreach (var ch in value) + { + switch (ch) + { + case '|': + sb.Append("||"); + break; // + case '\'': + sb.Append("|'"); + break; // + case '\n': + sb.Append("|n"); + break; // + case '\r': + sb.Append("|r"); + break; // + case '[': + sb.Append("|["); + break; // + case ']': + sb.Append("|]"); + break; // + case '\u0085': + sb.Append("|x"); + break; //\u0085 (next line)=>|x + case '\u2028': + sb.Append("|l"); + break; //\u2028 (line separator)=>|l + case '\u2029': + sb.Append("|p"); + break; // + default: + if (ch > 127) + { + sb.Append($"|0x{(ulong)ch:x4}"); + } + else + { + sb.Append(ch); + } + + break; + } + } + + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/NUnit.Retries/TeamCity/TestContextExtensions.cs b/NUnit.Retries/TeamCity/TestContextExtensions.cs new file mode 100644 index 0000000..7f06196 --- /dev/null +++ b/NUnit.Retries/TeamCity/TestContextExtensions.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; + +namespace SkbKontur.NUnit.Retries.TeamCity +{ + public static class TestContextExtensions + { + public static bool IsOnTeamCity() + { + return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION")); + } + + public static void WriteFailureForTeamCity(this TestExecutionContext context, DateTimeOffset start, int tryCount) + { + var props = GetTestProperties(context.CurrentTest); + + TestContext.Progress.TestStarted(props, start); + TestContext.Progress.TestStdOut(props, context.CurrentResult.Output, context.CurrentRepeatCount + 1, tryCount); + TestContext.Progress.TestFailed(props, context.CurrentResult.Message, context.CurrentResult.StackTrace); + TestContext.Progress.TestFinished(props); + } + + private static void TestStarted(this TextWriter writer, IDictionary props, DateTimeOffset start) + { + writer.WriteLine(ServiceMessageFormatter.FormatMessage("testStarted", new Dictionary(props) + { + ["timestamp"] = $"{start:yyyy-MM-dd'T'HH:mm:ss.fff}+0000", + })); + } + + private static void TestStdOut(this TextWriter writer, IDictionary props, string output, int attempt, int tryCount) + { + writer.WriteLine(ServiceMessageFormatter.FormatMessage("testStdOut", new Dictionary(props) + { + ["timestamp"] = $"{DateTimeOffset.UtcNow:yyyy-MM-dd'T'HH:mm:ss.fff}+0000", + ["out"] = $"Test failed on attempt {attempt}/{tryCount}, will retry\n\n" + output, + })); + } + + private static void TestFailed(this TextWriter writer, IDictionary props, string? message, string? stackTrace) + { + var testFailedProps = new Dictionary(props) + { + ["timestamp"] = $"{DateTimeOffset.UtcNow:yyyy-MM-dd'T'HH:mm:ss.fff}+0000", + }; + + if (message != null) + { + testFailedProps["message"] = message; + } + + if (stackTrace != null) + { + testFailedProps["details"] = stackTrace; + } + + writer.WriteLine(ServiceMessageFormatter.FormatMessage("testFailed", testFailedProps)); + } + + private static void TestFinished(this TextWriter writer, IDictionary props) + { + writer.WriteLine(ServiceMessageFormatter.FormatMessage("testFinished", new Dictionary(props) + { + ["timestamp"] = $"{DateTimeOffset.UtcNow:yyyy-MM-dd'T'HH:mm:ss.fff}+0000", + })); + } + + private static Dictionary GetTestProperties(ITest test) + { + var assembly = test.TypeInfo!.Assembly; + var suiteName = test.TypeInfo.Assembly.GetName().Name; + var testSource = assembly.Location; + var testId = Guid.NewGuid().ToString(); + + return new Dictionary + { + ["name"] = $"{suiteName}: {test.FullName}", + ["captureStandardOutput"] = "false", + ["suiteName"] = suiteName, + ["testSource"] = testSource, + ["displayName"] = test.Name, + ["fullyQualifiedName"] = test.FullName, + ["id"] = testId, + }; + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index 71fa646..3f48f87 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ A collection of extensions for NUnit. | | Build Status | |-------------------------------------|:--------------: | | NUnit.Middlewares | [![NuGet Status](https://img.shields.io/nuget/v/SkbKontur.NUnit.Middlewares.svg)](https://www.nuget.org/packages/SkbKontur.NUnit.Middlewares/) | -| Build | [![Build status](https://github.com/skbkontur/nunit-middlewares/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-middlewares/actions) | +| NUnit.Retries | [![NuGet Status](https://img.shields.io/nuget/v/SkbKontur.NUnit.Retries.svg)](https://www.nuget.org/packages/SkbKontur.NUnit.Retries/) | +| Build | [![Build status](https://github.com/skbkontur/nunit-extensions/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-extensions/actions) | ## Release Notes @@ -13,4 +14,5 @@ See [CHANGELOG](CHANGELOG.md). ## Projects -- [NUnit.Middlewares](NUnit.Middlewares) \ No newline at end of file +- [NUnit.Middlewares](NUnit.Middlewares) +- [NUnit.Retries](NUnit.Retries) \ No newline at end of file diff --git a/NUnit.Middlewares.sln b/nunit-extensions.sln similarity index 63% rename from NUnit.Middlewares.sln rename to nunit-extensions.sln index 49bd016..365740f 100644 --- a/NUnit.Middlewares.sln +++ b/nunit-extensions.sln @@ -2,7 +2,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnit.Middlewares", "NUnit.Middlewares\NUnit.Middlewares.csproj", "{2726FA5A-69A8-4F38-A09D-6C3150DCC565}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnit.Middlewares.Tests", "NUnit.Middlewares.Tests\NUnit.Middlewares.Tests.csproj", "{0695DF88-605F-401C-894D-24B40B9A71DD}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnit.Extensions.Tests", "NUnit.Extensions.Tests\NUnit.Extensions.Tests.csproj", "{0695DF88-605F-401C-894D-24B40B9A71DD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnit.Retries", "NUnit.Retries\NUnit.Retries.csproj", "{7EEE6F2E-F282-410D-A82A-6A5C4B9A9ECF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -18,5 +20,9 @@ Global {0695DF88-605F-401C-894D-24B40B9A71DD}.Debug|Any CPU.Build.0 = Debug|Any CPU {0695DF88-605F-401C-894D-24B40B9A71DD}.Release|Any CPU.ActiveCfg = Release|Any CPU {0695DF88-605F-401C-894D-24B40B9A71DD}.Release|Any CPU.Build.0 = Release|Any CPU + {7EEE6F2E-F282-410D-A82A-6A5C4B9A9ECF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7EEE6F2E-F282-410D-A82A-6A5C4B9A9ECF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7EEE6F2E-F282-410D-A82A-6A5C4B9A9ECF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7EEE6F2E-F282-410D-A82A-6A5C4B9A9ECF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/NUnit.Middlewares.sln.DotSettings b/nunit-extensions.sln.DotSettings similarity index 100% rename from NUnit.Middlewares.sln.DotSettings rename to nunit-extensions.sln.DotSettings