From 0b118ec63a52d351b879d51643f8ffa05a622237 Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Mon, 27 Dec 2021 20:36:09 +0100 Subject: [PATCH] prepare 4.2 (#928) * prepare 4.2 * Fix warning and set beta,2 * Adding more logging * filters added handling explicits * small fixes * Code cleanup * beta.4 * Addind filter acceptance tests * update with more logs * Adding nunit.where tester * FIx warnings * Added more tests and added csproj base class to make it easier to make more * fix tests * fix SA warnings * Oppdatert pakker o.a. * Reanebled output of console message, introducing another ConsoleOut level, so one can go back. Fixes #934 * fixing linux errors * Add consoletests * COnsole and propertytests * naming fixed * fix * FIx namespaces * sa fix * sa fix * accentance tests, with parantheses issue * Fixing Issue #919 * fix braces * fix braces * Version to 4.2.0 --- .editorconfig | 2 + build.cake | 6 +- nuget/NUnit3TestAdapter.nuspec | 6 +- src/Directory.Build.props | 4 +- .../AcceptanceTests.cs | 87 +++++++----- .../BundledDependencyTests.cs | 9 +- .../ConsoleOutTests.cs | 53 +++++++ .../CsProjAcceptanceTests.cs | 55 +++++++ .../FilterTests.cs | 90 ++++++++++++ .../FixtureTests.cs | 134 ++++++++++++++++++ .../NUnit.TestAdapter.Tests.Acceptance.csproj | 9 +- .../ParanthesisTests.cs | 86 +++++++++++ .../PropertyTests.cs | 58 ++++++++ .../SinglePassingTestResultTests.cs | 32 ++--- .../TestSourceWithCustomNames.cs | 5 +- .../IsolatedWorkspace.RunSettings.cs | 14 +- .../WorkspaceTools/IsolatedWorkspace.cs | 62 ++++++-- .../WorkspaceTools/ToolResolver.cs | 20 +-- .../WorkspaceTools/VSTestResult.cs | 10 ++ .../WorkspaceTools/VSTestResultCounters.cs | 2 + .../WorkspaceTools/VsTestFilter.cs | 49 +++++++ src/NUnitTestAdapter/AdapterSettings.cs | 22 ++- src/NUnitTestAdapter/Execution.cs | 100 ++++++++----- src/NUnitTestAdapter/NUnit.TestAdapter.csproj | 6 - src/NUnitTestAdapter/NUnit3TestExecutor.cs | 20 ++- .../NUnitEngine/DiscoveryConverter.cs | 10 +- .../NUnitEngine/Extensions.cs | 7 + .../NUnitEngine/NUnitDiscoveryTestClasses.cs | 4 +- .../NUnitEngine/NUnitEngineAdapter.cs | 9 +- .../NUnitEngine/NUnitResults.cs | 10 +- src/NUnitTestAdapter/NUnitEventListener.cs | 7 +- src/NUnitTestAdapter/NUnitTestAdapter.cs | 9 +- .../NUnitTestFilterBuilder.cs | 7 +- src/NUnitTestAdapter/TestConverter.cs | 21 +-- src/NUnitTestAdapter/TestConverterForXml.cs | 19 +-- .../TestFilterConverter/TestFilterParser.cs | 29 ++-- src/NUnitTestAdapter/TestLogger.cs | 5 + .../NUnit.TestAdapter.Tests.csproj | 6 +- .../NUnitEngineTests/NUnitTestCaseTests.cs | 2 +- .../NUnitEngineTests/NUnitTestEventsTests.cs | 2 +- .../NUnitEventListenerTests.cs | 4 +- src/NUnitTestAdapterTests/ProjectTests.cs | 2 +- src/NUnitTestAdapterTests/TempDirectory.cs | 2 +- .../TestConverterForXmlTests.cs | 2 +- .../TestExecutionTests.cs | 47 ++++-- .../TestFilterCombinerTests.cs | 34 +++++ src/mock-assembly/MockAssembly.cs | 41 ++++-- 47 files changed, 986 insertions(+), 234 deletions(-) create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/ConsoleOutTests.cs create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/CsProjAcceptanceTests.cs create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/FilterTests.cs create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/FixtureTests.cs create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/ParanthesisTests.cs create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/PropertyTests.cs create mode 100644 src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VsTestFilter.cs create mode 100644 src/NUnitTestAdapterTests/TestFilterConverterTests/TestFilterCombinerTests.cs diff --git a/.editorconfig b/.editorconfig index c6ed9419..d2333efa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -172,6 +172,8 @@ csharp_space_between_square_brackets = false # Wrapping preferences csharp_preserve_single_line_blocks = true csharp_preserve_single_line_statements = true +dotnet_diagnostic.SA1000.severity=silent +dotnet_diagnostic.SA1518.severity=silent #### Naming styles #### [*.{cs,vb}] diff --git a/build.cake b/build.cake index 18bc920c..82a49adc 100644 --- a/build.cake +++ b/build.cake @@ -1,5 +1,5 @@ -#tool vswhere&version=2.7.1 -#tool Microsoft.TestPlatform&version=16.11.0 +#tool vswhere&version=2.8.4 +#tool Microsoft.TestPlatform&version=17.0.0 ////////////////////////////////////////////////////////////////////// // ARGUMENTS @@ -13,7 +13,7 @@ var configuration = Argument("configuration", "Release"); ////////////////////////////////////////////////////////////////////// var version = "4.2.0"; -var modifier = "-alpha.4"; +var modifier = ""; var dbgSuffix = configuration.ToLower() == "debug" ? "-dbg" : ""; var packageVersion = version + modifier + dbgSuffix; diff --git a/nuget/NUnit3TestAdapter.nuspec b/nuget/NUnit3TestAdapter.nuspec index d6bfc8ae..ff5e0010 100644 --- a/nuget/NUnit3TestAdapter.nuspec +++ b/nuget/NUnit3TestAdapter.nuspec @@ -3,21 +3,19 @@ NUnit3TestAdapter $version$ - NUnit 3 Test Adapter for Visual Studio and DotNet + NUnit3 Test Adapter for Visual Studio and DotNet Charlie Poole, Terje Sandstrom MIT https://docs.nunit.org/articles/vs-test-adapter/Index.html https://cdn.rawgit.com/nunit/resources/master/images/icon/nunit_256.png false - NUnit 3 adapter for running tests in Visual Studio and DotNet. Works with NUnit 3.x, use the NUnit 2 adapter for 2.x tests. + NUnit3 adapter for running tests in Visual Studio and DotNet. Works with NUnit 3.x, use the NUnit 2 adapter for 2.x tests. The NUnit3 TestAdapter for Visual Studio, all versions from 2012 and onwards, and DotNet (incl. .Net core). Note that this package ONLY contains the adapter, not the NUnit framework. For VS 2017 and forward, you should add this package to every test project in your solution. (Earlier versions only require a single adapter package per solution.) - - Note that with this package you should not install the VSIX adapter package. See https://docs.nunit.org/articles/vs-test-adapter/Adapter-Release-Notes.html Copyright (c) 2011-2021 Charlie Poole, 2014-2021 Terje Sandstrom diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 1cdd466a..0c4b14d1 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -9,11 +9,11 @@ - + - + diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/AcceptanceTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/AcceptanceTests.cs index 007ca3d6..e1d844b2 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/AcceptanceTests.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/AcceptanceTests.cs @@ -3,18 +3,27 @@ using System.Diagnostics; using System.IO; using System.Linq; + using NUnit.Framework; using NUnit.Framework.Interfaces; using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance { + public class Frameworks + { + public const string NetCoreApp31 = "netcoreapp3.1"; + public const string NetCoreApp21 = "netcoreapp2.1"; + public const string Net50 = "net5.0"; + public const string Net60 = "net6.0"; + } + [Category("Acceptance")] public abstract class AcceptanceTests { - public static string NuGetPackageId { get; } = "NUnit3TestAdapter"; + public static string NuGetPackageId => "NUnit3TestAdapter"; - public static string NuGetPackageVersion => Initialization.Value.nupkgVersion; + public static string NuGetPackageVersion => Initialization.Value.NupkgVersion; public const string LowestNetfxTarget = "net35"; public const string LegacyProjectTargetFrameworkVersion = "v3.5"; @@ -22,60 +31,62 @@ public abstract class AcceptanceTests public static IEnumerable TargetFrameworks => new[] { LowestNetfxTarget, - "netcoreapp2.1" + Frameworks.NetCoreApp21 }; public static IEnumerable DotNetCliTargetFrameworks => new[] { - "netcoreapp2.1", - "netcoreapp3.1" + Frameworks.NetCoreApp21, + Frameworks.NetCoreApp31, + Frameworks.Net50 }; - private static readonly Lazy<(IsolatedWorkspaceManager manager, string nupkgVersion, bool keepWorkspaces)> Initialization = new (() => - { - var directory = TestContext.Parameters["ProjectWorkspaceDirectory"] - ?? TryAutoDetectProjectWorkspaceDirectory() - ?? throw new InvalidOperationException("The test parameter ProjectWorkspaceDirectory must be set in order to run this test."); + private static readonly Lazy<(IsolatedWorkspaceManager Manager, string NupkgVersion, bool KeepWorkspaces)> Initialization = new(() => + { + var directory = TestContext.Parameters["ProjectWorkspaceDirectory"] + ?? TryAutoDetectProjectWorkspaceDirectory() + ?? throw new InvalidOperationException("The test parameter ProjectWorkspaceDirectory must be set in order to run this test."); - var nupkgDirectory = TestContext.Parameters["TestNupkgDirectory"] - ?? TryAutoDetectTestNupkgDirectory(NuGetPackageId) - ?? throw new InvalidOperationException("The test parameter TestNupkgDirectory must be set in order to run this test."); + var nupkgDirectory = TestContext.Parameters["TestNupkgDirectory"] + ?? TryAutoDetectTestNupkgDirectory(NuGetPackageId) + ?? throw new InvalidOperationException("The test parameter TestNupkgDirectory must be set in order to run this test."); - var nupkgVersion = TryGetTestNupkgVersion(nupkgDirectory, packageId: NuGetPackageId) - ?? throw new InvalidOperationException($"No NuGet package with the ID {NuGetPackageId} was found in {nupkgDirectory}."); + var nupkgVersion = TryGetTestNupkgVersion(nupkgDirectory, packageId: NuGetPackageId) + ?? throw new InvalidOperationException($"No NuGet package with the ID {NuGetPackageId} was found in {nupkgDirectory}."); - var keepWorkspaces = TestContext.Parameters.Get("KeepWorkspaces", defaultValue: false); + var keepWorkspaces = TestContext.Parameters.Get("KeepWorkspaces", defaultValue: false); - var packageCachePath = Path.Combine(directory, ".isolatednugetcache"); - ClearCachedTestNupkgs(packageCachePath); + var packageCachePath = Path.Combine(directory, ".isolatednugetcache"); + ClearCachedTestNupkgs(packageCachePath); - var manager = new IsolatedWorkspaceManager( - reason: string.Join( - Environment.NewLine, - "Test assembly: " + typeof(AcceptanceTests).Assembly.Location, - "Runner process: " + Process.GetCurrentProcess().MainModule.FileName), - directory, - nupkgDirectory, - packageCachePath, - downloadCachePath: Path.Combine(directory, ".toolcache")); + var manager = new IsolatedWorkspaceManager( + reason: string.Join( + Environment.NewLine, + "Test assembly: " + typeof(AcceptanceTests).Assembly.Location, + "Runner process: " + Process.GetCurrentProcess().MainModule.FileName), + directory, + nupkgDirectory, + packageCachePath, + downloadCachePath: Path.Combine(directory, ".toolcache")); - if (keepWorkspaces) manager.PreserveDirectory("The KeepWorkspaces test parameter was set to true."); + if (keepWorkspaces) manager.PreserveDirectory("The KeepWorkspaces test parameter was set to true."); - return (manager, nupkgVersion, keepWorkspaces); - }); + return (manager, nupkgVersion, keepWorkspaces); + }); private static void ClearCachedTestNupkgs(string packageCachePath) { Utils.DeleteDirectoryRobust(Path.Combine(packageCachePath, NuGetPackageId)); } - private static readonly Dictionary> WorkspacesByTestId = new (); + private static readonly Dictionary> WorkspacesByTestId = new(); protected static IsolatedWorkspace CreateWorkspace() { var test = TestContext.CurrentContext?.Test ?? throw new InvalidOperationException("There is no current test."); - - var workspace = Initialization.Value.manager.CreateWorkspace(test.Name); + const string chars = "=()!,~-"; + string name = chars.Aggregate(test.Name, (current, ch) => current.Replace(ch, '_')); + var workspace = Initialization.Value.Manager.CreateWorkspace(name); lock (WorkspacesByTestId) { @@ -83,7 +94,6 @@ protected static IsolatedWorkspace CreateWorkspace() WorkspacesByTestId.Add(test.ID, workspaces = new List()); workspaces.Add(workspace); } - return workspace; } @@ -111,11 +121,11 @@ public static void TearDown() if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed) { - Initialization.Value.manager.PreserveDirectory( + Initialization.Value.Manager.PreserveDirectory( test.FullName + " failed:" + Environment.NewLine + TestContext.CurrentContext.Result.Message.TrimEnd() + Environment.NewLine); } - else if (!Initialization.Value.keepWorkspaces) + else if (!Initialization.Value.KeepWorkspaces) { foreach (var workspace in workspaces) Utils.DeleteDirectoryRobust(workspace.Directory); @@ -126,7 +136,7 @@ internal static void OnGlobalTeardown() { if (!Initialization.IsValueCreated) return; - Initialization.Value.manager.Dispose(); + Initialization.Value.Manager.Dispose(); } private static string TryAutoDetectProjectWorkspaceDirectory() @@ -146,6 +156,7 @@ private static string TryAutoDetectTestNupkgDirectory(string packageId) { // Keep in sync with build.cake. + // Search for it for (var directory = TestContext.CurrentContext.TestDirectory; directory != null; directory = Path.GetDirectoryName(directory)) { var packagePath = Path.Combine(directory, "package"); @@ -153,7 +164,9 @@ private static string TryAutoDetectTestNupkgDirectory(string packageId) try { if (Directory.EnumerateFiles(Path.Combine(directory, "package"), packageId + ".*.nupkg").Any()) + { return packagePath; + } } catch (DirectoryNotFoundException) { diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/BundledDependencyTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/BundledDependencyTests.cs index 1da6fdf1..cf8111d5 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/BundledDependencyTests.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/BundledDependencyTests.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance { @@ -49,11 +50,11 @@ public static void User_tests_get_the_version_of_Mono_Cecil_referenced_from_the_ } }"); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); foreach (var targetFramework in TargetFrameworks) { - workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll"); + workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter); } } @@ -127,11 +128,11 @@ public void OnTestEvent(string report) .AddFile("test.addins", @" Test.dll"); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); foreach (var targetFramework in TargetFrameworks) { - workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll"); + workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter); } } } diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/ConsoleOutTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/ConsoleOutTests.cs new file mode 100644 index 00000000..33090440 --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/ConsoleOutTests.cs @@ -0,0 +1,53 @@ +using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; + +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance +{ + public sealed class ConsoleOutTests : CsProjAcceptanceTests + { + protected override void AddTestsCs(IsolatedWorkspace workspace) + { + workspace.AddFile("Issue774.cs", @" + using System; + using NUnit.Framework; + + namespace Issue774 + { + public class ConsoleOutTest + { + [Test] + public void Test1() + { + Console.WriteLine(); // Did not work pre-Issue774 fix + Assert.Pass(); + } + + [Test] + public void Test2() + { + Console.WriteLine(""Does work""); + Assert.Pass(); + } + } + }"); + } + + protected override string Framework => Frameworks.NetCoreApp31; + + [Test, Platform("Win")] + public void DotNetTest() + { + var workspace = Build(); + var results = workspace.DotNetTest("", true, true, TestContext.WriteLine); + Verify(2, 2, results); + } + + [Test, Platform("Win")] + public void VsTest() + { + var workspace = Build(); + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", VsTestFilter.NoFilter); + Verify(2, 2, results); + } + } +} \ No newline at end of file diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/CsProjAcceptanceTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/CsProjAcceptanceTests.cs new file mode 100644 index 00000000..e111849f --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/CsProjAcceptanceTests.cs @@ -0,0 +1,55 @@ +using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; + +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance +{ + public abstract class CsProjAcceptanceTests : AcceptanceTests + { + protected abstract void AddTestsCs(IsolatedWorkspace workspace); + + protected abstract string Framework { get; } + protected const string NoFilter = ""; + protected IsolatedWorkspace CreateTestWorkspace(string framework) + { + var workspace = CreateWorkspace() + .AddProject("Test.csproj", $@" + + + + {framework} + + + + + + + + + "); + return workspace; + } + + protected IsolatedWorkspace Build() + { + var workspace = CreateTestWorkspace(Framework); + AddTestsCs(workspace); + workspace.MsBuild(restore: true); + return workspace; + } + + protected void Verify(int executed, int total, VSTestResult results) + { + TestContext.WriteLine(" "); + foreach (var error in results.RunErrors) + TestContext.WriteLine(error); + Assert.Multiple(() => + { + Assert.That(results.Counters.Total, Is.EqualTo(total), + $"Total tests counter did not match expectation\n{results.ProcessRunResult.StdOut}"); + Assert.That(results.Counters.Executed, Is.EqualTo(executed), + "Executed tests counter did not match expectation"); + Assert.That(results.Counters.Passed, Is.EqualTo(executed), "Passed tests counter did not match expectation"); + }); + } + } +} diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/FilterTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/FilterTests.cs new file mode 100644 index 00000000..e339c409 --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/FilterTests.cs @@ -0,0 +1,90 @@ +using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; + +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance +{ + public sealed class FilterTests : CsProjAcceptanceTests + { + protected override void AddTestsCs(IsolatedWorkspace workspace) + { + workspace.AddFile("Tests.cs", @" + using NUnit.Framework; + + namespace Filter + { + public class Tests + { + + [Test,Category(""FooGroup"")] + public void Foo() + { + Assert.Pass(); + } + + [Test,Explicit,Category(""IsExplicit""),Category(""FooGroup"")] + public void FooExplicit() + { + Assert.Pass(); + } + + [Test, Category(""BarGroup"")] + public void Bar() + { + Assert.Pass(); + } + } + }"); + } + + protected override string Framework => Frameworks.NetCoreApp31; + + [Test, Platform("Win")] + [TestCase(NoFilter, 2, 3)] + [TestCase(@"TestCategory=FooGroup", 1, 2)] + [TestCase(@"TestCategory!=BarGroup", 1, 2)] + [TestCase(@"TestCategory=IsExplicit", 1, 1)] + [TestCase(@"FullyQualifiedName=Filter.Tests.Foo", 1, 1)] + [TestCase(@"FullyQualifiedName!=Filter.Tests.Foo", 1, 1)] + // [TestCase(@"FullyQualifiedName~Filter.Tests.Foo", 1, 1)] + // [TestCase(@"FullyQualifiedName~Foo", 1, 1)] + public void Filter_DotNetTest(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.DotNetTest(filter, true, true, TestContext.WriteLine); + Verify(executed, total, results); + } + + [Test, Platform("Win")] + [TestCase(NoFilter, 2, 3)] + [TestCase(@"TestCategory=FooGroup", 1, 2)] + [TestCase(@"TestCategory!=BarGroup", 1, 2)] + [TestCase(@"TestCategory=IsExplicit", 1, 1)] + [TestCase(@"FullyQualifiedName=Filter.Tests.Foo", 1, 1)] + [TestCase(@"TestCategory=XXXX", 0, 0)] + public void Filter_VSTest(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", new VsTestTestCaseFilter(filter)); + Verify(executed, total, results); + } + + [Test, Platform("Win")] + [TestCase(NoFilter, 2, 3)] + [TestCase("Category=FooGroup", 1, 1)] + [TestCase("cat==FooGroup", 1, 2)] + [TestCase("cat!=FooGroup", 1, 1)] + [TestCase("Category!=BarGroup", 1, 1)] + [TestCase("Category=IsExplicit", 1, 1)] + [TestCase("test==Filter.Tests.Foo", 1, 1)] + [TestCase("name==Foo", 1, 1)] + [TestCase("name!=Bar", 1, 1)] + // [TestCase("test=~Foo", 1, 1)] + public void Filter_DotNetTest_NUnitWhere(string filter, int executed, int total) + { + var workspace = Build(); + var nunitWhere = $@"NUnit.Where={filter}"; + var results = workspace.DotNetTest(nunitWhere, true, true, TestContext.WriteLine); + Verify(executed, total, results); + } + } +} diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/FixtureTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/FixtureTests.cs new file mode 100644 index 00000000..0b894760 --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/FixtureTests.cs @@ -0,0 +1,134 @@ +using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; + +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance +{ + public sealed class FixtureTests : CsProjAcceptanceTests + { + protected override void AddTestsCs(IsolatedWorkspace workspace) + { + workspace.AddFile("Issue918.cs", @" + using System; + using NUnit.Framework; + + namespace Issue918 + { + [Category(""918"")] + [TestFixtureSource(typeof(FixtureSources), nameof(FixtureSources.Types))] + public class SomeTest + { + [Test] + public void Foo() + { + Assert.Pass(); + } + } + + public static class FixtureSources + { + public static Type[] Types = + { + typeof(object) + }; + } + }"); + + workspace.AddFile("Issue869.cs", @" + using NUnit.Framework; + using System; + using System.Collections.Generic; + + namespace Issue869 + { + public static class Sources + { + public static IEnumerable GetTypes() => new List + { + typeof(string), + typeof(bool) + }; + } + [Category(""869"")] + [TestFixtureSource(typeof(Sources), nameof(Sources.GetTypes))] + public class Tests + { + [Test] + public void SomeRandomTest() + { + } + } + }"); + + workspace.AddFile("Issue884.SetupFixture.cs", @" + using NUnit.Framework; + + namespace NUnitTestAdapterIssueRepro + { + [SetUpFixture] + public class SetupFixture + { + [OneTimeSetUp] + public void OneTimeSetup() + { + } + + [OneTimeTearDown] + public void OneTimeTeardown() + { + } + } + }"); + workspace.AddFile("Issue884.Tests.cs", @" + using NUnit.Framework; + + namespace NUnitTestAdapterIssueRepro + { + [Category(""884"")] + [TestFixture(1)] + [TestFixture(2)] + [TestFixture(3)] + public class Fixture + { + public Fixture(int n) + { + } + + [SetUp] + public void Setup() + { + } + + [Test] + public void Test() + { + Assert.Pass(); + } + } + }"); + } + + protected override string Framework => Frameworks.NetCoreApp31; + + [Test, Platform("Win")] + [TestCase("TestCategory=869", 2, 2)] + [TestCase("TestCategory=884", 3, 3)] + [TestCase("TestCategory=918", 1, 1)] + public void DotNetTest(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.DotNetTest(filter, true, true, TestContext.WriteLine); + Verify(executed, total, results); + } + + [Test, Platform("Win")] + [TestCase("TestCategory=869", 2, 2)] + [TestCase("TestCategory=884", 3, 3)] + [TestCase("TestCategory=918", 1, 1)] + public void VsTest(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", new VsTestTestCaseFilter(filter)); + Verify(executed, total, results); + } + } +} diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/NUnit.TestAdapter.Tests.Acceptance.csproj b/src/NUnit.TestAdapter.Tests.Acceptance/NUnit.TestAdapter.Tests.Acceptance.csproj index 2831c01f..11f1c71f 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/NUnit.TestAdapter.Tests.Acceptance.csproj +++ b/src/NUnit.TestAdapter.Tests.Acceptance/NUnit.TestAdapter.Tests.Acceptance.csproj @@ -12,8 +12,15 @@ - + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/ParanthesisTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/ParanthesisTests.cs new file mode 100644 index 00000000..232646b2 --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/ParanthesisTests.cs @@ -0,0 +1,86 @@ +using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; + +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance +{ + public sealed class ParanthesisTests : CsProjAcceptanceTests + { + protected override void AddTestsCs(IsolatedWorkspace workspace) + { + workspace.AddFile("Issue919.cs", @" + using System; + using NUnit.Framework; + + namespace Issue919 + { + public class Foo + { + [TestCase(1)] + public void Baz(int a) + { + Assert.Pass(); + } + + [Test] + public void Bzzt() + { + Assert.Pass(); + } + } + }"); + } + + protected override string Framework => Frameworks.NetCoreApp31; + + [Test, Platform("Win")] + [TestCase] + public void VsTestNoFilter() + { + var workspace = Build(); + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", VsTestFilter.NoFilter); + Verify(2, 2, results); + } + + [Test, Platform("Win")] + [TestCase(@"FullyQualifiedName=Issue919.Foo.Bzzt", 1, 1)] // Sanity check + [TestCase(@"FullyQualifiedName=Issue919.Foo.Bar\(1\)", 0, 0)] + [TestCase(@"FullyQualifiedName=Issue919.Foo.Baz\(1\)", 1, 1)] + [TestCase(@"Name=Bzzt", 1, 1)] // Sanity check + [TestCase(@"Name=Bar\(1\)", 0, 0)] + [TestCase(@"Name=Baz\(1\)", 1, 1)] + [TestCase(@"", 2, 2)] + public void VsTestTestCases(string filter, int executed, int total) + { + var workspace = Build(); + workspace.DumpTestExecution = true; + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", new VsTestTestCaseFilter(filter)); + Verify(executed, total, results); + } + + [Test, Platform("Win")] + [TestCase(@"Bzzt", 1, 1)] // Sanity check + [TestCase(@"Bar\(1\)", 0, 0)] + [TestCase(@"Baz\(1\)", 1, 1)] + public void VsTestTests(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", new VsTestTestsFilter(filter)); + Verify(executed, total, results); + } + + [Test, Platform("Win")] + [TestCase(@"FullyQualifiedName=Issue919.Foo.Bzzt", 1, 1)] // Sanity check + [TestCase(@"FullyQualifiedName=Issue919.Foo.Bar\(1\)", 0, 0)] + [TestCase(@"FullyQualifiedName=Issue919.Foo.Baz\(1\)", 1, 1)] + [TestCase(@"Name=Bzzt", 1, 1)] // Sanity check + [TestCase(@"Name=Bar\(1\)", 0, 0)] + [TestCase(@"Name=Baz\(1\)", 1, 1)] + [TestCase(@"", 2, 2)] + public void DotnetTestCases(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.DotNetTest(filter, true, true, TestContext.WriteLine); + Verify(executed, total, results); + } + } +} \ No newline at end of file diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/PropertyTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/PropertyTests.cs new file mode 100644 index 00000000..95afad49 --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/PropertyTests.cs @@ -0,0 +1,58 @@ +using NUnit.Framework; +using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools; + +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance +{ + public sealed class PropertyTests : CsProjAcceptanceTests + { + protected override void AddTestsCs(IsolatedWorkspace workspace) + { + workspace.AddFile("Issue779.cs", @" + using System; + using NUnit.Framework; + + namespace Issue779 + { + public class PropertyTest + { + [Property(""Bug"", ""12345"")] + [Test] + public void Test1() + { + Assert.Pass(); + } + + [Test] + public void Test2() + { + Assert.Pass(); + } + } + }"); + } + + protected override string Framework => Frameworks.NetCoreApp31; + + [Test, Platform("Win")] + [TestCase("Bug=99999", 0, 0)] + [TestCase("Bug=12345", 1, 1)] + [TestCase("Bug!=12345", 1, 1)] + public void DotNetTest(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.DotNetTest(filter, true, true, TestContext.WriteLine); + Verify(executed, total, results); + } + + [Test, Platform("Win")] + [TestCase("Bug=99999", 0, 0)] + [TestCase("Bug=12345", 1, 1)] + [TestCase("Bug!=12345", 1, 1)] + public void VsTest(string filter, int executed, int total) + { + var workspace = Build(); + var results = workspace.VSTest($@"bin\Debug\{Framework}\Test.dll", new VsTestTestCaseFilter(filter)); + Verify(executed, total, results); + } + } +} \ No newline at end of file diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/SinglePassingTestResultTests.cs b/src/NUnit.TestAdapter.Tests.Acceptance/SinglePassingTestResultTests.cs index 326c141d..7225136c 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/SinglePassingTestResultTests.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/SinglePassingTestResultTests.cs @@ -61,9 +61,9 @@ public static void Single_target_csproj(string targetFramework) AddTestsCs(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); - workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll") + workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } @@ -112,9 +112,9 @@ public static void Single_target_vbproj(string targetFramework) AddTestsVb(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); - workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll") + workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } @@ -163,11 +163,11 @@ public static void Multi_target_csproj() AddTestsCs(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); foreach (var targetFramework in TargetFrameworks) { - workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll") + workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } } @@ -217,11 +217,11 @@ public static void Multi_target_vbproj() AddTestsVb(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); foreach (var targetFramework in TargetFrameworks) { - workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll") + workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } } @@ -315,9 +315,9 @@ public static void Legacy_csproj_with_PackageReference() AddTestsCs(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); - var result = workspace.VSTest(@"bin\Debug\Test.dll"); + var result = workspace.VSTest(@"bin\Debug\Test.dll", VsTestFilter.NoFilter); result.AssertSinglePassingTest(); } @@ -408,9 +408,9 @@ public static void Legacy_vbproj_with_PackageReference() AddTestsVb(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); - workspace.VSTest(@"bin\Debug\Test.dll") + workspace.VSTest(@"bin\Debug\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } @@ -510,9 +510,9 @@ public static void Legacy_csproj_with_packages_config() workspace.NuGetRestore(packagesDirectory: "packages"); - workspace.MSBuild(); + workspace.MsBuild(); - workspace.VSTest(@"bin\Debug\Test.dll") + workspace.VSTest(@"bin\Debug\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } @@ -623,9 +623,9 @@ public static void Legacy_vbproj_with_packages_config() workspace.NuGetRestore(packagesDirectory: "packages"); - workspace.MSBuild(); + workspace.MsBuild(); - workspace.VSTest(@"bin\Debug\Test.dll") + workspace.VSTest(@"bin\Debug\Test.dll", VsTestFilter.NoFilter) .AssertSinglePassingTest(); } } diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/TestSourceWithCustomNames.cs b/src/NUnit.TestAdapter.Tests.Acceptance/TestSourceWithCustomNames.cs index db14ea64..226b9451 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/TestSourceWithCustomNames.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/TestSourceWithCustomNames.cs @@ -84,6 +84,7 @@ public class Case [Test, Platform("Win")] [TestCase("net48")] // test code requires ValueTuple support, so can't got to net35 [TestCase("netcoreapp2.1")] + [TestCase("net5.0")] public static void Single_target_csproj(string targetFramework) { var workspace = CreateWorkspace() @@ -104,9 +105,9 @@ public static void Single_target_csproj(string targetFramework) AddTestsCs(workspace); - workspace.MSBuild(restore: true); + workspace.MsBuild(restore: true); - var results = workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll"); + var results = workspace.VSTest($@"bin\Debug\{targetFramework}\Test.dll", VsTestFilter.NoFilter); // Total Tests = // 3 from PassingTestStr/TestCaseSourceMethod diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.RunSettings.cs b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.RunSettings.cs index 8521961b..a0e0eba2 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.RunSettings.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.RunSettings.cs @@ -7,11 +7,13 @@ partial class IsolatedWorkspace { private sealed class RunSettings { - private readonly List arguments = new (); + private List Arguments { get; } = new(); public string WorkingDirectory { get; } public string FileName { get; } + public string ArgumentsAsEscapedString => ProcessUtils.EscapeProcessArguments(Arguments); + public RunSettings(string workingDirectory, string fileName) { WorkingDirectory = workingDirectory; @@ -20,34 +22,34 @@ public RunSettings(string workingDirectory, string fileName) public ProcessRunResult Run(bool throwOnError = true) { - var result = ProcessUtils.Run(WorkingDirectory, FileName, arguments); + var result = ProcessUtils.Run(WorkingDirectory, FileName, Arguments); if (throwOnError) result.ThrowIfError(); return result; } public RunSettings Add(string argument) { - arguments.Add(argument); + Arguments.Add(argument); return this; } public RunSettings AddRange(IEnumerable arguments) { if (arguments is null) throw new ArgumentNullException(nameof(arguments)); - this.arguments.AddRange(arguments); + Arguments.AddRange(arguments); return this; } public RunSettings AddIf(bool condition, string argument) { - if (condition) arguments.Add(argument); + if (condition) Arguments.Add(argument); return this; } public RunSettings AddRangeIf(bool condition, IEnumerable arguments) { if (arguments is null) throw new ArgumentNullException(nameof(arguments)); - if (condition) this.arguments.AddRange(arguments); + if (condition) Arguments.AddRange(arguments); return this; } } diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.cs b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.cs index 0bd140d8..2a193897 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.cs @@ -8,10 +8,12 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools [DebuggerDisplay("{Directory,nq}")] public sealed partial class IsolatedWorkspace : IDisposable { - private readonly List projectPaths = new (); + private readonly List projectPaths = new(); private readonly ToolResolver toolResolver; private readonly DirectoryMutex directoryMutex; + public bool DumpTestExecution { get; set; } = false; + public string Directory => directoryMutex.DirectoryPath; public IsolatedWorkspace(DirectoryMutex directoryMutex, ToolResolver toolResolver) @@ -56,15 +58,41 @@ public void DotNetBuild(bool noRestore = false) .Run(); } - public VSTestResult DotNetTest(bool noBuild = false) + /// + /// Runs dotnet test. + /// + /// Possible filter statement. + /// if you run MSBuild or dotnet build first, set to false. + /// Set NUnit verbosity to 5, enables seing more info from the run in StdOut. + /// VSTestResults. + public VSTestResult DotNetTest(string filterArgument = "", bool noBuild = false, bool verbose = false, Action log = null) { using var tempTrxFile = new TempFile(); - var result = ConfigureRun("dotnet") + var dotnettest = ConfigureRun("dotnet") .Add("test") .AddIf(noBuild, "--no-build") - .Add("--logger").Add("trx;LogFileName=" + tempTrxFile) - .Run(throwOnError: false); + .Add("-v:n") + .Add("--logger").Add("trx;LogFileName=" + tempTrxFile); + + bool hasNUnitWhere = filterArgument.StartsWith("NUnit.Where"); + + if (filterArgument.Length > 0 && !hasNUnitWhere) + { + dotnettest.Add("--filter").Add($"{filterArgument}"); + } + else if (hasNUnitWhere) + { + dotnettest.Add("--").Add(filterArgument); + } + if (verbose) + { + if (!hasNUnitWhere) + dotnettest.Add("--"); + dotnettest.Add("NUnit.Verbosity=5"); + } + log?.Invoke($"\n{dotnettest.ArgumentsAsEscapedString}"); + var result = dotnettest.Run(throwOnError: false); if (new FileInfo(tempTrxFile).Length == 0) result.ThrowIfError(); @@ -88,7 +116,7 @@ public void NuGetRestore(string packagesDirectory = null) .Run(); } - public void MSBuild(string target = null, bool restore = false) + public void MsBuild(string target = null, bool restore = false) { ConfigureRun(toolResolver.MSBuild) .AddIf(target != null, "/t:" + target) @@ -96,21 +124,33 @@ public void MSBuild(string target = null, bool restore = false) .Run(); } - public VSTestResult VSTest(string testAssemblyPath) + public VSTestResult VSTest(string testAssemblyPath, IFilterArgument filter) { using var tempTrxFile = new TempFile(); - var result = ConfigureRun(toolResolver.VSTest) + var vstest = ConfigureRun(toolResolver.VSTest) .Add(testAssemblyPath) - .Add("/logger:trx;LogFileName=" + tempTrxFile) - .Run(throwOnError: false); + .Add("/logger:trx;LogFileName=" + tempTrxFile); + + if (filter.HasArguments) + { + vstest.Add(filter.CompletedArgument()); + } + + if (DumpTestExecution) + vstest.Add("--").Add("NUnit.DumpXmlTestResults=true"); + + var result = vstest.Run(throwOnError: false); if (new FileInfo(tempTrxFile).Length == 0) + { result.ThrowIfError(); + return new VSTestResult(result); + } return VSTestResult.Load(result, tempTrxFile); } - private RunSettings ConfigureRun(string filename) => new (Directory, filename); + private RunSettings ConfigureRun(string filename) => new(Directory, filename); } } diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/ToolResolver.cs b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/ToolResolver.cs index 8f1f71f2..52013a30 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/ToolResolver.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/ToolResolver.cs @@ -28,10 +28,7 @@ public ToolResolver(string downloadCacheDirectory) this.downloadCacheDirectory = downloadCacheDirectory; - nuGet = new Lazy(() => - { - return FindDownloadedTool("NuGet", "nuget.exe", "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe"); - }); + nuGet = new Lazy(() => FindDownloadedTool("NuGet", "nuget.exe", "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe")); msBuild = new Lazy(() => { @@ -57,12 +54,9 @@ public ToolResolver(string downloadCacheDirectory) return Path.Combine(vsInstallation, @"Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"); }); - vsWhere = new Lazy(() => - { - return Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), - @"Microsoft Visual Studio\Installer\vswhere.exe"); - }); + vsWhere = new Lazy(() => Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), + @"Microsoft Visual Studio\Installer\vswhere.exe")); } private string FindDownloadedTool(string id, string fileName, string downloadUrl) @@ -74,10 +68,8 @@ private string FindDownloadedTool(string id, string fileName, string downloadUrl { Directory.CreateDirectory(directory); - using (var client = new WebClient()) - { - client.DownloadFile(downloadUrl, toolPath); - } + using var client = new WebClient(); + client.DownloadFile(downloadUrl, toolPath); } return toolPath; diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResult.cs b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResult.cs index ceaefa7f..b023aaba 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResult.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResult.cs @@ -17,6 +17,16 @@ public readonly struct VSTestResult public IReadOnlyList RunErrors { get; } public IReadOnlyList RunWarnings { get; } + public VSTestResult(ProcessRunResult processRunResult) + { + ProcessRunResult = processRunResult; + Outcome = ""; + Counters = VSTestResultCounters.CreateEmptyCounters(); + RunErrors = Array.Empty(); + RunWarnings = Array.Empty(); + } + + public VSTestResult(ProcessRunResult processRunResult, string outcome, VSTestResultCounters counters, IReadOnlyList runErrors = null, IReadOnlyList runWarnings = null) { ProcessRunResult = processRunResult; diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResultCounters.cs b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResultCounters.cs index 5fde67a8..c91a6f8c 100644 --- a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResultCounters.cs +++ b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VSTestResultCounters.cs @@ -22,6 +22,8 @@ public readonly struct VSTestResultCounters public int InProgress { get; } public int Pending { get; } + public static VSTestResultCounters CreateEmptyCounters() => new (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + public VSTestResultCounters( int total, int executed, diff --git a/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VsTestFilter.cs b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VsTestFilter.cs new file mode 100644 index 00000000..442b89ad --- /dev/null +++ b/src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/VsTestFilter.cs @@ -0,0 +1,49 @@ +namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools +{ + public interface IFilterArgument + { + string CompletedArgument(); + bool HasArguments { get; } + } + + public abstract class VsTestFilter : IFilterArgument + { + protected string Arguments { get; } + public bool HasArguments => Arguments.Length > 0; + protected VsTestFilter(string arguments) + { + Arguments = arguments; + } + + public abstract string CompletedArgument(); + + public static IFilterArgument NoFilter => new VsTestTestCaseFilter(""); + } + + + + public class VsTestTestCaseFilter : VsTestFilter + { + public VsTestTestCaseFilter(string arguments) : base(arguments) + { + } + + public override string CompletedArgument() + { + var completeFilterStatement = $"/TestCaseFilter:{Arguments}"; + return completeFilterStatement; + } + } + + public class VsTestTestsFilter : VsTestFilter + { + public VsTestTestsFilter(string arguments) : base(arguments) + { + } + public override string CompletedArgument() + { + var completeFilterStatement = $"/Tests:{Arguments}"; + return completeFilterStatement; + } + } +} diff --git a/src/NUnitTestAdapter/AdapterSettings.cs b/src/NUnitTestAdapter/AdapterSettings.cs index a9771841..eadeed36 100644 --- a/src/NUnitTestAdapter/AdapterSettings.cs +++ b/src/NUnitTestAdapter/AdapterSettings.cs @@ -25,6 +25,7 @@ using System.Collections.Generic; using System.IO; using System.Xml; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; @@ -125,6 +126,10 @@ public interface IAdapterSettings bool Debug { get; } bool DebugExecution { get; } bool DebugDiscovery { get; } + + // Filter control + ExplicitModeEnum ExplicitMode { get; } + bool SkipExecutionWhenNoTests { get; } } public enum VsTestCategoryType @@ -263,7 +268,8 @@ public AdapterSettings(ITestLogger logger) public char FullnameSeparator { get; private set; } = ':'; - + public ExplicitModeEnum ExplicitMode { get; private set; } = ExplicitModeEnum.Strict; + public bool SkipExecutionWhenNoTests { get; private set; } #region NUnit Diagnostic properties @@ -332,11 +338,12 @@ public void Load(string settingsXml) ShowInternalProperties = GetInnerTextAsBool(nunitNode, nameof(ShowInternalProperties), false); UseParentFQNForParametrizedTests = GetInnerTextAsBool(nunitNode, nameof(UseParentFQNForParametrizedTests), false); UseNUnitIdforTestCaseId = GetInnerTextAsBool(nunitNode, nameof(UseNUnitIdforTestCaseId), false); - ConsoleOut = GetInnerTextAsInt(nunitNode, nameof(ConsoleOut), 1); // 0 no output to console, 1 : output to console + ConsoleOut = GetInnerTextAsInt(nunitNode, nameof(ConsoleOut), 2); // 0 no output to console, 1 : output to console StopOnError = GetInnerTextAsBool(nunitNode, nameof(StopOnError), false); UseNUnitFilter = GetInnerTextAsBool(nunitNode, nameof(UseNUnitFilter), true); IncludeStackTraceForSuites = GetInnerTextAsBool(nunitNode, nameof(IncludeStackTraceForSuites), true); - EnsureAttachmentFileScheme = GetInnerTextAsBool(nunitNode, nameof(IncludeStackTraceForSuites), false); + EnsureAttachmentFileScheme = GetInnerTextAsBool(nunitNode, nameof(EnsureAttachmentFileScheme), false); + SkipExecutionWhenNoTests = GetInnerTextAsBool(nunitNode, nameof(SkipExecutionWhenNoTests), false); // Engine settings DiscoveryMethod = MapEnum(GetInnerText(nunitNode, nameof(DiscoveryMethod), Verbosity > 0), DiscoveryMethod.Current); @@ -344,6 +351,9 @@ public void Load(string settingsXml) AssemblySelectLimit = GetInnerTextAsInt(nunitNode, nameof(AssemblySelectLimit), 2000); + ExplicitMode = MapEnum(GetInnerText(nunitNode, nameof(ExplicitMode), Verbosity > 0), ExplicitModeEnum.Strict); + + ExtractNUnitDiagnosticSettings(nunitNode); // Adapter Display Options @@ -615,4 +625,10 @@ public T MapEnum(string setting, T defaultValue) #endregion } + + public enum ExplicitModeEnum + { + Strict, + Relaxed + } } diff --git a/src/NUnitTestAdapter/Execution.cs b/src/NUnitTestAdapter/Execution.cs index 80780d1f..b38f4bb6 100644 --- a/src/NUnitTestAdapter/Execution.cs +++ b/src/NUnitTestAdapter/Execution.cs @@ -1,4 +1,5 @@ using System; + using NUnit.Engine; using NUnit.VisualStudio.TestAdapter.Dump; using NUnit.VisualStudio.TestAdapter.NUnitEngine; @@ -62,43 +63,73 @@ public virtual bool Run(TestFilter filter, DiscoveryConverter discovery, NUnit3T public abstract TestFilter CheckFilterInCurrentMode(TestFilter filter, IDiscoveryConverter discovery); protected NUnitTestFilterBuilder CreateTestFilterBuilder() - => new (NUnitEngineAdapter.GetService(), Settings); + => new(NUnitEngineAdapter.GetService(), Settings); protected ITestConverterCommon CreateConverter(DiscoveryConverter discovery) => Settings.DiscoveryMethod == DiscoveryMethod.Current ? discovery.TestConverter : discovery.TestConverterForXml; - protected TestFilter CheckFilter(IDiscoveryConverter discovery) + protected TestFilter CheckFilter(TestFilter testFilter, IDiscoveryConverter discovery) { - TestFilter filter; - if (discovery.NoOfLoadedTestCasesAboveLimit) + if (discovery.NoOfLoadedTestCasesAboveLimit && !testFilter.IsCategoryFilter()) { TestLog.Debug("Setting filter to empty due to number of testcases"); - filter = TestFilter.Empty; + var filter = TestFilter.Empty; + return filter; } - else + if (testFilter.IsCategoryFilter()) { - var filterBuilder = CreateTestFilterBuilder(); - filter = filterBuilder.FilterByList(discovery.LoadedTestCases); + if (!discovery.IsExplicitRun && discovery.HasExplicitTests && Settings.ExplicitMode == ExplicitModeEnum.Strict) + { + var filterExt = new TestFilter($"true"); + var combiner = new TestFilterCombiner(testFilter, filterExt); + return combiner.GetFilter(); + } + return testFilter; } - return filter; + var filterBuilder = CreateTestFilterBuilder(); + return filterBuilder.FilterByList(discovery.LoadedTestCases); } } - public class IdeExecution : Execution + public class TestFilterCombiner { - public IdeExecution(IExecutionContext ctx) : base(ctx) + private readonly TestFilter _a; + private readonly TestFilter _b; + + public TestFilterCombiner(TestFilter a, TestFilter b) { + _a = a; + _b = b; } - public override bool Run(TestFilter filter, DiscoveryConverter discovery, NUnit3TestExecutor nUnit3TestExecutor) + + public TestFilter GetFilter() { - return base.Run(filter, discovery, nUnit3TestExecutor); + var innerA = StripFilter(_a); + var innerB = StripFilter(_b); + var inner = $"{innerA}{innerB}"; + return new TestFilter(inner); } + private string StripFilter(TestFilter x) + { + var s = x.Text.Replace("", ""); + var s2 = s.Replace("", ""); + return s2; + } + } + + + + public class IdeExecution : Execution + { + public IdeExecution(IExecutionContext ctx) : base(ctx) + { + } public override TestFilter CheckFilterInCurrentMode(TestFilter filter, IDiscoveryConverter discovery) { if (!discovery.IsDiscoveryMethodCurrent) return filter; if (filter.IsEmpty()) return filter; - filter = CheckFilter(discovery); + filter = CheckFilter(filter, discovery); return filter; } } @@ -126,23 +157,16 @@ public TestFilter CheckVsTestFilter(TestFilter filter, IDiscoveryConverter disco // If we have a VSTest TestFilter, convert it to an nunit filter if (vsTestFilter == null || vsTestFilter.IsEmpty) return filter; - TestLog.Debug( - $"TfsFilter used, length: {vsTestFilter.TfsTestCaseFilterExpression?.TestCaseFilterValue.Length}"); + TestLog.Debug($"TfsFilter used, length: {vsTestFilter.TfsTestCaseFilterExpression?.TestCaseFilterValue.Length}"); // NOTE This overwrites filter used in call var filterBuilder = CreateTestFilterBuilder(); - if (Settings.DiscoveryMethod == DiscoveryMethod.Current) - { - filter = Settings.UseNUnitFilter + filter = Settings.DiscoveryMethod == DiscoveryMethod.Current + ? Settings.UseNUnitFilter ? filterBuilder.ConvertVsTestFilterToNUnitFilter(vsTestFilter) - : filterBuilder.ConvertTfsFilterToNUnitFilter(vsTestFilter, discovery); - } - else - { - filter = filterBuilder - .ConvertTfsFilterToNUnitFilter(vsTestFilter, discovery.LoadedTestCases); - } + : filterBuilder.ConvertTfsFilterToNUnitFilter(vsTestFilter, discovery) + : filterBuilder.ConvertTfsFilterToNUnitFilter(vsTestFilter, discovery.LoadedTestCases); - Dump?.AddString($"\n\nTFSFilter: {vsTestFilter.TfsTestCaseFilterExpression.TestCaseFilterValue}\n"); + Dump?.AddString($"\n\nTFSFilter: {vsTestFilter.TfsTestCaseFilterExpression?.TestCaseFilterValue}\n"); Dump?.DumpVSInputFilter(filter, "(At Execution (TfsFilter)"); return filter; @@ -151,22 +175,28 @@ public override TestFilter CheckFilterInCurrentMode(TestFilter filter, IDiscover { if (!discovery.IsDiscoveryMethodCurrent) return filter; - if ((VsTestFilter == null || VsTestFilter.IsEmpty) && filter != TestFilter.Empty) + if (filter != TestFilter.Empty) { - filter = CheckFilter(discovery); + filter = CheckFilter(filter, discovery); } - else if (VsTestFilter != null && !VsTestFilter.IsEmpty && !Settings.UseNUnitFilter) + else if (VsTestFilter is { IsEmpty: false } && !Settings.UseNUnitFilter) { var s = VsTestFilter.TfsTestCaseFilterExpression.TestCaseFilterValue; var scount = s.Split('|', '&').Length; - if (scount > Settings.AssemblySelectLimit) - { - TestLog.Debug("Setting filter to empty due to TfsFilter size"); - filter = TestFilter.Empty; - } + filter = CheckAssemblySelectLimit(filter, scount); } return filter; } + + private TestFilter CheckAssemblySelectLimit(TestFilter filter, int scount) + { + if (scount <= Settings.AssemblySelectLimit) + { + return filter; + } + TestLog.Debug("Setting filter to empty due to TfsFilter size"); + return TestFilter.Empty; + } } } diff --git a/src/NUnitTestAdapter/NUnit.TestAdapter.csproj b/src/NUnitTestAdapter/NUnit.TestAdapter.csproj index 836ff568..1869fa83 100644 --- a/src/NUnitTestAdapter/NUnit.TestAdapter.csproj +++ b/src/NUnitTestAdapter/NUnit.TestAdapter.csproj @@ -12,12 +12,6 @@ true - - Program - $(DevEnvDir)\devenv.exe - /rootsuffix Exp - - NUnit3TestAdapter Charlie Poole, Terje Sandstrom diff --git a/src/NUnitTestAdapter/NUnit3TestExecutor.cs b/src/NUnitTestAdapter/NUnit3TestExecutor.cs index ed057d17..25fc5f8c 100644 --- a/src/NUnitTestAdapter/NUnit3TestExecutor.cs +++ b/src/NUnitTestAdapter/NUnit3TestExecutor.cs @@ -143,7 +143,8 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame Unload(); } - private void SetRunTypeByStrings() => + private void SetRunTypeByStrings() + { RunType = !Settings.DesignMode ? Settings.DiscoveryMethod == DiscoveryMethod.Legacy ? RunType.CommandLineLegacy @@ -151,9 +152,11 @@ private void SetRunTypeByStrings() => ? RunType.CommandLineCurrentNUnit : RunType.CommandLineCurrentVSTest : RunType.Ide; + TestLog.Debug($"Runtype: {RunType}"); + } /// - /// Called by the VisualStudio IDE when selected tests are to be run. Never called from TFS Build. + /// Called by the VisualStudio IDE when selected tests are to be run. Never called from TFS Build, except (at least 2022, probably also 2019) when vstest.console uses /test: then this is being used. /// /// The tests to be run. /// The RunContext. @@ -162,9 +165,9 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame { Initialize(runContext, frameworkHandle); CheckIfDebug(); - TestLog.Debug("RunTests by IEnumerable"); InitializeForExecution(runContext, frameworkHandle); RunType = RunType.Ide; + TestLog.Debug("RunTests by IEnumerable. RunType = Ide"); var timing = new TimingLogger(Settings, TestLog); Debug.Assert(NUnitEngineAdapter != null, "NUnitEngineAdapter is null"); Debug.Assert(NUnitEngineAdapter.EngineEnabled, "NUnitEngineAdapter TestEngine is null"); @@ -280,8 +283,15 @@ private void RunAssembly(string assemblyPath, IGrouping testCa { var discovery = new DiscoveryConverter(TestLog, Settings); discovery.Convert(discoveryResults, assemblyPath); - var ea = ExecutionFactory.Create(this); - ea.Run(filter, discovery, this); + if (!Settings.SkipExecutionWhenNoTests || discovery.AllTestCases.Any()) + { + var ea = ExecutionFactory.Create(this); + ea.Run(filter, discovery, this); + } + else + { + TestLog.InfoNoTests(assemblyPath); + } } else { diff --git a/src/NUnitTestAdapter/NUnitEngine/DiscoveryConverter.cs b/src/NUnitTestAdapter/NUnitEngine/DiscoveryConverter.cs index 2573418d..a031b2f1 100644 --- a/src/NUnitTestAdapter/NUnitEngine/DiscoveryConverter.cs +++ b/src/NUnitTestAdapter/NUnitEngine/DiscoveryConverter.cs @@ -46,7 +46,9 @@ public interface IDiscoveryConverter bool IsDiscoveryMethodCurrent { get; } bool NoOfLoadedTestCasesAboveLimit { get; } - IEnumerable CheckTestCasesExplicit(IEnumerable filteredTestCases); + int NoOfExplicitTestCases { get; } + bool HasExplicitTests { get; } + IEnumerable GetExplicitTestCases(IEnumerable filteredTestCases); } public class DiscoveryConverter : IDiscoveryConverter @@ -92,6 +94,10 @@ internal static class NUnitXmlAttributeNames public bool IsExplicitRun => CurrentTestAssembly?.IsExplicit ?? false; + public int NoOfExplicitTestCases => CurrentTestAssembly.NoOfExplicitTestCases; + + public bool HasExplicitTests => NoOfExplicitTestCases > 0; + private readonly List loadedTestCases = new (); public IList LoadedTestCases => loadedTestCases; @@ -103,7 +109,7 @@ internal static class NUnitXmlAttributeNames private ITestLogger TestLog { get; } public bool NoOfLoadedTestCasesAboveLimit => NoOfLoadedTestCases > Settings.AssemblySelectLimit; - public IEnumerable CheckTestCasesExplicit(IEnumerable filteredTestCases) + public IEnumerable GetExplicitTestCases(IEnumerable filteredTestCases) { var explicitCases = new List(); foreach (var tc in filteredTestCases) diff --git a/src/NUnitTestAdapter/NUnitEngine/Extensions.cs b/src/NUnitTestAdapter/NUnitEngine/Extensions.cs index 38e01a4b..b717cced 100644 --- a/src/NUnitTestAdapter/NUnitEngine/Extensions.cs +++ b/src/NUnitTestAdapter/NUnitEngine/Extensions.cs @@ -38,5 +38,12 @@ public static bool AllWithEmptyFalse(this IEnumerable list, Func list.All(pred) && list.Any(); public static bool IsEmpty(this TestFilter filter) => filter == TestFilter.Empty; + + + public static bool IsCategoryFilter(this TestFilter filter) => + filter != TestFilter.Empty && filter.Text.Contains(""); + + public static bool IsNegativeCategoryFilter(this TestFilter filter) => + filter.IsCategoryFilter() && filter.Text.Contains(""); } } diff --git a/src/NUnitTestAdapter/NUnitEngine/NUnitDiscoveryTestClasses.cs b/src/NUnitTestAdapter/NUnitEngine/NUnitDiscoveryTestClasses.cs index f1da9719..ceea0925 100644 --- a/src/NUnitTestAdapter/NUnitEngine/NUnitDiscoveryTestClasses.cs +++ b/src/NUnitTestAdapter/NUnitEngine/NUnitDiscoveryTestClasses.cs @@ -145,6 +145,8 @@ public NUnitDiscoveryTestAssembly(BaseProperties theBase, NUnitDiscoveryTestRun /// public IEnumerable RunnableTestCases => allTestCases.Where(c => !c.IsExplicitReverse); + public int NoOfExplicitTestCases => allTestCases.Count(c => c.IsExplicitReverse); + public void AddTestSuiteToAssembly(NUnitDiscoveryTestSuite ts) { AddTestSuite(ts); @@ -227,7 +229,7 @@ public interface INUnitCommonTestCase string FullName { get; } string ClassName { get; } string MethodName { get; } - long Seed { get; } + long Seed { get; } } public sealed class NUnitDiscoveryTestCase : NUnitDiscoverySuiteBase, INUnitDiscoveryTestCase, INUnitCommonTestCase diff --git a/src/NUnitTestAdapter/NUnitEngine/NUnitEngineAdapter.cs b/src/NUnitTestAdapter/NUnitEngine/NUnitEngineAdapter.cs index 9ab7666e..a366ec43 100644 --- a/src/NUnitTestAdapter/NUnitEngine/NUnitEngineAdapter.cs +++ b/src/NUnitTestAdapter/NUnitEngine/NUnitEngineAdapter.cs @@ -87,7 +87,10 @@ public NUnitResults Explore(TestFilter filter) { var timing = new TimingLogger(settings, logger); var results = new NUnitResults(Runner.Explore(filter)); - timing.LogTime("Execution engine discovery time "); + timing.LogTime($"Execution engine discovery time with filter length {filter.Text.Length}"); + if (filter.Text.Length < 300) + logger.Debug($"Filter: {filter.Text}"); + return results; } @@ -95,7 +98,9 @@ public NUnitResults Run(ITestEventListener listener, TestFilter filter) { var timing = new TimingLogger(settings, logger); var results = new NUnitResults(Runner.Run(listener, filter)); - timing.LogTime("Execution engine run time "); + timing.LogTime($"Execution engine run time with filter length {filter.Text.Length}"); + if (filter.Text.Length < 300) + logger.Debug($"Filter: {filter.Text}"); return results; } diff --git a/src/NUnitTestAdapter/NUnitEngine/NUnitResults.cs b/src/NUnitTestAdapter/NUnitEngine/NUnitResults.cs index 84e5c238..ba432c02 100644 --- a/src/NUnitTestAdapter/NUnitEngine/NUnitResults.cs +++ b/src/NUnitTestAdapter/NUnitEngine/NUnitResults.cs @@ -56,11 +56,11 @@ public NUnitResults(XmlNode results) public SkipReason WhatSkipReason() { var msgNode = TopNode.SelectSingleNode("properties/property[@name='_SKIPREASON']"); - if (msgNode != null && - new[] { "contains no tests", "Has no TestFixtures" }.Any(msgNode.GetAttribute("value") - .Contains)) - return SkipReason.NoNUnitTests; - return SkipReason.LoadFailure; + return msgNode != null && + new[] { "contains no tests", "Has no TestFixtures" }.Any(msgNode.GetAttribute("value") + .Contains) + ? SkipReason.NoNUnitTests + : SkipReason.LoadFailure; } public bool HasNoNUnitTests => WhatSkipReason() == SkipReason.NoNUnitTests; diff --git a/src/NUnitTestAdapter/NUnitEventListener.cs b/src/NUnitTestAdapter/NUnitEventListener.cs index 180f4add..081a812b 100644 --- a/src/NUnitTestAdapter/NUnitEventListener.cs +++ b/src/NUnitTestAdapter/NUnitEventListener.cs @@ -158,14 +158,17 @@ public void TestFinished(INUnitTestEventTestCase resultNode) } var result = testConverter.GetVsTestResults(resultNode, outputNodes ?? EmptyNodes); - if (settings.ConsoleOut == 1) + if (settings.ConsoleOut >= 1) { if (!result.ConsoleOutput.IsNullOrWhiteSpace() && result.ConsoleOutput != NL) { string msg = result.ConsoleOutput; if (settings.UseTestNameInConsoleOutput) msg = $"{resultNode.Name}: {msg}"; - recorder.SendMessage(TestMessageLevel.Informational, msg); + var messageLevel = settings.ConsoleOut == 1 + ? TestMessageLevel.Informational + : TestMessageLevel.Warning; + recorder.SendMessage(messageLevel, msg); } if (!resultNode.ReasonMessage.IsNullOrWhiteSpace()) { diff --git a/src/NUnitTestAdapter/NUnitTestAdapter.cs b/src/NUnitTestAdapter/NUnitTestAdapter.cs index 754f662c..ef879998 100644 --- a/src/NUnitTestAdapter/NUnitTestAdapter.cs +++ b/src/NUnitTestAdapter/NUnitTestAdapter.cs @@ -242,14 +242,7 @@ protected TestPackage CreateTestPackage(string assemblyName, IGrouping testCases) { if (testCases.Count() > settings.AssemblySelectLimit) + { + // Need to log that filter has been set to empty due to AssemblySelectLimit return TestFilter.Empty; - var filterBuilder = _filterService.GetTestFilterBuilder(); + } + var filterBuilder = _filterService.GetTestFilterBuilder(); foreach (var testCase in testCases) { filterBuilder.AddTest(testCase.FullyQualifiedName); diff --git a/src/NUnitTestAdapter/TestConverter.cs b/src/NUnitTestAdapter/TestConverter.cs index c19be76f..00d259d5 100644 --- a/src/NUnitTestAdapter/TestConverter.cs +++ b/src/NUnitTestAdapter/TestConverter.cs @@ -90,7 +90,7 @@ public TestCase GetCachedTestCase(string id) if (_vsTestCaseMap.ContainsKey(id)) return _vsTestCaseMap[id]; - _logger.Warning("Test " + id + " not found in cache"); + _logger.Debug("Test " + id + " not found in cache"); return null; } @@ -227,15 +227,16 @@ bool CheckCodeFilePathOverride() private string CreateDisplayName(string fullyQualifiedName, string testNodeName) { - if (adapterSettings.FreakMode) - return "N:Name -> MS:DisplayName (default)"; - return adapterSettings.DisplayName switch - { - DisplayNameOptions.Name => testNodeName, - DisplayNameOptions.FullName => fullyQualifiedName, - DisplayNameOptions.FullNameSep => fullyQualifiedName.Replace('.', adapterSettings.FullnameSeparator), - _ => throw new ArgumentOutOfRangeException(), - }; + return adapterSettings.FreakMode + ? "N:Name -> MS:DisplayName (default)" + : adapterSettings.DisplayName switch + { + DisplayNameOptions.Name => testNodeName, + DisplayNameOptions.FullName => fullyQualifiedName, + DisplayNameOptions.FullNameSep => + fullyQualifiedName.Replace('.', adapterSettings.FullnameSeparator), + _ => throw new ArgumentOutOfRangeException(), + }; } private VSTestResult MakeTestResultFromLegacyXmlNode(INUnitTestEventTestCase resultNode, IEnumerable outputNodes) diff --git a/src/NUnitTestAdapter/TestConverterForXml.cs b/src/NUnitTestAdapter/TestConverterForXml.cs index 61527c78..2e9f3396 100644 --- a/src/NUnitTestAdapter/TestConverterForXml.cs +++ b/src/NUnitTestAdapter/TestConverterForXml.cs @@ -262,15 +262,16 @@ bool CheckCodeFilePathOverride() private string CreateDisplayName(string fullyQualifiedName, string testNodeName) { - if (adapterSettings.FreakMode) - return "N:Name -> MS:DisplayName (default)"; - return adapterSettings.DisplayName switch - { - DisplayNameOptions.Name => testNodeName, - DisplayNameOptions.FullName => fullyQualifiedName, - DisplayNameOptions.FullNameSep => fullyQualifiedName.Replace('.', adapterSettings.FullnameSeparator), - _ => throw new ArgumentOutOfRangeException(), - }; + return adapterSettings.FreakMode + ? "N:Name -> MS:DisplayName (default)" + : adapterSettings.DisplayName switch + { + DisplayNameOptions.Name => testNodeName, + DisplayNameOptions.FullName => fullyQualifiedName, + DisplayNameOptions.FullNameSep => + fullyQualifiedName.Replace('.', adapterSettings.FullnameSeparator), + _ => throw new ArgumentOutOfRangeException(), + }; } private VSTestResult MakeTestResultFromLegacyXmlNode(INUnitTestEventTestCase resultNode, IEnumerable outputNodes) diff --git a/src/NUnitTestAdapter/TestFilterConverter/TestFilterParser.cs b/src/NUnitTestAdapter/TestFilterConverter/TestFilterParser.cs index 48f38eef..94470833 100644 --- a/src/NUnitTestAdapter/TestFilterConverter/TestFilterParser.cs +++ b/src/NUnitTestAdapter/TestFilterConverter/TestFilterParser.cs @@ -35,23 +35,23 @@ public class TestFilterParser { private Tokenizer _tokenizer; - private static readonly Token LPAREN = new (TokenKind.Symbol, "("); - private static readonly Token RPAREN = new (TokenKind.Symbol, ")"); - private static readonly Token AND_OP = new (TokenKind.Symbol, "&"); - private static readonly Token OR_OP = new (TokenKind.Symbol, "|"); - private static readonly Token NOT_OP = new (TokenKind.Symbol, "!"); + private static readonly Token LPAREN = new(TokenKind.Symbol, "("); + private static readonly Token RPAREN = new(TokenKind.Symbol, ")"); + private static readonly Token AND_OP = new(TokenKind.Symbol, "&"); + private static readonly Token OR_OP = new(TokenKind.Symbol, "|"); + private static readonly Token NOT_OP = new(TokenKind.Symbol, "!"); - private static readonly Token EQ_OP = new (TokenKind.Symbol, "="); - private static readonly Token NE_OP = new (TokenKind.Symbol, "!="); - private static readonly Token CONTAINS_OP = new (TokenKind.Symbol, "~"); - private static readonly Token NOTCONTAINS_OP = new (TokenKind.Symbol, "!~"); + private static readonly Token EQ_OP = new(TokenKind.Symbol, "="); + private static readonly Token NE_OP = new(TokenKind.Symbol, "!="); + private static readonly Token CONTAINS_OP = new(TokenKind.Symbol, "~"); + private static readonly Token NOTCONTAINS_OP = new(TokenKind.Symbol, "!~"); private static readonly Token[] AND_OPS = { AND_OP }; private static readonly Token[] OR_OPS = { OR_OP }; private static readonly Token[] EQ_OPS = { EQ_OP }; private static readonly Token[] REL_OPS = { EQ_OP, NE_OP, CONTAINS_OP, NOTCONTAINS_OP }; - private static readonly Token EOF = new (TokenKind.Eof); + private static readonly Token EOF = new(TokenKind.Eof); public string Parse(string input) { @@ -140,7 +140,7 @@ public string ParseFilterCondition() { case "FullyQualifiedName": rhs = Expect(TokenKind.FQN, TokenKind.Word); - return EmitFullNameFilter(op, rhs.Text); + return EmitFullNameFilter(op, UnEscape(rhs.Text)); case "TestCategory": rhs = Expect(TokenKind.Word); return EmitCategoryFilter(op, rhs.Text); @@ -149,7 +149,7 @@ public string ParseFilterCondition() return EmitPropertyFilter(op, lhs.Text, rhs.Text); case "Name": rhs = Expect(TokenKind.FQN, TokenKind.Word); - return EmitNameFilter(op, rhs.Text); + return EmitNameFilter(op, UnEscape(rhs.Text)); default: // Assume it's a property name @@ -158,6 +158,11 @@ public string ParseFilterCondition() } } + private string UnEscape(string rhs) + { + return rhs.Replace(@"\(", "(").Replace(@"\)", ")"); + } + private static string EmitFullNameFilter(Token op, string value) { return EmitFilter("test", op, value); diff --git a/src/NUnitTestAdapter/TestLogger.cs b/src/NUnitTestAdapter/TestLogger.cs index 1341ab14..a0102d35 100644 --- a/src/NUnitTestAdapter/TestLogger.cs +++ b/src/NUnitTestAdapter/TestLogger.cs @@ -166,6 +166,11 @@ public void InfoNoTests(bool discoveryResultsHasNoNUnitTests, string assemblyPat ? " NUnit couldn't find any tests in " + assemblyPath : " NUnit failed to load " + assemblyPath); } + + public void InfoNoTests(string assemblyPath) + { + Info($" NUnit couldn't find any tests in {assemblyPath}"); + } #endregion } } diff --git a/src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj b/src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj index fe79fb80..3b53375e 100644 --- a/src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj +++ b/src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj @@ -15,7 +15,7 @@ - + @@ -24,8 +24,8 @@ - - + + diff --git a/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestCaseTests.cs b/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestCaseTests.cs index 4c971a0c..b04acfc5 100644 --- a/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestCaseTests.cs +++ b/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestCaseTests.cs @@ -55,4 +55,4 @@ public void ThatRunStateIsHandledForNone() Assert.That(sut.RunState, Is.EqualTo(RunStateEnum.NA)); } } -} +} \ No newline at end of file diff --git a/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestEventsTests.cs b/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestEventsTests.cs index 88fd0c74..77311458 100644 --- a/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestEventsTests.cs +++ b/src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestEventsTests.cs @@ -342,4 +342,4 @@ public void ThatExplicitTestFixtureWorksWithZeroStartTime() Assert.DoesNotThrow(code: () => sut.StartTime()); } } -} +} \ No newline at end of file diff --git a/src/NUnitTestAdapterTests/NUnitEventListenerTests.cs b/src/NUnitTestAdapterTests/NUnitEventListenerTests.cs index be70e2fb..12f3c72c 100644 --- a/src/NUnitTestAdapterTests/NUnitEventListenerTests.cs +++ b/src/NUnitTestAdapterTests/NUnitEventListenerTests.cs @@ -107,7 +107,7 @@ public void TestFinished_CallsRecordEndCorrectly() } /// - /// Issue516 + /// Issue516. /// [TestCase(null)] [TestCase("")] @@ -217,4 +217,4 @@ private void VerifyTestResult(VSTestResult ourResult) #endregion } -} +} \ No newline at end of file diff --git a/src/NUnitTestAdapterTests/ProjectTests.cs b/src/NUnitTestAdapterTests/ProjectTests.cs index 73cd526e..cd8816a4 100644 --- a/src/NUnitTestAdapterTests/ProjectTests.cs +++ b/src/NUnitTestAdapterTests/ProjectTests.cs @@ -68,4 +68,4 @@ public static DirectoryInfo MoveUp(this DirectoryInfo di, int noOfLevels) return grandParent; } } -} +} \ No newline at end of file diff --git a/src/NUnitTestAdapterTests/TempDirectory.cs b/src/NUnitTestAdapterTests/TempDirectory.cs index b9da643b..0199290c 100644 --- a/src/NUnitTestAdapterTests/TempDirectory.cs +++ b/src/NUnitTestAdapterTests/TempDirectory.cs @@ -44,4 +44,4 @@ public void Dispose() public static implicit operator string(TempDirectory tempDirectory) => tempDirectory.Path; } -} +} \ No newline at end of file diff --git a/src/NUnitTestAdapterTests/TestConverterForXmlTests.cs b/src/NUnitTestAdapterTests/TestConverterForXmlTests.cs index d57b22ce..bddf8fff 100644 --- a/src/NUnitTestAdapterTests/TestConverterForXmlTests.cs +++ b/src/NUnitTestAdapterTests/TestConverterForXmlTests.cs @@ -86,7 +86,7 @@ public void CanMakeTestCaseShouldBuildTraitsCache() foreach (XmlNode node in xmlNodeList) { - var testCase = testConverterForXml.ConvertTestCase(new NUnitEventTestCase(node)); + testConverterForXml.ConvertTestCase(new NUnitEventTestCase(node)); } var traitsCache = testConverterForXml.TraitsCache; diff --git a/src/NUnitTestAdapterTests/TestExecutionTests.cs b/src/NUnitTestAdapterTests/TestExecutionTests.cs index 083f799c..68fbc8d2 100644 --- a/src/NUnitTestAdapterTests/TestExecutionTests.cs +++ b/src/NUnitTestAdapterTests/TestExecutionTests.cs @@ -25,8 +25,10 @@ using System.Collections.Generic; using System.IO; using System.Linq; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + using NUnit.Framework; using NUnit.Tests; using NUnit.Tests.Assemblies; @@ -75,19 +77,23 @@ public void LoadMockassembly() "The reference to mock-assembly.dll appears to be the wrong version"); } - [TestCase("", 35)] - [TestCase(null, 35)] + [TestCase("", 38, 5, 5)] + [TestCase(null, 38, 5, 5)] [TestCase("cat == Special", 1)] [TestCase("cat == MockCategory", 2)] - [TestCase("method =~ MockTest?", 5)] - [TestCase("method =~ MockTest? and cat != MockCategory", 3)] + [TestCase("method =~ MockTest?", 5, 1)] + [TestCase("method =~ MockTest? and cat != MockCategory", 3, 1)] [TestCase("namespace == ThisNamespaceDoesNotExist", 0)] - [TestCase("test==NUnit.Tests.Assemblies.MockTestFixture", MockTestFixture.Tests - MockTestFixture.Explicit, TestName = "{m}_MockTestFixture")] - [TestCase("test==NUnit.Tests.IgnoredFixture and method == Test2", 1, TestName = "{m}_IgnoredFixture")] - [TestCase("class==NUnit.Tests.Assemblies.MockTestFixture", MockTestFixture.Tests - MockTestFixture.Explicit)] - [TestCase("name==MockTestFixture", MockTestFixture.Tests + NUnit.Tests.TestAssembly.MockTestFixture.Tests - MockTestFixture.Explicit)] - [TestCase("cat==FixtureCategory", MockTestFixture.Tests - MockTestFixture.Explicit)] - public void TestsWhereShouldFilter(string filter, int expectedCount) + [TestCase("test==NUnit.Tests.Assemblies.MockTestFixture", MockTestFixture.Tests - MockTestFixture.Explicit, 1, 1, TestName = "{m}_MockTestFixture")] + [TestCase("test==NUnit.Tests.IgnoredFixture and method == Test2", 1, 1, TestName = "{m}_IgnoredFixture")] + [TestCase("class==NUnit.Tests.Assemblies.MockTestFixture", MockTestFixture.Tests - MockTestFixture.Explicit, 1, 1)] + [TestCase("name==MockTestFixture", MockTestFixture.Tests + NUnit.Tests.TestAssembly.MockTestFixture.Tests - MockTestFixture.Explicit, 1, 1)] + [TestCase("cat==FixtureCategory", MockTestFixture.Tests, 1, 2)] + [TestCase("cat!=Special", 38 - 1, 5, 4)] + [TestCase("class==NUnit.Tests.Assemblies.MockTestFixture", MockTestFixture.Tests - MockTestFixture.Explicit, MockTestFixture.Ignored, 1)] + [TestCase("class==NUnit.Tests.OneOfEach", 2, 1, 0)] + [TestCase("cat==OneOfEachCat", 3, 1, 1)] + public void TestsWhereShouldFilter(string filter, int expectedCount, int noOfSkipped = 0, int noOfNone = 0) { // Create a fake environment. var context = new FakeRunContext(new FakeRunSettingsForWhere(filter)); @@ -96,9 +102,19 @@ public void TestsWhereShouldFilter(string filter, int expectedCount) var executor = TestAdapterUtils.CreateExecutor(); executor.RunTests(new[] { mockAssemblyPath }, context, fakeFramework); - var completedRuns = fakeFramework.Events.Where(e => e.EventType == FakeFrameworkHandle.EventType.RecordEnd); + var completedRuns = fakeFramework.Events.Where(e => e.EventType == FakeFrameworkHandle.EventType.RecordEnd).ToList(); + TestContext.WriteLine(" "); + foreach (var test in completedRuns) + { + TestContext.WriteLine($"{test.TestCase.DisplayName}:{test.TestOutcome}"); + } - Assert.That(completedRuns, Has.Exactly(expectedCount).Items); + Assert.Multiple(() => + { + Assert.That(completedRuns, Has.Exactly(expectedCount).Items, "Total number wrong"); + Assert.That(completedRuns.Count(o => o.TestOutcome == TestOutcome.Skipped), Is.EqualTo(noOfSkipped), "Skipped number wrong"); + Assert.That(completedRuns.Count(o => o.TestOutcome == TestOutcome.None), Is.EqualTo(noOfNone), "Explicit and inconclusive number wrong"); + }); } } @@ -154,11 +170,12 @@ public void DumpEvents() public void CorrectNumberOfTestCasesWereStarted() { const FakeFrameworkHandle.EventType eventType = FakeFrameworkHandle.EventType.RecordStart; + Console.WriteLine(" "); foreach (var ev in testLog.Events.FindAll(e => e.EventType == eventType)) - Console.WriteLine(ev.TestCase.DisplayName); + Console.WriteLine($"{ev.TestCase.DisplayName}"); Assert.That( testLog.Events.FindAll(e => e.EventType == eventType).Count, - Is.EqualTo(MockAssembly.ResultCount - BadFixture.Tests - IgnoredFixture.Tests - ExplicitFixture.Tests - MockTestFixture.Explicit)); + Is.EqualTo(MockAssembly.ResultCount - BadFixture.Tests - IgnoredFixture.Tests - ExplicitFixture.Tests - MockTestFixture.Explicit - OneOfEach.ExplicitTests)); } [Test] @@ -192,6 +209,8 @@ public void CorrectNumberOfResultsWereReceived() [TestCaseSource(nameof(Outcomes))] public int TestOutcomeTotalsAreCorrect(TestOutcome outcome) { + TestContext.WriteLine(" "); + TestContext.WriteLine($"Looking for outcome: {outcome}"); return testLog.Events .Count(e => e.EventType == FakeFrameworkHandle.EventType.RecordResult && e.TestResult.Outcome == outcome); } diff --git a/src/NUnitTestAdapterTests/TestFilterConverterTests/TestFilterCombinerTests.cs b/src/NUnitTestAdapterTests/TestFilterConverterTests/TestFilterCombinerTests.cs new file mode 100644 index 00000000..b9da0856 --- /dev/null +++ b/src/NUnitTestAdapterTests/TestFilterConverterTests/TestFilterCombinerTests.cs @@ -0,0 +1,34 @@ +using NUnit.Engine; +using NUnit.Framework; + +namespace NUnit.VisualStudio.TestAdapter.Tests.TestFilterConverterTests +{ + public class TestFilterCombinerTests + { + [Test] + public void TestCombiningCategories() + { + var builder = new TestFilterBuilder(); + builder.SelectWhere("cat==FOO"); + + var t1 = builder.GetFilter(); + builder = new TestFilterBuilder(); + builder.SelectWhere("cat!=BOO"); + var t2 = builder.GetFilter(); + var combiner = new TestFilterCombiner(t1, t2); + var tRes = combiner.GetFilter(); + Assert.Multiple(() => + { + Assert.That(t1.Text, Does.StartWith("")); + Assert.That(t1.Text, Is.EqualTo("FOO")); + Assert.That(t2.Text, Does.StartWith("")); + Assert.That(t2.Text, Is.EqualTo("BOO")); + Assert.That(tRes.Text, Does.StartWith("")); + Assert.That(tRes.Text, Does.Not.StartWith("")); + Assert.That(tRes.Text, Is.EqualTo("FOOBOO")); + }); + TestContext.Out.WriteLine(" "); + TestContext.Out.WriteLine(tRes.Text); + } + } +} diff --git a/src/mock-assembly/MockAssembly.cs b/src/mock-assembly/MockAssembly.cs index a0158ee0..9e57cd48 100644 --- a/src/mock-assembly/MockAssembly.cs +++ b/src/mock-assembly/MockAssembly.cs @@ -23,6 +23,7 @@ using System; using System.IO; + using NUnit.Framework; using NUnit.Framework.Internal; @@ -35,7 +36,7 @@ namespace Assemblies /// public class MockAssembly { - public const int Classes = 10; + public const int Classes = 11; public const int NamespaceSuites = 6; // assembly, NUnit, Tests, Assemblies, Singletons, TestAssembly // While const values are copied to other projects at compile time, @@ -52,7 +53,8 @@ public class MockAssembly + ParameterizedFixture.Tests + GenericFixtureConstants.Tests + ParentClass.Tests - + FixtureWithAttachment.Tests; + + FixtureWithAttachment.Tests + + OneOfEach.Tests; public const int Suites = MockTestFixture.Suites + Singletons.OneTestCase.Suites @@ -70,8 +72,8 @@ public class MockAssembly public const int ExplicitFixtures = 1; public const int SuitesRun = Suites - ExplicitFixtures; - public const int Ignored = MockTestFixture.Ignored + IgnoredFixture.Tests; - public const int Explicit = MockTestFixture.Explicit + ExplicitFixture.Tests; + public const int Ignored = MockTestFixture.Ignored + IgnoredFixture.Tests + OneOfEach.IgnoredTests; + public const int Explicit = MockTestFixture.Explicit + ExplicitFixture.Tests + OneOfEach.ExplicitTests; public const int NotRun = Ignored + Explicit + NotRunnable; public const int TestsRun = Tests - NotRun; public const int ResultCount = Tests; @@ -86,7 +88,7 @@ public class MockAssembly public const int Categories = MockTestFixture.Categories; } - [TestFixture(Description="Fake Test Fixture")] + [TestFixture(Description = "Fake Test Fixture")] [Category("FixtureCategory")] public class MockTestFixture { @@ -110,7 +112,7 @@ public class MockTestFixture public const int Categories = 5; public const int MockCategoryTests = 2; - [Test(Description="Mock Test #1")] + [Test(Description = "Mock Test #1")] public void MockTest1() { } @@ -253,8 +255,8 @@ public class FixtureWithTestCases public const int Tests = 4; public const int Suites = 3; - [TestCase(2, 2, ExpectedResult=4)] - [TestCase(9, 11, ExpectedResult=20)] + [TestCase(2, 2, ExpectedResult = 4)] + [TestCase(9, 11, ExpectedResult = 20)] public int MethodWithParameters(int x, int y) { return x + y; @@ -344,4 +346,27 @@ public void AttachmentTest() TestContext.AddTestAttachment(filepath2, Attachment2Description); } } + + [Category("OneOfEachCat")] + [TestFixture] + public class OneOfEach + { + public const int Tests = 3; + public const int ExplicitTests = 1; + public const int IgnoredTests = 1; + + [Test] + public void NormalTest() { } + + [Explicit] + [Test] + public void ExplicitTest() + { + Environment.Exit(42); + } + + [Ignore("")] + [Test] + public void IgnoredTest() { } + } }