Combine two NUnit3 Drivers

.github/workflows/NUnitConsoleAndEngine.CI.yml

@@ -36,7 +36,8 @@ jobs:
         with:
           dotnet-version: |
             3.1.x
-            6.0.x
+            6.0.x 
+            7.0.x
             8.0.100
 
       - name: 🔧 Install dotnet tools

NUnitConsole.sln

@@ -139,6 +139,16 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WpfTest", "src\TestData\wpf
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WpfApp", "src\TestData\WpfApp\WpfApp.csproj", "{93D182B7-2F63-4661-BB32-94F1716CF03F}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "OlderNUnit", "OlderNUnit", "{D31607D2-D689-488B-9F9F-92F47AC1D7F6}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.0", "src\TestData\NUnit3.0\NUnit3.0.csproj", "{2448CE80-59AA-41B7-83A0-768BE5520F7A}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.0.1", "src\TestData\NUnit3.0.1\NUnit3.0.1.csproj", "{56416AD9-8E7B-4F72-B0E1-E75A024431F8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.2", "src\TestData\NUnit3.2\NUnit3.2.csproj", "{8AFBB856-700A-4CCC-AE0E-9B622178E1E0}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.10", "src\TestData\NUnit3.10\NUnit3.10.csproj", "{0555B97D-E918-455B-951C-74EFCDA8790A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -241,6 +251,22 @@ Global
 		{93D182B7-2F63-4661-BB32-94F1716CF03F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{93D182B7-2F63-4661-BB32-94F1716CF03F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{93D182B7-2F63-4661-BB32-94F1716CF03F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2448CE80-59AA-41B7-83A0-768BE5520F7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2448CE80-59AA-41B7-83A0-768BE5520F7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2448CE80-59AA-41B7-83A0-768BE5520F7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2448CE80-59AA-41B7-83A0-768BE5520F7A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0555B97D-E918-455B-951C-74EFCDA8790A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0555B97D-E918-455B-951C-74EFCDA8790A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0555B97D-E918-455B-951C-74EFCDA8790A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0555B97D-E918-455B-951C-74EFCDA8790A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -280,6 +306,11 @@ Global
 		{48DF1E40-93BA-436A-B460-5D1130316ADA} = {37D508B2-91E0-4B32-869B-DFF9E68EA213}
 		{2F9D8932-2186-464F-BED6-7D7979C8FFA6} = {37D508B2-91E0-4B32-869B-DFF9E68EA213}
 		{93D182B7-2F63-4661-BB32-94F1716CF03F} = {37D508B2-91E0-4B32-869B-DFF9E68EA213}
+		{D31607D2-D689-488B-9F9F-92F47AC1D7F6} = {37D508B2-91E0-4B32-869B-DFF9E68EA213}
+		{2448CE80-59AA-41B7-83A0-768BE5520F7A} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6}
+		{56416AD9-8E7B-4F72-B0E1-E75A024431F8} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6}
+		{8AFBB856-700A-4CCC-AE0E-9B622178E1E0} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6}
+		{0555B97D-E918-455B-951C-74EFCDA8790A} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D8E4FC26-5422-4C51-8BBC-D1AC0A578711}

package-tests.cake

@@ -35,7 +35,7 @@ class MockAssemblyExpectedResult : ExpectedResult
 StandardRunnerTests.Add(new PackageTest(
     1, "Net462Test",
     "Run mock-assembly.dll under .NET 4.6.2",
-    "testdata/net462/mock-assembly.dll",
+    "testdata/net462/mock-assembly.dll --trace:Debug",
     new MockAssemblyExpectedResult("net-4.6.2")));
 
 AddToBothLists(new PackageTest(
@@ -142,6 +142,46 @@ StandardRunnerTests.Add(new PackageTest(
     "testdata/net462/mock-assembly.dll testdata/net6.0/mock-assembly.dll",
     new MockAssemblyExpectedResult("net-4.6.2", "netcore-6.0")));
 
+//////////////////////////////////////////////////////////////////////
+// TEST OLDER VERSIONS OF NUNIT SWITCHING API IF NEEDED
+//////////////////////////////////////////////////////////////////////
+
+StandardRunnerTests.Add(new PackageTest(
+    1, "NUnit30Test",
+    "Run a test under NUnit 3.0 using 2009 API",
+    "testdata/NUnit3.0/net462/NUnit3.0.dll",
+    new ExpectedResult("Passed")
+    {
+        Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.0.dll", "net462") }
+    }));
+
+StandardRunnerTests.Add(new PackageTest(
+    1, "NUnit301Test",
+    "Run a test under NUnit 3.0.1 using 2009 API",
+    "testdata/NUnit3.0.1/net462/NUnit3.0.1.dll",
+    new ExpectedResult("Passed")
+    {
+        Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.0.1.dll", "net462") }
+    }));
+
+StandardRunnerTests.Add(new PackageTest(
+    1, "NUnit32Test",
+    "Run a test under NUnit 3.2 using 20018 API",
+    "testdata/NUnit3.2/net462/NUnit3.2.dll",
+    new ExpectedResult("Passed")
+    {
+        Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.2.dll", "net462") }
+    }));
+
+StandardRunnerTests.Add(new PackageTest(
+    1, "NUnit310Test",
+    "Run a test under NUnit 3.10 using 2018 API",
+    "testdata/NUnit3.10/net462/NUnit3.10.dll",
+    new ExpectedResult("Passed")
+    {
+        Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.10.dll", "net462") }
+    }));
+
 //////////////////////////////////////////////////////////////////////
 // ASP.NETCORE TESTS
 //////////////////////////////////////////////////////////////////////

src/NUnitEngine/nunit.engine.api/Extensibility/IDriverFactory.cs

@@ -25,17 +25,19 @@ public interface IDriverFactory
         /// which the assembly is already known to reference.
         /// </summary>
         /// <param name="domain">The domain in which the assembly will be loaded</param>
+        /// <param name="id">The driver id.</param>
         /// <param name="reference">An AssemblyName referring to the test framework.</param>
         /// <returns></returns>
-        IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference);
+        IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference);
 #else
         /// <summary>
         /// Gets a driver for a given test assembly and a framework
         /// which the assembly is already known to reference.
         /// </summary>
+        /// <param name="id">The driver id.</param>
         /// <param name="reference">An AssemblyName referring to the test framework.</param>
         /// <returns></returns>
-        IFrameworkDriver GetDriver(AssemblyName reference);
+        IFrameworkDriver GetDriver(string id, AssemblyName reference);
 #endif
     }
 }

src/NUnitEngine/nunit.engine.api/Extensibility/IFrameworkDriver.cs

@@ -18,7 +18,7 @@ public interface IFrameworkDriver
         /// Gets and sets the unique identifier for this driver,
         /// used to ensure that test ids are unique across drivers.
         /// </summary>
-        string ID { get; set; }
+        string ID { get; }
 
         /// <summary>
         /// Loads the tests in an assembly.

src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnit3FrameworkDriverTests.cs → src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnitFrameworkDriverTests.cs

@@ -1,6 +1,5 @@
 // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt
 
-#if NETFRAMEWORK
 using System;
 using System.Collections.Generic;
 using System.Reflection;
@@ -13,22 +12,39 @@
 namespace NUnit.Engine.Drivers
 {
     // Functional tests of the NUnitFrameworkDriver calling into the framework.
-    public class NUnit3FrameworkDriverTests
+#if NETFRAMEWORK
+    [TestFixture("2009")]
+    [TestFixture("2018")]
+#endif
+    public class NUnitFrameworkDriverTests
     {
         private const string MOCK_ASSEMBLY = "mock-assembly.dll";
-        private const string LOAD_MESSAGE = "Method called without calling Load first";
+        private const string LOAD_MESSAGE = "Method called without calling Load first. Possible error in runner.";
 
         private IDictionary<string, object> _settings = new Dictionary<string, object>();
 
-        private NUnit3FrameworkDriver _driver;
+        private NUnitFrameworkDriver _driver;
         private string _mockAssemblyPath;
 
+#if NETFRAMEWORK
+        private string _whichApi;
+        public NUnitFrameworkDriverTests(string whichApi)
+        {
+            _whichApi = whichApi;
+        }
+#endif
+
         [SetUp]
         public void CreateDriver()
         {
-            var assemblyName = typeof(NUnit.Framework.TestAttribute).Assembly.GetName();
+            var nunitRef = typeof(TestAttribute).Assembly.GetName();
             _mockAssemblyPath = System.IO.Path.Combine(TestContext.CurrentContext.TestDirectory, MOCK_ASSEMBLY);
-            _driver = new NUnit3FrameworkDriver(AppDomain.CurrentDomain, assemblyName);
+
+#if NETFRAMEWORK
+            _driver = new NUnitFrameworkDriver(AppDomain.CurrentDomain, _whichApi, "99", nunitRef);
+#else
+            _driver = new NUnitFrameworkDriver("99", nunitRef);
+#endif
         }
 
         [Test]
@@ -109,14 +125,42 @@ public void RunTestsAction_WithoutLoad_ThrowsInvalidOperationException()
             Assert.That(ex.Message, Is.EqualTo(LOAD_MESSAGE));
         }
 
+#if NETFRAMEWORK
+        // Nested Class tests Api Selection in the driver
+        public class ApiSelectionTests()
+        {
+            [TestCase("4.2.2", "2018")]
+            [TestCase("3.14.0", "2018")]
+            [TestCase("3.2.0", "2018")]
+            [TestCase("3.0.1", "2009")]
+            [TestCase("3.0.0", "2009")]
+            public void CorrectApiIsSelected(string nunitVersion, string apiVersion)
+            {
+                var driver = new NUnitFrameworkDriver(AppDomain.CurrentDomain, "99", new AssemblyName()
+                {
+                    Name = "nunit.framework",
+                    Version = new Version(nunitVersion)
+                });
+
+                Assert.That(driver.API, Is.EqualTo(apiVersion));
+            }
+        }
+
         [Test]
-        public void RunTestsAction_WithInvalidFilterElement_ThrowsNUnitEngineException()
+        public void RunTestsAction_WithInvalidFilterElement_ThrowsException()
         {
             _driver.Load(_mockAssemblyPath, _settings);
 
             var invalidFilter = "<filter><invalidElement>foo</invalidElement></filter>";
             var ex = Assert.Catch(() => _driver.Run(new NullListener(), invalidFilter));
-            Assert.That(ex, Is.TypeOf<NUnitEngineException>());
+
+            if (_whichApi == "2018")
+            {
+                Assert.That(ex, Is.TypeOf<TargetInvocationException>());
+                Assert.That(ex.InnerException, Is.TypeOf<ArgumentException>());
+            }
+            else
+                Assert.That(ex, Is.TypeOf<NUnitEngineException>());
         }
 
         private class CallbackEventHandler : System.Web.UI.ICallbackEventHandler
@@ -133,6 +177,7 @@ public void RaiseCallbackEvent(string eventArgument)
                 _result = eventArgument;
             }
         }
+#endif
 
         public class NullListener : ITestEventListener
         {
@@ -143,4 +188,3 @@ public void OnTestEvent(string testEvent)
         }
     }
 }
-#endif

src/NUnitEngine/nunit.engine.core.tests/Drivers/NotRunnableFrameworkDriverTests.cs

@@ -14,7 +14,6 @@ namespace NUnit.Engine.Drivers
     // Functional tests of the NUnitFrameworkDriver calling into the framework.
     public abstract class NotRunnableFrameworkDriverTests
     {
-        private const string DRIVER_ID = "99";
         private const string EXPECTED_ID = "99-1";
 
         protected string? _expectedRunState;
@@ -95,7 +94,6 @@ public void Run(string filePath, string expectedType)
         private IFrameworkDriver GetDriver(string filePath)
         {
             IFrameworkDriver driver = CreateDriver(filePath);
-            driver.ID = DRIVER_ID;
             return driver;
         }
 
@@ -126,7 +124,7 @@ public InvalidAssemblyFrameworkDriverTests()
 
         protected override IFrameworkDriver CreateDriver(string filePath)
         {
-            return new InvalidAssemblyFrameworkDriver(filePath, _expectedReason ?? "Not Specified");
+            return new InvalidAssemblyFrameworkDriver(filePath, "99", _expectedReason ?? "Not Specified");
         }
     }
 
@@ -142,7 +140,7 @@ public SkippedAssemblyFrameworkDriverTests()
 
         protected override IFrameworkDriver CreateDriver(string filePath)
         {
-            return new SkippedAssemblyFrameworkDriver(filePath);
+            return new SkippedAssemblyFrameworkDriver(filePath, "99");
         }
     }
 }

src/NUnitEngine/nunit.engine.core.tests/DummyExtensions.cs

@@ -12,10 +12,10 @@ namespace NUnit.Engine
     [Extension]
     public class DummyFrameworkDriverExtension : IDriverFactory
     {
-#if !NETFRAMEWORK
-        public IFrameworkDriver GetDriver(AssemblyName reference)
+#if NETFRAMEWORK
+        public IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference)
 #else
-        public IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference)
+        public IFrameworkDriver GetDriver(string id, AssemblyName reference)
 #endif
         {
             throw new NotImplementedException();

src/NUnitEngine/nunit.engine.core.tests/Runners/TestAgentRunnerExceptionTests.cs

@@ -22,8 +22,9 @@ public void Initialize()
             var driverService = Substitute.For<Drivers.IDriverService>();
             driverService.GetDriver(
                 AppDomain.CurrentDomain,
+                new TestPackage(),
+                string.Empty,
                 string.Empty,
-                string.Empty, 
                 false).ReturnsForAnyArgs(_driver);
 
             _runner = new FakeTestAgentRunner(new TestPackage("mock-assembly.dll").SubPackages[0])

src/NUnitEngine/nunit.engine.core.tests/Services/DriverServiceTests.cs

@@ -7,7 +7,7 @@
 using NUnit.Engine.Drivers;
 using NUnit.Engine.Extensibility;
 
-namespace NUnit.Engine.Services.Tests
+namespace NUnit.Engine.Services
 {
     [TestFixture]
     public class DriverServiceTests
@@ -23,36 +23,25 @@ public void CreateDriverFactory()
         [TestCaseSource(nameof(DriverSelectionTestCases))]
         public void CorrectDriverIsUsed(string fileName, bool skipNonTestAssemblies, Type expectedType)
         {
-            var driver = _driverService.GetDriver(AppDomain.CurrentDomain, Path.Combine(TestContext.CurrentContext.TestDirectory, fileName), null, skipNonTestAssemblies);
+            var assemblyPath = Path.Combine(TestContext.CurrentContext.TestDirectory, fileName);
+            var driver = _driverService.GetDriver(AppDomain.CurrentDomain, new TestPackage(assemblyPath), assemblyPath, null, skipNonTestAssemblies);
             Assert.That(driver, Is.InstanceOf(expectedType));
         }
 
         static TestCaseData[] DriverSelectionTestCases = new[]
         {
-            // TODO: make commented tests work
-#if NETFRAMEWORK
-            new TestCaseData("mock-assembly.dll", false, typeof(NUnit3FrameworkDriver)),
-            new TestCaseData("mock-assembly.dll", true, typeof(NUnit3FrameworkDriver)),
-            //new TestCaseData("notest-assembly.dll", false, typeof(NUnit3FrameworkDriver)),
-#elif NET5_0_OR_GREATER
-            new TestCaseData("mock-assembly.dll", false, typeof(NUnitNetCore31Driver)),
-            new TestCaseData("mock-assembly.dll", true, typeof(NUnitNetCore31Driver)),
-            //new TestCaseData("notest-assembly.dll", false, typeof(NUnitNetCore31Driver)),
-#else
-            new TestCaseData("mock-assembly.dll", false, typeof(NUnitNetCore31Driver)),
-            new TestCaseData("mock-assembly.dll", true, typeof(NUnitNetCore31Driver)),
-            //new TestCaseData("notest-assembly.dll", false, typeof(NUnitNetCore31Driver)),
-#endif
-// Invalid cases should work with all target runtimes
+            new TestCaseData("mock-assembly.dll", false, typeof(NUnitFrameworkDriver)),
+            new TestCaseData("mock-assembly.dll", true, typeof(NUnitFrameworkDriver)),
+            new TestCaseData("notest-assembly.dll", false, typeof(NUnitFrameworkDriver)).Ignore("Assembly not present"),
+            new TestCaseData("notest-assembly.dll", true, typeof(SkippedAssemblyFrameworkDriver)).Ignore("Assembly not present"),
+
+            // Invalid cases should work with all target runtimes
             new TestCaseData("mock-assembly.pdb", false, typeof(InvalidAssemblyFrameworkDriver)),
             new TestCaseData("mock-assembly.pdb", true, typeof(InvalidAssemblyFrameworkDriver)),
             new TestCaseData("junk.dll", false, typeof(InvalidAssemblyFrameworkDriver)),
             new TestCaseData("junk.dll", true, typeof(InvalidAssemblyFrameworkDriver)),
             new TestCaseData("nunit.engine.core.dll", false, typeof(InvalidAssemblyFrameworkDriver)),
             new TestCaseData("nunit.engine.core.dll", true, typeof(SkippedAssemblyFrameworkDriver))
-//#if !NET5_0_OR_GREATER // Not yet working
-//            new TestCaseData"notest-assembly.dll", true, typeof(SkippedAssemblyFrameworkDriver))
-//#endif
         };
 
         [Test]