Ignore ignored tests in pure suites

disneystreaming · Nov 20, 2022 · 7679ae1 · 7679ae1
1 parent 473203b
commit 7679ae1
Showing 3 changed files with 110 additions and 52 deletions.
diff --git a/modules/core/src/weaver/suites.scala b/modules/core/src/weaver/suites.scala
@@ -67,8 +67,56 @@ abstract class RunnableSuite[F[_]] extends EffectSuite[F] {
   def plan : List[TestName]
   private[weaver] def runUnsafe(args: List[String])(report: TestOutcome => Unit) : Unit =
     effectCompat.unsafeRunSync(run(args)(outcome => effectCompat.effect.delay(report(outcome))))
+
+  def isCI: Boolean = System.getenv("CI") == "true"
+
+  private[weaver] def analyze[Res, F1[_]](testSeq: Seq[(TestName, Res => F1[TestOutcome])], args: List[String]): TagAnalysisResult[Res, F1] = {
+    val testsNotIgnored: Seq[(TestName, Res => F1[TestOutcome])] =
+      testSeq.filterNot(_._1.tags(TestName.Tags.ignore))
+
+    val testsTaggedOnly: Seq[(TestName, Res => F1[TestOutcome])] =
+      testSeq.filter(_._1.tags(TestName.Tags.only))
+
+    val onlyTestsNotIgnored =
+      testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly))
+
+    val filteredTests = if (onlyTestsNotIgnored.isEmpty) {
+      val argsFilter = Filters.filterTests(this.name)(args)
+      testsNotIgnored.collect {
+        case (name, test) if argsFilter(name) => test
+      }
+    } else onlyTestsNotIgnored.map(_._2)
+
+    if (testsTaggedOnly.nonEmpty && isCI) {
+      val failureOutcomes = testsTaggedOnly.map(_._1).map(onlyNotOnCiFailure)
+      TagAnalysisResult.Outcomes(failureOutcomes)
+    } else TagAnalysisResult.FilteredTests(filteredTests)
+  }
+
+
+  private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = {
+    val result = Result.Failure(
+      msg = "'Only' tag is not allowed when `isCI=true`",
+      source = None,
+      location = List(test.location)
+    )
+    TestOutcome(
+      name = test.name,
+      duration = FiniteDuration(0, "ns"),
+      result = result,
+      log = Chain.empty
+    )
+  }
+
+}
+
+private[weaver] sealed trait TagAnalysisResult[Res, F[_]]
+object TagAnalysisResult {
+  case class Outcomes[Res, F[_]](outcomes: Seq[TestOutcome]) extends TagAnalysisResult[Res, F]
+  case class FilteredTests[Res, F[_]](tests: Seq[Res => F[TestOutcome]]) extends TagAnalysisResult[Res, F]
 }
 
+
 abstract class MutableFSuite[F[_]] extends RunnableSuite[F]  {
 
   type Res
@@ -95,53 +143,28 @@ abstract class MutableFSuite[F[_]] extends RunnableSuite[F]  {
     def usingRes(run : Res => F[Expectations]) : Unit = apply(run)
   }
 
-  def isCI: Boolean = "true" == System.getenv("CI")
-
-  override def spec(args: List[String]) : Stream[F, TestOutcome] =
+  override def spec(args: List[String]): Stream[F, TestOutcome] =
     synchronized {
       if (!isInitialized) isInitialized = true
-      val testsNotIgnored: Seq[(TestName, Res => F[TestOutcome])] = testSeq.filterNot(_._1.tags(TestName.Tags.ignore))
-      val testsTaggedOnly: Seq[(TestName, Res => F[TestOutcome])]  = testSeq.filter(_._1.tags(TestName.Tags.only))
-      val onlyTestsNotIgnored = testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly))
-      val filteredTests = if (onlyTestsNotIgnored.isEmpty) {
-        val argsFilter = Filters.filterTests(this.name)(args)
-        testsNotIgnored.collect {
-          case (name, test) if argsFilter(name) => test
-        }
-      } else onlyTestsNotIgnored.map(_._2)
-      val parallism = math.max(1, maxParallelism)
-
-      if (testsTaggedOnly.nonEmpty && isCI) {
-        val failureOutcomes = testsTaggedOnly
-          .map(_._1)
-          .map(onlyNotOnCiFailure)
-        Stream.emits(failureOutcomes).lift[F](effectCompat.effect)
+      val parallelism = math.max(1, maxParallelism)
+
+      analyze(testSeq, args) match {
+        case TagAnalysisResult.Outcomes(outcomes) => fs2.Stream.emits(outcomes)
+        case TagAnalysisResult.FilteredTests(filteredTests)
+            if filteredTests.isEmpty =>
+          Stream.empty // no need to allocate resources
+        case TagAnalysisResult.FilteredTests(filteredTests) => for {
+            resource <- Stream.resource(sharedResource)
+            tests      = filteredTests.map(_.apply(resource))
+            testStream = Stream.emits(tests).covary[F]
+            result <- if (parallelism > 1)
+              testStream.parEvalMap(parallelism)(identity)(effectCompat.effect)
+            else testStream.evalMap(identity)
+          } yield result
       }
-      else if (filteredTests.isEmpty) Stream.empty // no need to allocate resources
-      else for {
-        resource <- Stream.resource(sharedResource)
-        tests = filteredTests.map(_.apply(resource))
-        testStream = Stream.emits(tests).lift[F](effectCompat.effect)
-        result <- if (parallism > 1 ) testStream.parEvalMap(parallism)(identity)(effectCompat.effect)
-                  else testStream.evalMap(identity)
-      } yield result
     }
 
-  private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = {
-    val result = Result.Failure(
-      msg = "'Only' tag is not allowed when `isCI=true`",
-      source = None,
-      location = List(test.location)
-    )
-    TestOutcome(
-      name = test.name,
-      duration = FiniteDuration(0, "ns"),
-      result = result,
-      log = Chain.empty
-    )
-  }
-
-  private[this] var testSeq = Seq.empty[(TestName, Res => F[TestOutcome])]
+  private[this] var testSeq: Seq[(TestName, Res => F[TestOutcome])] = Seq.empty
 
   def plan: List[TestName] = testSeq.map(_._1).toList
 
@@ -161,19 +184,18 @@ trait FunSuiteAux {
 abstract class FunSuiteF[F[_]] extends RunnableSuite[F] with FunSuiteAux { self =>
   override def test(name: TestName)(run: => Expectations): Unit = synchronized {
     if(isInitialized) throw initError
-    testSeq = testSeq :+ (name -> (() => Test.pure(name.name)(() => run)))
+    testSeq = testSeq :+ (name -> ((_: Unit) => Test.pure(name.name)(() => run)))
   }
 
   override def name : String = self.getClass.getName.replace("$", "")
-  private def pureSpec(args: List[String]) = synchronized {
+
+  private def pureSpec(args: List[String]): fs2.Stream[fs2.Pure, TestOutcome] = synchronized {
     if(!isInitialized) isInitialized = true
-    val argsFilter = Filters.filterTests(this.name)(args)
-    val filteredTests = if (testSeq.exists(_._1.tags(TestName.Tags.only))){
-        testSeq.filter(_._1.tags(TestName.Tags.only)).map { case (_, test) => test}
-      } else testSeq.collect {
-        case (name, test) if argsFilter(name) => test
-      }
-    fs2.Stream.emits(filteredTests.map(execute => execute()))
+    analyze[Unit, cats.Id](testSeq, args) match {
+      case TagAnalysisResult.Outcomes(outcomes) => fs2.Stream.emits(outcomes)
+      case TagAnalysisResult.FilteredTests(filteredTests) =>
+        fs2.Stream.emits(filteredTests.map(execute => execute(())))
+    }
   }
 
   override def spec(args: List[String]) = pureSpec(args).covary[F]
@@ -182,7 +204,7 @@ abstract class FunSuiteF[F[_]] extends RunnableSuite[F] with FunSuiteAux { self
     pureSpec(args).compile.toVector.foreach(report)
 
 
-  private[this] var testSeq = Seq.empty[(TestName, () => TestOutcome)]
+  private[this] var testSeq = Seq.empty[(TestName, Unit => TestOutcome)]
   def plan: List[TestName] = testSeq.map(_._1).toList
 
   private[this] var isInitialized = false

diff --git a/modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala b/modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala
@@ -114,6 +114,21 @@ object JUnitRunnerTests extends IOSuite {
     }
   }
 
+  test("Tests tagged with ignore are ignored (FunSuite)") { blocker =>
+    runPure(blocker, Meta.IgnorePure).map { notifications =>
+      val expected = List(
+        TestSuiteStarted("weaver.junit.Meta$IgnorePure$"),
+        TestIgnored("is ignored(weaver.junit.Meta$IgnorePure$)"),
+        TestStarted("not ignored 1(weaver.junit.Meta$IgnorePure$)"),
+        TestFinished("not ignored 1(weaver.junit.Meta$IgnorePure$)"),
+        TestStarted("not ignored 2(weaver.junit.Meta$IgnorePure$)"),
+        TestFinished("not ignored 2(weaver.junit.Meta$IgnorePure$)"),
+        TestSuiteFinished("weaver.junit.Meta$IgnorePure$")
+      )
+      expect.same(notifications, expected)
+    }
+  }
+
   test(
     "Even if all tests are ignored, will fail if a test is tagged with only") {
     blocker =>
@@ -172,6 +187,11 @@ object JUnitRunnerTests extends IOSuite {
       suite: SimpleIOSuite): IO[List[Notification]] =
     run(blocker, suite.getClass())
 
+  def runPure(
+      blocker: BlockerCompat[IO],
+      suite: FunSuite): IO[List[Notification]] =
+    run(blocker, suite.getClass())
+
   sealed trait Notification
   case class TestSuiteStarted(name: String)             extends Notification
   case class TestAssumptionFailure(failure: Failure)    extends Notification

diff --git a/modules/framework/cats/test/src-jvm/junit/Meta.scala b/modules/framework/cats/test/src-jvm/junit/Meta.scala
@@ -113,6 +113,22 @@ object Meta {
 
   }
 
+  object IgnorePure extends FunSuite {
+
+    test("not ignored 1") {
+      success
+    }
+
+    test("not ignored 2") {
+      success
+    }
+
+    test("is ignored".ignore) {
+      failure("foo")
+    }
+
+  }
+
   class Sharing(global: GlobalRead) extends IOSuite {
 
     type Res = Unit