Skip to content

Commit

Permalink
Merge pull request #609 from kubukoz/ignore-in-funsuite
Browse files Browse the repository at this point in the history 
Ignore ignored tests in pure suites
  • Loading branch information
@Baccata
Baccata authored Nov 20, 2022
2 parents 473203b + 7679ae1 commit 734ae54
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 52 deletions.
126 changes: 74 additions & 52 deletions modules/core/src/weaver/suites.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,56 @@ abstract class RunnableSuite[F[_]] extends EffectSuite[F] {
def plan : List[TestName]
private[weaver] def runUnsafe(args: List[String])(report: TestOutcome => Unit) : Unit =
effectCompat.unsafeRunSync(run(args)(outcome => effectCompat.effect.delay(report(outcome))))

def isCI: Boolean = System.getenv("CI") == "true"

private[weaver] def analyze[Res, F1[_]](testSeq: Seq[(TestName, Res => F1[TestOutcome])], args: List[String]): TagAnalysisResult[Res, F1] = {
val testsNotIgnored: Seq[(TestName, Res => F1[TestOutcome])] =
testSeq.filterNot(_._1.tags(TestName.Tags.ignore))

val testsTaggedOnly: Seq[(TestName, Res => F1[TestOutcome])] =
testSeq.filter(_._1.tags(TestName.Tags.only))

val onlyTestsNotIgnored =
testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly))

val filteredTests = if (onlyTestsNotIgnored.isEmpty) {
val argsFilter = Filters.filterTests(this.name)(args)
testsNotIgnored.collect {
case (name, test) if argsFilter(name) => test
}
} else onlyTestsNotIgnored.map(_._2)

if (testsTaggedOnly.nonEmpty && isCI) {
val failureOutcomes = testsTaggedOnly.map(_._1).map(onlyNotOnCiFailure)
TagAnalysisResult.Outcomes(failureOutcomes)
} else TagAnalysisResult.FilteredTests(filteredTests)
}


private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = {
val result = Result.Failure(
msg = "'Only' tag is not allowed when `isCI=true`",
source = None,
location = List(test.location)
)
TestOutcome(
name = test.name,
duration = FiniteDuration(0, "ns"),
result = result,
log = Chain.empty
)
}

}

private[weaver] sealed trait TagAnalysisResult[Res, F[_]]
object TagAnalysisResult {
case class Outcomes[Res, F[_]](outcomes: Seq[TestOutcome]) extends TagAnalysisResult[Res, F]
case class FilteredTests[Res, F[_]](tests: Seq[Res => F[TestOutcome]]) extends TagAnalysisResult[Res, F]
}


abstract class MutableFSuite[F[_]] extends RunnableSuite[F] {

type Res
Expand All @@ -95,53 +143,28 @@ abstract class MutableFSuite[F[_]] extends RunnableSuite[F] {
def usingRes(run : Res => F[Expectations]) : Unit = apply(run)
}

def isCI: Boolean = "true" == System.getenv("CI")

override def spec(args: List[String]) : Stream[F, TestOutcome] =
override def spec(args: List[String]): Stream[F, TestOutcome] =
synchronized {
if (!isInitialized) isInitialized = true
val testsNotIgnored: Seq[(TestName, Res => F[TestOutcome])] = testSeq.filterNot(_._1.tags(TestName.Tags.ignore))
val testsTaggedOnly: Seq[(TestName, Res => F[TestOutcome])] = testSeq.filter(_._1.tags(TestName.Tags.only))
val onlyTestsNotIgnored = testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly))
val filteredTests = if (onlyTestsNotIgnored.isEmpty) {
val argsFilter = Filters.filterTests(this.name)(args)
testsNotIgnored.collect {
case (name, test) if argsFilter(name) => test
}
} else onlyTestsNotIgnored.map(_._2)
val parallism = math.max(1, maxParallelism)

if (testsTaggedOnly.nonEmpty && isCI) {
val failureOutcomes = testsTaggedOnly
.map(_._1)
.map(onlyNotOnCiFailure)
Stream.emits(failureOutcomes).lift[F](effectCompat.effect)
val parallelism = math.max(1, maxParallelism)

analyze(testSeq, args) match {
case TagAnalysisResult.Outcomes(outcomes) => fs2.Stream.emits(outcomes)
case TagAnalysisResult.FilteredTests(filteredTests)
if filteredTests.isEmpty =>
Stream.empty // no need to allocate resources
case TagAnalysisResult.FilteredTests(filteredTests) => for {
resource <- Stream.resource(sharedResource)
tests = filteredTests.map(_.apply(resource))
testStream = Stream.emits(tests).covary[F]
result <- if (parallelism > 1)
testStream.parEvalMap(parallelism)(identity)(effectCompat.effect)
else testStream.evalMap(identity)
} yield result
}
else if (filteredTests.isEmpty) Stream.empty // no need to allocate resources
else for {
resource <- Stream.resource(sharedResource)
tests = filteredTests.map(_.apply(resource))
testStream = Stream.emits(tests).lift[F](effectCompat.effect)
result <- if (parallism > 1 ) testStream.parEvalMap(parallism)(identity)(effectCompat.effect)
else testStream.evalMap(identity)
} yield result
}

private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = {
val result = Result.Failure(
msg = "'Only' tag is not allowed when `isCI=true`",
source = None,
location = List(test.location)
)
TestOutcome(
name = test.name,
duration = FiniteDuration(0, "ns"),
result = result,
log = Chain.empty
)
}

private[this] var testSeq = Seq.empty[(TestName, Res => F[TestOutcome])]
private[this] var testSeq: Seq[(TestName, Res => F[TestOutcome])] = Seq.empty

def plan: List[TestName] = testSeq.map(_._1).toList

Expand All @@ -161,19 +184,18 @@ trait FunSuiteAux {
abstract class FunSuiteF[F[_]] extends RunnableSuite[F] with FunSuiteAux { self =>
override def test(name: TestName)(run: => Expectations): Unit = synchronized {
if(isInitialized) throw initError
testSeq = testSeq :+ (name -> (() => Test.pure(name.name)(() => run)))
testSeq = testSeq :+ (name -> ((_: Unit) => Test.pure(name.name)(() => run)))
}

override def name : String = self.getClass.getName.replace("$", "")
private def pureSpec(args: List[String]) = synchronized {

private def pureSpec(args: List[String]): fs2.Stream[fs2.Pure, TestOutcome] = synchronized {
if(!isInitialized) isInitialized = true
val argsFilter = Filters.filterTests(this.name)(args)
val filteredTests = if (testSeq.exists(_._1.tags(TestName.Tags.only))){
testSeq.filter(_._1.tags(TestName.Tags.only)).map { case (_, test) => test}
} else testSeq.collect {
case (name, test) if argsFilter(name) => test
}
fs2.Stream.emits(filteredTests.map(execute => execute()))
analyze[Unit, cats.Id](testSeq, args) match {
case TagAnalysisResult.Outcomes(outcomes) => fs2.Stream.emits(outcomes)
case TagAnalysisResult.FilteredTests(filteredTests) =>
fs2.Stream.emits(filteredTests.map(execute => execute(())))
}
}

override def spec(args: List[String]) = pureSpec(args).covary[F]
Expand All @@ -182,7 +204,7 @@ abstract class FunSuiteF[F[_]] extends RunnableSuite[F] with FunSuiteAux { self
pureSpec(args).compile.toVector.foreach(report)


private[this] var testSeq = Seq.empty[(TestName, () => TestOutcome)]
private[this] var testSeq = Seq.empty[(TestName, Unit => TestOutcome)]
def plan: List[TestName] = testSeq.map(_._1).toList

private[this] var isInitialized = false
Expand Down
20 changes: 20 additions & 0 deletions modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,21 @@ object JUnitRunnerTests extends IOSuite {
}
}

test("Tests tagged with ignore are ignored (FunSuite)") { blocker =>
runPure(blocker, Meta.IgnorePure).map { notifications =>
val expected = List(
TestSuiteStarted("weaver.junit.Meta$IgnorePure$"),
TestIgnored("is ignored(weaver.junit.Meta$IgnorePure$)"),
TestStarted("not ignored 1(weaver.junit.Meta$IgnorePure$)"),
TestFinished("not ignored 1(weaver.junit.Meta$IgnorePure$)"),
TestStarted("not ignored 2(weaver.junit.Meta$IgnorePure$)"),
TestFinished("not ignored 2(weaver.junit.Meta$IgnorePure$)"),
TestSuiteFinished("weaver.junit.Meta$IgnorePure$")
)
expect.same(notifications, expected)
}
}

test(
"Even if all tests are ignored, will fail if a test is tagged with only") {
blocker =>
Expand Down Expand Up @@ -172,6 +187,11 @@ object JUnitRunnerTests extends IOSuite {
suite: SimpleIOSuite): IO[List[Notification]] =
run(blocker, suite.getClass())

def runPure(
blocker: BlockerCompat[IO],
suite: FunSuite): IO[List[Notification]] =
run(blocker, suite.getClass())

sealed trait Notification
case class TestSuiteStarted(name: String) extends Notification
case class TestAssumptionFailure(failure: Failure) extends Notification
Expand Down
16 changes: 16 additions & 0 deletions modules/framework/cats/test/src-jvm/junit/Meta.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,22 @@ object Meta {

}

object IgnorePure extends FunSuite {

test("not ignored 1") {
success
}

test("not ignored 2") {
success
}

test("is ignored".ignore) {
failure("foo")
}

}

class Sharing(global: GlobalRead) extends IOSuite {

type Res = Unit
Expand Down

0 comments on commit 734ae54

Please sign in to comment.