diff --git a/docs/extensions.adoc b/docs/extensions.adoc index 004a3c72b5..2366ab1baa 100644 --- a/docs/extensions.adoc +++ b/docs/extensions.adoc @@ -44,6 +44,18 @@ runner { See the <> section for a detailed description. +=== Random Test Order Configuration + +[source,groovy] +---- +runner { + randomizeSpecRunOrder true // randomize specification run order + randomizeFeatureRunOrder true // randomize feature run order within each specification +} +---- + +See the <<#_randomize_run_order>> section for more details. + == Built-In Extensions Most of Spock's built-in extensions are _annotation-driven_. In other words, they are triggered by annotating a @@ -709,6 +721,24 @@ runner { } ---- +=== Randomize Run Order + +Ideally, automated tests in general and Spock specifications in particular should be independent of each other. The same +applies to feature methods within a specification. One heuristic way to gain confidence that this is indeed the case, is +to randomize the order in which specifications and/or features within each specification are executed. + +By default, Spock executes both specifications and features in deterministic order, even though users should not rely on +any particular run order. You can explicitly activate run order randomization separately for specifications and features +or combine both randomization modes, using the <>: + +[source,groovy] +---- +runner { + randomizeSpecRunOrder true // randomize specification run order + randomizeFeatureRunOrder true // randomize feature run order within each specification +} +---- + == Third-Party Extensions You can find a list of third-party extensions in the https://github.com/spockframework/spock/wiki/Third-Party-Extensions[Spock Wiki]. diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index c2708d81ff..ba2e45ec33 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -6,6 +6,8 @@ include::include.adoc[] == 2.4 (tbd) * Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions +* New built-in `RandomRunOrderExtension` supports run order randomization for specifications, features or a combination of both. + See manual section <>. == 2.4-M1 (2022-11-30) diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java new file mode 100644 index 0000000000..2646eb9d5a --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin; + +import org.spockframework.runtime.extension.IGlobalExtension; +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; +import spock.config.RunnerConfiguration; + +import java.util.Random; + +public class RandomRunOrderExtension implements IGlobalExtension { + private static final Random RANDOM = new Random(); + + private final RunnerConfiguration config; + + public RandomRunOrderExtension(RunnerConfiguration config) { + this.config = config; + } + + @Override + public void visitSpec(SpecInfo spec) { + if (config.randomizeSpecRunOrder) + spec.setExecutionOrder(RANDOM.nextInt()); + if (config.randomizeFeatureRunOrder) { + for (FeatureInfo featureInfo : spec.getAllFeatures()) + featureInfo.setExecutionOrder(RANDOM.nextInt()); + } + } +} diff --git a/spock-core/src/main/java/spock/config/RunnerConfiguration.java b/spock-core/src/main/java/spock/config/RunnerConfiguration.java index 5a01be9887..73ffd08bfb 100644 --- a/spock-core/src/main/java/spock/config/RunnerConfiguration.java +++ b/spock-core/src/main/java/spock/config/RunnerConfiguration.java @@ -28,7 +28,9 @@ * annotation some.pkg.Slow * baseClass IntegrationSpec * } - * filterStackTrace true // this is the default + * filterStackTrace true // this is the default + * randomizeSpecRunOrder false // this is the default + * randomizeFeatureRunOrder false // this is the default * } * */ @@ -39,4 +41,6 @@ public class RunnerConfiguration { public ParallelConfiguration parallel = new ParallelConfiguration(); public boolean filterStackTrace = true; public boolean optimizeRunOrder = false; + public boolean randomizeSpecRunOrder = false; + public boolean randomizeFeatureRunOrder = false; } diff --git a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension index 778826130b..f78e62228f 100644 --- a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension +++ b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension @@ -1,3 +1,4 @@ org.spockframework.runtime.extension.builtin.IncludeExcludeExtension org.spockframework.runtime.extension.builtin.OptimizeRunOrderExtension +org.spockframework.runtime.extension.builtin.RandomRunOrderExtension org.spockframework.runtime.extension.builtin.UnrollExtension diff --git a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy index a0d72daf58..6fdfd5ed9f 100644 --- a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy @@ -15,7 +15,7 @@ class ExtensionClassesLoaderSpec extends Specification { def result = new ExtensionClassesLoader().loadExtensionClassesFromDefaultLocation() then: - result == [IncludeExcludeExtension, OptimizeRunOrderExtension, UnrollExtension] + result == [IncludeExcludeExtension, OptimizeRunOrderExtension, RandomRunOrderExtension, UnrollExtension] } def "loads global ConfigObjects"() {