Skip to content

Commit 8c2b05a

Browse files
AndrewFossAWSAndrewFossAWS
committed
Add selector compliance tests
1 parent fe77f68 commit 8c2b05a

File tree

15 files changed

+1009
-0
lines changed

15 files changed

+1009
-0
lines changed

‎docs/source-1.0/spec/core/selectors.rst

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1646,5 +1646,75 @@ Selectors are defined by the following ABNF_ grammar.
16461646
SelectorVariableSet :"$" `smithy:Identifier` "(" `Selector` ")"
16471647
SelectorVariableGet :"${" `smithy:Identifier` "}"
16481648
1649+
1650+
Compliance Tests
1651+
================
1652+
1653+
Selector compliance tests are used to verify the behavior of selectors. Each compliance test is written as a Smithy file
1654+
and includes a :ref:`metadata <metadata>` called ``selectorTests``. This metadata contains a list of test cases, each including a selector,
1655+
the expected matched shapes, and additional configuration options. The test case contains the following properties:
1656+
1657+
.. list-table::
1658+
:header-rows: 1
1659+
:widths: 10 20 70
1660+
1661+
* - Property
1662+
- Type
1663+
- Description
1664+
* - selector
1665+
- ``string``
1666+
- **REQUIRED** The selector to match shapes within the smithy model
1667+
* - matches
1668+
- ``list<shape ID>``
1669+
- **REQUIRED** The expected shapes ID of the matched shapes
1670+
* - skipPreludeShapes
1671+
- ``boolean``
1672+
- Skip :ref:`prelude shapes <prelude>` when comparing the expected shapes and the actual shapes returned from the selector. Default value is ``false``
1673+
1674+
Below is an example selector compliance test:
1675+
1676+
.. code-block:: smithy
1677+
1678+
$version: "1.0"
1679+
1680+
metadata selectorTests = [
1681+
{
1682+
selector: "[trait|length|min > 1]"
1683+
matches: [
1684+
smithy.example#AtLeastTen
1685+
]
1686+
}
1687+
{
1688+
selector: "[trait|length|min >= 1]"
1689+
skipPreludeShapes: true
1690+
matches: [
1691+
smithy.example#AtLeastOne
1692+
smithy.example#AtLeastTen
1693+
]
1694+
}
1695+
{
1696+
selector: "[trait|length|min < 2]"
1697+
skipPreludeShapes: true
1698+
matches: [
1699+
smithy.example#AtLeastOne
1700+
]
1701+
}
1702+
]
1703+
1704+
namespace smithy.example
1705+
1706+
@length(min: 1)
1707+
string AtLeastOne
1708+
1709+
@length(max: 5)
1710+
string AtMostFive
1711+
1712+
@length(min: 10)
1713+
string AtLeastTen
1714+
1715+
The compliance tests can also be accessed in this
1716+
`directory <https://github.com/awslabs/smithy/tree/main/smithy-model/src/test/resources/software/amazon/smithy/model/selector/cases>`__
1717+
of the Smithy Github repository.
1718+
16491719
.. _ABNF: https://tools.ietf.org/html/rfc5234
16501720
.. _set: https://en.wikipedia.org/wiki/Set_(abstract_data_type)

‎docs/source-2.0/spec/selectors.rst

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1663,4 +1663,75 @@ Selectors are defined by the following :rfc:`ABNF <5234>` grammar.
16631663
SelectorVariableSet :"$" `smithy:Identifier` "(" `Selector` ")"
16641664
SelectorVariableGet :"${" `smithy:Identifier` "}"
16651665
1666+
1667+
Compliance Tests
1668+
================
1669+
1670+
Selector compliance tests are used to verify the behavior of selectors. Each compliance test is written as a Smithy file
1671+
and includes a :ref:`metadata <metadata>` called ``selectorTests``. This metadata contains a list of test cases, each including a selector,
1672+
the expected matched shapes, and additional configuration options. The test case contains the following properties:
1673+
1674+
.. list-table::
1675+
:header-rows: 1
1676+
:widths: 10 20 70
1677+
1678+
* - Property
1679+
- Type
1680+
- Description
1681+
* - selector
1682+
- ``string``
1683+
- **REQUIRED** The selector to match shapes within the smithy model
1684+
* - matches
1685+
- ``list<shape ID>``
1686+
- **REQUIRED** The expected shapes ID of the matched shapes
1687+
* - skipPreludeShapes
1688+
- ``boolean``
1689+
- Skip :ref:`prelude shapes <prelude>` when comparing the expected shapes and the actual shapes returned from the selector. Default value is ``false``
1690+
1691+
Below is an example selector compliance test:
1692+
1693+
.. code-block:: smithy
1694+
1695+
$version: "2.0"
1696+
1697+
metadata selectorTests = [
1698+
{
1699+
selector: "[trait|length|min > 1]"
1700+
matches: [
1701+
smithy.example#AtLeastTen
1702+
]
1703+
}
1704+
{
1705+
selector: "[trait|length|min >= 1]"
1706+
skipPreludeShapes: true
1707+
matches: [
1708+
smithy.example#AtLeastOne
1709+
smithy.example#AtLeastTen
1710+
]
1711+
}
1712+
{
1713+
selector: "[trait|length|min < 2]"
1714+
skipPreludeShapes: true
1715+
matches: [
1716+
smithy.example#AtLeastOne
1717+
]
1718+
}
1719+
]
1720+
1721+
namespace smithy.example
1722+
1723+
@length(min: 1)
1724+
string AtLeastOne
1725+
1726+
@length(max: 5)
1727+
string AtMostFive
1728+
1729+
@length(min: 10)
1730+
string AtLeastTen
1731+
1732+
The compliance tests can also be accessed in this
1733+
`directory <https://github.com/awslabs/smithy/tree/main/smithy-model/src/test/resources/software/amazon/smithy/model/selector/cases>`__
1734+
of the Smithy Github repository.
1735+
1736+
16661737
.. _set: https://en.wikipedia.org/wiki/Set_(abstract_data_type)

‎smithy-model/src/test/java/software/amazon/smithy/model/selector/SelectorRunnerTest.java

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package software.amazon.smithy.model.selector;
2+
3+
import java.io.IOException;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
import java.nio.file.Paths;
7+
import java.util.ArrayList;
8+
import java.util.HashSet;
9+
import java.util.List;
10+
import java.util.Set;
11+
import java.util.TreeSet;
12+
import java.util.stream.Collectors;
13+
import java.util.stream.Stream;
14+
import org.junit.jupiter.api.Assertions;
15+
import org.junit.jupiter.params.ParameterizedTest;
16+
import org.junit.jupiter.params.provider.MethodSource;
17+
import software.amazon.smithy.model.Model;
18+
import software.amazon.smithy.model.loader.Prelude;
19+
import software.amazon.smithy.model.node.ObjectNode;
20+
import software.amazon.smithy.model.shapes.Shape;
21+
import software.amazon.smithy.model.shapes.ShapeId;
22+
23+
public final class SelectorRunnerTest {
24+
25+
@ParameterizedTest(name = "{0}")
26+
@MethodSource("source")
27+
public void selectorTest(Path filename) {
28+
Model model = Model.assembler().addImport(filename).assemble().unwrap();
29+
List<ObjectNode> tests = findTestCases(model);
30+
31+
for (ObjectNode test : tests) {
32+
Selector selector = Selector.parse(test.expectStringMember("selector").getValue());
33+
boolean skipPrelude = test.getBooleanMemberOrDefault("skipPreludeShapes", false);
34+
35+
Set<ShapeId> expectedMatches = test.expectArrayMember("matches")
36+
.getElementsAs(n -> n.expectStringNode("Each element of matches must be an ID").expectShapeId())
37+
.stream()
38+
.filter(shapeId -> {
39+
String namespace = shapeId.getNamespace();
40+
return !namespace.contains(Prelude.NAMESPACE)
41+
|| (!skipPrelude && namespace.contains(Prelude.NAMESPACE));
42+
})
43+
.collect(Collectors.toSet());
44+
45+
Set<ShapeId> actualMatches = selector.shapes(model)
46+
.map(Shape::getId)
47+
.filter(shapeId -> {
48+
String namespace = shapeId.getNamespace();
49+
return !namespace.contains(Prelude.NAMESPACE)
50+
|| (!skipPrelude && namespace.contains(Prelude.NAMESPACE));
51+
})
52+
.collect(Collectors.toSet());
53+
54+
if (!expectedMatches.equals(actualMatches)) {
55+
failTest(filename, test, expectedMatches, actualMatches);
56+
}
57+
}
58+
}
59+
60+
private List<ObjectNode> findTestCases(Model model) {
61+
return model.getMetadataProperty("selectorTests")
62+
.orElseThrow(() -> new IllegalArgumentException("Missing selectorTests metadata key"))
63+
.expectArrayNode("selectorTests must be an array")
64+
.getElementsAs(ObjectNode.class);
65+
}
66+
67+
private void failTest(Path filename, ObjectNode test, Set<ShapeId> expectedMatches, Set<ShapeId> actualMatches) {
68+
String selector = test.expectStringMember("selector").getValue();
69+
Set<ShapeId> missing = new TreeSet<>(expectedMatches);
70+
missing.removeAll(actualMatches);
71+
72+
Set<ShapeId> extra = new TreeSet<>(actualMatches);
73+
extra.removeAll(expectedMatches);
74+
75+
StringBuilder error = new StringBuilder("Selector ")
76+
.append(selector)
77+
.append(" test case failed.\n");
78+
79+
if (!missing.isEmpty()) {
80+
error.append("The following shapes were not matched: ").append(missing).append(".\n");
81+
}
82+
83+
if (!extra.isEmpty()) {
84+
error.append("The following shapes were matched unexpectedly: ").append(extra).append(".\n");
85+
}
86+
87+
test.getStringMember("documentation")
88+
.ifPresent(docs -> error.append('(').append(docs.getValue()).append(")"));
89+
90+
Assertions.fail(error.toString());
91+
}
92+
93+
public static List<Path> source() throws Exception {
94+
List<Path> paths = new ArrayList<>();
95+
try (Stream<Path> files = Files.walk(Paths.get(SelectorRunnerTest.class.getResource("cases").toURI()))) {
96+
files
97+
.filter(Files::isRegularFile)
98+
.filter(file -> {
99+
String filename = file.toString();
100+
return filename.endsWith(".smithy") || filename.endsWith(".json");
101+
})
102+
.forEach(paths::add);
103+
} catch (IOException e) {
104+
throw new RuntimeException("Error loading models for selector runner", e);
105+
}
106+
107+
return paths;
108+
}
109+
}

‎smithy-model/src/test/resources/software/amazon/smithy/model/selector/cases/attribute-existence.smithy

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
$version: "2.0"
2+
3+
metadata selectorTests = [
4+
{
5+
selector: "[trait|enum]"
6+
matches: [
7+
smithy.example#SimpleEnum
8+
smithy.example#EnumWithTags
9+
]
10+
},
11+
{
12+
selector: "[trait|enum|(values)|tags|(values)]"
13+
matches: [
14+
smithy.example#EnumWithTags
15+
]
16+
}
17+
]
18+
19+
namespace smithy.example
20+
21+
@deprecated
22+
string NoMatch
23+
24+
@enum([
25+
{name: "foo", value: "foo"}
26+
{name: "baz", value: "baz"}
27+
])
28+
string SimpleEnum
29+
30+
@enum([
31+
{name: "foo", value: "foo", tags: ["a"]}
32+
{name: "baz", value: "baz"}
33+
{name: "spam", value: "spam", tags: []}
34+
])
35+
string EnumWithTags

0 commit comments

Comments
 (0)