Skip to content

Commit

Permalink
make the report backward compatible
Browse files Browse the repository at this point in the history 
Make the output from test262 more compatible with the test reports from before we removed the optimization levels.

Take out the option to run the tests with fewer optimization levels -- everything will run in compiled and interpreted mode by default.
  • Loading branch information
@rbri
rbri authored Dec 23, 2024
1 parent 48182a1 commit d253ea8
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 101 deletions.
63 changes: 32 additions & 31 deletions tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,6 @@ public class Test262SuiteTest {
/** The test must be executed just once--in non-strict mode, only. */
private static final String FLAG_NO_STRICT = "noStrict";

static final int[] OPT_LEVELS;

private static final File testDir = new File("test262/test");
private static final String testHarnessDir = "test262/harness/";
private static final String testProperties;
Expand Down Expand Up @@ -132,35 +130,11 @@ public class Test262SuiteTest {
includeUnsupported =
updateProps.isEmpty() || updateProps.indexOf("unsupported") != -1;
}

if (getOverriddenLevel() != null) {
System.out.println(
"Ignoring custom optLevels because the updateTest262Properties param is set");
}

OPT_LEVELS = Utils.DEFAULT_OPT_LEVELS;
} else {
updateTest262Properties = rollUpEnabled = statsEnabled = includeUnsupported = false;

// Reduce the number of tests that we run by a factor of three...
String overriddenLevel = getOverriddenLevel();
if (overriddenLevel != null) {
OPT_LEVELS = new int[] {Integer.parseInt(overriddenLevel)};
} else {
OPT_LEVELS = Utils.DEFAULT_OPT_LEVELS;
}
}
}

private static String getOverriddenLevel() {
String optLevel = System.getProperty("TEST_OPTLEVEL");

if (optLevel == null || optLevel.isEmpty()) {
optLevel = System.getenv("TEST_262_OPTLEVEL");
}
return optLevel;
}

@BeforeAll
public static void setUpClass() {
CTX_FACTORY.setLanguageVersion(Context.VERSION_ES6);
Expand Down Expand Up @@ -279,7 +253,7 @@ public static void tearDownClass() {
}

if (!testFile.isDirectory()) {
testResult = tt.getResult(OPT_LEVELS, testCases[j]);
testResult = tt.getResult(testCases[j]);

if (testResult == null) {
// At least one passing test in currentParent directory, so prevent
Expand Down Expand Up @@ -420,7 +394,9 @@ public static void tearDownClass() {
*/
private static final Pattern LINE_SPLITTER =
Pattern.compile(
"(~|(?:\\s*)(?:!|#)(?:\\s*)|\\s+)?(\\S+)(?:[^\\S\\r\\n]+(?:strict|non-strict|non-interpreted|\\d+/\\d+ \\(\\d+(?:\\.\\d+)?%%\\)|\\{(?:non-strict|strict|unsupported): \\[.*\\],?\\}))?[^\\S\\r\\n]*(.*)");
"(~|(?:\\s*)(?:!|#)(?:\\s*)|\\s+)?(\\S+)(?:[^\\S\\r\\n]+"
+ "(?:strict|non-strict|compiled-strict|compiled-non-strict|interpreted-strict|interpreted-non-strict|compiled|interpreted|"
+ "\\d+/\\d+ \\(\\d+(?:\\.\\d+)?%%\\)|\\{(?:non-strict|strict|unsupported): \\[.*\\],?\\}))?[^\\S\\r\\n]*(.*)");

/**
* @see https://github.com/tc39/test262/blob/main/INTERPRETING.md#host-defined-functions
Expand Down Expand Up @@ -1051,7 +1027,7 @@ public void passes(TestMode mode, boolean useStrict) {
modes.remove(makeKey(mode, useStrict));
}

public String getResult(int[] optLevels, Test262Case tc) {
public String getResult(Test262Case tc) {
// success on all optLevels in both strict and non-strict mode
if (modes.isEmpty()) {
return null;
Expand Down Expand Up @@ -1083,8 +1059,33 @@ public String getResult(int[] optLevels, Test262Case tc) {
return "";
}

// mix of mode and optLevel successes and failures
return '{' + String.join(",", new ArrayList<>(modes)) + '}';
// simplify the output for some cases
ArrayList res = new ArrayList<>(modes);
if (res.contains("compiled-non-strict") && res.contains("interpreted-non-strict")) {
res.remove("compiled-non-strict");
res.remove("interpreted-non-strict");
res.add("non-strict");
}
if (res.contains("compiled-strict") && res.contains("interpreted-strict")) {
res.remove("compiled-strict");
res.remove("interpreted-strict");
res.add("strict");
}
if (res.contains("compiled-strict") && res.contains("compiled-non-strict")) {
res.remove("compiled-strict");
res.remove("compiled-non-strict");
res.add("compiled");
}
if (res.contains("interpreted-strict") && res.contains("interpreted-non-strict")) {
res.remove("interpreted-strict");
res.remove("interpreted-non-strict");
res.add("interpreted");
}

if (res.size() > 1) {
return '{' + String.join(",", res) + '}';
}
return String.join(",", res);
}
}
}
1 change: 0 additions & 1 deletion tests/testsrc/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ This behavior can be changed through different means:
```
./gradlew test -DoptLevel=9
```
3. Setting an explicit optimization level through the `TEST_262_OPTLEVEL` environment variable

## Running a specific TestSuite
```
Expand Down
138 changes: 69 additions & 69 deletions tests/testsrc/test262.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -674,19 +674,19 @@ built-ins/Function 184/508 (36.22%)
prototype/apply/argarray-not-object-realm.js
prototype/apply/not-a-constructor.js
prototype/apply/resizable-buffer.js {unsupported: [resizable-arraybuffer]}
prototype/apply/S15.3.4.3_A3_T1.js non-interpreted
prototype/apply/S15.3.4.3_A3_T2.js non-interpreted
prototype/apply/S15.3.4.3_A3_T3.js non-interpreted
prototype/apply/S15.3.4.3_A3_T4.js non-interpreted
prototype/apply/S15.3.4.3_A3_T5.js non-interpreted
prototype/apply/S15.3.4.3_A3_T6.js non-interpreted
prototype/apply/S15.3.4.3_A3_T7.js non-interpreted
prototype/apply/S15.3.4.3_A3_T8.js non-interpreted
prototype/apply/S15.3.4.3_A5_T4.js non-interpreted
prototype/apply/S15.3.4.3_A7_T1.js non-interpreted
prototype/apply/S15.3.4.3_A7_T2.js non-interpreted
prototype/apply/S15.3.4.3_A7_T5.js non-interpreted
prototype/apply/S15.3.4.3_A7_T7.js non-interpreted
prototype/apply/S15.3.4.3_A3_T1.js compiled
prototype/apply/S15.3.4.3_A3_T2.js compiled
prototype/apply/S15.3.4.3_A3_T3.js compiled
prototype/apply/S15.3.4.3_A3_T4.js compiled
prototype/apply/S15.3.4.3_A3_T5.js compiled
prototype/apply/S15.3.4.3_A3_T6.js compiled-non-strict
prototype/apply/S15.3.4.3_A3_T7.js compiled
prototype/apply/S15.3.4.3_A3_T8.js compiled-non-strict
prototype/apply/S15.3.4.3_A5_T4.js compiled-non-strict
prototype/apply/S15.3.4.3_A7_T1.js compiled
prototype/apply/S15.3.4.3_A7_T2.js compiled
prototype/apply/S15.3.4.3_A7_T5.js compiled
prototype/apply/S15.3.4.3_A7_T7.js compiled
prototype/apply/this-not-callable-realm.js
prototype/bind/BoundFunction_restricted-properties.js
prototype/bind/get-fn-realm.js
Expand All @@ -706,19 +706,19 @@ built-ins/Function 184/508 (36.22%)
prototype/call/15.3.4.4-2-s.js strict
prototype/call/15.3.4.4-3-s.js strict
prototype/call/not-a-constructor.js
prototype/call/S15.3.4.4_A3_T1.js non-interpreted
prototype/call/S15.3.4.4_A3_T2.js non-interpreted
prototype/call/S15.3.4.4_A3_T3.js non-interpreted
prototype/call/S15.3.4.4_A3_T4.js non-interpreted
prototype/call/S15.3.4.4_A3_T5.js non-interpreted
prototype/call/S15.3.4.4_A3_T6.js non-interpreted
prototype/call/S15.3.4.4_A3_T7.js non-interpreted
prototype/call/S15.3.4.4_A3_T8.js non-interpreted
prototype/call/S15.3.4.4_A5_T4.js non-interpreted
prototype/call/S15.3.4.4_A6_T1.js non-interpreted
prototype/call/S15.3.4.4_A6_T2.js non-interpreted
prototype/call/S15.3.4.4_A6_T5.js non-interpreted
prototype/call/S15.3.4.4_A6_T7.js non-interpreted
prototype/call/S15.3.4.4_A3_T1.js compiled
prototype/call/S15.3.4.4_A3_T2.js compiled
prototype/call/S15.3.4.4_A3_T3.js compiled
prototype/call/S15.3.4.4_A3_T4.js compiled
prototype/call/S15.3.4.4_A3_T5.js compiled
prototype/call/S15.3.4.4_A3_T6.js compiled-non-strict
prototype/call/S15.3.4.4_A3_T7.js compiled
prototype/call/S15.3.4.4_A3_T8.js compiled-non-strict
prototype/call/S15.3.4.4_A5_T4.js compiled-non-strict
prototype/call/S15.3.4.4_A6_T1.js compiled
prototype/call/S15.3.4.4_A6_T2.js compiled
prototype/call/S15.3.4.4_A6_T5.js compiled
prototype/call/S15.3.4.4_A6_T7.js compiled
prototype/Symbol.hasInstance/length.js
prototype/Symbol.hasInstance/name.js
prototype/Symbol.hasInstance/prop-desc.js
Expand Down Expand Up @@ -843,15 +843,15 @@ built-ins/Function 184/508 (36.22%)
~built-ins/GeneratorFunction

built-ins/GeneratorPrototype 38/60 (63.33%)
next/from-state-executing.js non-interpreted
next/from-state-executing.js compiled
next/length.js
next/name.js
next/not-a-constructor.js
next/property-descriptor.js
next/this-val-not-generator.js
next/this-val-not-object.js
return 22/22 (100.0%)
throw/from-state-executing.js non-interpreted
throw/from-state-executing.js compiled
throw/length.js
throw/name.js
throw/not-a-constructor.js
Expand Down Expand Up @@ -3438,23 +3438,23 @@ language/destructuring 8/18 (44.44%)
binding/typedarray-backed-by-resizable-buffer.js {unsupported: [resizable-arraybuffer]}

language/directive-prologue 18/62 (29.03%)
14.1-1-s.js {non-strict: [-1]}
14.1-10-s.js {non-strict: [-1]}
14.1-11-s.js {non-strict: [-1]}
14.1-12-s.js {non-strict: [-1]}
14.1-13-s.js {non-strict: [-1]}
14.1-14-s.js {non-strict: [-1]}
14.1-15-s.js {non-strict: [-1]}
14.1-16-s.js non-interpreted
14.1-17-s.js non-interpreted
14.1-2-s.js {non-strict: [-1]}
14.1-3-s.js non-interpreted
14.1-4-s.js non-interpreted
14.1-5-s.js non-interpreted
14.1-6-s.js non-interpreted
14.1-7-s.js non-interpreted
14.1-8-s.js {non-strict: [-1]}
14.1-9-s.js {non-strict: [-1]}
14.1-1-s.js interpreted-non-strict
14.1-10-s.js interpreted-non-strict
14.1-11-s.js interpreted-non-strict
14.1-12-s.js interpreted-non-strict
14.1-13-s.js interpreted-non-strict
14.1-14-s.js interpreted-non-strict
14.1-15-s.js interpreted-non-strict
14.1-16-s.js compiled-non-strict
14.1-17-s.js compiled-non-strict
14.1-2-s.js interpreted-non-strict
14.1-3-s.js compiled-non-strict
14.1-4-s.js compiled-non-strict
14.1-5-s.js compiled-non-strict
14.1-6-s.js compiled-non-strict
14.1-7-s.js compiled-non-strict
14.1-8-s.js interpreted-non-strict
14.1-9-s.js interpreted-non-strict
func-decl-inside-func-decl-parse.js non-strict

language/eval-code 241/347 (69.45%)
Expand Down Expand Up @@ -4684,7 +4684,7 @@ language/expressions/generators 185/290 (63.79%)
prototype-value.js
rest-param-strict-body.js
scope-body-lex-distinct.js non-strict
scope-name-var-close.js non-interpreted
scope-name-var-close.js compiled
scope-name-var-open-non-strict.js non-strict
scope-name-var-open-strict.js strict
scope-param-rest-elem-var-close.js non-strict
Expand Down Expand Up @@ -5522,10 +5522,10 @@ language/expressions/object 719/1169 (61.51%)
__proto__-permitted-dup-shorthand.js
accessor-name-computed-in.js
accessor-name-computed-yield-id.js non-strict
accessor-name-literal-numeric-binary.js {strict: [-1], non-strict: [-1]}
accessor-name-literal-numeric-exponent.js {strict: [-1], non-strict: [-1]}
accessor-name-literal-numeric-hex.js {strict: [-1], non-strict: [-1]}
accessor-name-literal-numeric-octal.js {strict: [-1], non-strict: [-1]}
accessor-name-literal-numeric-binary.js interpreted
accessor-name-literal-numeric-exponent.js interpreted
accessor-name-literal-numeric-hex.js interpreted
accessor-name-literal-numeric-octal.js interpreted
computed-__proto__.js
cpn-obj-lit-computed-property-name-from-async-arrow-function-expression.js
cpn-obj-lit-computed-property-name-from-await-expression.js {unsupported: [module, async]}
Expand Down Expand Up @@ -5775,17 +5775,17 @@ language/function-code 122/217 (56.22%)
10.4.3-1-65gs.js
10.4.3-1-66-s.js
10.4.3-1-66gs.js
10.4.3-1-67-s.js {strict: [-1], non-strict: [-1]}
10.4.3-1-67gs.js {strict: [-1], non-strict: [-1]}
10.4.3-1-68-s.js {strict: [-1], non-strict: [-1]}
10.4.3-1-68gs.js {strict: [-1], non-strict: [-1]}
10.4.3-1-67-s.js interpreted
10.4.3-1-67gs.js interpreted
10.4.3-1-68-s.js interpreted
10.4.3-1-68gs.js interpreted
10.4.3-1-7-s.js strict
10.4.3-1-71-s.js
10.4.3-1-71gs.js
10.4.3-1-72-s.js {strict: [-1], non-strict: [-1]}
10.4.3-1-72gs.js {strict: [-1], non-strict: [-1]}
10.4.3-1-73-s.js {strict: [-1], non-strict: [-1]}
10.4.3-1-73gs.js {strict: [-1], non-strict: [-1]}
10.4.3-1-72-s.js interpreted
10.4.3-1-72gs.js interpreted
10.4.3-1-73-s.js interpreted
10.4.3-1-73gs.js interpreted
10.4.3-1-76-s.js
10.4.3-1-76gs.js
10.4.3-1-77-s.js
Expand All @@ -5794,18 +5794,18 @@ language/function-code 122/217 (56.22%)
10.4.3-1-78gs.js
10.4.3-1-7gs.js strict
10.4.3-1-8-s.js non-strict
10.4.3-1-86-s.js non-interpreted
10.4.3-1-86gs.js non-interpreted
10.4.3-1-87-s.js non-interpreted
10.4.3-1-87gs.js non-interpreted
10.4.3-1-86-s.js compiled-non-strict
10.4.3-1-86gs.js compiled-non-strict
10.4.3-1-87-s.js compiled-non-strict
10.4.3-1-87gs.js compiled-non-strict
10.4.3-1-8gs.js non-strict
10.4.3-1-9-s.js strict
10.4.3-1-90-s.js non-interpreted
10.4.3-1-90gs.js non-interpreted
10.4.3-1-91-s.js non-interpreted
10.4.3-1-91gs.js non-interpreted
10.4.3-1-92-s.js non-interpreted
10.4.3-1-92gs.js non-interpreted
10.4.3-1-90-s.js compiled-non-strict
10.4.3-1-90gs.js compiled-non-strict
10.4.3-1-91-s.js compiled-non-strict
10.4.3-1-91gs.js compiled-non-strict
10.4.3-1-92-s.js compiled-non-strict
10.4.3-1-92gs.js compiled-non-strict
10.4.3-1-9gs.js strict
block-decl-onlystrict.js strict
eval-param-env-with-computed-key.js non-strict
Expand Down Expand Up @@ -7522,7 +7522,7 @@ language/statements/try 113/201 (56.22%)
12.14-15.js non-strict
12.14-16.js non-strict
completion-values.js
completion-values-fn-finally-abrupt.js non-interpreted
completion-values-fn-finally-abrupt.js compiled
cptn-catch.js
cptn-catch-empty-break.js
cptn-catch-empty-continue.js
Expand Down

0 comments on commit d253ea8

Please sign in to comment.