Merge pull request #465 from wasmx/ci_spectests

CI: Convert WAST test to JSON

Author: chfast

circle.yml

@@ -27,6 +27,20 @@ commands:
           name: "Install system dependencies"
           command: HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install cmake ninja
 
+  install_wabt:
+    description: "Install WABT tools"
+    steps:
+      - run:
+          name: "Install WABT tools"
+          command: |
+            if type wast2json; then
+              wast2json --version
+            else
+              [[ $OSTYPE = darwin* ]] && os=macos || os=ubuntu
+              cd /usr/local
+              curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.19/wabt-1.0.19-$os.tar.gz | sudo tar xz --strip 1
+            fi
+
   build:
     description: "Build"
     parameters:
@@ -174,20 +188,24 @@ commands:
         default: 477
 
     steps:
+      - install_wabt
       - run:
           name: "Download spectest files"
           working_directory: ~/build
           command: |
             if [ ! -d wasm-spec ]; then
-              git clone https://github.com/wasmx/wasm-spec --branch w3c-1.0-jsontests-20200813 --depth 1
+              git clone https://github.com/wasmx/wasm-spec --branch w3c-1.0-jsontests-20200813 --depth 1 wasm-spec
+              mkdir json && cd json
+              options='--disable-saturating-float-to-int --disable-sign-extension --disable-multi-value'
+              find ../wasm-spec/test/core -name '*.wast' -exec wast2json $options {} \;
             fi
       - run:
           name: "Run spectest<<#parameters.skip_validation>> (skip validation)<</parameters.skip_validation>>"
           working_directory: ~/build
           command: |
             set +e
             expected="  PASSED <<parameters.expected_passed>>, FAILED <<parameters.expected_failed>>, SKIPPED <<parameters.expected_skipped>>."
-            result=$(bin/fizzy-spectests <<#parameters.skip_validation>>--skip-validation<</parameters.skip_validation>> wasm-spec/test/core/json | tail -1)
+            result=$(bin/fizzy-spectests <<#parameters.skip_validation>>--skip-validation<</parameters.skip_validation>> json | tail -1)
             echo $result
             if [ "$expected" != "$result" ]; then exit 1; fi
 
@@ -207,10 +225,7 @@ jobs:
           name: "Run codespell"
           command: |
             codespell --quiet-level=4 -I .codespell-whitelist
-      - run:
-          name: "Install wabt"
-          working_directory: ~/bin
-          command: curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.15/wabt-1.0.15-linux.tar.gz | tar xz --strip=1
+      - install_wabt
       - run:
           name: "Check wat2wasm4cpp"
           command: |

test/spectests/README.md

@@ -3,34 +3,39 @@
 ## Build and run
 
 Fizzy must be built with the `FIZZY_TESTING` option turned on:
-```sh
-$ mkdir build && cd build
-$ cmake -DFIZZY_TESTING=ON ..
-$ cmake --build .
+```shell script
+mkdir build && cd build
+cmake -DFIZZY_TESTING=ON ..
+cmake --build .
 ```
 
 It can then be executed:
-```sh
-$ bin/fizzy-spectests <test directory>
-```
-
-This will execute all test cases, but since Fizzy does not implement the complete validation specification, most of
-those cases will fail. It is possible to skip them:
-```sh
-$ bin/fizzy-spectests --skip-validation <test directory>
+```shell script
+bin/fizzy-spectests <test directory>
 ```
 
 ## Preparing tests
 
 Fizzy uses the official WebAssembly "[spec tests]", albeit not directly.
 It requires the Wast files to be translated into a JSON format using [wabt]'s `wast2json` tool.
 
-The reason for this is a design decision of Fizzy to not support the WebAssembly text format –– and unfortunately
+The reason for this is a design decision of Fizzy to not support the WebAssembly text format, and unfortunately
 the official test cases are in a text format.
 
 In order to prepare the tests, run the following command for each file:
-```sh
-$ wast2json <file.wast> -o <file.json>
+```shell script
+wast2json <file.wast> -o <file.json>
+```
+
+Make sure to disable all WebAssembly extensions when using `wast2json`:
+```shell script
+wast2json --disable-saturating-float-to-int --disable-sign-extension --disable-multi-value
+```
+
+To convert all files at once:
+```shell script
+# Make sure $options here contains the above settings
+find test/core -name '*.wast' -exec wast2json $options {} \;
 ```
 
 For ease of use, we have placed the JSON files of the spec tests [w3c-v1.0 branch] here:

wat2wasm4cpp.py

@@ -29,6 +29,8 @@
 DEBUG = False
 
 WAT2WASM_TOOL = 'wat2wasm'
+WAT2WASM_DEFAULT_OPTIONS = ['--disable-saturating-float-to-int',
+                            '--disable-sign-extension', '--disable-multi-value']
 FORMAT_TOOL = 'clang-format'
 
 WAT_RE = re.compile(r'/\* wat2wasm(.*)\n([^*]*)\*/', re.MULTILINE)
@@ -88,6 +90,7 @@ def report_wat_errors(errmsg, source, source_path, wat_pos):
         with open(TMP_WAT_FILE, 'w') as f:
             f.write(wat)
 
+        options = WAT2WASM_DEFAULT_OPTIONS + options
         r = subprocess.run([WAT2WASM_TOOL, TMP_WAT_FILE] + options,
                            capture_output=True, text=True)
         if r.returncode != 0: