Introduce gtest (#4466)

Add gtest as a git submodule in third_party and integrate it into the build the
same way WABT does. Adds a new executable, `binaryen-unittests`, to execute
`gtest_main`. As a nontrivial example test, port one of the `TypeBuilder` tests
from example/ to gtest/.

Using gtest has a number of advantages over the current example tests:

 - Tests are compiled and linked at build time rather than runtime, surfacing
   errors earlier and speeding up test execution.

 - Tests are all built into a single binary, reducing overall link time and
   further reducing test overhead.

 - Tests are built from the same CMake project as the rest of Binaryen, so
   compiler settings (e.g. sanitizers) are applied uniformly rather than having
   to be separately set via the COMPILER_FLAGS environment variable.

 - Using the industry-standard gtest rather than our own script reduces our
   maintenance burden.

Using gtest will lower the barrier to writing C++ tests and will hopefully lead
to us having more proper unit tests.

.flake8

@@ -3,4 +3,4 @@ ignore =
   E501, # line too long
   E241, # space after comma (ignored for list in gen-s-parser.py)
   W504  # line break after binary operator
-exclude = ./test/emscripten,./test/spec,./test/wasm-install,./test/lit
+exclude = third_party,./test/emscripten,./test/spec,./test/wasm-install,./test/lit

.github/workflows/ci.yml

@@ -20,6 +20,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: install tools
       run: |
         sudo pip3 install -r requirements-dev.txt
@@ -45,6 +47,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
 
     - name: install Python dev dependencies
       run: pip3 install -r requirements-dev.txt
@@ -111,6 +115,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: install ninja
       run: sudo apt-get install ninja-build
     - name: install Python dev dependencies
@@ -138,6 +144,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: install ninja
       run: sudo apt-get install ninja-build
     - name: install Python dev dependencies
@@ -162,6 +170,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: start docker
       run: |
         docker run -w /src -dit --name alpine -v $PWD:/src node:lts-alpine
@@ -200,6 +210,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: install ninja
       run: sudo apt-get install ninja-build
     - name: install Python dev dependencies
@@ -225,6 +237,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: install ninja
       run: sudo apt-get install ninja-build
     - name: install Python dev dependencies
@@ -247,6 +261,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: install ninja
       run: sudo apt-get install ninja-build
     - name: emsdk install
@@ -272,6 +288,8 @@ jobs:
       with:
         python-version: '3.x'
     - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: cmake
       run: |
         mkdir -p out

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -105,6 +105,16 @@ function(binaryen_setup_rpath name)
                         ${_install_name_dir})
 endfunction()
 
+function(binaryen_add_executable name sources)
+  add_executable(${name} ${sources})
+  target_link_libraries(${name} ${CMAKE_THREAD_LIBS_INIT})
+  target_link_libraries(${name} binaryen)
+  set_property(TARGET ${name} PROPERTY CXX_STANDARD ${CXX_STANDARD})
+  set_property(TARGET ${name} PROPERTY CXX_STANDARD_REQUIRED ON)
+  binaryen_setup_rpath(${name})
+  install(TARGETS ${name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()
+
 # Options
 
 option(BUILD_STATIC_LIB "Build as a static library" OFF)
@@ -306,6 +316,9 @@ add_subdirectory(third_party)
 # Configure lit tests
 add_subdirectory(test/lit)
 
+# Configure GTest unit tests
+add_subdirectory(test/gtest)
+
 # Object files
 set(binaryen_objs
     $<TARGET_OBJECTS:passes>

check.py

@@ -42,9 +42,12 @@ def get_changelog_version():
 def run_version_tests():
     print('[ checking --version ... ]\n')
 
-    not_executable_suffix = ['.DS_Store', '.txt', '.js', '.ilk', '.pdb', '.dll', '.wasm', '.manifest', 'binaryen-lit']
+    not_executable_suffix = ['.DS_Store', '.txt', '.js', '.ilk', '.pdb', '.dll', '.wasm', '.manifest']
+    not_executable_prefix = ['binaryen-lit', 'binaryen-unittests']
     bin_files = [os.path.join(shared.options.binaryen_bin, f) for f in os.listdir(shared.options.binaryen_bin)]
-    executables = [f for f in bin_files if os.path.isfile(f) and not any(f.endswith(s) for s in not_executable_suffix)]
+    executables = [f for f in bin_files if os.path.isfile(f) and
+                   not any(f.endswith(s) for s in not_executable_suffix) and
+                   not any(os.path.basename(f).startswith(s) for s in not_executable_prefix)]
     executables = sorted(executables)
     assert len(executables)
 
@@ -337,6 +340,18 @@ def run():
     shared.with_pass_debug(run)
 
 
+def run_gtest():
+    def run():
+        gtest = os.path.join(shared.options.binaryen_bin, 'binaryen-unittests')
+        result = subprocess.run(gtest)
+        if result.returncode != 0:
+            shared.num_failures += 1
+        if shared.options.abort_on_first_failure and shared.num_failures:
+            raise Exception("gtest test failed")
+
+    shared.with_pass_debug(run)
+
+
 TEST_SUITES = OrderedDict([
     ('version', run_version_tests),
     ('wasm-opt', wasm_opt.test_wasm_opt),
@@ -355,6 +370,7 @@ def run():
     ('binaryenjs', binaryenjs.test_binaryen_js),
     ('binaryenjs_wasm', binaryenjs.test_binaryen_wasm),
     ('lit', run_lit),
+    ('gtest', run_gtest),
 ])
 
 

src/tools/CMakeLists.txt

@@ -1,13 +1,3 @@
-function(binaryen_add_executable name sources)
-  add_executable(${name} ${sources})
-  target_link_libraries(${name} ${CMAKE_THREAD_LIBS_INIT})
-  target_link_libraries(${name} binaryen)
-  set_property(TARGET ${name} PROPERTY CXX_STANDARD ${CXX_STANDARD})
-  set_property(TARGET ${name} PROPERTY CXX_STANDARD_REQUIRED ON)
-  binaryen_setup_rpath(${name})
-  install(TARGETS ${name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
-
 include_directories(fuzzing)
 FILE(GLOB fuzzing_HEADERS fuzzing/*h)
 set(fuzzing_SOURCES

test/example/type-builder.cpp

@@ -5,70 +5,6 @@
 
 using namespace wasm;
 
-// Construct Signature, Struct, and Array heap types using undefined types.
-void test_builder() {
-  std::cout << ";; Test TypeBuilder\n";
-
-  // (type $sig (func (param (ref $struct)) (result (ref $array) i32)))
-  // (type $struct (struct (field (ref null $array) (mut rtt 0 $array))))
-  // (type $array (array (mut externref)))
-
-  TypeBuilder builder;
-  assert(builder.size() == 0);
-  builder.grow(3);
-  assert(builder.size() == 3);
-
-  Type refSig = builder.getTempRefType(builder[0], NonNullable);
-  Type refStruct = builder.getTempRefType(builder[1], NonNullable);
-  Type refArray = builder.getTempRefType(builder[2], NonNullable);
-  Type refNullArray = builder.getTempRefType(builder[2], Nullable);
-  Type rttArray = builder.getTempRttType(Rtt(0, builder[2]));
-  Type refNullExt(HeapType::ext, Nullable);
-
-  Signature sig(refStruct, builder.getTempTupleType({refArray, Type::i32}));
-  Struct struct_({Field(refNullArray, Immutable), Field(rttArray, Mutable)});
-  Array array(Field(refNullExt, Mutable));
-
-  std::cout << "Before setting heap types:\n";
-  std::cout << "(ref $sig) => " << refSig << "\n";
-  std::cout << "(ref $struct) => " << refStruct << "\n";
-  std::cout << "(ref $array) => " << refArray << "\n";
-  std::cout << "(ref null $array) => " << refNullArray << "\n";
-  std::cout << "(rtt 0 $array) => " << rttArray << "\n\n";
-
-  builder[0] = sig;
-  builder[1] = struct_;
-  builder[2] = array;
-
-  std::cout << "After setting heap types:\n";
-  std::cout << "(ref $sig) => " << refSig << "\n";
-  std::cout << "(ref $struct) => " << refStruct << "\n";
-  std::cout << "(ref $array) => " << refArray << "\n";
-  std::cout << "(ref null $array) => " << refNullArray << "\n";
-  std::cout << "(rtt 0 $array) => " << rttArray << "\n\n";
-
-  std::vector<HeapType> built = builder.build();
-
-  Type newRefSig = Type(built[0], NonNullable);
-  Type newRefStruct = Type(built[1], NonNullable);
-  Type newRefArray = Type(built[2], NonNullable);
-  Type newRefNullArray = Type(built[2], Nullable);
-  Type newRttArray = Type(Rtt(0, built[2]));
-
-  assert(refSig != newRefSig);
-  assert(refStruct != newRefStruct);
-  assert(refArray != newRefArray);
-  assert(refNullArray != newRefNullArray);
-  assert(rttArray != newRttArray);
-
-  std::cout << "After building types:\n";
-  std::cout << "(ref $sig) => " << newRefSig << "\n";
-  std::cout << "(ref $struct) => " << newRefStruct << "\n";
-  std::cout << "(ref $array) => " << newRefArray << "\n";
-  std::cout << "(ref null $array) => " << newRefNullArray << "\n";
-  std::cout << "(rtt 0 $array) => " << newRttArray << "\n\n";
-}
-
 // Check that the builder works when there are duplicate definitions
 void test_canonicalization() {
   std::cout << ";; Test canonicalization\n";
@@ -496,7 +432,6 @@ int main() {
   // Run the tests twice to ensure things still work when the global stores are
   // already populated.
   for (size_t i = 0; i < 2; ++i) {
-    test_builder();
     test_canonicalization();
     test_basic();
     test_recursive();

test/example/type-builder.txt

@@ -1,25 +1,3 @@
-;; Test TypeBuilder
-Before setting heap types:
-(ref $sig) => [T](ref [T](func))
-(ref $struct) => [T](ref [T](func))
-(ref $array) => [T](ref [T](func))
-(ref null $array) => [T](ref null [T](func))
-(rtt 0 $array) => [T](rtt 0 [T](func))
-
-After setting heap types:
-(ref $sig) => [T](ref [T](func (param [T](ref [T](struct (field [T](ref null [T](array (mut externref))) (mut [T](rtt 0 [T](array (mut externref)))))))) (result [T](ref [T](array (mut externref))) i32)))
-(ref $struct) => [T](ref [T](struct (field [T](ref null [T](array (mut externref))) (mut [T](rtt 0 [T](array (mut externref)))))))
-(ref $array) => [T](ref [T](array (mut externref)))
-(ref null $array) => [T](ref null [T](array (mut externref)))
-(rtt 0 $array) => [T](rtt 0 [T](array (mut externref)))
-
-After building types:
-(ref $sig) => (ref (func (param (ref (struct (field (ref null (array (mut externref))) (mut (rtt 0 (array (mut externref)))))))) (result (ref (array (mut externref))) i32)))
-(ref $struct) => (ref (struct (field (ref null (array (mut externref))) (mut (rtt 0 (array (mut externref)))))))
-(ref $array) => (ref (array (mut externref)))
-(ref null $array) => (ref null (array (mut externref)))
-(rtt 0 $array) => (rtt 0 (array (mut externref)))
-
 ;; Test canonicalization
 ;; Test basic
 ;; Test recursive types
@@ -48,28 +26,6 @@ After building types:
 (func (param anyref) (result (ref null ...1)))
 
 ;; Test LUBs
-;; Test TypeBuilder
-Before setting heap types:
-(ref $sig) => [T](ref [T](func))
-(ref $struct) => [T](ref [T](func))
-(ref $array) => [T](ref [T](func))
-(ref null $array) => [T](ref null [T](func))
-(rtt 0 $array) => [T](rtt 0 [T](func))
-
-After setting heap types:
-(ref $sig) => [T](ref [T](func (param [T](ref [T](struct (field [T](ref null [T](array (mut externref))) (mut [T](rtt 0 [T](array (mut externref)))))))) (result [T](ref [T](array (mut externref))) i32)))
-(ref $struct) => [T](ref [T](struct (field [T](ref null [T](array (mut externref))) (mut [T](rtt 0 [T](array (mut externref)))))))
-(ref $array) => [T](ref [T](array (mut externref)))
-(ref null $array) => [T](ref null [T](array (mut externref)))
-(rtt 0 $array) => [T](rtt 0 [T](array (mut externref)))
-
-After building types:
-(ref $sig) => (ref (func (param (ref (struct (field (ref null (array (mut externref))) (mut (rtt 0 (array (mut externref)))))))) (result (ref (array (mut externref))) i32)))
-(ref $struct) => (ref (struct (field (ref null (array (mut externref))) (mut (rtt 0 (array (mut externref)))))))
-(ref $array) => (ref (array (mut externref)))
-(ref null $array) => (ref null (array (mut externref)))
-(rtt 0 $array) => (rtt 0 (array (mut externref)))
-
 ;; Test canonicalization
 ;; Test basic
 ;; Test recursive types

test/gtest/CMakeLists.txt

@@ -0,0 +1,8 @@
+include_directories(../../third_party/googletest/googletest/include)
+
+set(unittest_SOURCES
+  type-builder.cpp
+)
+
+binaryen_add_executable(binaryen-unittests "${unittest_SOURCES}")
+target_link_libraries(binaryen-unittests gtest gtest_main)

test/gtest/type-builder.cpp

@@ -0,0 +1,64 @@
+#include <gtest/gtest.h>
+#include <wasm-type.h>
+
+using namespace wasm;
+
+TEST(TypeBuilder, Growth) {
+  TypeBuilder builder;
+  EXPECT_EQ(builder.size(), size_t{0});
+  builder.grow(3);
+  EXPECT_EQ(builder.size(), size_t{3});
+}
+
+TEST(TypeBuilder, Basics) {
+  // (type $sig (func (param (ref $struct)) (result (ref $array) i32)))
+  // (type $struct (struct (field (ref null $array) (mut rtt 0 $array))))
+  // (type $array (array (mut externref)))
+
+  TypeBuilder builder(3);
+  ASSERT_EQ(builder.size(), size_t{3});
+
+  Type refSig = builder.getTempRefType(builder[0], NonNullable);
+  Type refStruct = builder.getTempRefType(builder[1], NonNullable);
+  Type refArray = builder.getTempRefType(builder[2], NonNullable);
+  Type refNullArray = builder.getTempRefType(builder[2], Nullable);
+  Type rttArray = builder.getTempRttType(Rtt(0, builder[2]));
+  Type refNullExt(HeapType::ext, Nullable);
+
+  Signature sig(refStruct, builder.getTempTupleType({refArray, Type::i32}));
+  Struct struct_({Field(refNullArray, Immutable), Field(rttArray, Mutable)});
+  Array array(Field(refNullExt, Mutable));
+
+  builder[0] = sig;
+  builder[1] = struct_;
+  builder[2] = array;
+
+  std::vector<HeapType> built = builder.build();
+  ASSERT_EQ(built.size(), size_t{3});
+
+  // The built types should have the correct kinds.
+  ASSERT_TRUE(built[0].isSignature());
+  ASSERT_TRUE(built[1].isStruct());
+  ASSERT_TRUE(built[2].isArray());
+
+  // The built types should have the correct structure.
+  Type newRefSig = Type(built[0], NonNullable);
+  Type newRefStruct = Type(built[1], NonNullable);
+  Type newRefArray = Type(built[2], NonNullable);
+  Type newRefNullArray = Type(built[2], Nullable);
+  Type newRttArray = Type(Rtt(0, built[2]));
+
+  EXPECT_EQ(built[0].getSignature(),
+            Signature(newRefStruct, {newRefArray, Type::i32}));
+  EXPECT_EQ(
+    built[1].getStruct(),
+    Struct({Field(newRefNullArray, Immutable), Field(newRttArray, Mutable)}));
+  EXPECT_EQ(built[2].getArray(), Array(Field(refNullExt, Mutable)));
+
+  // The built types should be different from the temporary types.
+  EXPECT_NE(newRefSig, refSig);
+  EXPECT_NE(newRefStruct, refStruct);
+  EXPECT_NE(newRefArray, refArray);
+  EXPECT_NE(newRefNullArray, refNullArray);
+  EXPECT_NE(newRttArray, rttArray);
+}

third_party/CMakeLists.txt

@@ -1,3 +1,16 @@
 if(BUILD_LLVM_DWARF)
   add_subdirectory(llvm-project)
 endif()
+
+include_directories(
+  googletest/googletest
+  googletest/googletest/include
+)
+
+add_library(gtest STATIC
+  googletest/googletest/src/gtest-all.cc
+)
+
+add_library(gtest_main STATIC
+  googletest/googletest/src/gtest_main.cc
+)