ci: run unittests on clean Chromium (electron#15125)

* ci: run unittests on clean Chromium

* test: add '--run-only-disabled-tests' flag to "script/native-tests.py"

* ci: add a job to run only disabled unittests

* test: use a different GN config for the native tests

* test: enable blink_common_unittests

* test: disable WeakPtrDeathTest*

.circleci/config.yml

@@ -42,13 +42,13 @@ env-stack-dumping: &env-stack-dumping
   ELECTRON_ENABLE_STACK_DUMPING: '1'
 
 env-browsertests: &env-browsertests
-  GN_EXTRA_ARGS: 'is_component_ffmpeg = false'
-  BUILD_TARGET: electron:chromium_browsertests
+  GN_CONFIG: //electron/build/args/native_tests.gn
+  BUILD_TARGET: electron/spec:chromium_browsertests
   TESTS_CONFIG: src/electron/spec/configs/browsertests.yml
 
 env-unittests: &env-unittests
-  GN_EXTRA_ARGS: 'is_component_ffmpeg = false'
-  BUILD_TARGET: electron:chromium_unittests
+  GN_CONFIG: //electron/build/args/native_tests.gn
+  BUILD_TARGET: electron/spec:chromium_unittests
   TESTS_CONFIG: src/electron/spec/configs/unittests.yml
 
 # Build targets options.
@@ -598,7 +598,8 @@ steps-native-tests: &steps-native-tests
           python src/electron/script/native-tests.py run \
             --config $TESTS_CONFIG \
             --tests-dir src/out/Default \
-            --output-dir test_results
+            --output-dir test_results  \
+            $TESTS_ARGS
 
     - store_artifacts:
         path: test_results
@@ -723,6 +724,12 @@ jobs:
     <<: *machine-linux-2xlarge
     <<: *steps-checkout
 
+  linux-x64-checkout-no-patches:
+    <<: *machine-linux-2xlarge
+    environment:
+      GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False'
+    <<: *steps-checkout
+
   # Layer 2: Builds.
   linux-x64-debug:
     <<: *machine-linux-2xlarge
@@ -956,7 +963,23 @@ jobs:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-unittests
-      <<: *env-testing-build
+      <<: *env-enable-sccache
+      <<: *env-headless-testing
+    <<: *steps-native-tests
+
+  linux-x64-disabled-unittests:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-unittests
+      <<: *env-enable-sccache
+      <<: *env-headless-testing
+      TESTS_ARGS: '--run-only-disabled-tests'
+    <<: *steps-native-tests
+
+  linux-x64-chromium-unittests:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-unittests
       <<: *env-enable-sccache
       <<: *env-headless-testing
     <<: *steps-native-tests
@@ -1234,3 +1257,13 @@ workflows:
       - linux-x64-unittests:
           requires:
             - linux-x64-checkout
+
+      - linux-x64-disabled-unittests:
+          requires:
+            - linux-x64-checkout
+
+      - linux-x64-checkout-no-patches
+
+      - linux-x64-chromium-unittests:
+          requires:
+            - linux-x64-checkout-no-patches

BUILD.gn

@@ -810,75 +810,6 @@ if (is_mac) {
   }
 }
 
-group("electron_tests") {
-  testonly = true
-
-  deps = [
-    ":chromium_browsertests",
-    ":chromium_unittests",
-  ]
-}
-
-group("chromium_unittests") {
-  testonly = true
-
-  deps = [
-    "//base:base_unittests",
-    "//cc:cc_unittests",
-    "//content/test:content_unittests",
-    "//crypto:crypto_unittests",
-    "//device:device_unittests",
-    "//gin:gin_unittests",
-    "//gpu:gpu_unittests",
-    "//ipc:ipc_tests",
-    "//media:media_unittests",
-    "//media/capture:capture_unittests",
-    "//media/midi:midi_unittests",
-    "//media/mojo:media_mojo_unittests",
-    "//mojo:mojo_unittests",
-    "//net:net_unittests",
-    "//ppapi:ppapi_unittests",
-    "//skia:skia_unittests",
-    "//sql:sql_unittests",
-    "//storage:storage_unittests",
-    "//third_party/angle/src/tests:angle_unittests",
-    "//third_party/blink/public:all_blink",
-    "//third_party/blink/public:test_support",
-    "//third_party/leveldatabase:env_chromium_unittests",
-    "//ui/base:ui_base_unittests",
-    "//ui/compositor:compositor_unittests",
-    "//ui/display:display_unittests",
-    "//ui/events:events_unittests",
-    "//ui/gl:gl_unittests",
-    "//url:url_unittests",
-    "//url/ipc:url_ipc_unittests",
-    "//v8/test/unittests:unittests",
-  ]
-
-  if (enable_basic_printing) {
-    deps += [ "//printing:printing_unittests" ]
-  }
-
-  if (is_linux) {
-    deps += [
-      "//net:disk_cache_memory_test",
-      "//sandbox/linux:sandbox_linux_unittests",
-    ]
-
-    if (use_dbus) {
-      deps += [ "//dbus:dbus_unittests" ]
-    }
-  }
-}
-
-group("chromium_browsertests") {
-  testonly = true
-
-  deps = [
-    "//content/test:content_browsertests",
-  ]
-}
-
 template("dist_zip") {
   _runtime_deps_target = "${target_name}__deps"
   _runtime_deps_file =

build/args/native_tests.gn

@@ -0,0 +1,8 @@
+root_extra_deps = [ "//electron/spec" ]
+
+dcheck_always_on = true
+is_debug = false
+is_component_build = false
+is_component_ffmpeg = false
+symbol_level = 1
+use_jumbo_build = true

script/lib/native_tests.py

@@ -87,7 +87,8 @@ def get_for_current_platform(self):
     supported_binaries = filter(self.__platform_supports, all_binaries)
     return supported_binaries
 
-  def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY):
+  def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
+      run_only_disabled_tests=False):
     # Don't run anything twice.
     binaries = set(binaries)
 
@@ -105,14 +106,19 @@ def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY):
                 binary_name, Platform.get_current()))
 
     suite_returncode = sum(
-        [self.__run(binary, output_dir, verbosity) for binary in binaries])
+        [self.__run(binary, output_dir, verbosity, run_only_disabled_tests)
+        for binary in binaries])
     return suite_returncode
 
-  def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY):
-    return self.run([binary_name], output_dir, verbosity)
+  def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY,
+      run_only_disabled_tests=False):
+    return self.run([binary_name], output_dir, verbosity,
+                    run_only_disabled_tests)
 
-  def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY):
-    return self.run(self.get_for_current_platform(), output_dir, verbosity)
+  def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY,
+      run_only_disabled_tests=False):
+    return self.run(self.get_for_current_platform(), output_dir, verbosity,
+                    run_only_disabled_tests)
 
   @staticmethod
   def __get_tests_list(config_path):
@@ -169,7 +175,7 @@ def __get_test_data(data_item):
 
     binary_name = data_item.keys()[0]
     test_data = {
-      'excluded_tests': None,
+      'excluded_tests': [],
       'platforms': Platform.get_all()
     }
 
@@ -193,16 +199,26 @@ def __get_test_data(data_item):
 
     return (binary_name, test_data)
 
-  def __run(self, binary_name, output_dir, verbosity):
+  def __run(self, binary_name, output_dir, verbosity,
+      run_only_disabled_tests):
     binary_path = os.path.join(self.tests_dir, binary_name)
     test_binary = TestBinary(binary_path)
 
     test_data = self.tests[binary_name]
+    included_tests = []
     excluded_tests = test_data['excluded_tests']
 
+    if run_only_disabled_tests and len(excluded_tests) == 0:
+      # There is nothing to run.
+      return 0
+
+    if run_only_disabled_tests:
+      included_tests, excluded_tests = excluded_tests, included_tests
+
     output_file_path = TestsList.__get_output_path(binary_name, output_dir)
 
-    return test_binary.run(excluded_tests=excluded_tests,
+    return test_binary.run(included_tests=included_tests,
+                           excluded_tests=excluded_tests,
                            output_file_path=output_file_path,
                            verbosity=verbosity)
 
@@ -221,9 +237,10 @@ class TestBinary():
   def __init__(self, binary_path):
     self.binary_path = binary_path
 
-  def run(self, excluded_tests=None, output_file_path=None,
-      verbosity=Verbosity.CHATTY):
-    gtest_filter = TestBinary.__get_gtest_filter(excluded_tests)
+  def run(self, included_tests=None, excluded_tests=None,
+      output_file_path=None, verbosity=Verbosity.CHATTY):
+    gtest_filter = TestBinary.__get_gtest_filter(included_tests,
+                                                 excluded_tests)
     gtest_output = TestBinary.__get_gtest_output(output_file_path)
 
     args = [self.binary_path, gtest_filter, gtest_output]
@@ -241,12 +258,12 @@ def run(self, excluded_tests=None, output_file_path=None,
     return returncode
 
   @staticmethod
-  def __get_gtest_filter(excluded_tests):
-    gtest_filter = ""
-    if excluded_tests is not None and len(excluded_tests) > 0:
-      excluded_tests_string = TestBinary.__format_excluded_tests(
-          excluded_tests)
-      gtest_filter = "--gtest_filter={}".format(excluded_tests_string)
+  def __get_gtest_filter(included_tests, excluded_tests):
+    included_tests_string = TestBinary.__list_tests(included_tests)
+    excluded_tests_string = TestBinary.__list_tests(excluded_tests)
+
+    gtest_filter = "--gtest_filter={}-{}".format(included_tests_string,
+                                                 excluded_tests_string)
     return gtest_filter
 
   @staticmethod
@@ -258,8 +275,10 @@ def __get_gtest_output(output_file_path):
     return gtest_output
 
   @staticmethod
-  def __format_excluded_tests(excluded_tests):
-    return "-" + ":".join(excluded_tests)
+  def __list_tests(tests):
+    if tests is None:
+      return ''
+    return ':'.join(tests)
 
   @staticmethod
   def __get_stdout_and_stderr(verbosity):

script/native-tests.py

@@ -25,6 +25,9 @@ def parse_args():
                       help='binaries to run')
   parser.add_argument('-c', '--config', required=True,
                       help='path to a tests config')
+  parser.add_argument('--run-only-disabled-tests',
+                      action='store_true', default=False,
+                      help='if disabled tests should be run')
   parser.add_argument('-t', '--tests-dir', required=False,
                       help='path to a directory with test binaries')
   parser.add_argument('-o', '--output-dir', required=False,
@@ -82,9 +85,11 @@ def main():
 
   if args.command == Command.RUN:
     if args.binary is not None:
-      return tests_list.run(args.binary, args.output_dir, args.verbosity)
+      return tests_list.run(args.binary, args.output_dir, args.verbosity,
+                            args.run_only_disabled_tests)
     else:
-      return tests_list.run_all(args.output_dir, args.verbosity)
+      return tests_list.run_all(args.output_dir, args.verbosity,
+                                args.run_only_disabled_tests)
 
   assert False, "unexpected command '{}'".format(args.command)
 

spec/BUILD.gn

@@ -0,0 +1,72 @@
+import("//build/config/features.gni")
+import("//printing/buildflags/buildflags.gni")
+
+group("spec") {
+  testonly = true
+
+  deps = [
+    ":chromium_browsertests",
+    ":chromium_unittests",
+  ]
+}
+
+group("chromium_unittests") {
+  testonly = true
+
+  deps = [
+    "//base:base_unittests",
+    "//cc:cc_unittests",
+    "//content/test:content_unittests",
+    "//crypto:crypto_unittests",
+    "//device:device_unittests",
+    "//gin:gin_unittests",
+    "//gpu:gpu_unittests",
+    "//ipc:ipc_tests",
+    "//media:media_unittests",
+    "//media/capture:capture_unittests",
+    "//media/midi:midi_unittests",
+    "//media/mojo:media_mojo_unittests",
+    "//mojo:mojo_unittests",
+    "//net:net_unittests",
+    "//ppapi:ppapi_unittests",
+    "//printing:printing_unittests",
+    "//skia:skia_unittests",
+    "//sql:sql_unittests",
+    "//storage:storage_unittests",
+    "//third_party/angle/src/tests:angle_unittests",
+    "//third_party/blink/public:all_blink",
+    "//third_party/blink/public:test_support",
+    "//third_party/leveldatabase:env_chromium_unittests",
+    "//ui/base:ui_base_unittests",
+    "//ui/compositor:compositor_unittests",
+    "//ui/display:display_unittests",
+    "//ui/events:events_unittests",
+    "//ui/gl:gl_unittests",
+    "//url:url_unittests",
+    "//url/ipc:url_ipc_unittests",
+    "//v8/test/unittests:unittests",
+  ]
+
+  if (enable_basic_printing) {
+    deps += [ "//printing:printing_unittests" ]
+  }
+
+  if (is_linux) {
+    deps += [
+      "//net:disk_cache_memory_test",
+      "//sandbox/linux:sandbox_linux_unittests",
+    ]
+
+    if (use_dbus) {
+      deps += [ "//dbus:dbus_unittests" ]
+    }
+  }
+}
+
+group("chromium_browsertests") {
+  testonly = true
+
+  deps = [
+    "//content/test:content_browsertests",
+  ]
+}

spec/configs/unittests.yml

@@ -10,6 +10,7 @@ tests:
           - RTLTest*
           - SysStrings*
           - UTFOffsetStringConversionsTest*
+          - WeakPtrDeathTest*
   - cc_unittests
   - cc_blink_unittests
   - content_unittests:
@@ -75,8 +76,7 @@ tests:
         # TODO(alexeykuzmin): Should it be fixed?
         - LiveRangeUnitTest*
   # Blink
-# TODO: Enable in Ch68.
-#  - blink_common_unittests
+  - blink_common_unittests
   - blink_heap_unittests
   - blink_platform_unittests:
       disabled: