ci: run more Chromium unit tests (electron#15363)

* test: add an option to native tests runner to run also disabled tests

* ci: run all native tests on clean Chromium

* ci: update the disabled tests list for native unittests

.circleci/config.yml

@@ -974,7 +974,7 @@ jobs:
       <<: *env-unittests
       <<: *env-enable-sccache
       <<: *env-headless-testing
-      TESTS_ARGS: '--run-only-disabled-tests'
+      TESTS_ARGS: '--only-disabled-tests'
     <<: *steps-native-tests
 
   linux-x64-chromium-unittests:
@@ -983,6 +983,7 @@ jobs:
       <<: *env-unittests
       <<: *env-enable-sccache
       <<: *env-headless-testing
+      TESTS_ARGS: '--include-disabled-tests'
     <<: *steps-native-tests
 
   linux-x64-browsertests:

script/lib/native_tests.py

@@ -44,6 +44,12 @@ def le(a, b):
     return a_index <= b_index
 
 
+class DisabledTestsPolicy:
+  DISABLE = 'disable'  # Disabled tests are disabled. Wow. Much sense.
+  ONLY = 'only'  # Only disabled tests should be run.
+  INCLUDE = 'include'  # Do not disable any tests.
+
+
 class Platform:
   LINUX = 'linux'
   MAC = 'mac'
@@ -88,7 +94,7 @@ def get_for_current_platform(self):
     return supported_binaries
 
   def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
     # Don't run anything twice.
     binaries = set(binaries)
 
@@ -106,19 +112,19 @@ def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
                 binary_name, Platform.get_current()))
 
     suite_returncode = sum(
-        [self.__run(binary, output_dir, verbosity, run_only_disabled_tests)
+        [self.__run(binary, output_dir, verbosity, disabled_tests_policy)
         for binary in binaries])
     return suite_returncode
 
   def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
     return self.run([binary_name], output_dir, verbosity,
-                    run_only_disabled_tests)
+                    disabled_tests_policy)
 
   def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
     return self.run(self.get_for_current_platform(), output_dir, verbosity,
-                    run_only_disabled_tests)
+                    disabled_tests_policy)
 
   @staticmethod
   def __get_tests_list(config_path):
@@ -200,21 +206,23 @@ def __get_test_data(data_item):
     return (binary_name, test_data)
 
   def __run(self, binary_name, output_dir, verbosity,
-      run_only_disabled_tests):
+      disabled_tests_policy):
     binary_path = os.path.join(self.tests_dir, binary_name)
     test_binary = TestBinary(binary_path)
 
     test_data = self.tests[binary_name]
     included_tests = []
     excluded_tests = test_data['excluded_tests']
 
-    if run_only_disabled_tests and len(excluded_tests) == 0:
-      # There is nothing to run.
-      return 0
-
-    if run_only_disabled_tests:
+    if disabled_tests_policy == DisabledTestsPolicy.ONLY:
+      if len(excluded_tests) == 0:
+        # There is nothing to run.
+        return 0
       included_tests, excluded_tests = excluded_tests, included_tests
 
+    if disabled_tests_policy == DisabledTestsPolicy.INCLUDE:
+      excluded_tests = []
+
     output_file_path = TestsList.__get_output_path(binary_name, output_dir)
 
     return test_binary.run(included_tests=included_tests,

script/native-tests.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-from lib.native_tests import TestsList, Verbosity
+from lib.native_tests import TestsList, Verbosity, DisabledTestsPolicy
 
 
 class Command:
@@ -25,14 +25,24 @@ def parse_args():
                       help='binaries to run')
   parser.add_argument('-c', '--config', required=True,
                       help='path to a tests config')
-  parser.add_argument('--run-only-disabled-tests',
-                      action='store_true', default=False,
-                      help='if disabled tests should be run')
   parser.add_argument('-t', '--tests-dir', required=False,
                       help='path to a directory with test binaries')
   parser.add_argument('-o', '--output-dir', required=False,
                       help='path to a folder to save tests results')
 
+  disabled_tests = parser.add_mutually_exclusive_group()
+  disabled_tests.add_argument('--only-disabled-tests',
+                              dest='disabled_tests_policy',
+                              action='store_const',
+                              const=DisabledTestsPolicy.ONLY,
+                              help='run disabled tests only')
+  disabled_tests.add_argument('--include-disabled-tests',
+                              dest='disabled_tests_policy',
+                              action='store_const',
+                              const=DisabledTestsPolicy.INCLUDE,
+                              help='if disabled tests should be run as well')
+  parser.set_defaults(disabled_tests_policy=DisabledTestsPolicy.DISABLE)
+
   verbosity = parser.add_mutually_exclusive_group()
   verbosity.add_argument('-v', '--verbosity', required=False,
                          default=Verbosity.CHATTY,
@@ -86,10 +96,10 @@ def main():
   if args.command == Command.RUN:
     if args.binary is not None:
       return tests_list.run(args.binary, args.output_dir, args.verbosity,
-                            args.run_only_disabled_tests)
+                            args.disabled_tests_policy)
     else:
       return tests_list.run_all(args.output_dir, args.verbosity,
-                                args.run_only_disabled_tests)
+                                args.disabled_tests_policy)
 
   assert False, "unexpected command '{}'".format(args.command)
 

spec/configs/unittests.yml

@@ -1,23 +1,34 @@
+# Note: a "disabled" dict can have arbitrary named properties
+# with lists of disabled tests as a value. Those properties' names
+# are for humans only and are there only to explain
+# why a group of tests is disabled.
+
 tests:
   - base_unittests:
-      # "disabled" dict can have arbitrary named properties
-      # with lists of disabled tests. Those properties' names
-      # are used only to explain why a group of tests is disabled.
       disabled:
-        undecided:
-          - FilePathTest*
+        upstream_failures:
+          - FilePathTest.FromUTF8Unsafe_And_AsUTF8Unsafe
+          - RTLTest.WrapPathWithLTRFormatting
+          - SysStrings.SysNativeMBAndWide
+          - SysStrings.SysNativeMBToWide
+          - SysStrings.SysWideToNativeMB
+        to_fix:
           - PartitionReallocReturnNullTest.RepeatedReturnNullDirect
-          - RTLTest*
-          - SysStrings*
-          - UTFOffsetStringConversionsTest*
           - WeakPtrDeathTest*
   - cc_unittests
   - cc_blink_unittests
   - content_unittests:
       disabled:
-        # TODO(alexeykuzmin): Should those be fixed?
-        - _/DOMStorageMapParamTest.EnforcesQuota/0
-        - _/DOMStorageMapParamTest.EnforcesQuota/1
+        upstream_failures:
+          - BackgroundSyncManagerTest.*
+          - FrameServiceBaseTest.ConnectionError
+          - MediaSessionControllerTest.*
+          - MediaStreamUIProxyFeaturePolicyTest.FeaturePolicy
+          - PresentationServiceImplTest.*
+        to_fix:
+          - _/DOMStorageMapParamTest.EnforcesQuota/0
+          - _/DOMStorageMapParamTest.EnforcesQuota/1
+          - RenderFrameAudioInputStreamFactoryTest.ConstructDestruct
   - crypto_unittests
   - device_unittests
   - gin_unittests
@@ -31,39 +42,19 @@ tests:
   - mojo_common_unittests
   - net_unittests:
       disabled:
-        # TODO(alexeykuzmin): Should those be fixed?
-        - EndToEndTestsWithTls*
-        - FilenameUtilTest*
-        - HTTPSAIATest.AIAFetching
-        - HTTPSCRLSetTest*
-        - HTTPSEVCRLSetTest*
-        - HTTPSFallbackTest*
-        - HTTPSHardFailTest.FailsOnOCSPInvalid
-        - HTTPSOCSPTest*
-        - HTTPSRequestTest*
-        - HTTPSSessionTest*
-        - OCSPVerify*
-        - ProxyConfigServiceLinuxTest*
-        - SSLClientSocketCertRequestInfoTest*
-        - SSLClientSocketChannelIDTest*
-        - SSLClientSocketFalseStartTest*
-        - SSLClientSocketReadTest*
-        - SSLClientSocketTest*
-        - TokenBindingURLRequestTest*
-        - UDPSocketTest*
-        - URLRequestTestFTP*
-        - URLRequestTestHTTP*
-        - WebSocketEndToEndTest*
+        upstream_failures:
+          - EndToEndTestsWithTls/EndToEndTestWithTls.SimpleRequestResponsev6/*
+          - FilenameUtilTest.GenerateFileName
+          - SSLClientSocketTest.*
+          - UDPSocketTest.ClientGetLocalPeerAddresses
+          - UDPSocketTest.ClientSetDoNotFragment
   - ppapi_unittests
   - printing_unittests
   - skia_unittests
   - sql_unittests
   - storage_unittests
   - gn_unittests
-  - ui_base_unittests:
-      disabled:
-        # TODO(alexeykuzmin): Should it be fixed?
-        - AcceleratorTest.GetShortcutText
+  - ui_base_unittests
   - compositor_unittests
   - display_unittests
   - events_unittests
@@ -73,31 +64,43 @@ tests:
   # V8
   - unittests:
       disabled:
-        # TODO(alexeykuzmin): Should it be fixed?
-        - LiveRangeUnitTest*
+        upstream_failures:
+          - LiveRangeUnitTest.InvalidConstruction
+          - LiveRangeUnitTest.SplinterMultipleIntervalsInside
+          - LiveRangeUnitTest.SplitInvalidStart
   # Blink
   - blink_common_unittests
   - blink_heap_unittests
   - blink_platform_unittests:
       disabled:
         # TODO(alexeykuzmin): Should those be fixed?
-        - AnimatedPNGTests*
-        - AnimatedWebPTests*
+        - AnimatedPNGTests.*
+        - AnimatedWebPTests.*
         - AnimatedWEBPTests.clearCacheExceptFrameWithAncestors
-        - BitmapImageTest*
+        - BitmapImageTest.*
         - BMPImageDecoderTest*
         - DecodedImageOrientationHistogramTest*
         - DecodedImageTypeHistogramTest*
+        - DeferredImageDecoderTest.decodeOnOtherThread
         - DeferredImageDecoderTestWoPlatform*
         - GIFImageDecoderTest*
         - HighContrastImageClassifierTest*
         - ICOImageDecoderTests*
         - JPEGImageDecoderTest*
         - FilePathConversionTest*
+        - PaintPropertyNodeTest.InitialStateAndReset
         - PNGTests*
         - StaticPNGTests*
         - StaticWebPTests*
-  - webkit_unit_tests
+  - webkit_unit_tests:
+      disabled:
+        upstream_failures:
+          - Document/ThreadableLoaderTest.CancelAfterStart/0
+          - ModernMediaControlsImplTest.ControlsShouldUseSafeAreaInsets
+          - ScriptRunnerTest.QueueReentrantScript_InOrder
+          - ScrollbarsTestWithVirtualTimer.PressScrollbarButtonOnInfiniteScrolling
+          - VirtualTimeTest.AllowVirtualTimeToAdvance
+          - VirtualTimeTest.VirtualTimeNotAllowedToAdvanceWhileResourcesLoading
   - wtf_unittests
   # Third party
   - angle_unittests
@@ -108,13 +111,16 @@ tests:
   - sandbox_linux_unittests:
       platform: linux
       disabled:
-        # TODO(alexeykuzmin): Should these be fixed?
-        - BrokerProcess*
-        - SandboxBPF.SigBus
+        upstream_failures:
+          - BrokerProcess.OpenFilePermsWithClientCheck
+          - BrokerProcess.OpenFilePermsWithClientCheckNoEnt
+          - BrokerProcess.OpenOpenFilePermsNoClientCheck
+          - BrokerProcess.OpenOpenFilePermsNoClientCheckNoEnt
+          - SandboxBPF.SigBus
   - dbus_unittests:
       platform: linux
       disabled:
-        # TODO(alexeykuzmin): Should these be fixed?
-        - EndToEndAsyncTest.InvalidServiceName
-        - EndToEndSyncTest.InvalidServiceName
-        - MessageTest.SetInvalidHeaders
+        upstream_failures:
+          - EndToEndAsyncTest.InvalidServiceName
+          - EndToEndSyncTest.InvalidServiceName
+          - MessageTest.SetInvalidHeaders