[ci] Add LUCI web platform tests (flutter#4391)

Adds web platform tests for LUCI in bringup mode.

The Cirrus version of the test starts `chromedriver` at the beginning and just leaves it running, which is fine since it's a fresh docker image each run. For LUCI we don't want that behavior though, so instead this adds tooling support for running chromedriver as part of running the integration tests, controllable with a flag. This will also be useful for running locally.

Part of flutter/flutter#114373

Author: stuartmorgan

-

@@ -437,6 +437,50 @@ targets:
       target_file: web_build_all_packages.yaml
       channel: stable
 
+  - name: Linux_web web_platform_tests_shard_1 master
+    bringup: true # New target
+    recipe: packages/packages
+    timeout: 60
+    properties:
+      target_file: web_platform_tests.yaml
+      cores: "32"
+      version_file: flutter_master.version
+      channel: master
+      package_sharding: "--shardIndex 0 --shardCount 2"
+
+  - name: Linux_web web_platform_tests_shard_2 master
+    bringup: true # New target
+    recipe: packages/packages
+    timeout: 60
+    properties:
+      target_file: web_platform_tests.yaml
+      cores: "32"
+      version_file: flutter_master.version
+      channel: master
+      package_sharding: "--shardIndex 1 --shardCount 2"
+
+  - name: Linux_web web_platform_tests_shard_1 stable
+    bringup: true # New target
+    recipe: packages/packages
+    timeout: 60
+    properties:
+      target_file: web_platform_tests.yaml
+      cores: "32"
+      version_file: flutter_stable.version
+      channel: stable
+      package_sharding: "--shardIndex 0 --shardCount 2"
+
+  - name: Linux_web web_platform_tests_shard_2 stable
+    bringup: true # New target
+    recipe: packages/packages
+    timeout: 60
+    properties:
+      target_file: web_platform_tests.yaml
+      cores: "32"
+      version_file: flutter_stable.version
+      channel: stable
+      package_sharding: "--shardIndex 1 --shardCount 2"
+
   ### Linux desktop tasks
   - name: Linux_desktop build_all_packages master
     recipe: packages/packages

.ci.yaml

@@ -0,0 +1,9 @@
+tasks:
+  - name: prepare tool
+    script: .ci/scripts/prepare_tool.sh
+  - name: build examples
+    script: script/tool_runner.sh
+    args: ["build-examples", "--web"]
+  - name: drive examples
+    script: script/tool_runner.sh
+    args: ["drive-examples", "--web", "--run-chromedriver", "--exclude=script/configs/exclude_integration_web.yaml"]

.ci/targets/web_platform_tests.yaml

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
@@ -17,6 +18,9 @@ import 'common/repository_package.dart';
 const int _exitNoPlatformFlags = 2;
 const int _exitNoAvailableDevice = 3;
 
+// From https://docs.flutter.dev/testing/integration-tests#running-in-a-browser
+const int _chromeDriverPort = 4444;
+
 /// A command to run the integration tests for a package's example applications.
 class DriveExamplesCommand extends PackageLoopingCommand {
   /// Creates an instance of the drive command.
@@ -44,8 +48,13 @@ class DriveExamplesCommand extends PackageLoopingCommand {
       help:
           'Runs the driver tests in Dart VM with the given experiments enabled.',
     );
+    argParser.addFlag(_chromeDriverFlag,
+        help: 'Runs chromedriver for the duration of the test.\n\n'
+            'Requires the correct version of chromedriver to be in your path.');
   }
 
+  static const String _chromeDriverFlag = 'run-chromedriver';
+
   @override
   final String name = 'drive-examples';
 
@@ -197,6 +206,12 @@ class DriveExamplesCommand extends PackageLoopingCommand {
 
       testsRan = true;
       if (useFlutterDrive) {
+        Process? chromedriver;
+        if (getBoolArg(_chromeDriverFlag)) {
+          print('Starting chromedriver on port $_chromeDriverPort');
+          chromedriver = await processRunner
+              .start('chromedriver', <String>['--port=$_chromeDriverPort']);
+        }
         for (final File driver in drivers) {
           final List<File> failingTargets = await _driveTests(
               example, driver, testTargets,
@@ -206,6 +221,10 @@ class DriveExamplesCommand extends PackageLoopingCommand {
                 getRelativePosixPath(failingTarget, from: package.directory));
           }
         }
+        if (chromedriver != null) {
+          print('Stopping chromedriver');
+          chromedriver.kill();
+        }
       } else {
         if (!await _runTests(example, deviceFlags: deviceFlags)) {
           errors.add('Integration tests failed.');

script/tool/lib/src/drive_examples_command.dart

@@ -520,6 +520,54 @@ void main() {
           ]));
     });
 
+    test('runs chromedriver when requested', () async {
+      final RepositoryPackage plugin = createFakePlugin(
+        'plugin',
+        packagesDir,
+        extraFiles: <String>[
+          'example/integration_test/plugin_test.dart',
+          'example/test_driver/integration_test.dart',
+          'example/web/index.html',
+        ],
+        platformSupport: <String, PlatformDetails>{
+          platformWeb: const PlatformDetails(PlatformSupport.inline),
+        },
+      );
+
+      final Directory pluginExampleDirectory = getExampleDir(plugin);
+
+      final List<String> output = await runCapturingPrint(
+          runner, <String>['drive-examples', '--web', '--run-chromedriver']);
+
+      expect(
+        output,
+        containsAllInOrder(<Matcher>[
+          contains('Running for plugin'),
+          contains('No issues found!'),
+        ]),
+      );
+
+      expect(
+          processRunner.recordedCalls,
+          orderedEquals(<ProcessCall>[
+            const ProcessCall('chromedriver', <String>['--port=4444'], null),
+            ProcessCall(
+                getFlutterCommand(mockPlatform),
+                const <String>[
+                  'drive',
+                  '-d',
+                  'web-server',
+                  '--web-port=7357',
+                  '--browser-name=chrome',
+                  '--driver',
+                  'test_driver/integration_test.dart',
+                  '--target',
+                  'integration_test/plugin_test.dart',
+                ],
+                pluginExampleDirectory.path),
+          ]));
+    });
+
     test('drives a web plugin with CHROME_EXECUTABLE', () async {
       final RepositoryPackage plugin = createFakePlugin(
         'plugin',

script/tool/test/drive_examples_command_test.dart

@@ -79,6 +79,9 @@ class MockProcess extends Mock implements io.Process {
 
   @override
   IOSink get stdin => stdinMock;
+
+  @override
+  bool kill([io.ProcessSignal signal = io.ProcessSignal.sigterm]) => true;
 }
 
 class MockIOSink extends Mock implements IOSink {