Better error case handling, auto snapshotting, minor cleanup

Author: MichaelRFairhurst

CHANGELOG.md

@@ -4,6 +4,11 @@
 - Simplifier improvements: bugfixes, faster fixed-point search.
 - Fixed issue treating failures as executions (printing . & double counting)
 - New simplifier constraint "contains output"
+- Better error when port is already taken and observatory fails to start
+- Auto-snapshot scripts for users
+- Better error when script is erroneous (though erroneous snapshots may not give
+  a good error message)
+- switch from `vm_service_lib` to `vm_service`
 
 * 1.0.0-beta.8
 

README.md

@@ -188,18 +188,23 @@ The fuzzer works like so:
 - [ ] improve error handling for cases where the dart VM crashes etc
 - [ ] auto-snapshot scripts for users
 - [ ] improve coverage %
+- [ ] provide coverage report in standardized format
 - [ ] some renames in the code: Library/Corpus
 - [ ] targeted scoring for paths through certain files/packages/etc
-- [ ] better (different?) simplification algorithm(s?)
+- [ ] entropy based simplifier algorithm
 - [ ] automatic detection of simpler seeds during fuzzing?
 - [ ] store fuzzing options in script file (such as custom mutators, timeouts)
 - [ ] use locality sensitive hashing to dedupe failures with different messages
       (or in the case of timeouts, the same messages) by jaccard index of their
       code coverage sets. Perhaps from: https://arxiv.org/pdf/1811.04633
 - [ ] customizable limits & timeouts for simplifier?
-- [ ] can we benefit from kernel transformers? For instance, break apart string
-      checks?
-- [ ] service extensions?
-- [ ] special value recording? (perhaps via service extensions)?
+- [ ] special value recording via service extensions + kernel transformer?
+- [ ] break apart string comparisons with kernel transformer?
+- [ ] other service extensions?
+- [ ] other kernel transformer?
+- [ ] semantically-valid-dart transformer
+- [ ] generations of seeds?
+- [ ] way to exclude or score seeds when found?
+- [ ] change scoring of failed seeds?
 
 etc.

lib/src/cli.dart

@@ -92,6 +92,11 @@ class Cli {
         abbr: 'l',
         help: 'Whether to simplify input & discovered seeds.',
         defaultsTo: true)
+    ..addFlag('snapshot',
+        abbr: 'a',
+        help: 'Whether to generate a snapshot before fuzzing. Defaults to true'
+            ' for scripts that end in .dart',
+        defaultsTo: null)
     ..addCommand(
         'simplify',
         ArgParser()
@@ -132,7 +137,7 @@ class Cli {
       _usageAndExit();
     }
 
-    final script = args.rest[0];
+    var script = args.rest[0];
 
     int batchSize;
     int vms;
@@ -167,6 +172,12 @@ class Cli {
       await failurePersistence.load();
     }
 
+    if (args['snapshot'] ?? script.endsWith('.dart')) {
+      final snapshot = '$script.snapshot';
+      await VmController.snapshot(script, snapshot);
+      script = snapshot;
+    }
+
     List<VmController> vmControllers;
     _CliStats cliStats;
     List<IsolateMutator> isolateMutators;
@@ -183,8 +194,7 @@ class Cli {
       await Future.wait(
           vmControllers.map((vmController) => vmController.prestart()));
 
-      final statsCollector =
-          StatsCollector(ProgramStats(await vmControllers[0].countPaths()));
+      final statsCollector = StatsCollector(ProgramStats(0));
       isolateMutators = await _getIsolateMutators(args);
       final mutators = _getMutators(args, isolateMutators, seedLibrary);
       final driver = Driver(seedLibrary, failureLibrary, batchSize,

lib/src/vm_controller.dart

@@ -5,15 +5,14 @@
 import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
-import 'dart:math';
 
 import 'package:dust/src/path.dart';
 import 'package:dust/src/path_canonicalizer.dart';
 import 'package:dust/src/vm_result.dart';
 import 'package:path/path.dart' as path;
 import 'package:pedantic/pedantic.dart';
-import 'package:vm_service_lib/vm_service_lib.dart';
-import 'package:vm_service_lib/vm_service_lib_io.dart';
+import 'package:vm_service/vm_service.dart';
+import 'package:vm_service/vm_service_io.dart';
 
 /// A class to start a VM, connect to its service, and control its fuzz cases.
 ///
@@ -47,17 +46,6 @@ class VmController {
   /// Check if this VM is connected.
   bool get isConnected => _serviceClient != null;
 
-  /// Count all paths that could be exercised for the target script.
-  Future<int> countPaths() async {
-    final fuzzIsolate = await _preRunCase('');
-
-    final pathCount = await _countPath(fuzzIsolate);
-
-    await _finalizeOutput(fuzzIsolate);
-
-    return pathCount;
-  }
-
   /// Kill the process & close the service connection.
   Future<void> dispose() async {
     try {
@@ -85,8 +73,10 @@ class VmController {
     final fuzzIsolate = await _preRunCase(input);
 
     final paths = readCoverage ? await _getPath(fuzzIsolate) : null;
+    await _serviceClient.resume(fuzzIsolate.id);
+    await _fuzzIsolateDead();
 
-    final jsonOut = await _finalizeOutput(fuzzIsolate);
+    final jsonOut = await _finalizeOutput();
 
     final bool succeeded = jsonOut['success'];
     if (succeeded) {
@@ -122,19 +112,6 @@ class VmController {
     return _serviceClient;
   }
 
-  Future<int> _countPath(Isolate isolate) async {
-    final report = await _serviceClient
-        .getSourceReport(isolate.id, [SourceReportKind.kCoverage]);
-    var sum = 0;
-    for (final range in report.ranges) {
-      if (range.coverage == null) {
-        continue;
-      }
-      sum += range.coverage.hits.length + range.coverage.misses.length;
-    }
-    return sum;
-  }
-
   Future<T> _exponentialBackoff<T>(
       Future<T> Function() action, bool Function(T) accept,
       {Duration limit = const Duration(seconds: 5)}) async {
@@ -166,14 +143,9 @@ class VmController {
     throw Exception('$limit tries exceeded: $reason');
   }
 
-  /// Continue [fuzzIsolate] to unblock the main isolate and get json output.
-  Future<Map<String, dynamic>> _finalizeOutput(Isolate fuzzIsolate) async {
-    await _serviceClient.resume(fuzzIsolate.id);
-    await _fuzzIsolateDead();
-
-    return _exponentialBackoff(
-        () async => jsonDecode(_outputBuffer.toString().trim()), (_) => true);
-  }
+  /// Read the json output from the main isolate.
+  Future<Map<String, dynamic>> _finalizeOutput() async => _exponentialBackoff(
+      () async => jsonDecode(_outputBuffer.toString().trim()), (_) => true);
 
   Future<Isolate> _fuzzIsolateComplete() async {
     final isolateRef = await _exponentialBackoff(
@@ -184,7 +156,7 @@ class VmController {
 
     final isolate = await _exponentialBackoff(
         () async => _serviceClient.getIsolate(isolateRef.id),
-        (isolate) => isolate.pauseEvent.kind == 'PauseExit',
+        (isolate) => isolate == null || isolate.pauseEvent.kind == 'PauseExit',
         limit: Duration(seconds: _timeout + 2));
 
     _timeElapsed ??= DateTime.now().difference(_startTime);
@@ -234,7 +206,7 @@ class VmController {
       path.join(Platform.environment['HOME'],
           '.pub-cache/global_packages/dust/bin/controller.dart.snapshot.dart2'),
       _script,
-      "$_timeout",
+      '$_timeout',
     ]);
 
     final vmCompleter = Completer();
@@ -246,12 +218,39 @@ class VmController {
     }));
     _processExit = vmCompleter.future;
 
+    _outputBuffer = StringBuffer();
     _process.stdout
         .transform(utf8.decoder)
         .listen((output) => _outputBuffer?.write(output));
 
     _stdErrBuffer = StringBuffer();
     _process.stderr.transform(utf8.decoder).listen(_stdErrBuffer.write);
+
+    // Ensure observatory starts up properly.
+    // TODO: Try a different port on failure.
+    try {
+      await _exponentialBackoff(
+          () async => _outputBuffer.toString(),
+          (output) =>
+              output.contains('listening on') && output.contains(':$_port/'));
+    } catch (_) {
+      throw 'Observatory did not start on $_port\noutput:\n$_outputBuffer';
+    }
+  }
+
+  /// Generate a snapshot for the script, for faster fuzzing.
+  static Future<void> snapshot(String script, String snapshotPath) async {
+    final sdk = path.dirname(path.dirname(Platform.resolvedExecutable));
+
+    final result = await Process.run('$sdk/bin/dart', [
+      '--snapshot=$snapshotPath',
+      '--snapshot-kind=kernel',
+      script,
+    ]);
+
+    if (result.exitCode != 0) {
+      throw '${result.stdout}${result.stderr}';
+    }
   }
 }
 

pubspec.yaml

@@ -11,7 +11,7 @@ dependencies:
   pedantic: '^1.8.0+1'
   crypto: '^2.0.6'
   args: '^1.5.2'
-  vm_service_lib: '^3.21.0'
+  vm_service: '^1.2.0'
   test: '^1.6.5'
   quiver: '^2.0.3'
   collection: '^1.14.11'