From 183bc158165dff729f439b01b79f3b43e07dff3c Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Thu, 2 May 2024 23:09:03 -0700 Subject: [PATCH] Move snippets package back into flutter repo (#147690) ## Description This moves the snippets package back into the Flutter repo so that API documentation generation can happen without the use of `dart pub global run` because `pub run` doesn't handle concurrency well. The change modifies the dartdoc building process to include building an executable from the snippets tool and installing that in the cache directory for use during docs generation. The snippets tool will reside in dev/snippets, where it originally resided before being moved to https://github.com/flutter/assets-for-api-docs. The snippets code itself is unchanged from the code that is in https://github.com/flutter/assets-for-api-docs/packages/snippets. ## Related Issues - https://github.com/flutter/flutter/issues/144408 - https://github.com/flutter/flutter/issues/147609 - https://github.com/flutter/flutter/pull/147645 ## Tests - Added snippets tests to the overall testing build. --- dartdoc_options.yaml | 6 +- dev/bots/docs.sh | 17 +- dev/bots/test.dart | 16 + dev/snippets/README.md | 231 +++++++ dev/snippets/bin/snippets.dart | 295 +++++++++ dev/snippets/lib/snippets.dart | 11 + dev/snippets/lib/src/analysis.dart | 361 +++++++++++ dev/snippets/lib/src/configuration.dart | 63 ++ dev/snippets/lib/src/data_types.dart | 567 ++++++++++++++++++ dev/snippets/lib/src/import_sorter.dart | 434 ++++++++++++++ dev/snippets/lib/src/snippet_generator.dart | 497 +++++++++++++++ dev/snippets/lib/src/snippet_parser.dart | 426 +++++++++++++ dev/snippets/lib/src/util.dart | 292 +++++++++ dev/snippets/pubspec.yaml | 66 ++ dev/snippets/test/configuration_test.dart | 55 ++ dev/snippets/test/fake_process_manager.dart | 33 + .../test/filesystem_resource_provider.dart | 428 +++++++++++++ dev/snippets/test/import_sorter_test.dart | 103 ++++ dev/snippets/test/snippet_parser_test.dart | 334 +++++++++++ dev/snippets/test/snippets_test.dart | 438 ++++++++++++++ dev/snippets/test/util_test.dart | 87 +++ dev/tools/create_api_docs.dart | 8 +- 22 files changed, 4757 insertions(+), 11 deletions(-) create mode 100644 dev/snippets/README.md create mode 100644 dev/snippets/bin/snippets.dart create mode 100644 dev/snippets/lib/snippets.dart create mode 100644 dev/snippets/lib/src/analysis.dart create mode 100644 dev/snippets/lib/src/configuration.dart create mode 100644 dev/snippets/lib/src/data_types.dart create mode 100644 dev/snippets/lib/src/import_sorter.dart create mode 100644 dev/snippets/lib/src/snippet_generator.dart create mode 100644 dev/snippets/lib/src/snippet_parser.dart create mode 100644 dev/snippets/lib/src/util.dart create mode 100644 dev/snippets/pubspec.yaml create mode 100644 dev/snippets/test/configuration_test.dart create mode 100644 dev/snippets/test/fake_process_manager.dart create mode 100644 dev/snippets/test/filesystem_resource_provider.dart create mode 100644 dev/snippets/test/import_sorter_test.dart create mode 100644 dev/snippets/test/snippet_parser_test.dart create mode 100644 dev/snippets/test/snippets_test.dart create mode 100644 dev/snippets/test/util_test.dart diff --git a/dartdoc_options.yaml b/dartdoc_options.yaml index 35b37f5184d05..54a2a69ecadac 100644 --- a/dartdoc_options.yaml +++ b/dartdoc_options.yaml @@ -5,13 +5,13 @@ dartdoc: # The dev/bots/docs.sh script does this automatically. tools: snippet: - command: ["bin/cache/dart-sdk/bin/dart", "pub", "global", "run", "snippets", "--output-directory=doc/snippets", "--type=snippet"] + command: ["bin/cache/artifacts/snippets/snippets", "--output-directory=doc/snippets", "--type=snippet"] description: "Creates sample code documentation output from embedded documentation samples." sample: - command: ["bin/cache/dart-sdk/bin/dart", "pub", "global", "run", "snippets", "--output-directory=doc/snippets", "--type=sample"] + command: ["bin/cache/artifacts/snippets/snippets", "--output-directory=doc/snippets", "--type=sample"] description: "Creates full application sample code documentation output from embedded documentation samples." dartpad: - command: ["bin/cache/dart-sdk/bin/dart", "pub", "global", "run", "snippets", "--output-directory=doc/snippets", "--type=dartpad"] + command: ["bin/cache/artifacts/snippets/snippets", "--output-directory=doc/snippets", "--type=dartpad"] description: "Creates full application sample code documentation output from embedded documentation samples and displays it in an embedded DartPad." errors: ## Default errors of dartdoc: diff --git a/dev/bots/docs.sh b/dev/bots/docs.sh index e82ed0b1a4f4d..a91cc0086e50a 100755 --- a/dev/bots/docs.sh +++ b/dev/bots/docs.sh @@ -107,16 +107,25 @@ function parse_args() { fi } +function build_snippets_tool() ( + local snippets_dir="$FLUTTER_ROOT/dev/snippets" + local output_dir="$FLUTTER_BIN/cache/artifacts/snippets" + echo "Building snippets tool executable." + command cd "$snippets_dir" + mkdir -p "$output_dir" + dart pub get + dart compile exe -o "$output_dir/snippets" bin/snippets.dart +) + function generate_docs() { # Install and activate dartdoc. # When updating to a new dartdoc version, please also update # `dartdoc_options.yaml` to include newly introduced error and warning types. "$DART" pub global activate dartdoc 8.0.6 - # Install and activate the snippets tool, which resides in the - # assets-for-api-docs repo: - # https://github.com/flutter/assets-for-api-docs/tree/main/packages/snippets - "$DART" pub global activate snippets 0.4.3 + # Build and install the snippets tool, which resides in + # the dev/docs/snippets directory. + build_snippets_tool # This script generates a unified doc set, and creates # a custom index.html, placing everything into DOC_DIR. diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 86312350ea266..0cfe0e962f809 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -146,6 +146,7 @@ Future main(List args) async { 'customer_testing': customerTestingRunner, 'analyze': analyzeRunner, 'fuchsia_precache': fuchsiaPrecacheRunner, + 'snippets': _runSnippetsTests, 'docs': docsRunner, 'verify_binaries_codesigned': verifyCodesignedTestRunner, kTestHarnessShardName: testHarnessTestsRunner, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. @@ -236,6 +237,21 @@ Future _runToolTests() async { }); } +Future _runSnippetsTests() async { + final String snippetsPath = path.join(flutterRoot, 'dev', 'snippets'); + final List allTests = Directory(path.join(snippetsPath, 'test')) + .listSync(recursive: true).whereType() + .map((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath)) + .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList(); + + await runDartTest( + snippetsPath, + forceSingleCore: true, + testPaths: selectIndexOfTotalSubshard(allTests), + collectMetrics: true, + ); +} + Future runForbiddenFromReleaseTests() async { // Build a release APK to get the snapshot json. final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.'); diff --git a/dev/snippets/README.md b/dev/snippets/README.md new file mode 100644 index 0000000000000..a28e50db97230 --- /dev/null +++ b/dev/snippets/README.md @@ -0,0 +1,231 @@ +# Dartdoc Sample Generation + +The Flutter API documentation contains code blocks that help provide context or +a good starting point when learning to use any of Flutter's APIs. + +To generate these code blocks, Flutter uses dartdoc tools to turn documentation +in the source code into API documentation, as seen on [https://api.flutter.dev/] + +## Table of Contents + +- [Types of code blocks](#types-of-code-blocks) + - [Snippet tool](#snippet-tool) + - [Sample tool](#sample-tool) +- [Skeletons](#skeletons) +- [Test Doc Generation Workflow](#test-doc-generation-workflow) + +## Types of code blocks + +There are three kinds of code blocks. + +- A `snippet`, which is a more or less context-free code snippet that we + magically determine how to analyze. + +- A `dartpad` sample, which gets placed into a full-fledged application, and can + be executed inline in the documentation on the web page using + DartPad. + +- A `sample`, which gets placed into a full-fledged application, but isn't + placed into DartPad in the documentation because it doesn't make sense to do + so. + +Ideally, every sample is a DartPad sample, but some samples don't have any visual +representation and some just don't make sense that way (for example, sample +code for setting the system UI's notification area color on Android won't do +anything on the web). + +### Snippet Tool + +![Code snippet image](assets/code_snippet.png) + +The code `snippet` tool generates a block containing a description and example +code. Here is an example of the code `snippet` tool in use: + +```dart +/// {@tool snippet} +/// +/// If the avatar is to have an image, the image should be specified in the +/// [backgroundImage] property: +/// +/// ```dart +/// CircleAvatar( +/// backgroundImage: NetworkImage(userAvatarUrl), +/// ) +/// ``` +/// {@end-tool} +``` + +This will generate sample code that can be copied to the clipboard and added to +existing applications. + +This uses the skeleton for `snippet` snippets when generating the HTML to put +into the Dart docs. You can find this [template in the Flutter +repo](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/snippet.html). + +#### Analysis + +The +[`analyze_sample_code.dart`](https://github.com/flutter/flutter/blob/main/dev/bots/analyze_sample_code.dart) +script finds code inside the `@tool +snippet` sections and uses the Dart analyzer to check them. + +There are several kinds of sample code you can specify: + +- Constructor calls, typically showing what might exist in a build method. These + will be inserted into an assignment expression assigning to a variable of type + "dynamic" and followed by a semicolon, for analysis. + +- Class definitions. These start with "class", and are analyzed verbatim. + +- Other code. It gets included verbatim, though any line that says `// ...` is + considered to separate the block into multiple blocks to be processed + individually. + +The above means that it's tricky to include verbatim imperative code (e.g. a +call to a method) since it won't be valid to have such code at the top level. +Instead, wrap it in a function or even a whole class, or make it a valid +variable declaration. + +You can declare code that should be included in the analysis but not shown in +the API docs by adding a comment "// Examples can assume:" to the file (usually +at the top of the file, after the imports), following by one or more +commented-out lines of code. That code is included verbatim in the analysis. For +example: + +```dart +// Examples can assume: +// final BuildContext context; +// final String userAvatarUrl; +``` + +You can assume that the entire Flutter framework and most common +`dart:*` packages are imported and in scope; `dart:math` as `math` and +`dart:ui` as `ui`. + +### Sample Tool + +![Code sample image](assets/code_sample.png) + +The code `sample` and `dartpad` tools can expand sample code into full Flutter +applications. These sample applications can be directly copied and used to +demonstrate the API's functionality in a sample application, or used with the +`flutter create` command to create a local project with the sample code. The +`dartpad` samples are embedded into the API docs web page and are live +applications in the API documentation. + +```dart +/// {@tool sample --template=stateless_widget_material} +/// This example shows how to make a simple [FloatingActionButton] in a +/// [Scaffold], with a pink [backgroundColor] and a thumbs up [Icon]. +/// +/// ```dart +/// Widget build(BuildContext context) { +/// return Scaffold( +/// appBar: AppBar( +/// title: Text('Floating Action Button Sample'), +/// ), +/// body: Center( +/// child: Text('Press the button below!') +/// ), +/// floatingActionButton: FloatingActionButton( +/// onPressed: () { +/// // Add your onPressed code here! +/// }, +/// child: Icon(Icons.thumb_up), +/// backgroundColor: Colors.pink, +/// ), +/// ); +/// } +/// ``` +/// {@end-tool} +``` + +This uses the skeleton for [application](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/sample.html) +snippets in the Flutter repo. + +The `sample` and `dartpad` tools also allow for quick Flutter app generation +using the following command: + +```bash +flutter create --sample=[directory.File.sampleNumber] [name_of_project_directory] +``` + +This command is displayed as part of the sample in the API docs. + +#### Sample Analysis + +The [`../bots/analyze_sample_code.dart`](../bots/analyze_sample_code.dart) +script finds code inside the `@tool sample` sections and uses the Dart analyzer +to check the sample code. + +## Skeletons + +A skeleton (concerning this tool) is an HTML template into which the Dart +code blocks and descriptions are interpolated. + +There is currently one skeleton for +[application](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/sample.html) +samples, one for +[dartpad](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/dartpad-sample.html), +and one for +[snippet](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/snippet.html) +code samples, but there could be more. + +Skeletons use mustache notation (e.g. `{{code}}`) to mark where components will +be interpolated into the template. It doesn't use the mustache +package since these are simple string substitutions, but it uses the same +syntax. + +The code block generation tools that process the source input and emit HTML for +output, which dartdoc places back into the documentation. Any options given to +the `{@tool ...}` directive are passed on verbatim to the tool. + +The `snippets` tool renders these examples through a combination of markdown +and HTML using the `{@inject-html}` dartdoc directive. + +## Test Doc Generation Workflow + +If you are making changes to an existing code block or are creating a new code +block, follow these steps to generate a local copy of the API docs and verify +that your code blocks are showing up correctly: + +1. Make an update to a code block or create a new code block. +2. From the root directory, run `./dev/bots/docs.sh`. This should start + generating a local copy of the API documentation. + Supplying the "--output" argument allows you to specify the output zip file + for the completed documentation. Defaults to `api_docs.zip`` in the current + directory. +3. Once complete, unzip the files to the desired location and open the `index.html` + within. + +Note that generating the sample output will not allow you to run your code in +DartPad, because DartPad pulls the code it runs from the appropriate docs server +(main or stable). + +Copy the generated code and paste it into a regular DartPad instance to test if +it runs in DartPad. To get the code that will be produced by your documentation +changes, run sample analysis locally (see the next section) and paste the output +into a DartPad at [https://dartpad.dartlang.org]. + +## Running sample analysis locally + +If all you want to do is analyze the sample code you have written locally, then +generating the entire docs output takes a long time. + +Instead, you can run the analysis locally with this command from the Flutter root: + +```bash +TMPDIR=/tmp bin/cache/dart-sdk/bin/dart dev/bots/analyze_sample_code.dart --temp=samples +``` + +This will analyze the samples, and leave the generated files in `/tmp/samples` + +You can find the sample you are working on in `/tmp/samples`. It is named using the +path to the file it is in, and the line of the file that the `{@tool ...}` directive +is on. + +For example, the file `sample.src.widgets.animated_list.52.dart` points to the sample +in `packages/flutter/src/widgets/animated_list.dart` at line 52. You can then take the +contents of that file, and paste it into [Dartpad](https://dartpad.dev) and see if it +works. If the sample relies on new features that have just landed, it may not work +until the features make it into the `dev` branch. diff --git a/dev/snippets/bin/snippets.dart b/dev/snippets/bin/snippets.dart new file mode 100644 index 0000000000000..8590cadf4ba23 --- /dev/null +++ b/dev/snippets/bin/snippets.dart @@ -0,0 +1,295 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' show ProcessResult, exitCode, stderr; + +import 'package:args/args.dart'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; +import 'package:snippets/snippets.dart'; + +const String _kElementOption = 'element'; +const String _kFormatOutputOption = 'format-output'; +const String _kHelpOption = 'help'; +const String _kInputOption = 'input'; +const String _kLibraryOption = 'library'; +const String _kOutputDirectoryOption = 'output-directory'; +const String _kOutputOption = 'output'; +const String _kPackageOption = 'package'; +const String _kSerialOption = 'serial'; +const String _kTemplateOption = 'template'; +const String _kTypeOption = 'type'; + +class GitStatusFailed implements Exception { + GitStatusFailed(this.gitResult); + + final ProcessResult gitResult; + + @override + String toString() { + return 'git status exited with a non-zero exit code: ' + '${gitResult.exitCode}:\n${gitResult.stderr}\n${gitResult.stdout}'; + } +} + +/// A singleton filesystem that can be set by tests to a memory filesystem. +FileSystem filesystem = const LocalFileSystem(); + +/// A singleton snippet generator that can be set by tests to a mock, so that +/// we can test the command line parsing. +SnippetGenerator snippetGenerator = SnippetGenerator(); + +/// A singleton platform that can be set by tests for use in testing command line +/// parsing. +Platform platform = const LocalPlatform(); + +/// A singleton process manager that can be set by tests for use in testing. +ProcessManager processManager = const LocalProcessManager(); + +/// Get the name of the channel these docs are from. +/// +/// First check env variable LUCI_BRANCH, then refer to the currently +/// checked out git branch. +String getChannelName({ + Platform platform = const LocalPlatform(), + ProcessManager processManager = const LocalProcessManager(), +}) { + final String? envReleaseChannel = platform.environment['LUCI_BRANCH']?.trim(); + if (['master', 'stable', 'main'].contains(envReleaseChannel)) { + // Backward compatibility: Still support running on "master", but pretend it is "main". + if (envReleaseChannel == 'master') { + return 'main'; + } + return envReleaseChannel!; + } + + final RegExp gitBranchRegexp = RegExp(r'^## (?.*)'); + final ProcessResult gitResult = processManager.runSync( + ['git', 'status', '-b', '--porcelain'], + // Use the FLUTTER_ROOT, if defined. + workingDirectory: platform.environment['FLUTTER_ROOT']?.trim() ?? + filesystem.currentDirectory.path, + // Adding extra debugging output to help debug why git status inexplicably fails + // (random non-zero error code) about 2% of the time. + environment: {'GIT_TRACE': '2', 'GIT_TRACE_SETUP': '2'}); + if (gitResult.exitCode != 0) { + throw GitStatusFailed(gitResult); + } + + final RegExpMatch? gitBranchMatch = gitBranchRegexp + .firstMatch((gitResult.stdout as String).trim().split('\n').first); + return gitBranchMatch == null + ? '' + : gitBranchMatch.namedGroup('branch')!.split('...').first; +} + +const List sampleTypes = [ + 'snippet', + 'sample', + 'dartpad', +]; + +// This is a hack to workaround the fact that git status inexplicably fails +// (with random non-zero error code) about 2% of the time. +String getChannelNameWithRetries({ + Platform platform = const LocalPlatform(), + ProcessManager processManager = const LocalProcessManager(), +}) { + int retryCount = 0; + + while (retryCount < 2) { + try { + return getChannelName(platform: platform, processManager: processManager); + } on GitStatusFailed catch (e) { + retryCount += 1; + stderr.write( + 'git status failed, retrying ($retryCount)\nError report:\n$e'); + } + } + + return getChannelName(platform: platform, processManager: processManager); +} + +/// Generates snippet dartdoc output for a given input, and creates any sample +/// applications needed by the snippet. +void main(List argList) { + final Map environment = platform.environment; + final ArgParser parser = ArgParser(); + + parser.addOption( + _kTypeOption, + defaultsTo: 'dartpad', + allowed: sampleTypes, + allowedHelp: { + 'dartpad': + 'Produce a code sample application complete with embedding the sample in an ' + 'application template for using in Dartpad.', + 'sample': + 'Produce a code sample application complete with embedding the sample in an ' + 'application template.', + 'snippet': + 'Produce a nicely formatted piece of sample code. Does not embed the ' + 'sample into an application template.', + }, + help: 'The type of snippet to produce.', + ); + // TODO(goderbauer): Remove template support, this is no longer used. + parser.addOption( + _kTemplateOption, + help: 'The name of the template to inject the code into.', + ); + parser.addOption( + _kOutputOption, + help: 'The output name for the generated sample application. Overrides ' + 'the naming generated by the --$_kPackageOption/--$_kLibraryOption/--$_kElementOption ' + 'arguments. Metadata will be written alongside in a .json file. ' + 'The basename of this argument is used as the ID. If this is a ' + 'relative path, will be placed under the --$_kOutputDirectoryOption location.', + ); + parser.addOption( + _kOutputDirectoryOption, + defaultsTo: '.', + help: 'The output path for the generated sample application.', + ); + parser.addOption( + _kInputOption, + defaultsTo: environment['INPUT'], + help: 'The input file containing the sample code to inject.', + ); + parser.addOption( + _kPackageOption, + defaultsTo: environment['PACKAGE_NAME'], + help: 'The name of the package that this sample belongs to.', + ); + parser.addOption( + _kLibraryOption, + defaultsTo: environment['LIBRARY_NAME'], + help: 'The name of the library that this sample belongs to.', + ); + parser.addOption( + _kElementOption, + defaultsTo: environment['ELEMENT_NAME'], + help: 'The name of the element that this sample belongs to.', + ); + parser.addOption( + _kSerialOption, + defaultsTo: environment['INVOCATION_INDEX'], + help: 'A unique serial number for this snippet tool invocation.', + ); + parser.addFlag( + _kFormatOutputOption, + defaultsTo: true, + help: 'Applies the Dart formatter to the published/extracted sample code.', + ); + parser.addFlag( + _kHelpOption, + negatable: false, + help: 'Prints help documentation for this command', + ); + + final ArgResults args = parser.parse(argList); + + if (args[_kHelpOption]! as bool) { + stderr.writeln(parser.usage); + exitCode = 0; + return; + } + + final String sampleType = args[_kTypeOption]! as String; + + if (args[_kInputOption] == null) { + stderr.writeln(parser.usage); + errorExit( + 'The --$_kInputOption option must be specified, either on the command ' + 'line, or in the INPUT environment variable.'); + return; + } + + final File input = filesystem.file(args['input']! as String); + if (!input.existsSync()) { + errorExit('The input file ${input.path} does not exist.'); + return; + } + + final bool formatOutput = args[_kFormatOutputOption]! as bool; + final String packageName = args[_kPackageOption] as String? ?? ''; + final String libraryName = args[_kLibraryOption] as String? ?? ''; + final String elementName = args[_kElementOption] as String? ?? ''; + final String serial = args[_kSerialOption] as String? ?? ''; + late String id; + File? output; + final Directory outputDirectory = + filesystem.directory(args[_kOutputDirectoryOption]! as String).absolute; + + if (args[_kOutputOption] != null) { + id = path.basenameWithoutExtension(args[_kOutputOption]! as String); + final File outputPath = filesystem.file(args[_kOutputOption]! as String); + if (outputPath.isAbsolute) { + output = outputPath; + } else { + output = + filesystem.file(path.join(outputDirectory.path, outputPath.path)); + } + } else { + final List idParts = []; + if (packageName.isNotEmpty && packageName != 'flutter') { + idParts.add(packageName.replaceAll(RegExp(r'\W'), '_').toLowerCase()); + } + if (libraryName.isNotEmpty) { + idParts.add(libraryName.replaceAll(RegExp(r'\W'), '_').toLowerCase()); + } + if (elementName.isNotEmpty) { + idParts.add(elementName); + } + if (serial.isNotEmpty) { + idParts.add(serial); + } + if (idParts.isEmpty) { + errorExit('Unable to determine ID. At least one of --$_kPackageOption, ' + '--$_kLibraryOption, --$_kElementOption, -$_kSerialOption, or the environment variables ' + 'PACKAGE_NAME, LIBRARY_NAME, ELEMENT_NAME, or INVOCATION_INDEX must be non-empty.'); + return; + } + id = idParts.join('.'); + output = outputDirectory.childFile('$id.dart'); + } + output.parent.createSync(recursive: true); + + final int? sourceLine = environment['SOURCE_LINE'] != null + ? int.tryParse(environment['SOURCE_LINE']!) + : null; + final String sourcePath = environment['SOURCE_PATH'] ?? 'unknown.dart'; + final SnippetDartdocParser sampleParser = SnippetDartdocParser(filesystem); + final SourceElement element = sampleParser.parseFromDartdocToolFile( + input, + startLine: sourceLine, + element: elementName, + sourceFile: filesystem.file(sourcePath), + type: sampleType, + ); + final Map metadata = { + 'channel': getChannelNameWithRetries( + platform: platform, processManager: processManager), + 'serial': serial, + 'id': id, + 'package': packageName, + 'library': libraryName, + 'element': elementName, + }; + + for (final CodeSample sample in element.samples) { + sample.metadata.addAll(metadata); + snippetGenerator.generateCode( + sample, + output: output, + formatOutput: formatOutput, + ); + print(snippetGenerator.generateHtml(sample)); + } + + exitCode = 0; +} diff --git a/dev/snippets/lib/snippets.dart b/dev/snippets/lib/snippets.dart new file mode 100644 index 0000000000000..116ae7970b4ec --- /dev/null +++ b/dev/snippets/lib/snippets.dart @@ -0,0 +1,11 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/analysis.dart'; +export 'src/configuration.dart'; +export 'src/data_types.dart'; +export 'src/import_sorter.dart'; +export 'src/snippet_generator.dart'; +export 'src/snippet_parser.dart'; +export 'src/util.dart'; diff --git a/dev/snippets/lib/src/analysis.dart b/dev/snippets/lib/src/analysis.dart new file mode 100644 index 0000000000000..73a871ae42784 --- /dev/null +++ b/dev/snippets/lib/src/analysis.dart @@ -0,0 +1,361 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:analyzer/dart/analysis/features.dart'; +import 'package:analyzer/dart/analysis/results.dart'; +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/token.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:analyzer/file_system/file_system.dart' as afs; +import 'package:analyzer/file_system/physical_file_system.dart' as afs; +import 'package:analyzer/source/line_info.dart'; +import 'package:file/file.dart'; + +import 'data_types.dart'; +import 'util.dart'; + +/// Gets an iterable over all of the blocks of documentation comments in a file +/// using the analyzer. +/// +/// Each entry in the list is a list of source lines corresponding to the +/// documentation comment block. +Iterable> getFileDocumentationComments(File file) { + return getDocumentationComments(getFileElements(file)); +} + +/// Gets an iterable over all of the blocks of documentation comments from an +/// iterable over the [SourceElement]s involved. +Iterable> getDocumentationComments( + Iterable elements) { + return elements + .where((SourceElement element) => element.comment.isNotEmpty) + .map>((SourceElement element) => element.comment); +} + +/// Gets an iterable over the comment [SourceElement]s in a file. +Iterable getFileCommentElements(File file) { + return getCommentElements(getFileElements(file)); +} + +/// Filters the source `elements` to only return the comment elements. +Iterable getCommentElements(Iterable elements) { + return elements.where((SourceElement element) => element.comment.isNotEmpty); +} + +/// Reads the file content from a string, to avoid having to read the file more +/// than once if the caller already has the content in memory. +/// +/// The `file` argument is used to tag the lines with a filename that they came from. +Iterable getElementsFromString(String content, File file) { + final ParseStringResult parseResult = parseString( + featureSet: FeatureSet.fromEnableFlags2( + sdkLanguageVersion: FlutterInformation.instance.getDartSdkVersion(), + flags: [], + ), + content: content); + final _SourceVisitor visitor = + _SourceVisitor(file); + visitor.visitCompilationUnit(parseResult.unit); + visitor.assignLineNumbers(); + return visitor.elements; +} + +/// Gets an iterable over the [SourceElement]s in the given `file`. +/// +/// Takes an optional [ResourceProvider] to allow reading from a memory +/// filesystem. +Iterable getFileElements(File file, + {afs.ResourceProvider? resourceProvider}) { + resourceProvider ??= afs.PhysicalResourceProvider.INSTANCE; + final ParseStringResult parseResult = parseFile( + featureSet: FeatureSet.fromEnableFlags2( + sdkLanguageVersion: FlutterInformation.instance.getDartSdkVersion(), + flags: [], + ), + path: file.absolute.path, + resourceProvider: resourceProvider); + final _SourceVisitor visitor = + _SourceVisitor(file); + visitor.visitCompilationUnit(parseResult.unit); + visitor.assignLineNumbers(); + return visitor.elements; +} + +class _SourceVisitor extends RecursiveAstVisitor { + _SourceVisitor(this.file) : elements = {}; + + final Set elements; + String enclosingClass = ''; + + File file; + + void assignLineNumbers() { + final String contents = file.readAsStringSync(); + final LineInfo lineInfo = LineInfo.fromContent(contents); + + final Set removedElements = {}; + final Set replacedElements = {}; + for (final SourceElement element in elements) { + final List newLines = []; + for (final SourceLine line in element.comment) { + final CharacterLocation intervalLine = + lineInfo.getLocation(line.startChar); + newLines.add(line.copyWith(line: intervalLine.lineNumber)); + } + final int elementLine = lineInfo.getLocation(element.startPos).lineNumber; + replacedElements + .add(element.copyWith(comment: newLines, startLine: elementLine)); + removedElements.add(element); + } + elements.removeAll(removedElements); + elements.addAll(replacedElements); + } + + List _processComment(String element, Comment comment) { + final List result = []; + if (comment.tokens.isNotEmpty) { + for (final Token token in comment.tokens) { + result.add(SourceLine( + token.toString(), + element: element, + file: file, + startChar: token.charOffset, + endChar: token.charEnd, + )); + } + } + return result; + } + + @override + T? visitCompilationUnit(CompilationUnit node) { + elements.clear(); + return super.visitCompilationUnit(node); + } + + static bool isPublic(String name) { + return !name.startsWith('_'); + } + + static bool isInsideMethod(AstNode startNode) { + AstNode? node = startNode.parent; + while (node != null) { + if (node is MethodDeclaration) { + return true; + } + node = node.parent; + } + return false; + } + + @override + T? visitTopLevelVariableDeclaration(TopLevelVariableDeclaration node) { + for (final VariableDeclaration declaration in node.variables.variables) { + if (!isPublic(declaration.name.lexeme)) { + continue; + } + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment( + declaration.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.topLevelVariableType, + declaration.name.lexeme, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + ), + ); + } + return super.visitTopLevelVariableDeclaration(node); + } + + @override + T? visitGenericTypeAlias(GenericTypeAlias node) { + if (isPublic(node.name.lexeme)) { + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.typedefType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + ), + ); + } + return super.visitGenericTypeAlias(node); + } + + @override + T? visitFieldDeclaration(FieldDeclaration node) { + for (final VariableDeclaration declaration in node.fields.variables) { + if (!isPublic(declaration.name.lexeme) || !isPublic(enclosingClass)) { + continue; + } + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + assert(enclosingClass.isNotEmpty); + comment = _processComment('$enclosingClass.${declaration.name.lexeme}', + node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.fieldType, + declaration.name.lexeme, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + override: _isOverridden(node), + ), + ); + return super.visitFieldDeclaration(node); + } + return null; + } + + @override + T? visitConstructorDeclaration(ConstructorDeclaration node) { + final String fullName = + '$enclosingClass${node.name == null ? '' : '.${node.name}'}'; + if (isPublic(enclosingClass) && + (node.name == null || isPublic(node.name!.lexeme))) { + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment( + '$enclosingClass.$fullName', node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.constructorType, + fullName, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + ), + ); + } + return super.visitConstructorDeclaration(node); + } + + @override + T? visitFunctionDeclaration(FunctionDeclaration node) { + if (isPublic(node.name.lexeme)) { + List comment = []; + // Skip functions that are defined inside of methods. + if (!isInsideMethod(node)) { + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = + _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.functionType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + override: _isOverridden(node), + ), + ); + } + } + return super.visitFunctionDeclaration(node); + } + + @override + T? visitMethodDeclaration(MethodDeclaration node) { + if (isPublic(node.name.lexeme) && isPublic(enclosingClass)) { + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + assert(enclosingClass.isNotEmpty); + comment = _processComment( + '$enclosingClass.${node.name.lexeme}', node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.methodType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + override: _isOverridden(node), + ), + ); + } + return super.visitMethodDeclaration(node); + } + + bool _isOverridden(AnnotatedNode node) { + return node.metadata.where((Annotation annotation) { + return annotation.name.name == 'override'; + }).isNotEmpty; + } + + @override + T? visitMixinDeclaration(MixinDeclaration node) { + enclosingClass = node.name.lexeme; + if (!node.name.lexeme.startsWith('_')) { + enclosingClass = node.name.lexeme; + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.classType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + ), + ); + } + final T? result = super.visitMixinDeclaration(node); + enclosingClass = ''; + return result; + } + + @override + T? visitClassDeclaration(ClassDeclaration node) { + enclosingClass = node.name.lexeme; + if (!node.name.lexeme.startsWith('_')) { + enclosingClass = node.name.lexeme; + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.classType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + ), + ); + } + final T? result = super.visitClassDeclaration(node); + enclosingClass = ''; + return result; + } +} diff --git a/dev/snippets/lib/src/configuration.dart b/dev/snippets/lib/src/configuration.dart new file mode 100644 index 0000000000000..37176ce0dc609 --- /dev/null +++ b/dev/snippets/lib/src/configuration.dart @@ -0,0 +1,63 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:path/path.dart' as path; + +// Represents the locations of all of the data for snippets. +class SnippetConfiguration { + const SnippetConfiguration({ + required this.configDirectory, + required this.skeletonsDirectory, + required this.templatesDirectory, + this.filesystem = const LocalFileSystem(), + }); + + final FileSystem filesystem; + + /// This is the configuration directory for the snippets system, containing + /// the skeletons and templates. + final Directory configDirectory; + + /// The directory containing the HTML skeletons to be filled out with metadata + /// and returned to dartdoc for insertion in the output. + final Directory skeletonsDirectory; + + /// The directory containing the code templates that can be referenced by the + /// dartdoc. + final Directory templatesDirectory; + + /// Gets the skeleton file to use for the given [SampleType] and DartPad + /// preference. + File getHtmlSkeletonFile(String type) { + final String filename = + type == 'dartpad' ? 'dartpad-sample.html' : '$type.html'; + return filesystem.file(path.join(skeletonsDirectory.path, filename)); + } +} + +/// A class to compute the configuration of the snippets input and output +/// locations based in the current location of the snippets main.dart. +class FlutterRepoSnippetConfiguration extends SnippetConfiguration { + FlutterRepoSnippetConfiguration({required this.flutterRoot, super.filesystem}) + : super( + configDirectory: _underRoot(filesystem, flutterRoot, + const ['dev', 'snippets', 'config']), + skeletonsDirectory: _underRoot(filesystem, flutterRoot, + const ['dev', 'snippets', 'config', 'skeletons']), + templatesDirectory: _underRoot( + filesystem, + flutterRoot, + const ['dev', 'snippets', 'config', 'templates'], + ), + ); + + final Directory flutterRoot; + + static Directory _underRoot( + FileSystem fs, Directory flutterRoot, List dirs) => + fs.directory(path.canonicalize( + path.joinAll([flutterRoot.absolute.path, ...dirs]))); +} diff --git a/dev/snippets/lib/src/data_types.dart b/dev/snippets/lib/src/data_types.dart new file mode 100644 index 0000000000000..fd5bac98020b0 --- /dev/null +++ b/dev/snippets/lib/src/data_types.dart @@ -0,0 +1,567 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/args.dart'; +import 'package:file/file.dart'; + +import 'util.dart'; + +/// A class to represent a line of input code, with associated line number, file +/// and element name. +class SourceLine { + const SourceLine( + this.text, { + this.file, + this.element, + this.line = -1, + this.startChar = -1, + this.endChar = -1, + this.indent = 0, + }); + final File? file; + final String? element; + final int line; + final int startChar; + final int endChar; + final int indent; + final String text; + + String toStringWithColumn(int column) => + '$file:$line:${column + indent}: $text'; + + SourceLine copyWith({ + String? element, + String? text, + File? file, + int? line, + int? startChar, + int? endChar, + int? indent, + }) { + return SourceLine( + text ?? this.text, + element: element ?? this.element, + file: file ?? this.file, + line: line ?? this.line, + startChar: startChar ?? this.startChar, + endChar: endChar ?? this.endChar, + indent: indent ?? this.indent, + ); + } + + bool get hasFile => file != null; + + @override + String toString() => '$file:${line == -1 ? '??' : line}: $text'; +} + +/// A class containing the name and contents associated with a code block inside of a +/// code sample, for named injection into a template. +class TemplateInjection { + TemplateInjection(this.name, this.contents, {this.language = ''}); + final String name; + final List contents; + final String language; + Iterable get stringContents => + contents.map((SourceLine line) => line.text.trimRight()); + String get mergedContent => stringContents.join('\n'); +} + +/// A base class to represent a block of any kind of sample code, marked by +/// "{@tool (snippet|sample|dartdoc) ...}...{@end-tool}". +abstract class CodeSample { + CodeSample( + this.args, + this.input, { + required this.index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + _lineProto = lineProto, + sourceFile = null; + + CodeSample.fromFile( + this.args, + this.input, + this.sourceFile, { + required this.index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + _lineProto = lineProto; + + final File? sourceFile; + final List args; + final List input; + final SourceLine _lineProto; + String? _sourceFileContents; + String get sourceFileContents { + if (sourceFile != null && _sourceFileContents == null) { + // Strip lines until the first non-comment line. This gets rid of the + // copyright and comment directing the reader to the original source file. + final List stripped = []; + bool doneStrippingHeaders = false; + try { + for (final String line in sourceFile!.readAsLinesSync()) { + if (!doneStrippingHeaders && + RegExp(r'^\s*(\/\/.*)?$').hasMatch(line)) { + continue; + } + // Stop skipping lines after the first line that isn't stripped. + doneStrippingHeaders = true; + stripped.add(line); + } + } on FileSystemException catch (e) { + throw SnippetException( + 'Unable to read linked source file ${sourceFile!}: $e', + file: _lineProto.file?.absolute.path, + ); + } + // Remove any section markers + final RegExp sectionMarkerRegExp = RegExp( + r'(\/\/\*\*+\n)?\/\/\* [▼▲]+.*$(\n\/\/\*\*+)?\n\n?', + multiLine: true, + ); + _sourceFileContents = + stripped.join('\n').replaceAll(sectionMarkerRegExp, ''); + } + return _sourceFileContents ?? ''; + } + + Iterable get inputStrings => + input.map((SourceLine line) => line.text); + String get inputAsString => inputStrings.join('\n'); + + /// The index of this sample within the dartdoc comment it came from. + final int index; + String description = ''; + String get element => start.element ?? ''; + String output = ''; + Map metadata = {}; + List parts = []; + SourceLine get start => input.isEmpty ? _lineProto : input.first; + + String get template { + final ArgParser parser = ArgParser(); + parser.addOption('template', defaultsTo: ''); + final ArgResults parsedArgs = parser.parse(args); + return parsedArgs['template']! as String; + } + + @override + String toString() { + final StringBuffer buf = StringBuffer('${args.join(' ')}:\n'); + for (final SourceLine line in input) { + buf.writeln( + '${(line.line == -1 ? '??' : line.line).toString().padLeft(4)}: ${line.text} ', + ); + } + return buf.toString(); + } + + String get type; +} + +/// A class to represent a snippet of sample code, marked by "{@tool +/// snippet}...{@end-tool}". +/// +/// Snippets are code that is not meant to be run as a complete application, but +/// rather as a code usage example. +class SnippetSample extends CodeSample { + SnippetSample( + List input, { + required int index, + required SourceLine lineProto, + }) : assumptions = [], + super( + ['snippet'], + input, + index: index, + lineProto: lineProto, + ); + + factory SnippetSample.combine( + List sections, { + required int index, + required SourceLine lineProto, + }) { + final List code = + sections.expand((SnippetSample section) => section.input).toList(); + return SnippetSample(code, index: index, lineProto: lineProto); + } + + factory SnippetSample.fromStrings(SourceLine firstLine, List code, + {required int index}) { + final List codeLines = []; + int startPos = firstLine.startChar; + for (int i = 0; i < code.length; ++i) { + codeLines.add( + firstLine.copyWith( + text: code[i], + line: firstLine.line + i, + startChar: startPos, + ), + ); + startPos += code[i].length + 1; + } + return SnippetSample( + codeLines, + index: index, + lineProto: firstLine, + ); + } + + factory SnippetSample.surround( + String prefix, + List code, + String postfix, { + required int index, + }) { + return SnippetSample( + [ + if (prefix.isNotEmpty) SourceLine(prefix), + ...code, + if (postfix.isNotEmpty) SourceLine(postfix), + ], + index: index, + lineProto: code.first, + ); + } + + List assumptions; + + @override + String get template => ''; + + @override + SourceLine get start => + input.firstWhere((SourceLine line) => line.file != null); + + @override + String get type => 'snippet'; +} + +/// A class to represent a plain application sample in the dartdoc comments, +/// marked by `{@tool sample ...}...{@end-tool}`. +/// +/// Application samples are processed separately from [SnippetSample]s, because +/// they must be injected into templates in order to be analyzed. Each +/// [ApplicationSample] represents one `{@tool sample ...}...{@end-tool}` block +/// in the source file. +class ApplicationSample extends CodeSample { + ApplicationSample({ + List input = const [], + required List args, + required int index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + super(args, input, index: index, lineProto: lineProto); + + ApplicationSample.fromFile({ + List input = const [], + required List args, + required File sourceFile, + required int index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + super.fromFile(args, input, sourceFile, + index: index, lineProto: lineProto); + + @override + String get type => 'sample'; +} + +/// A class to represent a Dartpad application sample in the dartdoc comments, +/// marked by `{@tool dartpad ...}...{@end-tool}`. +/// +/// Dartpad samples are processed separately from [SnippetSample]s, because they +/// must be injected into templates in order to be analyzed. Each +/// [DartpadSample] represents one `{@tool dartpad ...}...{@end-tool}` block in +/// the source file. +class DartpadSample extends ApplicationSample { + DartpadSample({ + super.input, + required super.args, + required super.index, + required super.lineProto, + }) : assert(args.isNotEmpty); + + DartpadSample.fromFile({ + super.input, + required super.args, + required super.sourceFile, + required super.index, + required super.lineProto, + }) : assert(args.isNotEmpty), + super.fromFile(); + + @override + String get type => 'dartpad'; +} + +/// The different types of Dart [SourceElement]s that can be found in a source file. +enum SourceElementType { + /// A class + classType, + + /// A field variable of a class. + fieldType, + + /// A constructor for a class. + constructorType, + + /// A method of a class. + methodType, + + /// A function typedef + typedefType, + + /// A top level (non-class) variable. + topLevelVariableType, + + /// A function, either top level, or embedded in another function. + functionType, + + /// An unknown type used for initialization. + unknownType, +} + +/// Converts the enun type [SourceElementType] to a human readable string. +String sourceElementTypeAsString(SourceElementType type) { + switch (type) { + case SourceElementType.classType: + return 'class'; + case SourceElementType.fieldType: + return 'field'; + case SourceElementType.methodType: + return 'method'; + case SourceElementType.constructorType: + return 'constructor'; + case SourceElementType.typedefType: + return 'typedef'; + case SourceElementType.topLevelVariableType: + return 'variable'; + case SourceElementType.functionType: + return 'function'; + case SourceElementType.unknownType: + return 'unknown'; + } +} + +/// A class that represents a Dart element in a source file. +/// +/// The element is one of the types in [SourceElementType]. +class SourceElement { + /// A factory constructor for SourceElements. + /// + /// This uses a factory so that the default for the `comment` and `samples` + /// lists can be modifiable lists. + factory SourceElement( + SourceElementType type, + String name, + int startPos, { + required File file, + String className = '', + List? comment, + int startLine = -1, + List? samples, + bool override = false, + }) { + comment ??= []; + samples ??= []; + final List commentLines = + comment.map((SourceLine line) => line.text).toList(); + final String commentString = commentLines.join('\n'); + return SourceElement._( + type, + name, + startPos, + file: file, + className: className, + comment: comment, + startLine: startLine, + samples: samples, + override: override, + commentString: commentString, + commentStringWithoutTools: _getCommentStringWithoutTools(commentString), + commentStringWithoutCode: _getCommentStringWithoutCode(commentString), + commentLines: commentLines, + ); + } + + const SourceElement._( + this.type, + this.name, + this.startPos, { + required this.file, + this.className = '', + this.comment = const [], + this.startLine = -1, + this.samples = const [], + this.override = false, + String commentString = '', + String commentStringWithoutTools = '', + String commentStringWithoutCode = '', + List commentLines = const [], + }) : _commentString = commentString, + _commentStringWithoutTools = commentStringWithoutTools, + _commentStringWithoutCode = commentStringWithoutCode, + _commentLines = commentLines; + + final String _commentString; + final String _commentStringWithoutTools; + final String _commentStringWithoutCode; + final List _commentLines; + + // Does not include the description of the sample code, just the text outside + // of any dartdoc tools. + static String _getCommentStringWithoutTools(String string) { + return string.replaceAll( + RegExp(r'(\{@tool ([^}]*)\}.*?\{@end-tool\}|/// ?)', dotAll: true), ''); + } + + // Includes the description text inside of an "@tool"-based sample, but not + // the code itself, or any dartdoc tags. + static String _getCommentStringWithoutCode(String string) { + return string.replaceAll( + RegExp(r'([`]{3}.*?[`]{3}|\{@\w+[^}]*\}|/// ?)', dotAll: true), ''); + } + + /// The type of the element + final SourceElementType type; + + /// The name of the element. + /// + /// For example, a method called "doSomething" that is part of the class + /// "MyClass" would have "doSomething" as its name. + final String name; + + /// The name of the class the element belongs to, if any. + /// + /// This is the empty string if it isn't part of a class. + /// + /// For example, a method called "doSomething" that is part of the class + /// "MyClass" would have "MyClass" as its `className`. + final String className; + + /// Whether or not this element has the "@override" annotation attached to it. + final bool override; + + /// The file that this [SourceElement] was parsed from. + final File file; + + /// The character position in the file that this [SourceElement] starts at. + final int startPos; + + /// The line in the file that the first position of [SourceElement] is on. + final int startLine; + + /// The list of [SourceLine]s that make up the documentation comment for this + /// [SourceElement]. + final List comment; + + /// The list of [CodeSample]s that are in the documentation comment for this + /// [SourceElement]. + /// + /// This field will be populated by calling [replaceSamples]. + final List samples; + + /// Get the comments as an iterable of lines. + Iterable get commentLines => _commentLines; + + /// Get the comments as a single string. + String get commentString => _commentString; + + /// Does not include the description of the sample code, just the text outside of any dartdoc tools. + String get commentStringWithoutTools => _commentStringWithoutTools; + + /// Includes the description text inside of an "@tool"-based sample, but not + /// the code itself, or any dartdoc tags. + String get commentStringWithoutCode => _commentStringWithoutCode; + + /// The number of samples in the dartdoc comment for this element. + int get sampleCount => samples.length; + + /// The number of [DartpadSample]s in the dartdoc comment for this element. + int get dartpadSampleCount => samples.whereType().length; + + /// The number of [ApplicationSample]s in the dartdoc comment for this element. + int get applicationSampleCount => samples.where((CodeSample sample) { + return sample is ApplicationSample && sample is! DartpadSample; + }).length; + + /// The number of [SnippetSample]s in the dartdoc comment for this element. + int get snippetCount => samples.whereType().length; + + /// Count of comment lines, not including lines of code in the comment. + int get lineCount => commentStringWithoutCode.split('\n').length; + + /// Count of comment words, not including words in any code in the comment. + int get wordCount { + return commentStringWithoutCode.split(RegExp(r'\s+')).length; + } + + /// Count of comment characters, not including any code samples in the + /// comment, after collapsing each run of whitespace to a single space. + int get charCount => + commentStringWithoutCode.replaceAll(RegExp(r'\s+'), ' ').length; + + /// Whether or not this element's documentation has a "See also:" section in it. + bool get hasSeeAlso => commentStringWithoutTools.contains('See also:'); + + int get referenceCount { + final RegExp regex = RegExp(r'\[[. \w]*\](?!\(.*\))'); + return regex.allMatches(commentStringWithoutCode).length; + } + + int get linkCount { + final RegExp regex = RegExp(r'\[[. \w]*\]\(.*\)'); + return regex.allMatches(commentStringWithoutCode).length; + } + + /// Returns the fully qualified name of this element. + /// + /// For example, a method called "doSomething" that is part of the class + /// "MyClass" would have "MyClass.doSomething" as its `elementName`. + String get elementName { + if (type == SourceElementType.constructorType) { + // Constructors already have the name of the class in them. + return name; + } + return className.isEmpty ? name : '$className.$name'; + } + + /// Returns the type of this element as a [String]. + String get typeAsString { + return '${override ? 'overridden ' : ''}${sourceElementTypeAsString(type)}'; + } + + void replaceSamples(Iterable samples) { + this.samples.clear(); + this.samples.addAll(samples); + } + + /// Copy the source element, with some attributes optionally replaced. + SourceElement copyWith({ + SourceElementType? type, + String? name, + int? startPos, + File? file, + String? className, + List? comment, + int? startLine, + List? samples, + bool? override, + }) { + return SourceElement( + type ?? this.type, + name ?? this.name, + startPos ?? this.startPos, + file: file ?? this.file, + className: className ?? this.className, + comment: comment ?? this.comment, + startLine: startLine ?? this.startLine, + samples: samples ?? this.samples, + override: override ?? this.override, + ); + } +} diff --git a/dev/snippets/lib/src/import_sorter.dart b/dev/snippets/lib/src/import_sorter.dart new file mode 100644 index 0000000000000..ef7fa29554179 --- /dev/null +++ b/dev/snippets/lib/src/import_sorter.dart @@ -0,0 +1,434 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:math'; + +import 'package:analyzer/dart/analysis/features.dart'; +import 'package:analyzer/dart/analysis/results.dart'; +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/token.dart'; +import 'package:analyzer/error/error.dart'; +import 'package:analyzer/source/line_info.dart'; +import 'package:meta/meta.dart'; + +import 'util.dart'; + +/// Read the given source code, and return the new contents after sorting the +/// imports. +String sortImports(String contents) { + final ParseStringResult parseResult = parseString( + content: contents, + featureSet: FeatureSet.fromEnableFlags2( + sdkLanguageVersion: FlutterInformation.instance.getDartSdkVersion(), + flags: [], + ), + ); + final List errors = []; + final _ImportOrganizer organizer = + _ImportOrganizer(contents, parseResult.unit, errors); + final List<_SourceEdit> edits = organizer.organize(); + // Sort edits in reverse order + edits.sort((_SourceEdit a, _SourceEdit b) { + return b.offset.compareTo(a.offset); + }); + // Apply edits + for (final _SourceEdit edit in edits) { + contents = contents.replaceRange(edit.offset, edit.end, edit.replacement); + } + return contents; +} + +/// Organizer of imports (and other directives) in the [unit]. +// Adapted from the analysis_server package. +// This code is largely copied from: +// https://github.com/dart-lang/sdk/blob/c7405b9d86b4b47cf7610667491f1db72723b0dd/pkg/analysis_server/lib/src/services/correction/organize_imports.dart#L15 +// TODO(gspencergoog): If ImportOrganizer ever becomes part of the public API, +// this class should probably be replaced. +// https://github.com/flutter/flutter/issues/86197 +class _ImportOrganizer { + _ImportOrganizer(this.initialCode, this.unit, this.errors) + : code = initialCode { + endOfLine = getEOL(code); + hasUnresolvedIdentifierError = errors.any((AnalysisError error) { + return error.errorCode.isUnresolvedIdentifier; + }); + } + + final String initialCode; + + final CompilationUnit unit; + + final List errors; + + String code; + + String endOfLine = '\n'; + + bool hasUnresolvedIdentifierError = false; + + /// Returns the number of characters common to the end of [a] and [b]. + int findCommonSuffix(String a, String b) { + final int aLength = a.length; + final int bLength = b.length; + final int n = min(aLength, bLength); + for (int i = 1; i <= n; i++) { + if (a.codeUnitAt(aLength - i) != b.codeUnitAt(bLength - i)) { + return i - 1; + } + } + return n; + } + + /// Return the [_SourceEdit]s that organize imports in the [unit]. + List<_SourceEdit> organize() { + _organizeDirectives(); + // prepare edits + final List<_SourceEdit> edits = <_SourceEdit>[]; + if (code != initialCode) { + final int suffixLength = findCommonSuffix(initialCode, code); + final _SourceEdit edit = _SourceEdit(0, initialCode.length - suffixLength, + code.substring(0, code.length - suffixLength)); + edits.add(edit); + } + return edits; + } + + /// Organize all [Directive]s. + void _organizeDirectives() { + final LineInfo lineInfo = unit.lineInfo; + bool hasLibraryDirective = false; + final List<_DirectiveInfo> directives = <_DirectiveInfo>[]; + for (final Directive directive in unit.directives) { + if (directive is LibraryDirective) { + hasLibraryDirective = true; + } + if (directive is UriBasedDirective) { + final _DirectivePriority? priority = getDirectivePriority(directive); + if (priority != null) { + int offset = directive.offset; + int end = directive.end; + + final Token? leadingComment = + getLeadingComment(unit, directive, lineInfo); + final Token? trailingComment = + getTrailingComment(unit, directive, lineInfo, end); + + String? leadingCommentText; + if (leadingComment != null) { + leadingCommentText = + code.substring(leadingComment.offset, directive.offset); + offset = leadingComment.offset; + } + String? trailingCommentText; + if (trailingComment != null) { + trailingCommentText = + code.substring(directive.end, trailingComment.end); + end = trailingComment.end; + } + String? documentationText; + final Comment? documentationComment = directive.documentationComment; + if (documentationComment != null) { + documentationText = code.substring( + documentationComment.offset, documentationComment.end); + } + String? annotationText; + final Token? beginToken = directive.metadata.beginToken; + final Token? endToken = directive.metadata.endToken; + if (beginToken != null && endToken != null) { + annotationText = code.substring(beginToken.offset, endToken.end); + } + final String text = code.substring( + directive.firstTokenAfterCommentAndMetadata.offset, + directive.end); + final String uriContent = directive.uri.stringValue ?? ''; + directives.add( + _DirectiveInfo( + directive, + priority, + leadingCommentText, + documentationText, + annotationText, + uriContent, + trailingCommentText, + offset, + end, + text, + ), + ); + } + } + } + // nothing to do + if (directives.isEmpty) { + return; + } + final int firstDirectiveOffset = directives.first.offset; + final int lastDirectiveEnd = directives.last.end; + + // Without a library directive, the library comment is the comment of the + // first directive. + _DirectiveInfo? libraryDocumentationDirective; + if (!hasLibraryDirective && directives.isNotEmpty) { + libraryDocumentationDirective = directives.first; + } + + // sort + directives.sort(); + // append directives with grouping + String directivesCode; + { + final StringBuffer sb = StringBuffer(); + if (libraryDocumentationDirective != null && + libraryDocumentationDirective.documentationText != null) { + sb.write(libraryDocumentationDirective.documentationText); + sb.write(endOfLine); + } + _DirectivePriority currentPriority = directives.first.priority; + for (final _DirectiveInfo directiveInfo in directives) { + if (currentPriority != directiveInfo.priority) { + sb.write(endOfLine); + currentPriority = directiveInfo.priority; + } + if (directiveInfo.leadingCommentText != null) { + sb.write(directiveInfo.leadingCommentText); + } + if (directiveInfo != libraryDocumentationDirective && + directiveInfo.documentationText != null) { + sb.write(directiveInfo.documentationText); + sb.write(endOfLine); + } + if (directiveInfo.annotationText != null) { + sb.write(directiveInfo.annotationText); + sb.write(endOfLine); + } + sb.write(directiveInfo.text); + if (directiveInfo.trailingCommentText != null) { + sb.write(directiveInfo.trailingCommentText); + } + sb.write(endOfLine); + } + directivesCode = sb.toString(); + directivesCode = directivesCode.trimRight(); + } + // prepare code + final String beforeDirectives = code.substring(0, firstDirectiveOffset); + final String afterDirectives = code.substring(lastDirectiveEnd); + code = beforeDirectives + directivesCode + afterDirectives; + } + + static _DirectivePriority? getDirectivePriority(UriBasedDirective directive) { + final String uriContent = directive.uri.stringValue ?? ''; + if (directive is ImportDirective) { + if (uriContent.startsWith('dart:')) { + return _DirectivePriority.IMPORT_SDK; + } else if (uriContent.startsWith('package:')) { + return _DirectivePriority.IMPORT_PKG; + } else if (uriContent.contains('://')) { + return _DirectivePriority.IMPORT_OTHER; + } else { + return _DirectivePriority.IMPORT_REL; + } + } + if (directive is ExportDirective) { + if (uriContent.startsWith('dart:')) { + return _DirectivePriority.EXPORT_SDK; + } else if (uriContent.startsWith('package:')) { + return _DirectivePriority.EXPORT_PKG; + } else if (uriContent.contains('://')) { + return _DirectivePriority.EXPORT_OTHER; + } else { + return _DirectivePriority.EXPORT_REL; + } + } + if (directive is PartDirective) { + return _DirectivePriority.PART; + } + return null; + } + + /// Return the EOL to use for [code]. + static String getEOL(String code) { + if (code.contains('\r\n')) { + return '\r\n'; + } else { + return '\n'; + } + } + + /// Gets the first comment token considered to be the leading comment for this + /// directive. + /// + /// Leading comments for the first directive in a file are considered library + /// comments and not returned unless they contain blank lines, in which case + /// only the last part of the comment will be returned. + static Token? getLeadingComment( + CompilationUnit unit, UriBasedDirective directive, LineInfo lineInfo) { + if (directive.beginToken.precedingComments == null) { + return null; + } + + Token? firstComment = directive.beginToken.precedingComments; + Token? comment = firstComment; + Token? nextComment = comment?.next; + // Don't connect comments that have a blank line between them + while (comment != null && nextComment != null) { + final int currentLine = lineInfo.getLocation(comment.offset).lineNumber; + final int nextLine = lineInfo.getLocation(nextComment.offset).lineNumber; + if (nextLine - currentLine > 1) { + firstComment = nextComment; + } + comment = nextComment; + nextComment = comment.next; + } + + // Check if the comment is the first comment in the document + if (firstComment != unit.beginToken.precedingComments) { + final int previousDirectiveLine = + lineInfo.getLocation(directive.beginToken.previous!.end).lineNumber; + + // Skip over any comments on the same line as the previous directive + // as they will be attached to the end of it. + Token? comment = firstComment; + while (comment != null && + previousDirectiveLine == + lineInfo.getLocation(comment.offset).lineNumber) { + comment = comment.next; + } + return comment; + } + return null; + } + + /// Gets the last comment token considered to be the trailing comment for this + /// directive. + /// + /// To be considered a trailing comment, the comment must be on the same line + /// as the directive. + static Token? getTrailingComment(CompilationUnit unit, + UriBasedDirective directive, LineInfo lineInfo, int end) { + final int line = lineInfo.getLocation(end).lineNumber; + Token? comment = directive.endToken.next!.precedingComments; + while (comment != null) { + if (lineInfo.getLocation(comment.offset).lineNumber == line) { + return comment; + } + comment = comment.next; + } + return null; + } +} + +class _DirectiveInfo implements Comparable<_DirectiveInfo> { + _DirectiveInfo( + this.directive, + this.priority, + this.leadingCommentText, + this.documentationText, + this.annotationText, + this.uri, + this.trailingCommentText, + this.offset, + this.end, + this.text, + ); + + final UriBasedDirective directive; + final _DirectivePriority priority; + final String? leadingCommentText; + final String? documentationText; + final String? annotationText; + final String uri; + final String? trailingCommentText; + + /// The offset of the first token, usually the keyword but may include leading comments. + final int offset; + + /// The offset after the last token, including the end-of-line comment. + final int end; + + /// The text excluding comments, documentation and annotations. + final String text; + + @override + int compareTo(_DirectiveInfo other) { + if (priority == other.priority) { + return _compareUri(uri, other.uri); + } + return priority.ordinal - other.priority.ordinal; + } + + @override + String toString() => '(priority=$priority; text=$text)'; + + static int _compareUri(String a, String b) { + final List aList = _splitUri(a); + final List bList = _splitUri(b); + int result; + if ((result = aList[0].compareTo(bList[0])) != 0) { + return result; + } + if ((result = aList[1].compareTo(bList[1])) != 0) { + return result; + } + return 0; + } + + /// Split the given [uri] like `package:some.name/and/path.dart` into a list + /// like `[package:some.name, and/path.dart]`. + static List _splitUri(String uri) { + final int index = uri.indexOf('/'); + if (index == -1) { + return [uri, '']; + } + return [uri.substring(0, index), uri.substring(index + 1)]; + } +} + +enum _DirectivePriority { + IMPORT_SDK('IMPORT_SDK', 0), + IMPORT_PKG('IMPORT_PKG', 1), + IMPORT_OTHER('IMPORT_OTHER', 2), + IMPORT_REL('IMPORT_REL', 3), + EXPORT_SDK('EXPORT_SDK', 4), + EXPORT_PKG('EXPORT_PKG', 5), + EXPORT_OTHER('EXPORT_OTHER', 6), + EXPORT_REL('EXPORT_REL', 7), + PART('PART', 8); + + const _DirectivePriority(this.name, this.ordinal); + + final String name; + final int ordinal; + + @override + String toString() => name; +} + +/// SourceEdit +/// +/// { +/// "offset": int +/// "length": int +/// "replacement": String +/// "id": optional String +/// } +/// +/// Clients may not extend, implement or mix-in this class. +@immutable +class _SourceEdit { + const _SourceEdit(this.offset, this.length, this.replacement); + + /// The offset of the region to be modified. + final int offset; + + /// The length of the region to be modified. + final int length; + + /// The end of the region to be modified. + int get end => offset + length; + + /// The code that is to replace the specified region in the original code. + final String replacement; +} diff --git a/dev/snippets/lib/src/snippet_generator.dart b/dev/snippets/lib/src/snippet_generator.dart new file mode 100644 index 0000000000000..f83cf7ec1083a --- /dev/null +++ b/dev/snippets/lib/src/snippet_generator.dart @@ -0,0 +1,497 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:dart_style/dart_style.dart'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:path/path.dart' as path; + +import 'configuration.dart'; +import 'data_types.dart'; +import 'import_sorter.dart'; +import 'util.dart'; + +/// Generates the snippet HTML, as well as saving the output snippet main to +/// the output directory. +class SnippetGenerator { + SnippetGenerator( + {SnippetConfiguration? configuration, + FileSystem filesystem = const LocalFileSystem(), + Directory? flutterRoot}) + : flutterRoot = + flutterRoot ?? FlutterInformation.instance.getFlutterRoot(), + configuration = configuration ?? + FlutterRepoSnippetConfiguration( + filesystem: filesystem, + flutterRoot: flutterRoot ?? + FlutterInformation.instance.getFlutterRoot()); + + final Directory flutterRoot; + + /// The configuration used to determine where to get/save data for the + /// snippet. + final SnippetConfiguration configuration; + + static const JsonEncoder jsonEncoder = JsonEncoder.withIndent(' '); + + /// A Dart formatted used to format the snippet code and finished application + /// code. + static DartFormatter formatter = + DartFormatter(pageWidth: 80, fixes: StyleFix.all); + + /// Gets the path to the template file requested. + File? getTemplatePath(String templateName, {Directory? templatesDir}) { + final Directory templateDir = + templatesDir ?? configuration.templatesDirectory; + final File templateFile = configuration.filesystem + .file(path.join(templateDir.path, '$templateName.tmpl')); + return templateFile.existsSync() ? templateFile : null; + } + + /// Returns an iterable over the template files available in the templates + /// directory in the configuration. + Iterable getAvailableTemplates() sync* { + final Directory templatesDir = configuration.templatesDirectory; + for (final File file in templatesDir.listSync().whereType()) { + if (file.basename.endsWith('.tmpl')) { + yield file; + } + } + } + + /// Interpolates the [injections] into an HTML skeleton file. + /// + /// Similar to interpolateTemplate, but we are only looking for `code-` + /// components, and we care about the order of the injections. + /// + /// Takes into account the [type] and doesn't substitute in the id and the app + /// if not a [SnippetType.sample] snippet. + String interpolateSkeleton( + CodeSample sample, + String skeleton, + ) { + final List codeParts = []; + const HtmlEscape htmlEscape = HtmlEscape(); + String? language; + for (final TemplateInjection injection in sample.parts) { + if (!injection.name.startsWith('code')) { + continue; + } + codeParts.addAll(injection.stringContents); + if (injection.language.isNotEmpty) { + language = injection.language; + } + codeParts.addAll(['', '// ...', '']); + } + if (codeParts.length > 3) { + codeParts.removeRange(codeParts.length - 3, codeParts.length); + } + // Only insert a div for the description if there actually is some text there. + // This means that the {{description}} marker in the skeleton needs to + // be inside of an {@inject-html} block. + final String description = sample.description.trim().isNotEmpty + ? '
{@end-inject-html}${sample.description.trim()}{@inject-html}
' + : ''; + + // DartPad only supports stable or main as valid channels. Use main + // if not on stable so that local runs will work (although they will + // still take their sample code from the master docs server). + final String channel = + sample.metadata['channel'] == 'stable' ? 'stable' : 'main'; + + final Map substitutions = { + 'description': description, + 'code': htmlEscape.convert(codeParts.join('\n')), + 'language': language ?? 'dart', + 'serial': '', + 'id': sample.metadata['id']! as String, + 'channel': channel, + 'element': sample.metadata['element'] as String? ?? sample.element, + 'app': '', + }; + if (sample is ApplicationSample) { + substitutions + ..['serial'] = sample.metadata['serial']?.toString() ?? '0' + ..['app'] = htmlEscape.convert(sample.output); + } + return skeleton.replaceAllMapped( + RegExp('{{(${substitutions.keys.join('|')})}}'), (Match match) { + return substitutions[match[1]]!; + }); + } + + /// Consolidates all of the snippets and the assumptions into one snippet, in + /// order to create a compilable result. + Iterable consolidateSnippets(List samples, + {bool addMarkers = false}) { + if (samples.isEmpty) { + return []; + } + final Iterable snippets = samples.whereType(); + final List snippetLines = [ + ...snippets.first.assumptions, + ]; + for (final SnippetSample sample in snippets) { + parseInput(sample); + snippetLines.addAll(_processBlocks(sample)); + } + return snippetLines; + } + + /// A RegExp that matches a Dart constructor. + static final RegExp _constructorRegExp = + RegExp(r'(const\s+)?_*[A-Z][a-zA-Z0-9<>._]*\('); + + /// A serial number so that we can create unique expression names when we + /// generate them. + int _expressionId = 0; + + List _surround( + String prefix, Iterable body, String suffix) { + return [ + if (prefix.isNotEmpty) SourceLine(prefix), + ...body, + if (suffix.isNotEmpty) SourceLine(suffix), + ]; + } + + /// Process one block of sample code (the part inside of "```" markers). + /// Splits any sections denoted by "// ..." into separate blocks to be + /// processed separately. Uses a primitive heuristic to make sample blocks + /// into valid Dart code. + List _processBlocks(CodeSample sample) { + final List block = sample.parts + .expand((TemplateInjection injection) => injection.contents) + .toList(); + if (block.isEmpty) { + return []; + } + return _processBlock(block); + } + + List _processBlock(List block) { + final String firstLine = block.first.text; + if (firstLine.startsWith('new ') || + firstLine.startsWith(_constructorRegExp)) { + _expressionId += 1; + return _surround('dynamic expression$_expressionId = ', block, ';'); + } else if (firstLine.startsWith('await ')) { + _expressionId += 1; + return _surround( + 'Future expression$_expressionId() async { ', block, ' }'); + } else if (block.first.text.startsWith('class ') || + block.first.text.startsWith('enum ')) { + return block; + } else if ((block.first.text.startsWith('_') || + block.first.text.startsWith('final ')) && + block.first.text.contains(' = ')) { + _expressionId += 1; + return _surround( + 'void expression$_expressionId() { ', block.toList(), ' }'); + } else { + final List buffer = []; + int blocks = 0; + SourceLine? subLine; + final List subsections = []; + for (int index = 0; index < block.length; index += 1) { + // Each section of the dart code that is either split by a blank line, or with + // '// ...' is treated as a separate code block. + if (block[index].text.trim().isEmpty || block[index].text == '// ...') { + if (subLine == null) { + continue; + } + blocks += 1; + subsections.addAll(_processBlock(buffer)); + buffer.clear(); + assert(buffer.isEmpty); + subLine = null; + } else if (block[index].text.startsWith('// ')) { + if (buffer.length > 1) { + // don't include leading comments + // so that it doesn't start with "// " and get caught in this again + buffer.add(SourceLine('/${block[index].text}')); + } + } else { + subLine ??= block[index]; + buffer.add(block[index]); + } + } + if (blocks > 0) { + if (subLine != null) { + subsections.addAll(_processBlock(buffer)); + } + // Combine all of the subsections into one section, now that they've been processed. + return subsections; + } else { + return block; + } + } + } + + /// Parses the input for the various code and description segments, and + /// returns a set of template injections in the order found. + List parseInput(CodeSample sample) { + bool inCodeBlock = false; + final List description = []; + final List components = []; + String? language; + final RegExp codeStartEnd = + RegExp(r'^\s*```(?[-\w]+|[-\w]+ (?
[-\w]+))?\s*$'); + for (final SourceLine line in sample.input) { + final RegExpMatch? match = codeStartEnd.firstMatch(line.text); + if (match != null) { + // If we saw the start or end of a code block + inCodeBlock = !inCodeBlock; + if (match.namedGroup('language') != null) { + language = match[1]; + if (match.namedGroup('section') != null) { + components.add(TemplateInjection( + 'code-${match.namedGroup('section')}', [], + language: language!)); + } else { + components.add( + TemplateInjection('code', [], language: language!)); + } + } else { + language = null; + } + continue; + } + if (!inCodeBlock) { + description.add(line); + } else { + assert(language != null); + components.last.contents.add(line); + } + } + final List descriptionLines = []; + bool lastWasWhitespace = false; + for (final String line in description + .map((SourceLine line) => line.text.trimRight())) { + final bool onlyWhitespace = line.trim().isEmpty; + if (onlyWhitespace && descriptionLines.isEmpty) { + // Don't add whitespace lines until we see something without whitespace. + lastWasWhitespace = onlyWhitespace; + continue; + } + if (onlyWhitespace && lastWasWhitespace) { + // Don't add more than one whitespace line in a row. + continue; + } + descriptionLines.add(line); + lastWasWhitespace = onlyWhitespace; + } + sample.description = descriptionLines.join('\n').trimRight(); + sample.parts = [ + if (sample is SnippetSample) + TemplateInjection('#assumptions', sample.assumptions), + ...components, + ]; + return sample.parts; + } + + String _loadFileAsUtf8(File file) { + return file.readAsStringSync(); + } + + /// Generate the HTML using the skeleton file for the type of the given sample. + /// + /// Returns a string with the HTML needed to embed in a web page for showing a + /// sample on the web page. + String generateHtml(CodeSample sample) { + final String skeleton = + _loadFileAsUtf8(configuration.getHtmlSkeletonFile(sample.type)); + return interpolateSkeleton(sample, skeleton); + } + + // Sets the description string on the sample and in the sample metadata to a + // comment version of the description. + // Trims lines of extra whitespace, and strips leading and trailing blank + // lines. + String _getDescription(CodeSample sample) { + return sample.description.splitMapJoin( + '\n', + onMatch: (Match match) => match.group(0)!, + onNonMatch: (String nonmatch) => + nonmatch.trimRight().isEmpty ? '//' : '// ${nonmatch.trimRight()}', + ); + } + + /// The main routine for generating code samples from the source code doc comments. + /// + /// The `sample` is the block of sample code from a dartdoc comment. + /// + /// The optional `output` is the file to write the generated sample code to. + /// + /// If `addSectionMarkers` is true, then markers will be added before and + /// after each template section in the output. This is intended to facilitate + /// editing of the sample during the authoring process. + /// + /// If `includeAssumptions` is true, then the block in the "Examples can + /// assume:" block will also be included in the output. + /// + /// Returns a string containing the resulting code sample. + String generateCode( + CodeSample sample, { + File? output, + String? copyright, + String? description, + bool formatOutput = true, + bool addSectionMarkers = false, + bool includeAssumptions = false, + }) { + sample.metadata['copyright'] ??= copyright; + final List snippetData = parseInput(sample); + sample.description = description ?? sample.description; + sample.metadata['description'] = _getDescription(sample); + switch (sample.runtimeType) { + case DartpadSample _: + case ApplicationSample _: + String app; + if (sample.sourceFile == null) { + final String templateName = sample.template; + if (templateName.isEmpty) { + io.stderr + .writeln('Non-linked samples must have a --template argument.'); + io.exit(1); + } + final Directory templatesDir = configuration.templatesDirectory; + File? templateFile; + templateFile = + getTemplatePath(templateName, templatesDir: templatesDir); + if (templateFile == null) { + io.stderr.writeln( + 'The template $templateName was not found in the templates ' + 'directory ${templatesDir.path}'); + io.exit(1); + } + final String templateContents = _loadFileAsUtf8(templateFile); + final String templateRelativePath = + templateFile.absolute.path.contains(flutterRoot.absolute.path) + ? path.relative(templateFile.absolute.path, + from: flutterRoot.absolute.path) + : templateFile.absolute.path; + final String templateHeader = ''' +// Template: $templateRelativePath +// +// Comment lines marked with "▼▼▼" and "▲▲▲" are used for authoring +// of samples, and may be ignored if you are just exploring the sample. +'''; + app = interpolateTemplate( + snippetData, + addSectionMarkers + ? '$templateHeader\n$templateContents' + : templateContents, + sample.metadata, + addSectionMarkers: addSectionMarkers, + addCopyright: copyright != null, + ); + } else { + app = sample.sourceFileContents; + } + sample.output = app; + if (formatOutput) { + final DartFormatter formatter = + DartFormatter(pageWidth: 80, fixes: StyleFix.all); + try { + sample.output = formatter.format(sample.output); + } on FormatterException catch (exception) { + io.stderr + .write('Code to format:\n${_addLineNumbers(sample.output)}\n'); + errorExit('Unable to format sample code: $exception'); + } + sample.output = sortImports(sample.output); + } + if (output != null) { + output.writeAsStringSync(sample.output); + + final File metadataFile = configuration.filesystem.file(path.join( + path.dirname(output.path), + '${path.basenameWithoutExtension(output.path)}.json')); + sample.metadata['file'] = path.basename(output.path); + final Map metadata = sample.metadata; + if (metadata.containsKey('description')) { + metadata['description'] = (metadata['description']! as String) + .replaceAll(RegExp(r'^// ?', multiLine: true), ''); + } + metadataFile.writeAsStringSync(jsonEncoder.convert(metadata)); + } + case SnippetSample _: + if (sample is SnippetSample) { + String app; + if (sample.sourceFile == null) { + String templateContents; + if (includeAssumptions) { + templateContents = + '${headers.map((SourceLine line) => line.text).join('\n')}\n{{#assumptions}}\n{{description}}\n{{code}}'; + } else { + templateContents = '{{description}}\n{{code}}'; + } + app = interpolateTemplate( + snippetData, + templateContents, + sample.metadata, + addSectionMarkers: addSectionMarkers, + addCopyright: copyright != null, + ); + } else { + app = sample.inputAsString; + } + sample.output = app; + } + } + return sample.output; + } + + String _addLineNumbers(String code) { + final StringBuffer buffer = StringBuffer(); + int count = 0; + for (final String line in code.split('\n')) { + count++; + buffer.writeln('${count.toString().padLeft(5)}: $line'); + } + return buffer.toString(); + } + + /// Computes the headers needed for each snippet file. + /// + /// Not used for "sample" and "dartpad" samples, which use their own template. + List get headers { + return _headers ??= [ + '// generated code', + '// ignore_for_file: unused_import', + '// ignore_for_file: unused_element', + '// ignore_for_file: unused_local_variable', + "import 'dart:async';", + "import 'dart:convert';", + "import 'dart:math' as math;", + "import 'dart:typed_data';", + "import 'dart:ui' as ui;", + "import 'package:flutter_test/flutter_test.dart';", + for (final File file in _listDartFiles(FlutterInformation.instance + .getFlutterRoot() + .childDirectory('packages') + .childDirectory('flutter') + .childDirectory('lib'))) ...[ + '', + '// ${file.path}', + "import 'package:flutter/${path.basename(file.path)}';", + ], + ].map((String code) => SourceLine(code)).toList(); + } + + List? _headers; + + static List _listDartFiles(Directory directory, + {bool recursive = false}) { + return directory + .listSync(recursive: recursive, followLinks: false) + .whereType() + .where((File file) => path.extension(file.path) == '.dart') + .toList(); + } +} diff --git a/dev/snippets/lib/src/snippet_parser.dart b/dev/snippets/lib/src/snippet_parser.dart new file mode 100644 index 0000000000000..9af60b483453e --- /dev/null +++ b/dev/snippets/lib/src/snippet_parser.dart @@ -0,0 +1,426 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/file.dart'; +import 'package:path/path.dart' as path; + +import 'data_types.dart'; +import 'util.dart'; + +/// Parses [CodeSample]s from the source file given to one of the parsing routines. +/// +/// - [parseFromDartdocToolFile] parses the output of the dartdoc `@tool` +/// directive, which contains the dartdoc comment lines (with comment markers +/// stripped) contained between the tool markers. +/// +/// - [parseAndAddAssumptions] parses the assumptions in the "Examples can +/// assume:" block at the top of the file and adds them to the code samples +/// contained in the given [SourceElement] iterable. +class SnippetDartdocParser { + SnippetDartdocParser(this.filesystem); + + final FileSystem filesystem; + + /// The prefix of each comment line + static const String _dartDocPrefix = '///'; + + /// The prefix of each comment line with a space appended. + static const String _dartDocPrefixWithSpace = '$_dartDocPrefix '; + + /// A RegExp that matches the beginning of a dartdoc snippet or sample. + static final RegExp _dartDocSampleBeginRegex = + RegExp(r'\{@tool (?sample|snippet|dartpad)(?:| (?[^}]*))\}'); + + /// A RegExp that matches the end of a dartdoc snippet or sample. + static final RegExp _dartDocSampleEndRegex = RegExp(r'\{@end-tool\}'); + + /// A RegExp that matches the start of a code block within dartdoc. + static final RegExp _codeBlockStartRegex = RegExp(r'///\s+```dart.*$'); + + /// A RegExp that matches the end of a code block within dartdoc. + static final RegExp _codeBlockEndRegex = RegExp(r'///\s+```\s*$'); + + /// A RegExp that matches a linked sample pointer. + static final RegExp _filePointerRegex = + RegExp(r'\*\* See code in (?[^\]]+) \*\*'); + + /// Parses the assumptions in the "Examples can assume:" block at the top of + /// the `assumptionsFile` and adds them to the code samples contained in the + /// given `elements` iterable. + void parseAndAddAssumptions( + Iterable elements, + File assumptionsFile, { + bool silent = true, + }) { + final List assumptions = parseAssumptions(assumptionsFile); + for (final CodeSample sample in elements + .expand((SourceElement element) => element.samples)) { + if (sample is SnippetSample) { + sample.assumptions = assumptions; + } + sample.metadata.addAll({ + 'id': '${sample.element}.${sample.index}', + 'element': sample.element, + 'sourcePath': assumptionsFile.path, + 'sourceLine': sample.start.line, + }); + } + } + + /// Parses a file containing the output of the dartdoc `@tool` directive, + /// which contains the dartdoc comment lines (with comment markers stripped) + /// between the tool markers. + /// + /// This is meant to be run as part of a dartdoc tool that handles snippets. + SourceElement parseFromDartdocToolFile( + File input, { + int? startLine, + String? element, + required File sourceFile, + String type = '', + bool silent = true, + }) { + final List lines = []; + int lineNumber = startLine ?? 0; + final List inputStrings = [ + // The parser wants to read the arguments from the input, so we create a new + // tool line to match the given arguments, so that we can use the same parser for + // editing and docs generation. + '/// {@tool $type}', + // Snippet input comes in with the comment markers stripped, so we add them + // back to make it conform to the source format, so we can use the same + // parser for editing samples as we do for processing docs. + ...input + .readAsLinesSync() + .map((String line) => '/// $line'.trimRight()), + '/// {@end-tool}', + ]; + for (final String line in inputStrings) { + lines.add( + SourceLine(line, + element: element ?? '', line: lineNumber, file: sourceFile), + ); + lineNumber++; + } + // No need to get assumptions: dartdoc won't give that to us. + final SourceElement newElement = SourceElement( + SourceElementType.unknownType, element!, -1, + file: input, comment: lines); + parseFromComments([newElement], silent: silent); + for (final CodeSample sample in newElement.samples) { + sample.metadata.addAll({ + 'id': '${sample.element}.${sample.index}', + 'element': sample.element, + 'sourcePath': sourceFile.path, + 'sourceLine': sample.start.line, + }); + } + return newElement; + } + + /// This parses the assumptions in the "Examples can assume:" block from the + /// given `file`. + List parseAssumptions(File file) { + // Whether or not we're in the file-wide preamble section ("Examples can assume"). + bool inPreamble = false; + final List preamble = []; + int lineNumber = 0; + int charPosition = 0; + for (final String line in file.readAsLinesSync()) { + if (inPreamble && line.trim().isEmpty) { + // Reached the end of the preamble. + break; + } + if (!line.startsWith('// ')) { + lineNumber++; + charPosition += line.length + 1; + continue; + } + if (line == '// Examples can assume:') { + inPreamble = true; + lineNumber++; + charPosition += line.length + 1; + continue; + } + if (inPreamble) { + preamble.add(SourceLine( + line.substring(3), + startChar: charPosition, + endChar: charPosition + line.length + 1, + element: '#assumptions', + file: file, + line: lineNumber, + )); + } + lineNumber++; + charPosition += line.length + 1; + } + return preamble; + } + + /// This parses the code snippets from the documentation comments in the given + /// `elements`, and sets the resulting samples as the `samples` member of + /// each element in the supplied iterable. + void parseFromComments( + Iterable elements, { + bool silent = true, + }) { + int dartpadCount = 0; + int sampleCount = 0; + int snippetCount = 0; + + for (final SourceElement element in elements) { + if (element.comment.isEmpty) { + continue; + } + parseComment(element); + for (final CodeSample sample in element.samples) { + switch (sample.runtimeType) { + case DartpadSample _: + dartpadCount++; + case ApplicationSample _: + sampleCount++; + case SnippetSample _: + snippetCount++; + } + } + } + + if (!silent) { + print('Found:\n' + ' $snippetCount snippet code blocks,\n' + ' $sampleCount non-dartpad sample code sections, and\n' + ' $dartpadCount dartpad sections.\n'); + } + } + + /// This parses the documentation comment on a single [SourceElement] and + /// assigns the resulting samples to the `samples` member of the given + /// `element`. + void parseComment(SourceElement element) { + // Whether or not we're in a snippet code sample. + bool inSnippet = false; + // Whether or not we're in a '```dart' segment. + bool inDart = false; + bool foundSourceLink = false; + bool foundDartSection = false; + File? linkedFile; + List block = []; + List snippetArgs = []; + final List samples = []; + final Directory flutterRoot = FlutterInformation.instance.getFlutterRoot(); + + int index = 0; + for (final SourceLine line in element.comment) { + final String trimmedLine = line.text.trim(); + if (inSnippet) { + if (!trimmedLine.startsWith(_dartDocPrefix)) { + throw SnippetException('Snippet section unterminated.', + file: line.file?.path, line: line.line); + } + if (_dartDocSampleEndRegex.hasMatch(trimmedLine)) { + switch (snippetArgs.first) { + case 'snippet': + samples.add( + SnippetSample( + block, + index: index++, + lineProto: line, + ), + ); + case 'sample': + if (linkedFile != null) { + samples.add( + ApplicationSample.fromFile( + input: block, + args: snippetArgs, + sourceFile: linkedFile, + index: index++, + lineProto: line, + ), + ); + break; + } + samples.add( + ApplicationSample( + input: block, + args: snippetArgs, + index: index++, + lineProto: line, + ), + ); + case 'dartpad': + if (linkedFile != null) { + samples.add( + DartpadSample.fromFile( + input: block, + args: snippetArgs, + sourceFile: linkedFile, + index: index++, + lineProto: line, + ), + ); + break; + } + samples.add( + DartpadSample( + input: block, + args: snippetArgs, + index: index++, + lineProto: line, + ), + ); + default: + throw SnippetException( + 'Unknown snippet type ${snippetArgs.first}'); + } + snippetArgs = []; + block = []; + inSnippet = false; + foundSourceLink = false; + foundDartSection = false; + linkedFile = null; + } else if (_filePointerRegex.hasMatch(trimmedLine)) { + foundSourceLink = true; + if (foundDartSection) { + throw SnippetException( + 'Snippet contains a source link and a dart section. Cannot contain both.', + file: line.file?.path, + line: line.line, + ); + } + if (linkedFile != null) { + throw SnippetException( + 'Found more than one linked sample. Only one linked file per sample is allowed.', + file: line.file?.path, + line: line.line, + ); + } + final RegExpMatch match = _filePointerRegex.firstMatch(trimmedLine)!; + linkedFile = filesystem.file( + path.join(flutterRoot.absolute.path, match.namedGroup('file'))); + } else { + block.add(line.copyWith( + text: line.text.replaceFirst(RegExp(r'\s*/// ?'), ''))); + } + } else { + if (_dartDocSampleEndRegex.hasMatch(trimmedLine)) { + if (inDart) { + throw SnippetException( + "Dart section didn't terminate before end of sample", + file: line.file?.path, + line: line.line); + } + } + if (inDart) { + if (_codeBlockEndRegex.hasMatch(trimmedLine)) { + inDart = false; + block = []; + } else if (trimmedLine == _dartDocPrefix) { + block.add(line.copyWith(text: '')); + } else { + final int index = line.text.indexOf(_dartDocPrefixWithSpace); + if (index < 0) { + throw SnippetException( + 'Dart section inexplicably did not contain "$_dartDocPrefixWithSpace" prefix.', + file: line.file?.path, + line: line.line, + ); + } + block.add(line.copyWith(text: line.text.substring(index + 4))); + } + } else if (_codeBlockStartRegex.hasMatch(trimmedLine)) { + if (foundSourceLink) { + throw SnippetException( + 'Snippet contains a source link and a dart section. Cannot contain both.', + file: line.file?.path, + line: line.line, + ); + } + assert(block.isEmpty); + inDart = true; + foundDartSection = true; + } + } + if (!inSnippet && !inDart) { + final RegExpMatch? sampleMatch = + _dartDocSampleBeginRegex.firstMatch(trimmedLine); + if (sampleMatch != null) { + inSnippet = sampleMatch.namedGroup('type') == 'snippet' || + sampleMatch.namedGroup('type') == 'sample' || + sampleMatch.namedGroup('type') == 'dartpad'; + if (inSnippet) { + if (sampleMatch.namedGroup('args') != null) { + // There are arguments to the snippet tool to keep track of. + snippetArgs = [ + sampleMatch.namedGroup('type')!, + ..._splitUpQuotedArgs(sampleMatch.namedGroup('args')!) + ]; + } else { + snippetArgs = [ + sampleMatch.namedGroup('type')!, + ]; + } + } + } + } + } + for (final CodeSample sample in samples) { + sample.metadata.addAll({ + 'id': '${sample.element}.${sample.index}', + 'element': sample.element, + 'sourcePath': sample.start.file?.path ?? '', + 'sourceLine': sample.start.line, + }); + } + element.replaceSamples(samples); + } + + // Helper to process arguments given as a (possibly quoted) string. + // + // First, this will split the given [argsAsString] into separate arguments, + // taking any quoting (either ' or " are accepted) into account, including + // handling backslash-escaped quotes. + // + // Then, it will prepend "--" to any args that start with an identifier + // followed by an equals sign, allowing the argument parser to treat any + // "foo=bar" argument as "--foo=bar" (which is a dartdoc-ism). + Iterable _splitUpQuotedArgs(String argsAsString) { + // This function is used because the arg parser package doesn't handle + // quoted args. + + // Regexp to take care of splitting arguments, and handling the quotes + // around arguments, if any. + // + // Match group 1 (option) is the "foo=" (or "--foo=") part of the option, if any. + // Match group 2 (quote) contains the quote character used (which is discarded). + // Match group 3 (value) is a quoted arg, if any, without the quotes. + // Match group 4 (unquoted) is the unquoted arg, if any. + final RegExp argMatcher = RegExp( + r'(?