From efa28cd4c8a39dfe2ca82b9c520aca2cce991836 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:25:20 -0800 Subject: [PATCH 01/16] added dt command to run license utility --- tool/lib/devtools_command_runner.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tool/lib/devtools_command_runner.dart b/tool/lib/devtools_command_runner.dart index fa528413857..41a55a48346 100644 --- a/tool/lib/devtools_command_runner.dart +++ b/tool/lib/devtools_command_runner.dart @@ -13,6 +13,7 @@ import 'package:devtools_tool/commands/serve.dart'; import 'package:devtools_tool/commands/sync.dart'; import 'package:devtools_tool/commands/tag_version.dart'; import 'package:devtools_tool/commands/update_flutter_sdk.dart'; +import 'package:devtools_tool/commands/update_licenses.dart'; import 'package:devtools_tool/commands/update_perfetto.dart'; import 'package:devtools_tool/model.dart'; @@ -47,6 +48,7 @@ class DevToolsCommandRunner extends CommandRunner { addCommand(UpdateDartSdkDepsCommand()); addCommand(UpdateDevToolsVersionCommand()); addCommand(UpdateFlutterSdkCommand()); + addCommand(UpdateLicensesCommand()); addCommand(UpdatePerfettoCommand()); argParser.addFlag( From 1fddcb8554e17ff3d4ebcd1a1d3f9b81a9d8e002 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:09:37 -0800 Subject: [PATCH 02/16] committing update licenses --- tool/lib/commands/update_licenses.dart | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tool/lib/commands/update_licenses.dart diff --git a/tool/lib/commands/update_licenses.dart b/tool/lib/commands/update_licenses.dart new file mode 100644 index 00000000000..a2a39886a8d --- /dev/null +++ b/tool/lib/commands/update_licenses.dart @@ -0,0 +1,78 @@ +// Copyright 2024 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:cli_util/cli_logging.dart'; +import 'package:path/path.dart' as p; + +import '../license_utils.dart'; + +const _argConfig = 'config'; +const _argDirectory = 'directory'; +const _dryRun = 'dry-run'; + +/// This command updates license headers for the configured files. +/// +/// The config file is a YAML file as defined in [LicenseConfig]. +/// +/// If directory is not set, it will default to the current directory. +/// +/// When the '--dry-run' flag is passed in, a list of files to update will +/// be logged, but no files will be modified. +/// +/// To run this script +/// `dt update-licenses [--f ] [--d ] [--dry-run]` +class UpdateLicensesCommand extends Command { + UpdateLicensesCommand() { + argParser + ..addOption( + _argConfig, + abbr: 'c', + defaultsTo: p.join(Directory.current.path, 'update_licenses.yaml'), + help: + 'The path to the YAML license config file. Defaults to ' + 'update_licenses.yaml', + ) + ..addOption( + _argDirectory, + defaultsTo: Directory.current.path, + abbr: 'd', + help: 'Update license headers for files in the directory.', + ) + ..addFlag( + _dryRun, + negatable: false, + defaultsTo: false, + help: + 'If set, log a list of files that require an update, but do not ' + 'modify any files.', + ); + } + + @override + String get description => 'Update license headers as configured.'; + + @override + String get name => 'update-licenses'; + + @override + Future run() async { + final config = LicenseConfig.fromYamlFile( + File(argResults![_argConfig] as String), + ); + final directory = Directory(argResults![_argDirectory] as String); + final dryRun = argResults![_dryRun] as bool; + final log = Logger.standard(); + final header = LicenseHeader(); + final results = await header.bulkUpdate( + directory: directory, + config: config, + dryRun: dryRun, + ); + final updatedPaths = results.updatedPaths; + final prefix = dryRun ? 'Requires update: ' : 'Updated: '; + log.stdout('$prefix ${updatedPaths.join(", ")}'); + } +} From f34d7fe980e862e584c2b28de827fa66c81c7061 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:29:25 -0800 Subject: [PATCH 03/16] improved yaml example to remove extra newlines and properly handle values --- tool/lib/license_utils.dart | 24 +++++++------- tool/test/license_utils_test.dart | 55 ++++++++++++++----------------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index c171d13bfa9..afa82beaaf6 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -25,27 +25,27 @@ import 'package:yaml/yaml.dart'; /// ```yaml /// # sequence of license text strings that should be matched against at the top of a file and removed. , which normally represents a date, will be stored. /// remove_licenses: -/// - | -/// // This is some multiline license +/// - |- +/// // This is some multiline license /// // text that should be removed from the file. -/// - | -/// /* This is other multiline license +/// - |- +/// /* This is other multiline license /// text that should be removed from the file. */ -/// - | -/// # This is more multiline license +/// - |- +/// # This is more multiline license /// # text that should be removed from the file. -/// - | +/// - |- /// // This is some multiline license text to /// // remove that does not contain a stored value. /// # sequence of license text strings that should be added to the top of a file. {value} will be replaced. /// add_licenses: -/// - | -/// // This is some multiline license +/// - |- +/// // This is some multiline license /// // text that should be added to the file. -/// - | -/// # This is other multiline license +/// - |- +/// # This is other multiline license /// # text that should be added to the file. -/// - | +/// - |- /// // This is some multiline license text to /// // add that does not contain a stored value. /// # defines which files should have license text added or updated. diff --git a/tool/test/license_utils_test.dart b/tool/test/license_utils_test.dart index 5cbd21725c9..f1fa4900ab3 100644 --- a/tool/test/license_utils_test.dart +++ b/tool/test/license_utils_test.dart @@ -73,24 +73,20 @@ void main() { expect(config.removeLicenses.length, equals(4)); - var expectedVal = '''// This is some multiline license -// text that should be removed from the file. -'''; + var expectedVal = '''// This is some multiline license +// text that should be removed from the file.'''; expect(config.removeLicenses[0], equals(expectedVal)); - expectedVal = '''/* This is other multiline license -text that should be removed from the file. */ -'''; + expectedVal = '''/* This is other multiline license +text that should be removed from the file. */'''; expect(config.removeLicenses[1], equals(expectedVal)); - expectedVal = '''# This is more multiline license -# text that should be removed from the file. -'''; + expectedVal = '''# This is more multiline license +# text that should be removed from the file.'''; expect(config.removeLicenses[2], equals(expectedVal)); expectedVal = '''// This is some multiline license text to -// remove that does not contain a stored value. -'''; +// remove that does not contain a stored value.'''; expect(config.removeLicenses[3], equals(expectedVal)); }); @@ -99,19 +95,16 @@ text that should be removed from the file. */ expect(config.addLicenses.length, equals(3)); - var expectedVal = '''// This is some multiline license -// text that should be added to the file. -'''; + var expectedVal = '''// This is some multiline license +// text that should be added to the file.'''; expect(config.addLicenses[0], equals(expectedVal)); - expectedVal = '''# This is other multiline license -# text that should be added to the file. -'''; + expectedVal = '''# This is other multiline license +# text that should be added to the file.'''; expect(config.addLicenses[1], equals(expectedVal)); expectedVal = '''// This is some multiline license text to -// add that does not contain a stored value. -'''; +// add that does not contain a stored value.'''; expect(config.addLicenses[2], equals(expectedVal)); }); @@ -483,27 +476,27 @@ Future _setupTestConfigFile() async { final contents = '''--- # sequence of license text strings that should be matched against at the top of a file and removed. , which normally represents a date, will be stored. remove_licenses: - - | - // This is some multiline license + - |- + // This is some multiline license // text that should be removed from the file. - - | - /* This is other multiline license + - |- + /* This is other multiline license text that should be removed from the file. */ - - | - # This is more multiline license + - |- + # This is more multiline license # text that should be removed from the file. - - | + - |- // This is some multiline license text to // remove that does not contain a stored value. # sequence of license text strings that should be added to the top of a file. {value} will be replaced. add_licenses: - - | - // This is some multiline license + - |- + // This is some multiline license // text that should be added to the file. - - | - # This is other multiline license + - |- + # This is other multiline license # text that should be added to the file. - - | + - |- // This is some multiline license text to // add that does not contain a stored value. # defines which files should have license text added or updated. From 3f0ecce3325f768551f527a6f56141891d8e9eaf Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:32:00 -0800 Subject: [PATCH 04/16] fix comment in yaml --- tool/test/license_utils_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/test/license_utils_test.dart b/tool/test/license_utils_test.dart index f1fa4900ab3..4f5fc1367fb 100644 --- a/tool/test/license_utils_test.dart +++ b/tool/test/license_utils_test.dart @@ -488,7 +488,7 @@ remove_licenses: - |- // This is some multiline license text to // remove that does not contain a stored value. -# sequence of license text strings that should be added to the top of a file. {value} will be replaced. +# sequence of license text strings that should be added to the top of a file. will be replaced. add_licenses: - |- // This is some multiline license From a2392dd76a4e438e345f19ea5edc44a77d810a2b Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:48:38 -0800 Subject: [PATCH 05/16] fixed another comment --- tool/lib/license_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index afa82beaaf6..f6bc47c005e 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -37,7 +37,7 @@ import 'package:yaml/yaml.dart'; /// - |- /// // This is some multiline license text to /// // remove that does not contain a stored value. -/// # sequence of license text strings that should be added to the top of a file. {value} will be replaced. +/// # sequence of license text strings that should be added to the top of a file. will be replaced. /// add_licenses: /// - |- /// // This is some multiline license From 384edc718af04677fc04eb5b561ba77d3606c9b5 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:52:17 -0800 Subject: [PATCH 06/16] handling unconfigured extension type --- tool/lib/license_utils.dart | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index f6bc47c005e..9f515747a10 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -121,17 +121,24 @@ class LicenseConfig { final YamlMap fileTypes; /// Returns the list of indices for the given [ext] of [removeLicenses] - /// containing the license text to remove. + /// containing the license text to remove if they exist or an empty YamlList. YamlList getRemoveIndicesForExtension(String ext) { + print(ext); final fileType = fileTypes[_removeDotFromExtension(ext)]; - return fileType['remove'] as YamlList; + if (fileType) { + return fileType['remove'] as YamlList; + } + return YamlList(); } /// Returns the index for the given [ext] of [addLicenses] containing the - /// license text to add. + /// license text to add if it exists or -1. int getAddIndexForExtension(String ext) { final fileType = fileTypes[_removeDotFromExtension(ext)]; - return fileType['add']; + if (fileType) { + return fileType['add']; + } + return -1; } /// Returns whether the file should be excluded according to the config. From 13ab73d1298201bdad411cf8c62454aac3f8c4c7 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:54:07 -0800 Subject: [PATCH 07/16] attempt #2 at handling null --- tool/lib/license_utils.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index 9f515747a10..a8bbaf723df 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -125,8 +125,8 @@ class LicenseConfig { YamlList getRemoveIndicesForExtension(String ext) { print(ext); final fileType = fileTypes[_removeDotFromExtension(ext)]; - if (fileType) { - return fileType['remove'] as YamlList; + if (fileType != Null) { + return fileType!['remove'] as YamlList; } return YamlList(); } @@ -135,7 +135,7 @@ class LicenseConfig { /// license text to add if it exists or -1. int getAddIndexForExtension(String ext) { final fileType = fileTypes[_removeDotFromExtension(ext)]; - if (fileType) { + if (fileType != Null) { return fileType['add']; } return -1; From 79914e9a465e730d4d148cf7033fff19a9eff21f Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:54:48 -0800 Subject: [PATCH 08/16] attempt #3 at handling null --- tool/lib/license_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index a8bbaf723df..60caee9b28f 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -126,7 +126,7 @@ class LicenseConfig { print(ext); final fileType = fileTypes[_removeDotFromExtension(ext)]; if (fileType != Null) { - return fileType!['remove'] as YamlList; + return fileType['remove'] as YamlList; } return YamlList(); } From 19aec48c1e9ea9a3bb62215c0cfb080dd0acdac1 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:55:48 -0800 Subject: [PATCH 09/16] attempt #4 at handling null --- tool/lib/license_utils.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index 60caee9b28f..ad29c6a7b01 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -125,7 +125,7 @@ class LicenseConfig { YamlList getRemoveIndicesForExtension(String ext) { print(ext); final fileType = fileTypes[_removeDotFromExtension(ext)]; - if (fileType != Null) { + if (fileType != null) { return fileType['remove'] as YamlList; } return YamlList(); @@ -135,7 +135,7 @@ class LicenseConfig { /// license text to add if it exists or -1. int getAddIndexForExtension(String ext) { final fileType = fileTypes[_removeDotFromExtension(ext)]; - if (fileType != Null) { + if (fileType != null) { return fileType['add']; } return -1; From 93feb895ff2a390257e562d9fbad1f931eb850f2 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:57:01 -0800 Subject: [PATCH 10/16] better error message --- tool/lib/license_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index ad29c6a7b01..9b90730b162 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -209,7 +209,7 @@ class LicenseHeader { .handleError( (e) => throw StateError( - 'License header expected, but error reading file - $e', + 'License header expected, but error reading file $file - $e', ), ); await for (final content in stream) { From ad669e6067bbb421353515c212b7bea4600ab5ee Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:58:43 -0800 Subject: [PATCH 11/16] removed print --- tool/lib/license_utils.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index 9b90730b162..2bad0ba7d00 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -123,7 +123,6 @@ class LicenseConfig { /// Returns the list of indices for the given [ext] of [removeLicenses] /// containing the license text to remove if they exist or an empty YamlList. YamlList getRemoveIndicesForExtension(String ext) { - print(ext); final fileType = fileTypes[_removeDotFromExtension(ext)]; if (fileType != null) { return fileType['remove'] as YamlList; From 3656569cf734645c540855dbba00c259914f4aac Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:59:07 -0800 Subject: [PATCH 12/16] improved logic for missing license header --- tool/lib/license_utils.dart | 78 ++++++++++++++++++++++++++----- tool/test/license_utils_test.dart | 30 ++++++++++-- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index 2bad0ba7d00..1782caa988f 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -208,7 +208,7 @@ class LicenseHeader { .handleError( (e) => throw StateError( - 'License header expected, but error reading file $file - $e', + 'License header expected, but error reading $file - $e', ), ); await for (final content in stream) { @@ -260,6 +260,24 @@ class LicenseHeader { return rewrittenFile; } + /// Returns a copy of the given [file] that is missing a license header + /// with the [replacementHeader] added to the top. + /// + /// Reads and writes the entire file contents all at once, so performance may + /// degrade for large files. + File addLicenseHeader({ + required File file, + required String replacementHeader, + }) { + final rewrittenFile = File('${file.path}.tmp'); + final contents = file.readAsStringSync(); + rewrittenFile.writeAsStringSync( + '$replacementHeader${Platform.lineTerminator}$contents', + flush: true, + ); + return rewrittenFile; + } + /// Bulk update license headers for files in the [directory] as configured /// in the [config] and return a processed paths Record containing: /// - list of included paths @@ -281,10 +299,10 @@ class LicenseHeader { includedPathsList.add(file.path); final extension = p.extension(file.path); final removeIndices = config.getRemoveIndicesForExtension(extension); + final addIndex = config.getAddIndexForExtension(extension); + final replacementLicenseText = config.addLicenses[addIndex]; for (final removeIndex in removeIndices) { final existingLicenseText = config.removeLicenses[removeIndex]; - final addIndex = config.getAddIndexForExtension(extension); - final replacementLicenseText = config.addLicenses[addIndex]; final fileLength = file.lengthSync(); const bufferSize = 20; // Assume that the license text will be near the start of the file, @@ -301,6 +319,7 @@ class LicenseHeader { ); if (replacementInfo.existingHeader.isNotEmpty && replacementInfo.replacementHeader.isNotEmpty) { + // Case 1: Existing header needs to be replaced if (dryRun) { updatedPathsList.add(file.path); } else { @@ -309,15 +328,27 @@ class LicenseHeader { existingHeader: replacementInfo.existingHeader, replacementHeader: replacementInfo.replacementHeader, ); - if (rewrittenFile.lengthSync() > 0) { - file.writeAsStringSync( - rewrittenFile.readAsStringSync(), - mode: FileMode.writeOnly, - flush: true, - ); - updatedPathsList.add(file.path); - } - rewrittenFile.deleteSync(); + _updateLicense(rewrittenFile, file, updatedPathsList); + } + } + } + if (!updatedPathsList.contains(file.path)) { + final licenseHeaders = _processHeaders( + storedName: '', + existingLicenseText: '', + replacementLicenseText: replacementLicenseText, + content: '', + ); + if (licenseHeaders.replacementHeader.isNotEmpty) { + // Case 2: Missing header needs to be added + if (dryRun) { + updatedPathsList.add(file.path); + } else { + final rewrittenFile = addLicenseHeader( + file: file, + replacementHeader: licenseHeaders.replacementHeader, + ); + _updateLicense(rewrittenFile, file, updatedPathsList); } } } @@ -326,12 +357,35 @@ class LicenseHeader { return (includedPaths: includedPathsList, updatedPaths: updatedPathsList); } + void _updateLicense( + File rewrittenFile, + File file, + List updatedPathsList, + ) { + if (rewrittenFile.lengthSync() > 0) { + file.writeAsStringSync( + rewrittenFile.readAsStringSync(), + mode: FileMode.writeOnly, + flush: true, + ); + updatedPathsList.add(file.path); + } + rewrittenFile.deleteSync(); + } + ({String existingHeader, String replacementHeader}) _processHeaders({ required String storedName, required String existingLicenseText, required String replacementLicenseText, required String content, }) { + if (existingLicenseText.isEmpty) { + final defaultReplacementHeader = replacementLicenseText.replaceAll( + '<$storedName>', + DateTime.now().year.toString(), + ); + return (existingHeader: '', replacementHeader: defaultReplacementHeader); + } final matchStr = RegExp.escape(existingLicenseText); final storedNameIndex = matchStr.indexOf('<$storedName>'); if (storedNameIndex != -1) { diff --git a/tool/test/license_utils_test.dart b/tool/test/license_utils_test.dart index 4f5fc1367fb..27a72ffc5c0 100644 --- a/tool/test/license_utils_test.dart +++ b/tool/test/license_utils_test.dart @@ -293,7 +293,7 @@ text that should be added to the file. */''', expect( errorMessage, contains( - 'Bad state: License header expected, but error reading file - PathNotFoundException', + 'Bad state: License header expected, but error reading File: \'bad.txt\' - PathNotFoundException', ), ); }); @@ -330,6 +330,24 @@ text that should be added to the file. */''', ); }); + test('license header can be added on disk', () async { + final header = LicenseHeader(); + const replacementHeader = '''// This is some 2015 multiline license +// text that should be added to the file.'''; + final rewrittenFile = header.addLicenseHeader( + file: testFile9, + replacementHeader: replacementHeader, + ); + + expect(rewrittenFile.lengthSync(), greaterThan(0)); + + final rewrittenContents = rewrittenFile.readAsStringSync(); + expect( + rewrittenContents.substring(0, replacementHeader.length), + equals(replacementHeader), + ); + }); + test('license headers can be updated in bulk', () async { await _setupTestConfigFile(); final config = LicenseConfig.fromYamlFile(configFile); @@ -357,15 +375,15 @@ text that should be added to the file. */''', final updatedPaths = results.updatedPaths; expect(updatedPaths, isNotNull); - // testFile9 and testFile10 are intentionally misconfigured and so they - // won't be updated even though they are on the include list. - expect(updatedPaths.length, equals(5)); + expect(updatedPaths.length, equals(7)); // Order is not guaranteed expect(updatedPaths.contains(testFile1.path), true); expect(updatedPaths.contains(testFile2.path), true); expect(updatedPaths.contains(testFile3.path), true); expect(updatedPaths.contains(testFile7.path), true); expect(updatedPaths.contains(testFile8.path), true); + expect(updatedPaths.contains(testFile9.path), true); + expect(updatedPaths.contains(testFile10.path), true); }); test('license headers bulk update can be dry run', () async { @@ -383,7 +401,7 @@ text that should be added to the file. */''', final updatedPaths = results.updatedPaths; expect(updatedPaths, isNotNull); - expect(updatedPaths.length, equals(5)); + expect(updatedPaths.length, equals(7)); expect(updatedPaths.contains(testFile1.path), true); expect(contentsBeforeUpdate, equals(contentsAfterUpdate)); }); @@ -617,6 +635,7 @@ Future _setupTestDirectoryStructure() async { p.join(repoRoot.path, 'sub_dir2', 'sub_dir4'), ).createSync(recursive: true); + // Missing license header testFile9 = File(p.join(repoRoot.path, 'sub_dir2', 'sub_dir4', 'test9.ext1')) ..createSync(recursive: true); testFile9.writeAsStringSync(extraText, flush: true); @@ -626,6 +645,7 @@ Future _setupTestDirectoryStructure() async { p.join(repoRoot.path, 'sub_dir2', 'sub_dir4', 'sub_dir5'), ).createSync(recursive: true); + // Will be treated like a missing license header since not configured testFile10 = File( p.join(repoRoot.path, 'sub_dir2', 'sub_dir4', 'sub_dir5', 'test10.ext2'), )..createSync(recursive: true); From 10ad6de1006864f4fb1a27657569d900aecd9895 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:57:55 -0800 Subject: [PATCH 13/16] commited config file --- update_licenses.yaml | 136 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 update_licenses.yaml diff --git a/update_licenses.yaml b/update_licenses.yaml new file mode 100644 index 00000000000..c1038efcc43 --- /dev/null +++ b/update_licenses.yaml @@ -0,0 +1,136 @@ +--- +remove_licenses: + - |- + // Copyright The Chromium Authors. All rights reserved. + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file. + - |- + /** + Copyright The Chromium Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. + **/ + - |- + # Copyright The Chromium Authors. All rights reserved. + # Use of this source code is governed by a BSD-style license that can be + # found in the LICENSE file. + - |- + + - |- + REM Copyright The Chromium Authors. All rights reserved. + REM Use of this source code is governed by a BSD-style license that can be + REM found in the LICENSE file. +add_licenses: + - |- + // Copyright The Flutter Authors + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. + - |- + /** + Copyright The Flutter Authors + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. + **/ + - |- + # Copyright The Flutter Authors + # Use of this source code is governed by a BSD-style license that can be + # found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. + - |- + + - |- + REM Copyright The Flutter Authors + REM Use of this source code is governed by a BSD-style license that can be + REM found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. +update_paths: + include: + - /Users/mossman/Development/devtools + exclude: + - /Users/mossman/Development/devtools/third_party + - /Users/mossman/Development/devtools/packages/devtools_app/build + - /Users/mossman/Development/devtools/tool/flutter-sdk + file_types: + bat: + remove: + - 4 + add: 4 + cc: + remove: + - 0 + - 1 + add: 0 + cpp: + remove: + - 0 + - 1 + add: 0 + css: + remove: + - 1 + add: 1 + dart: + remove: + - 0 + - 1 + add: 0 + gradle: + remove: + - 0 + - 1 + add: 0 + h: + remove: + - 0 + - 1 + add: 0 + html: + remove: + - 3 + add: 3 + java: + remove: + - 0 + - 1 + add: 0 + json: + remove: + - 0 + - 1 + add: 0 + last_build_id: + remove: + - 0 + - 1 + add: 0 + md: + remove: + - 3 + add: 3 + sh: + remove: + - 2 + add: 2 + swift: + remove: + - 0 + - 1 + add: 0 + xml: + remove: + - 3 + add: 3 + yaml: + remove: + - 2 + add: 2 + yml: + remove: + - 2 + add: 2 From 0c44b769adeaaa68cda4156fab83d46e64a8fe71 Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:55:57 -0800 Subject: [PATCH 14/16] improved handling for extensions that are not configured --- tool/lib/license_utils.dart | 6 +++++- tool/test/license_utils_test.dart | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index 1782caa988f..cf2ad3ac20b 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -298,9 +298,13 @@ class LicenseHeader { if (!config.shouldExclude(file)) { includedPathsList.add(file.path); final extension = p.extension(file.path); - final removeIndices = config.getRemoveIndicesForExtension(extension); final addIndex = config.getAddIndexForExtension(extension); + if (addIndex == -1) { + // skip if add index doesn't exist for extension + continue; + } final replacementLicenseText = config.addLicenses[addIndex]; + final removeIndices = config.getRemoveIndicesForExtension(extension); for (final removeIndex in removeIndices) { final existingLicenseText = config.removeLicenses[removeIndex]; final fileLength = file.lengthSync(); diff --git a/tool/test/license_utils_test.dart b/tool/test/license_utils_test.dart index 27a72ffc5c0..8eb0e731034 100644 --- a/tool/test/license_utils_test.dart +++ b/tool/test/license_utils_test.dart @@ -52,6 +52,7 @@ late File testFile9; late File testFile10; late File excludeFile1; late File excludeFile2; +late File skippedFile; void main() { group('config file tests', () { @@ -362,7 +363,7 @@ text that should be added to the file. */''', final includedPaths = results.includedPaths; expect(includedPaths, isNotNull); - expect(includedPaths.length, equals(7)); + expect(includedPaths.length, equals(8)); // Order is not guaranteed expect(includedPaths.contains(testFile1.path), true); expect(contentsBeforeUpdate, isNot(equals(contentsAfterUpdate))); @@ -372,6 +373,7 @@ text that should be added to the file. */''', expect(includedPaths.contains(testFile8.path), true); expect(includedPaths.contains(testFile9.path), true); expect(includedPaths.contains(testFile10.path), true); + expect(includedPaths.contains(skippedFile.path), true); final updatedPaths = results.updatedPaths; expect(updatedPaths, isNotNull); @@ -384,6 +386,7 @@ text that should be added to the file. */''', expect(updatedPaths.contains(testFile8.path), true); expect(updatedPaths.contains(testFile9.path), true); expect(updatedPaths.contains(testFile10.path), true); + expect(updatedPaths.contains(skippedFile.path), false); }); test('license headers bulk update can be dry run', () async { @@ -550,6 +553,7 @@ update_paths: /// repo_root/ /// test1.ext1 /// test2.ext2 +/// test.skip /// .hidden/ /// test3.ext1 /// sub_dir1/ @@ -584,6 +588,10 @@ Future _setupTestDirectoryStructure() async { ..createSync(recursive: true); testFile2.writeAsStringSync(licenseText3 + extraText, flush: true); + skippedFile = File(p.join(repoRoot.path, 'test.skip')) + ..createSync(recursive: true); + skippedFile.writeAsStringSync(extraText, flush: true); + // Setup /repo_root/.hidden directory structure Directory(p.join(repoRoot.path, '.hidden')).createSync(recursive: true); From a53026ace3309afe3031aa3abe1a57de4543971c Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:58:26 -0800 Subject: [PATCH 15/16] skip processing files if the license already exists --- tool/lib/license_utils.dart | 24 ++++++++++++++-- tool/test/license_utils_test.dart | 48 ++++++++++++++----------------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/tool/lib/license_utils.dart b/tool/lib/license_utils.dart index cf2ad3ac20b..c63338fe906 100644 --- a/tool/lib/license_utils.dart +++ b/tool/lib/license_utils.dart @@ -303,19 +303,37 @@ class LicenseHeader { // skip if add index doesn't exist for extension continue; } + final fileLength = file.lengthSync(); + const bufferSize = 20; final replacementLicenseText = config.addLicenses[addIndex]; + final byteCount = min( + bufferSize + replacementLicenseText.length, + fileLength, + ); + var replacementInfo = await getReplacementInfo( + file: file, + existingLicenseText: replacementLicenseText, + replacementLicenseText: replacementLicenseText, + byteCount: byteCount as int, + ); + if (replacementInfo.existingHeader.isNotEmpty && + replacementInfo.replacementHeader.isNotEmpty && + replacementInfo.existingHeader == + replacementInfo.replacementHeader) { + // Do nothing if the replacement header is the same as the + // existing header + continue; + } final removeIndices = config.getRemoveIndicesForExtension(extension); for (final removeIndex in removeIndices) { final existingLicenseText = config.removeLicenses[removeIndex]; - final fileLength = file.lengthSync(); - const bufferSize = 20; // Assume that the license text will be near the start of the file, // but add in some buffer. final byteCount = min( bufferSize + existingLicenseText.length, fileLength, ); - final replacementInfo = await getReplacementInfo( + replacementInfo = await getReplacementInfo( file: file, existingLicenseText: existingLicenseText, replacementLicenseText: replacementLicenseText, diff --git a/tool/test/license_utils_test.dart b/tool/test/license_utils_test.dart index 8eb0e731034..a149cade759 100644 --- a/tool/test/license_utils_test.dart +++ b/tool/test/license_utils_test.dart @@ -52,7 +52,8 @@ late File testFile9; late File testFile10; late File excludeFile1; late File excludeFile2; -late File skippedFile; +late File skipFile; +late File doNothingFile; void main() { group('config file tests', () { @@ -257,27 +258,6 @@ text that should be added to the file. */''', } }); - test('update skipped if license text not found', () async { - var errorMessage = ''; - final header = LicenseHeader(); - try { - await header.getReplacementInfo( - file: testFile9, - existingLicenseText: 'test', - replacementLicenseText: 'test', - byteCount: 50, - ); - } on StateError catch (e) { - errorMessage = e.toString(); - } - expect( - errorMessage, - equals( - 'Bad state: License header expected in ${testFile9.path}, but not found!', - ), - ); - }); - test("update skipped if file can't be read", () async { var errorMessage = ''; final header = LicenseHeader(); @@ -363,7 +343,7 @@ text that should be added to the file. */''', final includedPaths = results.includedPaths; expect(includedPaths, isNotNull); - expect(includedPaths.length, equals(8)); + expect(includedPaths.length, equals(9)); // Order is not guaranteed expect(includedPaths.contains(testFile1.path), true); expect(contentsBeforeUpdate, isNot(equals(contentsAfterUpdate))); @@ -373,7 +353,8 @@ text that should be added to the file. */''', expect(includedPaths.contains(testFile8.path), true); expect(includedPaths.contains(testFile9.path), true); expect(includedPaths.contains(testFile10.path), true); - expect(includedPaths.contains(skippedFile.path), true); + expect(includedPaths.contains(skipFile.path), true); + expect(includedPaths.contains(doNothingFile.path), true); final updatedPaths = results.updatedPaths; expect(updatedPaths, isNotNull); @@ -386,7 +367,8 @@ text that should be added to the file. */''', expect(updatedPaths.contains(testFile8.path), true); expect(updatedPaths.contains(testFile9.path), true); expect(updatedPaths.contains(testFile10.path), true); - expect(updatedPaths.contains(skippedFile.path), false); + expect(updatedPaths.contains(skipFile.path), false); + expect(updatedPaths.contains(doNothingFile.path), false); }); test('license headers bulk update can be dry run', () async { @@ -544,6 +526,10 @@ update_paths: ext2: remove: - 2 + add: 1 + ext3: + remove: + - 3 add: 1'''; configFile.writeAsStringSync(contents, flush: true); @@ -588,9 +574,17 @@ Future _setupTestDirectoryStructure() async { ..createSync(recursive: true); testFile2.writeAsStringSync(licenseText3 + extraText, flush: true); - skippedFile = File(p.join(repoRoot.path, 'test.skip')) + final licenseText = ''' +# This is other 2001 multiline license +# text that should be added to the file. +'''; + doNothingFile = File(p.join(repoRoot.path, 'doNothingFile.ext3')) + ..createSync(recursive: true); + doNothingFile.writeAsStringSync(licenseText + extraText, flush: true); + + skipFile = File(p.join(repoRoot.path, 'test.skip')) ..createSync(recursive: true); - skippedFile.writeAsStringSync(extraText, flush: true); + skipFile.writeAsStringSync(extraText, flush: true); // Setup /repo_root/.hidden directory structure Directory(p.join(repoRoot.path, '.hidden')).createSync(recursive: true); From cc43ac81ae38d60b12c693bc006781b784c5797a Mon Sep 17 00:00:00 2001 From: Ann Mossman <233583+mossmana@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:28:10 -0800 Subject: [PATCH 16/16] added additional ignore paths --- update_licenses.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/update_licenses.yaml b/update_licenses.yaml index c1038efcc43..0398234f0eb 100644 --- a/update_licenses.yaml +++ b/update_licenses.yaml @@ -56,6 +56,8 @@ update_paths: - /Users/mossman/Development/devtools/third_party - /Users/mossman/Development/devtools/packages/devtools_app/build - /Users/mossman/Development/devtools/tool/flutter-sdk + - /Users/mossman/Development/devtools/.github + - /Users/mossman/Development/devtools/.vscode file_types: bat: remove: