diff --git a/.github/workflows/dart.yml b/.github/workflows/dart.yml index 0e3b4708..037c7941 100644 --- a/.github/workflows/dart.yml +++ b/.github/workflows/dart.yml @@ -16,9 +16,35 @@ env: permissions: read-all jobs: + changedPackages: + name: Calculating affected packages + runs-on: ubuntu-latest + steps: + - id: filter + name: Produce variables based on which packages are affected + uses: dorny/paths-filter@v2 + with: + filters: | + mono_repo: + - mono_repo/** + test_flutter_pkg: + - test_flutter_pkg/** + test_flutter_pkg_example: + - test_flutter_pkg/example/** + - test_flutter_pkg/** + test_pkg: + - test_pkg/** + outputs: + mono_repo: "${{ steps.filter.outputs.mono_repo }}" + test_flutter_pkg: "${{ steps.filter.outputs.test_flutter_pkg }}" + test_flutter_pkg_example: "${{ steps.filter.outputs.test_flutter_pkg_example }}" + test_pkg: "${{ steps.filter.outputs.test_pkg }}" + permissions: + pull-requests: read job_001: name: "smoke_test; linux; Dart 2.16.0; PKG: test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -50,9 +76,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_002: name: "smoke_test; linux; Dart 2.17.0-69.1.beta; PKG: test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -84,9 +113,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_003: name: "smoke_test; linux; Dart 2.18.0-106.0.dev; PKG: test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -118,9 +150,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_004: name: "smoke_test; linux; Dart 2.18.0; PKG: mono_repo; `dart analyze`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -148,9 +183,12 @@ jobs: run: dart analyze if: "always() && steps.mono_repo_pub_upgrade.conclusion == 'success'" working-directory: mono_repo + needs: + - changedPackages job_005: name: "smoke_test; linux; Flutter beta; PKG: test_flutter_pkg; `dart format --output=none --set-exit-if-changed .`, `flutter analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_flutter_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -182,9 +220,12 @@ jobs: run: flutter analyze --fatal-infos . if: "always() && steps.test_flutter_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_flutter_pkg + needs: + - changedPackages job_006: name: "smoke_test; linux; Dart beta; PKG: test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -216,9 +257,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_007: name: "smoke_test; linux; Dart dev; PKG: mono_repo; `cd ../ && dart mono_repo/bin/mono_repo.dart generate --validate`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -246,9 +290,12 @@ jobs: run: "cd ../ && dart mono_repo/bin/mono_repo.dart generate --validate" if: "always() && steps.mono_repo_pub_upgrade.conclusion == 'success'" working-directory: mono_repo + needs: + - changedPackages job_008: name: "smoke_test; linux; Dart dev; PKGS: mono_repo, test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true' || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -293,9 +340,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_009: name: "smoke_test; linux; Dart main; PKG: test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -327,9 +377,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_010: name: "smoke_test; linux; Flutter master; PKG: test_flutter_pkg; `dart format --output=none --set-exit-if-changed .`, `flutter analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_flutter_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -361,9 +414,12 @@ jobs: run: flutter analyze --fatal-infos . if: "always() && steps.test_flutter_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_flutter_pkg + needs: + - changedPackages job_011: name: "smoke_test; linux; Flutter stable; PKG: test_flutter_pkg; `dart format --output=none --set-exit-if-changed .`, `flutter analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_flutter_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -395,9 +451,12 @@ jobs: run: flutter analyze --fatal-infos . if: "always() && steps.test_flutter_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_flutter_pkg + needs: + - changedPackages job_012: name: "smoke_test; linux; Dart stable; PKG: test_pkg; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos .`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -429,9 +488,12 @@ jobs: run: dart analyze --fatal-infos . if: "always() && steps.test_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_pkg + needs: + - changedPackages job_013: name: "test; linux; Dart 2.18.0; PKG: mono_repo; `dart test -x yaml -P presubmit --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -460,6 +522,7 @@ jobs: if: "always() && steps.mono_repo_pub_upgrade.conclusion == 'success'" working-directory: mono_repo needs: + - changedPackages - job_001 - job_002 - job_003 @@ -475,6 +538,7 @@ jobs: job_014: name: "test; linux; Flutter beta; PKG: test_flutter_pkg; `flutter test --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_flutter_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -503,6 +567,7 @@ jobs: if: "always() && steps.test_flutter_pkg_pub_upgrade.conclusion == 'success'" working-directory: test_flutter_pkg needs: + - changedPackages - job_001 - job_002 - job_003 @@ -518,6 +583,7 @@ jobs: job_015: name: "test; linux; Dart dev; PKG: mono_repo; `dart pub global run coverage:test_with_coverage -- -t yaml --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -561,6 +627,7 @@ jobs: fail_ci_if_error: true name: coverage_00 needs: + - changedPackages - job_001 - job_002 - job_003 @@ -576,6 +643,7 @@ jobs: job_016: name: "test; linux; Dart dev; PKG: mono_repo; `dart pub global run coverage:test_with_coverage -- -x yaml -P presubmit --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -619,6 +687,7 @@ jobs: fail_ci_if_error: true name: coverage_01 needs: + - changedPackages - job_001 - job_002 - job_003 @@ -634,6 +703,7 @@ jobs: job_017: name: "test; linux; Dart dev; PKG: mono_repo; `dart test -t yaml --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -662,6 +732,7 @@ jobs: if: "always() && steps.mono_repo_pub_upgrade.conclusion == 'success'" working-directory: mono_repo needs: + - changedPackages - job_001 - job_002 - job_003 @@ -677,6 +748,7 @@ jobs: job_018: name: "test; linux; Dart dev; PKG: test_pkg; `dart pub global run coverage:test_with_coverage -- --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_pkg == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -720,6 +792,7 @@ jobs: fail_ci_if_error: true name: coverage_02 needs: + - changedPackages - job_001 - job_002 - job_003 @@ -735,6 +808,7 @@ jobs: job_019: name: "test; linux; Flutter stable; PKG: test_flutter_pkg/example; `flutter test --test-randomize-ordering-seed=random`" runs-on: ubuntu-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.test_flutter_pkg_example == 'true')" steps: - name: Cache Pub hosted dependencies uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 @@ -763,6 +837,7 @@ jobs: if: "always() && steps.test_flutter_pkg_example_pub_upgrade.conclusion == 'success'" working-directory: test_flutter_pkg/example needs: + - changedPackages - job_001 - job_002 - job_003 @@ -778,6 +853,7 @@ jobs: job_020: name: "test; windows; Dart 2.18.0; PKG: mono_repo; `dart test -x yaml -P presubmit --test-randomize-ordering-seed=random`" runs-on: windows-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Setup Dart SDK uses: dart-lang/setup-dart@d6a63dab3335f427404425de0fbfed4686d93c4f @@ -796,6 +872,7 @@ jobs: if: "always() && steps.mono_repo_pub_upgrade.conclusion == 'success'" working-directory: mono_repo needs: + - changedPackages - job_001 - job_002 - job_003 @@ -811,6 +888,7 @@ jobs: job_021: name: "test; windows; Dart dev; PKG: mono_repo; `dart test -x yaml -P presubmit --test-randomize-ordering-seed=random`" runs-on: windows-latest + if: "(contains(fromJSON('[\"push\", \"schedule\"]'), github.event_name) || needs.changedPackages.outputs.mono_repo == 'true')" steps: - name: Setup Dart SDK uses: dart-lang/setup-dart@d6a63dab3335f427404425de0fbfed4686d93c4f @@ -829,6 +907,7 @@ jobs: if: "always() && steps.mono_repo_pub_upgrade.conclusion == 'success'" working-directory: mono_repo needs: + - changedPackages - job_001 - job_002 - job_003 @@ -853,6 +932,7 @@ jobs: env: CHAT_WEBHOOK_URL: "${{ secrets.CHAT_WEBHOOK_URL }}" needs: + - changedPackages - job_001 - job_002 - job_003 @@ -874,7 +954,7 @@ jobs: - job_019 - job_020 - job_021 - job_23: + job_24: name: Mark Coveralls job finished runs-on: ubuntu-latest steps: diff --git a/mono_repo.yaml b/mono_repo.yaml index b116e0d3..242b6687 100644 --- a/mono_repo.yaml +++ b/mono_repo.yaml @@ -6,6 +6,8 @@ coverage_service: - coveralls - codecov +smart_jobs: true + github: # Setting just `cron` keeps the defaults for `push` and `pull_request` cron: '0 0 * * 0' # “At 00:00 (UTC) on Sunday.” diff --git a/mono_repo/lib/src/ci_shared.dart b/mono_repo/lib/src/ci_shared.dart index 968d70c6..34b68e91 100644 --- a/mono_repo/lib/src/ci_shared.dart +++ b/mono_repo/lib/src/ci_shared.dart @@ -32,6 +32,7 @@ String get createdWith => '# Created with package:mono_repo v$_pkgVersion'; String get _pkgVersion => _isTesting ? '1.2.3' : packageVersion; +const calculateChangesJobName = 'Calculating affected packages'; const selfValidateJobName = 'mono_repo self validate'; final selfValidateCommands = [ diff --git a/mono_repo/lib/src/commands/github/action_info.dart b/mono_repo/lib/src/commands/github/action_info.dart index 663eba00..acf5fd05 100644 --- a/mono_repo/lib/src/commands/github/action_info.dart +++ b/mono_repo/lib/src/commands/github/action_info.dart @@ -38,6 +38,13 @@ enum ActionInfo implements Comparable { name: 'Upload coverage to codecov.io', repo: 'codecov/codecov-action', version: codecovCodecovActionVersion, + ), + + /// See https://github.com/marketplace/actions/paths-changes-filter + pathsFilter( + name: 'Produce variables based on which packages are affected', + repo: 'dorny/paths-filter', + version: dornyPathsFilterVersion, ); const ActionInfo({ diff --git a/mono_repo/lib/src/commands/github/action_versions.dart b/mono_repo/lib/src/commands/github/action_versions.dart index a0fdde49..4bce7a01 100644 --- a/mono_repo/lib/src/commands/github/action_versions.dart +++ b/mono_repo/lib/src/commands/github/action_versions.dart @@ -12,3 +12,4 @@ const actionsCheckoutVersion = '8e5e7e5ab8b370d6c329ec480221332ada57f0ab'; const subositoFlutterActionVersion = '48cafc24713cca54bbe03cdc3a423187d413aafa'; const coverallsappGithubActionVersion = 'master'; const codecovCodecovActionVersion = 'main'; +const dornyPathsFilterVersion = 'v2'; diff --git a/mono_repo/lib/src/commands/github/github_yaml.dart b/mono_repo/lib/src/commands/github/github_yaml.dart index bd75b458..8f7bd090 100644 --- a/mono_repo/lib/src/commands/github/github_yaml.dart +++ b/mono_repo/lib/src/commands/github/github_yaml.dart @@ -18,6 +18,7 @@ import 'action_info.dart'; import 'job.dart'; import 'step.dart'; +const _calculateChangesJobId = 'changedPackages'; const _onCompletionStage = '_on_completion'; const githubWorkflowDirectory = '.github/workflows'; @@ -38,6 +39,10 @@ Map generateGitHubYml(RootConfig rootConfig) { jobs.add(_SelfValidateJob(selfValidateStage)); } + if (rootConfig.monoConfig.smartJobs == true) { + jobs.add(const _CalculateChangesJob()); + } + final allJobStages = {for (var job in jobs) job.stageName}; final orderedStages = calculateOrderedStages( rootConfig, @@ -64,10 +69,13 @@ Map generateGitHubYml(RootConfig rootConfig) { .compareTo(orderedStages.indexOf(b.stageName)); if (value == 0) { - if (a is _SelfValidateJob) { + if (a is _CalculateChangesJob) { value = -1; - } - if (b is _SelfValidateJob) { + } else if (b is _CalculateChangesJob) { + value = 1; + } else if (a is _SelfValidateJob) { + value = -1; + } else if (b is _SelfValidateJob) { value = 1; } } @@ -105,7 +113,7 @@ Map generateGitHubYml(RootConfig rootConfig) { } currStageJobs.add(job.id); if (allPrevStageJobs.isNotEmpty) { - job.value.needs = allPrevStageJobs.toList(); + (job.value.needs ??= []).addAll(allPrevStageJobs); } // process post-run logic @@ -122,8 +130,8 @@ Map generateGitHubYml(RootConfig rootConfig) { Map.fromEntries(allJobs.map((e) => MapEntry(e.id, e.value))); for (var completion in completionMap.entries) { - final job = completion.key.completionJobFactory!(rootConfig) - ..needs = completion.value.toList(); + final job = completion.key.completionJobFactory!(rootConfig); + (job.needs ??= []).addAll(completion.value); jobList['job_${jobList.length + 1}'] = job; } @@ -190,13 +198,15 @@ Iterable<_MapEntryWithStage> _listJobs( String jobName(int jobNum) => 'job_${jobNum.toString().padLeft(3, '0')}'; - _MapEntryWithStage jobEntry(Job content, String stage) { + _MapEntryWithStage jobEntry(Job content, String stage, {String? id}) { final conditional = conditionalStages[stage]; - if (conditional != null) { - content.ifContent = conditional.ifCondition; + if (conditional != null && conditional.ifCondition != null) { + content.ifContent = + (content.ifContent == null ? '' : '${content.ifContent} && ') + + conditional.ifCondition!; } return _MapEntryWithStage( - jobName(++count), + id ?? jobName(++count), content, stage, ); @@ -205,11 +215,19 @@ Iterable<_MapEntryWithStage> _listJobs( for (var job in jobs) { if (job is _SelfValidateJob) { yield jobEntry( - _selfValidateJob(rootConfig.monoConfig, rootConfig), + job.job(rootConfig.monoConfig, rootConfig), job.stageName, ); continue; } + if (job is _CalculateChangesJob) { + yield jobEntry( + job.job(rootConfig.monoConfig, rootConfig), + job.stageName, + id: _calculateChangesJobId, + ); + continue; + } final ciJob = job as CIJob; @@ -349,6 +367,14 @@ extension on CIJobEntry { 'packages': packages.join('-'), 'commands': commands.join('-'), }, + packages: packages + .map( + (path) => rootConfig + .singleWhere((pkg) => pkg.relativePath == path) + .pubspec + .name, + ) + .toList(), ); } @@ -390,30 +416,48 @@ Job _githubJob( RootConfig rootConfig, { required BasicConfiguration config, Map? additionalCacheKeys, -}) => - Job( - name: jobName, - runsOn: runsOn, - steps: [ - if (!runsOn.startsWith('windows')) - _cacheEntries( - runsOn, - rootConfig: rootConfig, - additionalCacheKeys: { - 'sdk': sdkVersion, - if (additionalCacheKeys != null) ...additionalCacheKeys, - }, - ), - packageFlavor.setupStep(sdkVersion, rootConfig), - ..._beforeSteps(runCommands.whereType<_CommandEntry>()), - ActionInfo.checkout.usage( - id: 'checkout', - versionOverrides: rootConfig.existingActionVersions, - ), - for (var command in runCommands) - ...command.runContent(config, rootConfig), - ], + // Used for "smart jobs" configuration, only applies for packages. + List? packages, +}) { + StringBuffer? ifContent; + if (rootConfig.monoConfig.smartJobs && + packages != null && + packages.isNotEmpty) { + ifContent = StringBuffer( + '(contains(fromJSON(\'["push", "schedule"]\'), github.event_name)', ); + for (var package in packages) { + ifContent.write( + ' || needs.$_calculateChangesJobId.outputs.$package == \'true\'', + ); + } + ifContent.write(')'); + } + return Job( + name: jobName, + runsOn: runsOn, + ifContent: ifContent?.toString(), + steps: [ + if (!runsOn.startsWith('windows')) + _cacheEntries( + runsOn, + rootConfig: rootConfig, + additionalCacheKeys: { + 'sdk': sdkVersion, + if (additionalCacheKeys != null) ...additionalCacheKeys, + }, + ), + packageFlavor.setupStep(sdkVersion, rootConfig), + ..._beforeSteps(runCommands.whereType<_CommandEntry>()), + ActionInfo.checkout.usage( + id: 'checkout', + versionOverrides: rootConfig.existingActionVersions, + ), + for (var command in runCommands) + ...command.runContent(config, rootConfig), + ], + ); +} Set _orderedTypes(Iterable<_CommandEntry> commands) => SplayTreeSet.of(commands.map((e) => e.type).whereType()); @@ -511,20 +555,6 @@ String _maxLength(String input) { return input.substring(0, 512 - hash.length) + hash; } -Job _selfValidateJob(BasicConfiguration config, RootConfig rootConfig) => - _githubJob( - selfValidateJobName, - _ubuntuLatest, - PackageFlavor.dart, - 'stable', - [ - for (var command in selfValidateCommands) - _CommandEntryBase(selfValidateJobName, command), - ], - rootConfig, - config: config, - ); - const _ubuntuLatest = 'ubuntu-latest'; /// Used as a place-holder so we can treat all jobs the same in certain @@ -534,6 +564,58 @@ class _SelfValidateJob implements HasStageName { final String stageName; _SelfValidateJob(this.stageName); + + Job job(BasicConfiguration config, RootConfig rootConfig) => _githubJob( + selfValidateJobName, + _ubuntuLatest, + PackageFlavor.dart, + 'stable', + [ + for (var command in selfValidateCommands) + _CommandEntryBase(selfValidateJobName, command), + ], + rootConfig, + config: config, + ); +} + +/// Used a place-holder, actual job is created later. +class _CalculateChangesJob implements HasStageName { + const _CalculateChangesJob(); + + @override + // This should always go before any other stages. + String get stageName => ''; + + Job job(BasicConfiguration config, RootConfig rootConfig) { + final outputs = { + for (var package in rootConfig) + package.pubspec.name: + '\${{ steps.filter.outputs.${package.pubspec.name} }}', + }; + return Job( + name: calculateChangesJobName, + runsOn: _ubuntuLatest, + outputs: outputs, + permissions: {'pull-requests': 'read'}, + steps: [ + ActionInfo.pathsFilter.usage( + id: 'filter', + versionOverrides: rootConfig.existingActionVersions, + withContent: { + 'filters': [ + for (var package in rootConfig) ...[ + '${package.pubspec.name}:', + ' - ${package.relativePath}/**', + for (var dependencyPath in package.pathDependencies) + ' - $dependencyPath/**', + ] + ].join('\n'), + }, + ), + ], + ); + } } class _MapEntryWithStage { diff --git a/mono_repo/lib/src/commands/github/job.dart b/mono_repo/lib/src/commands/github/job.dart index 911b9e56..49b811ec 100644 --- a/mono_repo/lib/src/commands/github/job.dart +++ b/mono_repo/lib/src/commands/github/job.dart @@ -18,16 +18,27 @@ class Job implements YamlLike { @JsonKey(name: 'runs-on') final String? runsOn; + @JsonKey(name: 'if') String? ifContent; + @JsonKey(required: true) final List steps; + List? needs; + Map? outputs; + + Map? permissions; + Job({ this.name, this.runsOn, required this.steps, + this.ifContent, + this.needs, + this.outputs, + this.permissions, }); factory Job.fromJson(Map json) => _$JobFromJson(json); diff --git a/mono_repo/lib/src/commands/github/job.g.dart b/mono_repo/lib/src/commands/github/job.g.dart index 5998e982..2fd71130 100644 --- a/mono_repo/lib/src/commands/github/job.g.dart +++ b/mono_repo/lib/src/commands/github/job.g.dart @@ -24,12 +24,20 @@ Job _$JobFromJson(Map json) => $checkedCreate( (v) => (v as List) .map((e) => Step.fromJson(e as Map)) .toList()), + ifContent: $checkedConvert('if', (v) => v as String?), + needs: $checkedConvert('needs', + (v) => (v as List?)?.map((e) => e as String).toList()), + outputs: $checkedConvert( + 'outputs', + (v) => (v as Map?)?.map( + (k, e) => MapEntry(k as String, e as String), + )), + permissions: $checkedConvert( + 'permissions', + (v) => (v as Map?)?.map( + (k, e) => MapEntry(k as String, e as String), + )), ); - $checkedConvert('if', (v) => val.ifContent = v as String?); - $checkedConvert( - 'needs', - (v) => val.needs = - (v as List?)?.map((e) => e as String).toList()); return val; }, fieldKeyMap: const {'runsOn': 'runs-on', 'ifContent': 'if'}, @@ -49,5 +57,7 @@ Map _$JobToJson(Job instance) { writeNotNull('if', instance.ifContent); val['steps'] = instance.steps.map((e) => e.toJson()).toList(); writeNotNull('needs', instance.needs); + writeNotNull('outputs', instance.outputs); + writeNotNull('permissions', instance.permissions); return val; } diff --git a/mono_repo/lib/src/mono_config.dart b/mono_repo/lib/src/mono_config.dart index 8067465c..2d3f3f3d 100644 --- a/mono_repo/lib/src/mono_config.dart +++ b/mono_repo/lib/src/mono_config.dart @@ -21,6 +21,7 @@ const _allowedMonoConfigKeys = { 'pub_action', 'self_validate', 'coverage_service', + 'smart_jobs', }; const _defaultPubAction = 'upgrade'; @@ -37,6 +38,7 @@ class MonoConfig implements BasicConfiguration { final String pubAction; final String? selfValidateStage; final GitHubConfig github; + final bool smartJobs; @override final Set coverageProcessors; @@ -47,6 +49,7 @@ class MonoConfig implements BasicConfiguration { required this.selfValidateStage, required Map github, required this.coverageProcessors, + required this.smartJobs, }) : githubConditionalStages = _readConditionalStages(github), github = GitHubConfig.fromJson(github); @@ -126,6 +129,16 @@ class MonoConfig implements BasicConfiguration { final coverageServices = _asList(json, 'coverage_service'); + final smartJobs = json['smart_jobs'] ?? false; + if (smartJobs is! bool) { + throw CheckedFromJsonException( + json, + 'smart_jobs', + 'MonoConfig', + 'Value must be `true` or `false`.', + ); + } + return MonoConfig._( mergeStages: Set.from(mergeStages), prettyAnsi: prettyAnsi, @@ -135,6 +148,7 @@ class MonoConfig implements BasicConfiguration { coverageProcessors: coverageServices .map((e) => CoverageProcessor.values.byName(e)) .toSet(), + smartJobs: smartJobs, ); } @@ -150,6 +164,7 @@ class MonoConfig implements BasicConfiguration { selfValidateStage: null, github: {}, coverageProcessors: {CoverageProcessor.coveralls}, + smartJobs: false, ); } diff --git a/mono_repo/lib/src/package_config.dart b/mono_repo/lib/src/package_config.dart index 867e611d..9c078243 100644 --- a/mono_repo/lib/src/package_config.dart +++ b/mono_repo/lib/src/package_config.dart @@ -5,6 +5,7 @@ import 'package:checked_yaml/checked_yaml.dart'; import 'package:io/ansi.dart'; import 'package:json_annotation/json_annotation.dart'; +import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; import 'package:pubspec_parse/pubspec_parse.dart'; import 'package:yaml/yaml.dart'; @@ -22,23 +23,27 @@ const monoPkgFileName = 'mono_pkg.yaml'; class PackageConfig { final String relativePath; final Pubspec pubspec; + final Pubspec? pubspecOverrides; final List oses; final List? sdks; final List stageNames; final List jobs; final List cacheDirectories; + final List pathDependencies; final bool dartSdkConfigUsed; final bool osConfigUsed; PackageConfig( this.relativePath, this.pubspec, + this.pubspecOverrides, this.oses, this.sdks, this.stageNames, this.jobs, this.cacheDirectories, + this.pathDependencies, this.dartSdkConfigUsed, this.osConfigUsed, ) : assert(() { @@ -54,15 +59,22 @@ class PackageConfig { factory PackageConfig.parse( String relativePath, Pubspec pubspec, + Pubspec? pubspecOverrides, Map monoPkgYaml, ) => createWithCheck( - () => PackageConfig._parse(relativePath, pubspec, monoPkgYaml), + () => PackageConfig._parse( + relativePath, + pubspec, + pubspecOverrides, + monoPkgYaml, + ), ); factory PackageConfig._parse( String relativePath, Pubspec pubspec, + Pubspec? pubspecOverrides, Map monoPkgYaml, ) { if (monoPkgYaml.isEmpty) { @@ -71,6 +83,8 @@ class PackageConfig { return PackageConfig( relativePath, pubspec, + pubspecOverrides, + [], [], [], [], @@ -178,14 +192,27 @@ class PackageConfig { return stage.name; }).toList(); + final deps = { + ...pubspec.dependencies, + if (pubspecOverrides != null) ...pubspecOverrides.dependencies, + ...pubspec.dependencyOverrides, + if (pubspecOverrides != null) ...pubspecOverrides.dependencyOverrides, + }; + final pathDependencies = [ + for (final dep in deps.values) + if (dep is PathDependency) p.normalize(p.join(relativePath, dep.path)), + ]; + return PackageConfig( relativePath, pubspec, + pubspecOverrides, rawConfig.oses, rawConfig.sdks, stageNames, jobs, rawConfig.cache?.directories ?? const [], + pathDependencies, sdkConfigUsed, osConfigUsed, ); diff --git a/mono_repo/lib/src/root_config.dart b/mono_repo/lib/src/root_config.dart index e35d3629..bd50872d 100644 --- a/mono_repo/lib/src/root_config.dart +++ b/mono_repo/lib/src/root_config.dart @@ -18,6 +18,7 @@ import 'yaml.dart'; const _legacyPkgConfigFileName = '.mono_repo.yml'; const _pubspecFileName = 'pubspec.yaml'; +const _pubspecOverridesFileName = 'pubspec_overrides.yaml'; PackageConfig? _packageConfigFromDir( String rootDirectory, @@ -56,7 +57,22 @@ PackageConfig? _packageConfigFromDir( sourceUrl: Uri.parse(pubspecFile.path), ); - return PackageConfig.parse(pkgRelativePath, pubspec, pkgConfigYaml); + Pubspec? pubspecOverrides; + final pubspecOverridesFile = + File(p.join(rootDirectory, pkgRelativePath, _pubspecOverridesFileName)); + if (pubspecOverridesFile.existsSync()) { + pubspecOverrides = Pubspec.parse( + pubspecOverridesFile.readAsStringSync(), + sourceUrl: Uri.parse(pubspecOverridesFile.path), + ); + } + + return PackageConfig.parse( + pkgRelativePath, + pubspec, + pubspecOverrides, + pkgConfigYaml, + ); } class RootConfig extends ListBase { diff --git a/mono_repo/test/mono_config_test.dart b/mono_repo/test/mono_config_test.dart index 9953f3e1..28557ab0 100644 --- a/mono_repo/test/mono_config_test.dart +++ b/mono_repo/test/mono_config_test.dart @@ -21,6 +21,7 @@ String _encodeJson(Object? input) => PackageConfig _parse(map) => PackageConfig.parse( 'a', _dummyPubspec, + null, map is YamlMap ? map : loadYamlChecked( diff --git a/test_flutter_pkg/pubspec.yaml b/test_flutter_pkg/pubspec.yaml index 442e5d75..a2a2409d 100644 --- a/test_flutter_pkg/pubspec.yaml +++ b/test_flutter_pkg/pubspec.yaml @@ -1,5 +1,5 @@ name: test_flutter_pkg -description: A new Flutter project. +description: A new Flutter project!!. # The following line prevents the package from being accidentally published to # pub.dev using `flutter pub publish`. This is preferred for private packages.