Skip to content

Commit

Permalink
feat: Local pub server (#2919)
Browse files Browse the repository at this point in the history
* ci: Fix db_common tests on Windows

* Use powershell

* feat: Add `pub_server` package

Adds a minimal package for running a local, embedded pub server.

* chore: Improve license script

Running license script without `addlicense` installed should not leave a bunch of files around and should be runnable without sudo

* feat(pub_server): Add launcher

Adds a mechanism for seeding the local pub server with either the packages of a local directory or of a remote git repo.

* feat(aft): Add `serve` command

Adds a command for serving the repo's packages on a local pub server.

* chore(pub_server): Clean up

* ci: Add `pub_server` workflow

* chore(pub_server): Add more doc comments

* test(pub_server): Add e2e tests
  • Loading branch information
dnys1 authored Aug 17, 2023
1 parent b9a6c57 commit 028a96d
Show file tree
Hide file tree
Showing 26 changed files with 2,366 additions and 57 deletions.
12 changes: 12 additions & 0 deletions .github/dependabot.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ updates:
- "version-update:semver-patch"
- dependency-name: "aws_common"
- dependency-name: "amplify_lints"
- dependency-name: "pub_server"
- dependency-name: "smithy"
- dependency-name: "smithy_codegen"
- dependency-name: "aws_signature_v4"
Expand Down Expand Up @@ -1414,6 +1415,17 @@ updates:
- dependency-name: "aws_common"
- dependency-name: "amplify_lints"
- dependency-name: "aws_signature_v4"
- package-ecosystem: "pub"
directory: "packages/test/pub_server"
schedule:
interval: "daily"
ignore:
# Ignore patch version bumps
- dependency-name: "*"
update-types:
- "version-update:semver-patch"
- dependency-name: "aws_common"
- dependency-name: "amplify_lints"
- package-ecosystem: "pub"
directory: "packages/worker_bee/e2e"
schedule:
Expand Down
39 changes: 39 additions & 0 deletions .github/workflows/pub_server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Generated with aft. To update, run: `aft generate workflows`
name: pub_server
on:
push:
branches:
- main
- stable
pull_request:
paths:
- '.github/workflows/dart_native.yaml'
- '.github/workflows/dart_vm.yaml'
- '.github/workflows/pub_server.yaml'
- 'packages/amplify_lints/lib/**/*.yaml'
- 'packages/amplify_lints/pubspec.yaml'
- 'packages/aws_common/lib/**/*.dart'
- 'packages/aws_common/pubspec.yaml'
- 'packages/test/pub_server/**/*.dart'
- 'packages/test/pub_server/**/*.yaml'
- 'packages/test/pub_server/lib/**/*'
- 'packages/test/pub_server/test/**/*'
schedule:
- cron: "0 0 * * 0" # Every Sunday at 00:00
defaults:
run:
shell: bash
permissions: read-all

jobs:
test:
uses: ./.github/workflows/dart_vm.yaml
with:
package-name: pub_server
working-directory: packages/test/pub_server
native_test:
needs: test
uses: ./.github/workflows/dart_native.yaml
with:
package-name: pub_server
working-directory: packages/test/pub_server
1 change: 1 addition & 0 deletions packages/aft/lib/aft.dart
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export 'src/commands/link_command.dart';
export 'src/commands/list_packages_command.dart';
export 'src/commands/publish_command.dart';
export 'src/commands/run_command.dart';
export 'src/commands/serve_command.dart';
export 'src/commands/version_bump_command.dart';
export 'src/models.dart';
export 'src/pub/pub_runner.dart';
Expand Down
3 changes: 2 additions & 1 deletion packages/aft/lib/src/command_runner.dart
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ Future<void> run(List<String> args) async {
..addCommand(CreateCommand())
..addCommand(SaveRepoStateCommand())
..addCommand(RunCommand())
..addCommand(DocsCommand());
..addCommand(DocsCommand())
..addCommand(ServeCommand());

try {
final argResults = runner.argParser.parse(args);
Expand Down
120 changes: 69 additions & 51 deletions packages/aft/lib/src/commands/publish_command.dart
Original file line number Diff line number Diff line change
Expand Up @@ -11,38 +11,39 @@ import 'package:collection/collection.dart';
import 'package:graphs/graphs.dart';
import 'package:path/path.dart' as p;

/// Command to publish all Dart/Flutter packages in the repo.
class PublishCommand extends AmplifyCommand with GlobOptions {
PublishCommand() {
argParser
..addFlag(
'force',
abbr: 'f',
help: 'Ignores errors in pre-publishing commands and publishes '
'without prompt',
negatable: false,
)
..addFlag(
'dry-run',
help: 'Passes `--dry-run` flag to `dart` or `flutter` publish command',
negatable: false,
);
}
/// Helpers for commands which publish packages to `pub.dev`.
mixin PublishHelpers on AmplifyCommand {
bool get dryRun;
bool get force;

late final bool force = argResults!['force'] as bool;
late final bool dryRun = argResults!['dry-run'] as bool;
/// Gathers the subset of packages which are publishable and whose latest
/// version is not already available on `pub.dev`.
Future<List<PackageInfo>> unpublishedPackages(
List<PackageInfo> publishablePackages,
) async {
final unpublishedPackages = (await Future.wait([
for (final package in publishablePackages) checkPublishable(package),
]))
.whereType<PackageInfo>()
.toList();

@override
String get description =>
'Publishes all packages in the Amplify Flutter repo which '
'need publishing.';
try {
sortPackagesTopologically<PackageInfo>(
unpublishedPackages,
(pkg) => pkg.pubspecInfo.pubspec,
);
} on CycleException<dynamic> {
if (!force) {
exitError('Cannot sort packages with inter-dependencies.');
}
}

@override
String get name => 'publish';
return unpublishedPackages;
}

/// Checks if [package] can be published based on whether the local version
/// is newer than the one published to `pub.dev`.
Future<PackageInfo?> _checkPublishable(PackageInfo package) async {
Future<PackageInfo?> checkPublishable(PackageInfo package) async {
final publishTo = package.pubspecInfo.pubspec.publishTo;
if (publishTo == 'none') {
return null;
Expand All @@ -61,8 +62,8 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
final publishedVersion = maxBy(
[
if (versionInfo?.latestPrerelease != null)
versionInfo?.latestPrerelease!,
if (versionInfo?.latestVersion != null) versionInfo?.latestVersion!,
versionInfo!.latestPrerelease!,
if (versionInfo?.latestVersion != null) versionInfo!.latestVersion!,
],
(v) => v,
);
Expand All @@ -85,7 +86,7 @@ class PublishCommand extends AmplifyCommand with GlobOptions {

/// Runs pre-publish operations for [package], most importantly any necessary
/// `build_runner` tasks.
Future<void> _prePublish(PackageInfo package) async {
Future<void> prePublish(PackageInfo package) async {
logger.info('Running pre-publish checks for ${package.name}...');
if (!dryRun) {
// Remove any overrides so that `pub` commands resolve against
Expand Down Expand Up @@ -136,7 +137,7 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
static final _validationErrorRegex = RegExp(r'^\s*\*');

/// Publishes the package using `pub`.
Future<void> _publish(PackageInfo package) async {
Future<void> publish(PackageInfo package) async {
logger.info('Publishing ${package.name}${dryRun ? ' (dry run)' : ''}...');
final publishCmd = await Process.start(
package.flavor.entrypoint,
Expand Down Expand Up @@ -181,21 +182,49 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
}
}
}
}

/// Command to publish all Dart/Flutter packages in the repo.
class PublishCommand extends AmplifyCommand with GlobOptions, PublishHelpers {
PublishCommand() {
argParser
..addFlag(
'force',
abbr: 'f',
help: 'Ignores errors in pre-publishing commands and publishes '
'without prompt',
negatable: false,
)
..addFlag(
'dry-run',
help: 'Passes `--dry-run` flag to `dart` or `flutter` publish command',
negatable: false,
);
}

@override
late final bool force = argResults!['force'] as bool;

@override
late final bool dryRun = argResults!['dry-run'] as bool;

@override
String get description =>
'Publishes all packages in the Amplify Flutter repo which '
'need publishing.';

@override
String get name => 'publish';

@override
Future<void> run() async {
await super.run();
// Gather packages which can be published.
final publishablePackages = repo
.publishablePackages(commandPackages)
.where((pkg) => pkg.pubspecInfo.pubspec.publishTo != 'none');
final publishablePackages = repo.publishablePackages(commandPackages);

// Gather packages which need to be published.
final packagesNeedingPublish = (await Future.wait([
for (final package in publishablePackages) _checkPublishable(package),
]))
.whereType<PackageInfo>()
.toList();
final packagesNeedingPublish =
await unpublishedPackages(publishablePackages);

// Publishable packages which are being held back.
final unpublishablePackages = publishablePackages.where(
Expand All @@ -207,17 +236,6 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
return;
}

try {
sortPackagesTopologically<PackageInfo>(
packagesNeedingPublish,
(pkg) => pkg.pubspecInfo.pubspec,
);
} on CycleException<dynamic> {
if (!force) {
exitError('Cannot sort packages with inter-dependencies.');
}
}

stdout
..writeln('Preparing to publish${dryRun ? ' (dry run)' : ''}: ')
..writeln(
Expand Down Expand Up @@ -247,8 +265,8 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
// some packages will not be published, it also means that the command
// can be re-run to pick up where it left off.
for (final package in packagesNeedingPublish) {
await _prePublish(package);
await _publish(package);
await prePublish(package);
await publish(package);
}

stdout.writeln(
Expand Down
108 changes: 108 additions & 0 deletions packages/aft/lib/src/commands/serve_command.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

import 'dart:io';

import 'package:aft/aft.dart';
import 'package:aft/src/options/glob_options.dart';
import 'package:async/async.dart';
import 'package:pub_server/pub_server.dart';
import 'package:shelf/shelf_io.dart' as io;

/// A command for serving all packages in the Amplify Flutter repo on a local
/// pub server.
class ServeCommand extends AmplifyCommand with GlobOptions, PublishHelpers {
ServeCommand() {
argParser.addOption(
'port',
abbr: 'p',
defaultsTo: '0',
help: 'The port to serve on.',
);
}

/// The port to serve on.
late final int port = int.parse(argResults!['port'] as String);

@override
bool get force => true;

@override
bool get dryRun => false;

@override
String get description => 'Serves all packages in the Amplify Flutter repo '
'on a local pub server.';

@override
String get name => 'serve';

@override
Future<PackageInfo?> checkPublishable(PackageInfo package) async {
final publishTo = package.pubspecInfo.pubspec.publishTo;
if (publishTo == 'none') {
return null;
}
return package;
}

@override
Future<void> run() async {
await super.run();
await linkPackages();

final pubServer = await io.serve(
PubServer.prod().handler,
InternetAddress.anyIPv4,
port,
);
final pubServerUri = Uri.parse('http://localhost:${pubServer.port}');

// Gather packages which can be published.
final publishablePackages = repo.publishablePackages(commandPackages);

// Publish packages to local pub server.
final packagesNeedingPublish =
await unpublishedPackages(publishablePackages);
final launcherPackages = packagesNeedingPublish
.map(
(pkg) => LocalPackage(
name: pkg.name,
path: pkg.path,
pubspec: pkg.pubspecInfo.pubspec,
),
)
.toList();
final launcher = AmplifyPubLauncher(
pubServerUri,
launcherPackages,
prePublish: (LocalPackage pkg) {
final package = repo.allPackages[pkg.name]!;
return prePublish(package);
},
);
await launcher.run();

logger.info('Serving packages on $pubServerUri. Press Ctrl+C to exit.');

await StreamGroup.merge([
ProcessSignal.sigint.watch(),
ProcessSignal.sigterm.watch(),
]).first;

await pubServer.close(force: true);
}
}

class AmplifyPubLauncher extends PubLauncher {
AmplifyPubLauncher(
super.pubServerUri,
super.packages, {
required Future<void> Function(LocalPackage) prePublish,
}) : _prePublish = prePublish;

final Future<void> Function(LocalPackage) _prePublish;

@override
Future<void> prePublish(LocalPackage package) => _prePublish(package);
}
1 change: 1 addition & 0 deletions packages/aft/lib/src/models.dart
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const falsePositiveExamples = [
'amplify_auth_cognito_test',
'amplify_secure_storage_test',
'amplify_native_legacy_wrapper',
'pub_server',

// Smithy Golden packages
'aws_json1_0_v1',
Expand Down
2 changes: 2 additions & 0 deletions packages/aft/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ dependencies:
path: any
pub_api_client: ^2.4.0
pub_semver: ^2.1.1
pub_server:
path: ../test/pub_server
pubspec_parse: ^1.2.0
shelf: ^1.4.0
shelf_static: ^1.1.1
Expand Down
Loading

0 comments on commit 028a96d

Please sign in to comment.