Skip to content

Commit

Permalink
Use custom pulumi app synthesizer (#172)
Browse files Browse the repository at this point in the history 
> Recommended reading
https://github.com/aws/aws-cdk/wiki/Security-And-Safety-Dev-Guide#controlling-the-permissions-used-by-cdk-deployments

Each CDK stack has a
[synthesizer](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib-readme.html#stack-synthesizers)
which determines how the CDK stack should be synthesized and deployed.
By default stacks will use the `DefaultStackSynthesizer` which expects
that the AWS account has been

[bootstrapped](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping-env.html).
This default bootstrapping will create a bunch of IAM roles, an S3
Bucket for file assets, and an ECR Repository for image assets.

In our case we do not want the user to have to worry about bootstrapping
the account with a CloudFormation template or with the CDK CLI.

This PR creates a new synthesizer (`PulumiSynthesizer`) which will
create the required S3 Bucket and ECR Repository on-demand as needed.

Also, since the synthesizer is also responsible for registering assets
and writing those assets to the asset manfests, our synthesizer is able
to simplify the asset publishing. Instead of registering the assets and
writing the manifest and then post processing that manifest to create
the `BucketObjectV2` resources for each asset in the manifest, we are
able to create those resource as the assets are registered.

closes #108
  • Loading branch information
@corymhall
corymhall authored Oct 25, 2024
1 parent 7653ec1 commit 456439c
Show file tree
Hide file tree
Showing 19 changed files with 1,422 additions and 1,100 deletions.
68 changes: 8 additions & 60 deletions src/assembly/manifest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@ import * as path from 'path';
import { AssemblyManifest, Manifest, ArtifactType, ArtifactMetadataEntryType } from '@aws-cdk/cloud-assembly-schema';
import * as fs from 'fs-extra';
import { CloudFormationTemplate } from '../cfn';
import { ArtifactManifest, AssetManifestProperties, LogicalIdMetadataEntry } from 'aws-cdk-lib/cloud-assembly-schema';
import { AssetManifest, DockerImageManifestEntry, FileManifestEntry } from 'cdk-assets';
import { ArtifactManifest, LogicalIdMetadataEntry } from 'aws-cdk-lib/cloud-assembly-schema';
import { StackManifest } from './stack';
import { ConstructTree, StackAsset, StackMetadata } from './types';
import { warn } from '@pulumi/pulumi/log';
import { ConstructTree, StackMetadata } from './types';

/**
* Reads a Cloud Assembly manifest
Expand Down Expand Up @@ -76,20 +74,18 @@ export class AssemblyManifestReader {

const metadata = this.getMetadata(artifact);

const assets = this.getAssetsForStack(artifactId);
if (!this.tree.children) {
throw new Error('Invalid tree.json found');
}
const stackTree = this.tree.children[artifactId];
const stackManifest = new StackManifest(
this.directory,
artifactId,
templateFile,
const stackManifest = new StackManifest({
id: artifactId,
templatePath: templateFile,
metadata,
stackTree,
tree: stackTree,
template,
assets,
);
dependencies: artifact.dependencies ?? [],
});
this._stackManifests.set(artifactId, stackManifest);
}
}
Expand Down Expand Up @@ -123,52 +119,4 @@ export class AssemblyManifestReader {
public get stackManifests(): StackManifest[] {
return Array.from(this._stackManifests.values());
}

/**
* Return a list of assets for a given stack
*
* @param stackId - The artifactId of the stack to find assets for
* @returns a list of `StackAsset` for the given stack
*/
private getAssetsForStack(stackId: string): StackAsset[] {
const assets: (FileManifestEntry | DockerImageManifestEntry)[] = [];
for (const artifact of Object.values(this.manifest.artifacts ?? {})) {
if (
artifact.type === ArtifactType.ASSET_MANIFEST &&
(artifact.properties as AssetManifestProperties)?.file === `${stackId}.assets.json`
) {
assets.push(...this.assetsFromAssetManifest(artifact));
}
}
return assets;
}

/**
* Get a list of assets from the asset manifest.
*
* @param artifact - An ArtifactManifest to extract individual assets from
* @returns a list of file and docker assets found in the manifest
*/
private assetsFromAssetManifest(artifact: ArtifactManifest): StackAsset[] {
const assets: (FileManifestEntry | DockerImageManifestEntry)[] = [];
const fileName = (artifact.properties as AssetManifestProperties).file;
const assetManifest = AssetManifest.fromFile(path.join(this.directory, fileName));
assetManifest.entries.forEach((entry) => {
if (entry.type === 'file') {
const source = (entry as FileManifestEntry).source;
// This will ignore template assets
if (source.path && source.path.startsWith('asset.')) {
assets.push(entry as FileManifestEntry);
}
} else if (entry.type === 'docker-image') {
const source = (entry as DockerImageManifestEntry).source;
if (source.directory && source.directory.startsWith('asset.')) {
assets.push(entry as DockerImageManifestEntry);
}
} else {
warn(`found unexpected asset type: ${entry.type}`);
}
});
return assets;
}
}
77 changes: 44 additions & 33 deletions src/assembly/stack.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as path from 'path';
import { DestinationIdentifier, FileManifestEntry } from 'cdk-assets';
import { CloudFormationParameter, CloudFormationResource, CloudFormationTemplate } from '../cfn';
import { ConstructTree, StackAsset, StackMetadata } from './types';
import { ConstructTree, StackMetadata } from './types';
import { FileAssetPackaging, FileDestination } from 'aws-cdk-lib/cloud-assembly-schema';

/**
Expand Down Expand Up @@ -41,6 +41,38 @@ export class FileAssetManifest {
}
}

export interface StackManifestProps {
/**
* The artifactId of the stack
*/
readonly id: string;

/**
* The path to the CloudFormation template file within the assembly
*/
readonly templatePath: string;

/**
* The StackMetadata for the stack
*/
readonly metadata: StackMetadata;

/**
* The construct tree for the App
*/
readonly tree: ConstructTree;

/**
* The actual CloudFormation template being processed
*/
readonly template: CloudFormationTemplate;

/**
* A list of artifact ids that this stack depends on
*/
readonly dependencies: string[];
}

/**
* StackManifest represents a single Stack that needs to be converted
* It contains all the necessary information for this library to fully convert
Expand Down Expand Up @@ -81,42 +113,21 @@ export class StackManifest {
*
*/
private readonly metadata: StackMetadata;
private readonly assets: StackAsset[];
private readonly directory: string;
constructor(
directory: string,
id: string,
templatePath: string,
metadata: StackMetadata,
tree: ConstructTree,
template: CloudFormationTemplate,
assets: StackAsset[],
) {
this.directory = directory;
this.assets = assets;
this.outputs = template.Outputs;
this.parameters = template.Parameters;
this.metadata = metadata;
this.templatePath = templatePath;
this.id = id;
this.constructTree = tree;
if (!template.Resources) {
public readonly dependencies: string[];
constructor(props: StackManifestProps) {
this.dependencies = props.dependencies;
this.outputs = props.template.Outputs;
this.parameters = props.template.Parameters;
this.metadata = props.metadata;
this.templatePath = props.templatePath;
this.id = props.id;
this.constructTree = props.tree;
if (!props.template.Resources) {
throw new Error('CloudFormation template has no resources!');
}
this.resources = template.Resources;
this.resources = props.template.Resources;
}

public get fileAssets(): FileAssetManifest[] {
return this.assets
.filter((asset) => asset.type === 'file')
.flatMap((asset) => new FileAssetManifest(this.directory, asset));
}

// TODO: implement docker assets
// public get dockerAssets(): DockerAssetManifest[] {
//
// }

/**
* Get the CloudFormation logicalId for the CFN resource at the given Construct path
*
Expand Down
77 changes: 53 additions & 24 deletions src/converters/app-converter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as pulumi from '@pulumi/pulumi';
import { AssemblyManifestReader, StackManifest } from '../assembly';
import { ConstructInfo, GraphBuilder } from '../graph';
import { StackComponentResource, lift, Mapping } from '../types';
import { ArtifactConverter, FileAssetManifestConverter } from './artifact-converter';
import { ArtifactConverter } from './artifact-converter';
import { CdkConstruct, ResourceMapping } from '../interop';
import { debug } from '@pulumi/pulumi/log';
import {
Expand Down Expand Up @@ -35,30 +35,51 @@ export class AppConverter {
}

convert() {
const assetStackIds = this.host.dependencies.flatMap((dep) => dep.name);
const stackManifests: StackManifest[] = [];
for (const stackManifest of this.manifestReader.stackManifests) {
// Don't process artifact manifests
if (assetStackIds.includes(stackManifest.id)) continue;
stackManifests.push(stackManifest);

const stackConverter = new StackConverter(this.host, stackManifest);
this.stacks.set(stackManifest.id, stackConverter);
this.convertStackManifest(stackManifest);
}

for (const stack of stackManifests) {
const done: { [artifactId: string]: StackConverter } = {};
this.convertStackManifest(stack, done);
}
}

private convertStackManifest(artifact: StackManifest): void {
const dependencies = new Set<ArtifactConverter>();
for (const file of artifact.fileAssets) {
const converter = new FileAssetManifestConverter(this.host, file);
converter.convert();
dependencies.add(converter);
private convertStackManifest(
artifact: StackManifest,
done: { [artifactId: string]: StackConverter },
): StackConverter | undefined {
if (artifact.id in done) {
return done[artifact.id];
}

// TODO add docker asset converter
// for (const image of artifact.dockerAssets) {
// }
const dependencies = new Set<ArtifactConverter>();
for (const d of artifact.dependencies) {
const converter = this.stacks.get(d);
if (!converter) {
throw new Error(`Could not convert artifact with id ${d}`);
}
const c = this.convertStackManifest(converter.stack, done);
if (c !== undefined) {
debug(`${artifact.id} depends on ${d}`);
dependencies.add(c);
}
}

const stackConverter = this.stacks.get(artifact.id);
if (!stackConverter) {
throw new Error(`missing CDK Stack for artifact ${artifact.id}`);
}
stackConverter.convert(dependencies);
done[artifact.id] = stackConverter;
return stackConverter;
}
}

Expand All @@ -70,7 +91,16 @@ export class StackConverter extends ArtifactConverter {
readonly resources = new Map<string, Mapping<pulumi.Resource>>();
readonly constructs = new Map<ConstructInfo, pulumi.Resource>();

constructor(host: StackComponentResource, readonly stack: StackManifest) {
private _stackResource?: CdkConstruct;

public get stackResource(): CdkConstruct {
if (!this._stackResource) {
throw new Error('StackConverter has no stack resource');
}
return this._stackResource;
}

constructor(private readonly host: StackComponentResource, readonly stack: StackManifest) {
super(host);
}

Expand All @@ -84,19 +114,19 @@ export class StackConverter extends ArtifactConverter {

for (const n of dependencyGraphNodes) {
if (n.construct.id === this.stack.id) {
const stackResource = new CdkConstruct(
this._stackResource = new CdkConstruct(
`${this.stackComponent.name}/${n.construct.path}`,
n.construct.id,
{
parent: this.stackComponent,
parent: this.stackComponent.component,
// NOTE: Currently we make the stack depend on all the assets and then all resources
// have the parent as the stack. This means we deploy all assets before we deploy any resources
// we might be able better and have individual resources depend on individual assets, but CDK
// doesn't track asset dependencies at that level
dependsOn: this.stackDependsOn(dependencies),
},
);
this.constructs.set(n.construct, stackResource);
this.constructs.set(n.construct, this._stackResource);
continue;
}

Expand Down Expand Up @@ -147,12 +177,11 @@ export class StackConverter extends ArtifactConverter {

private stackDependsOn(dependencies: Set<ArtifactConverter>): pulumi.Resource[] {
const dependsOn: pulumi.Resource[] = [];
dependsOn.push(...this.host.dependencies);
for (const d of dependencies) {
if (d instanceof FileAssetManifestConverter) {
this.resources.set(d.id, { resource: d.file, resourceType: d.resourceType });
dependsOn.push(d.file);
if (d instanceof StackConverter) {
dependsOn.push(d.stackResource);
}
// TODO: handle docker images
}
return dependsOn;
}
Expand Down Expand Up @@ -180,7 +209,7 @@ export class StackConverter extends ArtifactConverter {
return key;
}

this.parameters.set(logicalId, parameterValue(this.stackComponent));
this.parameters.set(logicalId, parameterValue(this.stackComponent.component));
}

private mapResource(
Expand Down Expand Up @@ -341,15 +370,15 @@ export class StackConverter extends ArtifactConverter {

switch (target) {
case 'AWS::AccountId':
return getAccountId({ parent: this.stackComponent }).then((r) => r.accountId);
return getAccountId({ parent: this.stackComponent.component }).then((r) => r.accountId);
case 'AWS::NoValue':
return undefined;
case 'AWS::Partition':
return getPartition({ parent: this.stackComponent }).then((p) => p.partition);
return getPartition({ parent: this.stackComponent.component }).then((p) => p.partition);
case 'AWS::Region':
return getRegion({ parent: this.stackComponent }).then((r) => r.region);
return getRegion({ parent: this.stackComponent.component }).then((r) => r.region);
case 'AWS::URLSuffix':
return getUrlSuffix({ parent: this.stackComponent }).then((r) => r.urlSuffix);
return getUrlSuffix({ parent: this.stackComponent.component }).then((r) => r.urlSuffix);
case 'AWS::NotificationARNs':
case 'AWS::StackId':
case 'AWS::StackName':
Expand Down
Loading

0 comments on commit 456439c

Please sign in to comment.