Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: Further Parallelize Execution Of Jest Tests #4324

Merged
merged 16 commits into from
Oct 15, 2024
Merged

CI: Further Parallelize Execution Of Jest Tests #4324

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
e866244
fix: add `aws:lambda/eventSourceMapping:EventSourceMapping`
adrians5j Oct 9, 2024
bafdb7a
fix: add missing permissions
adrians5j Oct 9, 2024
581de35
ci: shard CMS tests
adrians5j Oct 9, 2024
2283c04
ci:run workflows
adrians5j Oct 9, 2024
53aa3ad
ci: shard CMS tests
adrians5j Oct 9, 2024
7ce12e6
fix: upgrade Pulumi packages
adrians5j Oct 9, 2024
9ecf67c
Merge branch 'refs/heads/adrian/fix-cf-tpl' into adrian/ci-jest-cms-s…
adrians5j Oct 9, 2024
d4a7d41
ci: tweak package ID
adrians5j Oct 9, 2024
f21bf46
ci: tweak package ID
adrians5j Oct 9, 2024
be2878c
ci: shard pb tests
adrians5j Oct 10, 2024
d566aff
ci: remove CWP test in PR workflow
adrians5j Oct 14, 2024
1c82fcf
ci: update comment
adrians5j Oct 14, 2024
cd8b0ca
ci: test with a smaller shards count
adrians5j Oct 14, 2024
d19596e
chore: undo unrelated changes
adrians5j Oct 15, 2024
02351ab
chore: undo unrelated changes
adrians5j Oct 15, 2024
dd86603
ci: test with a smaller shards count
adrians5j Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 109 additions & 157 deletions .github/workflows/pullRequests.yml
View file
Open in desktop

Large diffs are not rendered by default.

163 changes: 109 additions & 54 deletions .github/workflows/pushDev.yml
View file
Open in desktop

Large diffs are not rendered by default.

163 changes: 109 additions & 54 deletions .github/workflows/pushNext.yml
View file
Open in desktop

Large diffs are not rendered by default.

218 changes: 110 additions & 108 deletions .github/workflows/wac/pullRequests.wac.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,114 +187,116 @@ export const pullRequests = createWorkflow({
jestTestsNoStorage: createJestTestsJob(null),
jestTestsDdb: createJestTestsJob("ddb"),
jestTestsDdbEs: createJestTestsJob("ddb-es"),
jestTestsDdbOs: createJestTestsJob("ddb-os"),
jestTestsDdbOs: createJestTestsJob("ddb-os")

verdaccioPublish: createJob({
name: "Publish to Verdaccio",
needs: ["constants", "build"],
if: "needs.constants.outputs.is-fork-pr != 'true'",
checkout: {
"fetch-depth": 0,
ref: "${{ github.event.pull_request.head.ref }}",
path: DIR_WEBINY_JS
},
steps: [
...yarnCacheSteps,
...runBuildCacheSteps,
...installBuildSteps,
...withCommonParams(
[
{
name: "Start Verdaccio local server",
run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
},
{
name: "Configure NPM to use local registry",
run: "npm config set registry http://localhost:4873"
},
{
name: "Set git email",
run: 'git config --global user.email "[email protected]"'
},
{
name: "Set git username",
run: 'git config --global user.name "webiny-bot"'
},
{
name: 'Create ".npmrc" file in the project root, with a dummy auth token',
run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
},
{
name: "Version and publish to Verdaccio",
run: "yarn release --type=verdaccio"
}
],
{ "working-directory": DIR_WEBINY_JS }
),
{
name: "Upload verdaccio files",
uses: "actions/upload-artifact@v4",
with: {
name: "verdaccio-files",
"retention-days": 1,
"include-hidden-files": true,
path: [
DIR_WEBINY_JS + "/.verdaccio/",
DIR_WEBINY_JS + "/.verdaccio.yaml"
].join("\n")
}
}
]
}),
testCreateWebinyProject: createJob({
name: 'Test "create-webiny-project"',
needs: "verdaccioPublish",
strategy: {
"fail-fast": false,
matrix: {
os: ["ubuntu-latest"],
node: [NODE_VERSION]
}
},
"runs-on": "${{ matrix.os }}",
checkout: false,
setupNode: {
"node-version": "${{ matrix.node }}"
},
steps: [
{
uses: "actions/download-artifact@v4",
with: {
name: "verdaccio-files",
path: "verdaccio-files"
}
},
{
name: "Start Verdaccio local server",
"working-directory": "verdaccio-files",
run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
},
{
name: "Configure NPM to use local registry",
run: "npm config set registry http://localhost:4873"
},
{
name: "Set git email",
run: 'git config --global user.email "[email protected]"'
},
{
name: "Set git username",
run: 'git config --global user.name "webiny-bot"'
},
{
name: "Disable Webiny telemetry",
run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
},
{
name: "Create a new Webiny project",
run: 'npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"eu-central-1"}\'\n'
}
]
})
// We commented out these tests because, A) they don't bring much value, and B) these are
// run within "push" workflows anyway (we deploy a Webiny instance and run E2E tests there).
// verdaccioPublish: createJob({
// name: "Publish to Verdaccio",
// needs: ["constants", "build"],
// if: "needs.constants.outputs.is-fork-pr != 'true'",
// checkout: {
// "fetch-depth": 0,
// ref: "${{ github.event.pull_request.head.ref }}",
// path: DIR_WEBINY_JS
// },
// steps: [
// ...yarnCacheSteps,
// ...runBuildCacheSteps,
// ...installBuildSteps,
// ...withCommonParams(
// [
// {
// name: "Start Verdaccio local server",
// run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: 'Create ".npmrc" file in the project root, with a dummy auth token',
// run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
// },
// {
// name: "Version and publish to Verdaccio",
// run: "yarn release --type=verdaccio"
// }
// ],
// { "working-directory": DIR_WEBINY_JS }
// ),
// {
// name: "Upload verdaccio files",
// uses: "actions/upload-artifact@v4",
// with: {
// name: "verdaccio-files",
// "retention-days": 1,
// "include-hidden-files": true,
// path: [
// DIR_WEBINY_JS + "/.verdaccio/",
// DIR_WEBINY_JS + "/.verdaccio.yaml"
// ].join("\n")
// }
// }
// ]
// }),
// testCreateWebinyProject: createJob({
// name: 'Test "create-webiny-project"',
// needs: "verdaccioPublish",
// strategy: {
// "fail-fast": false,
// matrix: {
// os: ["ubuntu-latest"],
// node: [NODE_VERSION]
// }
// },
// "runs-on": "${{ matrix.os }}",
// checkout: false,
// setupNode: {
// "node-version": "${{ matrix.node }}"
// },
// steps: [
// {
// uses: "actions/download-artifact@v4",
// with: {
// name: "verdaccio-files",
// path: "verdaccio-files"
// }
// },
// {
// name: "Start Verdaccio local server",
// "working-directory": "verdaccio-files",
// run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: "Disable Webiny telemetry",
// run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
// },
// {
// name: "Create a new Webiny project",
// run: 'npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"eu-central-1"}\'\n'
// }
// ]
// })
}
});
52 changes: 40 additions & 12 deletions .github/workflows/wac/utils/listPackagesWithJestTests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
* Dictates how package tests will be executed. With this script, we achieve
* parallelization of execution of Jest tests. Note: do not use any 3rd party
* libraries because we need this script to be executed in our CI/CD, as fast as possible.
* Using 3rd party libraries would require `yarn install` to be run before this script is executed.
*/

import fs from "fs";
import path from "path";
import crypto from "crypto";

/**
* Some packages require custom handling.
Expand All @@ -20,6 +22,18 @@ interface PackageWithTestsWithId extends PackageWithTests {
id: string;
}

// Takes a PackageWithTests object and returns an array of commands, where each
// command is just running a subset of tests. This is achieved by using the
// Jest's `--shard` option.
const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount: number = 5) => {
const commands: PackageWithTests[] = [];
for (let currentShard = 1; currentShard <= shardsCount; currentShard++) {
commands.push({ ...pkg, cmd: pkg.cmd + ` --shard=${currentShard}/${shardsCount}` });
}

return commands;
};

const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
// Ignore "i18n" package.
i18n: () => [],
Expand Down Expand Up @@ -84,9 +98,18 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {

"api-page-builder": () => {
return [
{ cmd: "packages/api-page-builder --storage=ddb-es,ddb", storage: "ddb-es" },
{ cmd: "packages/api-page-builder --storage=ddb-os,ddb", storage: "ddb-os" },
{ cmd: "packages/api-page-builder --storage=ddb", storage: "ddb" }
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb-es,ddb",
storage: "ddb-es"
}),
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb-os,ddb",
storage: "ddb-os"
}),
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb",
storage: "ddb"
})
];
},
"api-page-builder-so-ddb-es": () => {
Expand Down Expand Up @@ -125,9 +148,18 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {

"api-headless-cms": () => {
return [
{ cmd: "packages/api-headless-cms --storage=ddb", storage: "ddb" },
{ cmd: "packages/api-headless-cms --storage=ddb-es,ddb", storage: "ddb-es" },
{ cmd: "packages/api-headless-cms --storage=ddb-os,ddb", storage: "ddb-os" }
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb",
storage: "ddb"
}),
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb-es,ddb",
storage: "ddb-es"
}),
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb-os,ddb",
storage: "ddb-os"
})
];
},
"api-headless-cms-ddb-es": () => {
Expand Down Expand Up @@ -270,12 +302,8 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
const testFilePattern = /test\.j?t?sx?$/;

const cmdToId = (cmd: string) => {
return cmd
.replace("packages/", "")
.replace("--storage=", "")
.replace(/[,\s]/g, "_")
.replace(/[\(\)\[\]]/g, "")
.toLowerCase();
// Just convert the command to kebab-case.
return crypto.createHash("md5").update(cmd).digest("hex");
};

/**
Expand Down
18 changes: 9 additions & 9 deletions docs/DEPLOY_WEBINY_PROJECT_CF_TEMPLATE.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,14 +255,14 @@ Resources:
# AWS Step Functions
- Effect: Allow
Action:
- states:CreateStateMachine
- states:DeleteStateMachine
- states:DescribeStateMachine
- states:ListTagsForResource
- states:TagResource
- states:UntagResource
- states:ListStateMachines
- states:UpdateStateMachine
- states:CreateStateMachine
- states:DeleteStateMachine
- states:DescribeStateMachine
- states:ListTagsForResource
- states:TagResource
- states:UntagResource
- states:ListStateMachines
- states:UpdateStateMachine
Resource: arn:aws:states:*:*:stateMachine:wby-*

# AWS Lambda
Expand Down Expand Up @@ -502,4 +502,4 @@ Resources:
GroupName:
Ref: DeployWebinyProjectGroup3
Users:
- Ref: Username
- Ref: Username
Loading
Loading