Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: Further Parallelize Execution Of Jest Tests (Propagating Changes From dev Branch) #4334

Merged
merged 4 commits into from
Oct 15, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
ci: further parallelize execution of Jest tests (#4324)
adrians5j authored Oct 15, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit cc419c1d17625d1fa6c25f287143922cf2a99cb4
278 changes: 121 additions & 157 deletions .github/workflows/pullRequests.yml

Large diffs are not rendered by default.

175 changes: 121 additions & 54 deletions .github/workflows/pushDev.yml

Large diffs are not rendered by default.

175 changes: 121 additions & 54 deletions .github/workflows/pushNext.yml

Large diffs are not rendered by default.

218 changes: 110 additions & 108 deletions .github/workflows/wac/pullRequests.wac.ts
Original file line number Diff line number Diff line change
@@ -187,114 +187,116 @@ export const pullRequests = createWorkflow({
jestTestsNoStorage: createJestTestsJob(null),
jestTestsDdb: createJestTestsJob("ddb"),
jestTestsDdbEs: createJestTestsJob("ddb-es"),
jestTestsDdbOs: createJestTestsJob("ddb-os"),
jestTestsDdbOs: createJestTestsJob("ddb-os")

verdaccioPublish: createJob({
name: "Publish to Verdaccio",
needs: ["constants", "build"],
if: "needs.constants.outputs.is-fork-pr != 'true'",
checkout: {
"fetch-depth": 0,
ref: "${{ github.event.pull_request.head.ref }}",
path: DIR_WEBINY_JS
},
steps: [
...yarnCacheSteps,
...runBuildCacheSteps,
...installBuildSteps,
...withCommonParams(
[
{
name: "Start Verdaccio local server",
run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
},
{
name: "Configure NPM to use local registry",
run: "npm config set registry http://localhost:4873"
},
{
name: "Set git email",
run: 'git config --global user.email "[email protected]"'
},
{
name: "Set git username",
run: 'git config --global user.name "webiny-bot"'
},
{
name: 'Create ".npmrc" file in the project root, with a dummy auth token',
run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
},
{
name: "Version and publish to Verdaccio",
run: "yarn release --type=verdaccio"
}
],
{ "working-directory": DIR_WEBINY_JS }
),
{
name: "Upload verdaccio files",
uses: "actions/upload-artifact@v4",
with: {
name: "verdaccio-files",
"retention-days": 1,
"include-hidden-files": true,
path: [
DIR_WEBINY_JS + "/.verdaccio/",
DIR_WEBINY_JS + "/.verdaccio.yaml"
].join("\n")
}
}
]
}),
testCreateWebinyProject: createJob({
name: 'Test "create-webiny-project"',
needs: "verdaccioPublish",
strategy: {
"fail-fast": false,
matrix: {
os: ["ubuntu-latest"],
node: [NODE_VERSION]
}
},
"runs-on": "${{ matrix.os }}",
checkout: false,
setupNode: {
"node-version": "${{ matrix.node }}"
},
steps: [
{
uses: "actions/download-artifact@v4",
with: {
name: "verdaccio-files",
path: "verdaccio-files"
}
},
{
name: "Start Verdaccio local server",
"working-directory": "verdaccio-files",
run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
},
{
name: "Configure NPM to use local registry",
run: "npm config set registry http://localhost:4873"
},
{
name: "Set git email",
run: 'git config --global user.email "[email protected]"'
},
{
name: "Set git username",
run: 'git config --global user.name "webiny-bot"'
},
{
name: "Disable Webiny telemetry",
run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
},
{
name: "Create a new Webiny project",
run: 'npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"eu-central-1"}\'\n'
}
]
})
// We commented out these tests because, A) they don't bring much value, and B) these are
// run within "push" workflows anyway (we deploy a Webiny instance and run E2E tests there).
// verdaccioPublish: createJob({
// name: "Publish to Verdaccio",
// needs: ["constants", "build"],
// if: "needs.constants.outputs.is-fork-pr != 'true'",
// checkout: {
// "fetch-depth": 0,
// ref: "${{ github.event.pull_request.head.ref }}",
// path: DIR_WEBINY_JS
// },
// steps: [
// ...yarnCacheSteps,
// ...runBuildCacheSteps,
// ...installBuildSteps,
// ...withCommonParams(
// [
// {
// name: "Start Verdaccio local server",
// run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: 'Create ".npmrc" file in the project root, with a dummy auth token',
// run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
// },
// {
// name: "Version and publish to Verdaccio",
// run: "yarn release --type=verdaccio"
// }
// ],
// { "working-directory": DIR_WEBINY_JS }
// ),
// {
// name: "Upload verdaccio files",
// uses: "actions/upload-artifact@v4",
// with: {
// name: "verdaccio-files",
// "retention-days": 1,
// "include-hidden-files": true,
// path: [
// DIR_WEBINY_JS + "/.verdaccio/",
// DIR_WEBINY_JS + "/.verdaccio.yaml"
// ].join("\n")
// }
// }
// ]
// }),
// testCreateWebinyProject: createJob({
// name: 'Test "create-webiny-project"',
// needs: "verdaccioPublish",
// strategy: {
// "fail-fast": false,
// matrix: {
// os: ["ubuntu-latest"],
// node: [NODE_VERSION]
// }
// },
// "runs-on": "${{ matrix.os }}",
// checkout: false,
// setupNode: {
// "node-version": "${{ matrix.node }}"
// },
// steps: [
// {
// uses: "actions/download-artifact@v4",
// with: {
// name: "verdaccio-files",
// path: "verdaccio-files"
// }
// },
// {
// name: "Start Verdaccio local server",
// "working-directory": "verdaccio-files",
// run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: "Disable Webiny telemetry",
// run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
// },
// {
// name: "Create a new Webiny project",
// run: 'npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"eu-central-1"}\'\n'
// }
// ]
// })
}
});
52 changes: 40 additions & 12 deletions .github/workflows/wac/utils/listPackagesWithJestTests.ts
Original file line number Diff line number Diff line change
@@ -2,10 +2,12 @@
* Dictates how package tests will be executed. With this script, we achieve
* parallelization of execution of Jest tests. Note: do not use any 3rd party
* libraries because we need this script to be executed in our CI/CD, as fast as possible.
* Using 3rd party libraries would require `yarn install` to be run before this script is executed.
*/

import fs from "fs";
import path from "path";
import crypto from "crypto";

/**
* Some packages require custom handling.
@@ -20,6 +22,18 @@ interface PackageWithTestsWithId extends PackageWithTests {
id: string;
}

// Takes a PackageWithTests object and returns an array of commands, where each
// command is just running a subset of tests. This is achieved by using the
// Jest's `--shard` option.
const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount: number = 6) => {
const commands: PackageWithTests[] = [];
for (let currentShard = 1; currentShard <= shardsCount; currentShard++) {
commands.push({ ...pkg, cmd: pkg.cmd + ` --shard=${currentShard}/${shardsCount}` });
}

return commands;
};

const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
// Ignore "i18n" package.
i18n: () => [],
@@ -84,9 +98,18 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {

"api-page-builder": () => {
return [
{ cmd: "packages/api-page-builder --storage=ddb-es,ddb", storage: "ddb-es" },
{ cmd: "packages/api-page-builder --storage=ddb-os,ddb", storage: "ddb-os" },
{ cmd: "packages/api-page-builder --storage=ddb", storage: "ddb" }
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb-es,ddb",
storage: "ddb-es"
}),
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb-os,ddb",
storage: "ddb-os"
}),
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb",
storage: "ddb"
})
];
},
"api-page-builder-so-ddb-es": () => {
@@ -125,9 +148,18 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {

"api-headless-cms": () => {
return [
{ cmd: "packages/api-headless-cms --storage=ddb", storage: "ddb" },
{ cmd: "packages/api-headless-cms --storage=ddb-es,ddb", storage: "ddb-es" },
{ cmd: "packages/api-headless-cms --storage=ddb-os,ddb", storage: "ddb-os" }
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb",
storage: "ddb"
}),
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb-es,ddb",
storage: "ddb-es"
}),
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb-os,ddb",
storage: "ddb-os"
})
];
},
"api-headless-cms-ddb-es": () => {
@@ -270,12 +302,8 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
const testFilePattern = /test\.j?t?sx?$/;

const cmdToId = (cmd: string) => {
return cmd
.replace("packages/", "")
.replace("--storage=", "")
.replace(/[,\s]/g, "_")
.replace(/[\(\)\[\]]/g, "")
.toLowerCase();
// Just convert the command to kebab-case.
return crypto.createHash("md5").update(cmd).digest("hex");
};

/**
18 changes: 9 additions & 9 deletions docs/DEPLOY_WEBINY_PROJECT_CF_TEMPLATE.yaml
Original file line number Diff line number Diff line change
@@ -255,14 +255,14 @@ Resources:
# AWS Step Functions
- Effect: Allow
Action:
- states:CreateStateMachine
- states:DeleteStateMachine
- states:DescribeStateMachine
- states:ListTagsForResource
- states:TagResource
- states:UntagResource
- states:ListStateMachines
- states:UpdateStateMachine
- states:CreateStateMachine
- states:DeleteStateMachine
- states:DescribeStateMachine
- states:ListTagsForResource
- states:TagResource
- states:UntagResource
- states:ListStateMachines
- states:UpdateStateMachine
Resource: arn:aws:states:*:*:stateMachine:wby-*

# AWS Lambda
@@ -502,4 +502,4 @@ Resources:
GroupName:
Ref: DeployWebinyProjectGroup3
Users:
- Ref: Username
- Ref: Username
Loading