Skip to content

Commit 2049cf4

Browse files
freemvmtfreemvmt
authored
[performance] add API load testing script (#4237)
1 parent 9a3ea93 commit 2049cf4

22 files changed

+332
-24
lines changed

‎.env.example

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ FILE_API_KEY_DONCASTER=👻
3939
FILE_API_KEY_GLOUCESTER=👻
4040
FILE_API_KEY_TEWKESBURY=👻
4141

42+
# Used to circumvent API rate limiting for development purposes (e.g. load testing)
43+
SKIP_RATE_LIMIT_SECRET=👻
4244

4345
# Editor
4446
EDITOR_URL_EXT=http://localhost:3000

‎api.planx.uk/modules/auth/fixtures/microsoft-openid-configuration.json

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"token_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/token",
3+
"token_endpoint_auth_methods_supported": [
4+
"client_secret_post",
5+
"private_key_jwt",
6+
"client_secret_basic"
7+
],
8+
"jwks_uri": "https://login.microsoftonline.com/common/discovery/v2.0/keys",
9+
"response_modes_supported": ["query", "fragment", "form_post"],
10+
"subject_types_supported": ["pairwise"],
11+
"id_token_signing_alg_values_supported": ["RS256"],
12+
"response_types_supported": [
13+
"code",
14+
"id_token",
15+
"code id_token",
16+
"id_token token"
17+
],
18+
"scopes_supported": ["openid", "profile", "email", "offline_access"],
19+
"issuer": "https://login.microsoftonline.com/common/v2.0",
20+
"request_uri_parameter_supported": false,
21+
"userinfo_endpoint": "https://graph.microsoft.com/oidc/userinfo",
22+
"authorization_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
23+
"device_authorization_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/devicecode",
24+
"http_logout_supported": true,
25+
"frontchannel_logout_supported": true,
26+
"end_session_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
27+
"claims_supported": [
28+
"sub",
29+
"iss",
30+
"cloud_instance_name",
31+
"cloud_instance_host_name",
32+
"cloud_graph_host_name",
33+
"msgraph_host",
34+
"aud",
35+
"exp",
36+
"iat",
37+
"auth_time",
38+
"acr",
39+
"nonce",
40+
"preferred_username",
41+
"name",
42+
"tid",
43+
"ver",
44+
"at_hash",
45+
"c_hash",
46+
"email"
47+
],
48+
"kerberos_endpoint": "https://login.microsoftonline.com/common/kerberos",
49+
"tenant_region_scope": null,
50+
"cloud_instance_name": "microsoftonline.com",
51+
"cloud_graph_host_name": "graph.windows.net",
52+
"msgraph_host": "graph.microsoft.com",
53+
"rbac_url": "https://pas.windows.net"
54+
}

‎api.planx.uk/modules/auth/passport.ts

+28-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1+
import fs from "fs";
2+
import path from "path";
3+
import { fileURLToPath } from "url";
4+
15
import { Issuer } from "openid-client";
6+
import type { IssuerMetadata } from "openid-client";
27
import type { Authenticator } from "passport";
38
import passport from "passport";
49

@@ -14,7 +19,29 @@ export default async (): Promise<Authenticator> => {
1419
const customPassport = new passport.Passport();
1520

1621
// instantiate Microsoft OIDC client, and use it to build the related strategy
17-
const microsoftIssuer = await Issuer.discover(MICROSOFT_OPENID_CONFIG_URL);
22+
// we also keep said config as a fixture to enable offline local development
23+
let microsoftIssuer;
24+
if (
25+
process.env.APP_ENVIRONMENT == "development" &&
26+
process.env.DEVELOP_OFFLINE
27+
) {
28+
console.info(
29+
"Working offline: using saved Microsoft OIDC configuration in auth/fixtures",
30+
);
31+
const __filename = fileURLToPath(import.meta.url);
32+
const __dirname = path.dirname(__filename);
33+
const fixturePath = path.resolve(
34+
__dirname,
35+
"fixtures",
36+
"microsoft-openid-configuration.json",
37+
);
38+
const microsoftIssuerConfig: IssuerMetadata = JSON.parse(
39+
fs.readFileSync(fixturePath, "utf-8"),
40+
);
41+
microsoftIssuer = new Issuer(microsoftIssuerConfig);
42+
} else {
43+
microsoftIssuer = await Issuer.discover(MICROSOFT_OPENID_CONFIG_URL);
44+
}
1845
console.debug("Discovered issuer %s", microsoftIssuer.issuer);
1946
const microsoftOidcClient = new microsoftIssuer.Client(
2047
getMicrosoftClientConfig(),

‎api.planx.uk/modules/file/controller.ts

+30-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import assert from "assert";
2+
import { deleteFilesByKey } from "./service/deleteFile.js";
23
import { uploadPrivateFile, uploadPublicFile } from "./service/uploadFile.js";
34
import { buildFilePath } from "./service/utils.js";
45
import { getFileFromS3 } from "./service/getFile.js";
@@ -71,15 +72,15 @@ export const publicUploadController: UploadController = async (
7172
}
7273
};
7374

74-
export const downloadFileSchema = z.object({
75+
export const hostedFileSchema = z.object({
7576
params: z.object({
7677
fileKey: z.string(),
7778
fileName: z.string(),
7879
}),
7980
});
8081

8182
export type DownloadController = ValidatedRequestHandler<
82-
typeof downloadFileSchema,
83+
typeof hostedFileSchema,
8384
Buffer | undefined
8485
>;
8586

@@ -124,3 +125,30 @@ export const privateDownloadController: DownloadController = async (
124125
);
125126
}
126127
};
128+
129+
export type DeleteController = ValidatedRequestHandler<
130+
typeof hostedFileSchema,
131+
Record<string, never>
132+
>;
133+
134+
export const publicDeleteController: DeleteController = async (
135+
_req,
136+
res,
137+
next,
138+
) => {
139+
const { fileKey, fileName } = res.locals.parsedReq.params;
140+
const filePath = buildFilePath(fileKey, fileName);
141+
142+
try {
143+
const { isPrivate } = await getFileFromS3(filePath);
144+
if (isPrivate) throw Error("Bad request");
145+
146+
// once we've established that the file is public, we can delete it
147+
await deleteFilesByKey([filePath]);
148+
res.status(204).send();
149+
} catch (error) {
150+
return next(
151+
new ServerError({ message: `Failed to delete public file: ${error}` }),
152+
);
153+
}
154+
};

‎api.planx.uk/modules/file/docs.yaml

+15-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ info:
44
version: 0.1.0
55
tags:
66
- name: file
7-
description: Endpoints for uploading and downloading files
7+
description: Endpoints for uploading, downloading and deleting files
88
components:
99
parameters:
1010
fileKey:
@@ -84,6 +84,18 @@ paths:
8484
$ref: "#/components/responses/DownloadFile"
8585
"500":
8686
$ref: "#/components/responses/ErrorMessage"
87+
delete:
88+
tags: ["file"]
89+
parameters:
90+
- $ref: "#/components/parameters/fileKey"
91+
- $ref: "#/components/parameters/fileName"
92+
security:
93+
- bearerAuth: []
94+
responses:
95+
"204":
96+
$ref: "#/components/responses/UploadFile"
97+
"500":
98+
$ref: "#/components/responses/ErrorMessage"
8799
/file/private/{fileKey}/{fileName}:
88100
get:
89101
tags: ["file"]
@@ -93,7 +105,7 @@ paths:
93105
security:
94106
- fileAPIKeyAuth: []
95107
responses:
96-
"200":
97-
$ref: "#/components/responses/DownloadFile"
108+
"204":
109+
description: Successful deletion
98110
"500":
99111
$ref: "#/components/responses/ErrorMessage"

‎api.planx.uk/modules/file/routes.ts

+13-5
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
import { Router } from "express";
22

3-
import multer from "multer";
43
import {
5-
useNoCache,
64
useFilePermission,
5+
useNoCache,
6+
usePlatformAdminAuth,
77
useTeamEditorAuth,
88
} from "../auth/middleware.js";
99
import {
10-
downloadFileSchema,
10+
hostedFileSchema,
1111
privateDownloadController,
1212
privateUploadController,
13+
publicDeleteController,
1314
publicDownloadController,
1415
publicUploadController,
1516
uploadFileSchema,
@@ -36,16 +37,23 @@ router.post(
3637

3738
router.get(
3839
"/file/public/:fileKey/:fileName",
39-
validate(downloadFileSchema),
40+
validate(hostedFileSchema),
4041
publicDownloadController,
4142
);
4243

4344
router.get(
4445
"/file/private/:fileKey/:fileName",
4546
useNoCache,
4647
useFilePermission,
47-
validate(downloadFileSchema),
48+
validate(hostedFileSchema),
4849
privateDownloadController,
4950
);
5051

52+
router.delete(
53+
"/file/public/:fileKey/:fileName",
54+
usePlatformAdminAuth,
55+
validate(hostedFileSchema),
56+
publicDeleteController,
57+
);
58+
5159
export default router;

‎api.planx.uk/modules/file/service/uploadFile.ts

+9-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
GetObjectCommand,
3+
type S3,
34
type PutObjectCommandInput,
45
} from "@aws-sdk/client-s3";
56
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -19,7 +20,7 @@ export const uploadPublicFile = async (
1920
const { params, key, fileType } = generateFileParams(file, filename, filekey);
2021

2122
await s3.putObject(params);
22-
const fileUrl = await buildFileUrl(key, "public");
23+
const fileUrl = await buildFileUrl(s3, key, "public");
2324

2425
return {
2526
fileType,
@@ -41,7 +42,7 @@ export const uploadPrivateFile = async (
4142
};
4243

4344
await s3.putObject(params);
44-
const fileUrl = await buildFileUrl(key, "private");
45+
const fileUrl = await buildFileUrl(s3, key, "private");
4546

4647
return {
4748
fileType,
@@ -50,8 +51,11 @@ export const uploadPrivateFile = async (
5051
};
5152

5253
// Construct an API URL for the uploaded file
53-
const buildFileUrl = async (key: string, path: "public" | "private") => {
54-
const s3 = s3Factory();
54+
const buildFileUrl = async (
55+
s3: S3,
56+
key: string,
57+
path: "public" | "private",
58+
) => {
5559
const s3Url = await getSignedUrl(
5660
s3,
5761
new GetObjectCommand({ Key: key, Bucket: process.env.AWS_S3_BUCKET }),
@@ -85,8 +89,8 @@ export function generateFileParams(
8589
};
8690

8791
return {
88-
fileType,
8992
params,
9093
key,
94+
fileType,
9195
};
9296
}

‎api.planx.uk/modules/file/service/utils.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ function useMinio() {
2323
// Points to Minio
2424
return {
2525
endpoint: `http://minio:${process.env.MINIO_PORT}`,
26-
s3ForcePathStyle: true,
26+
forcePathStyle: true,
2727
signatureVersion: "v4",
2828
};
2929
}

‎api.planx.uk/rateLimit.ts

+6
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ const apiLimiter = rateLimit({
77
max: 250,
88
standardHeaders: true,
99
legacyHeaders: false,
10+
skip: (req: Request, _res: Response) => {
11+
// add a mechanism (guarded by a secret) for skipping rate limit when load testing
12+
return (
13+
req.get("X-Skip-Rate-Limit-Secret") === process.env.SKIP_RATE_LIMIT_SECRET
14+
);
15+
},
1016
});
1117

1218
const HASURA_ONLY_SEND_EMAIL_TEMPLATES = ["reminder", "expiry"];

‎docker-compose.yml

+1
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ services:
156156
ORDNANCE_SURVEY_API_KEY: ${ORDNANCE_SURVEY_API_KEY}
157157
PORT: ${API_PORT}
158158
SESSION_SECRET: ${SESSION_SECRET}
159+
SKIP_RATE_LIMIT_SECRET: ${SKIP_RATE_LIMIT_SECRET}
159160
SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
160161
UNIFORM_SUBMISSION_URL: ${UNIFORM_SUBMISSION_URL}
161162
UNIFORM_TOKEN_URL: ${UNIFORM_TOKEN_URL}

‎infrastructure/application/Pulumi.production.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ config:
3434
secure: AAABAL5G2cNl6XIQA0vcP6El4Us7Vk8Cz9JViRon25crc8MC0ix4ox2mE+XawsxYbLRwGfaRIJo=
3535
application:file-api-key-tewkesbury:
3636
secure: AAABAA1cz4BbfrTaXhcERQD0MaMSky0fC6ej9/N0n1IBB2w4iBd3S2hNi0T39VEiAB6CNouB+yElaoR7ZWg1xaIIJX2K6TGl
37+
application:skip-rate-limit-secret:
38+
secure: AAABADPv0Fp55IK8xL0Kxx/Kq2sFeRIgbKn0sgHBFhTsWh09kV3/OyG0d/wkE3zm+15zbud0GR4TSl4z//MV5XgxLWDhJgGJ
3739
application:google-client-id: 987324067365-vpsk3kgeq5n32ihjn760ihf8l7m5rhh8.apps.googleusercontent.com
3840
application:google-client-secret:
3941
secure: AAABAN5E+De3A3HtpLVaSNTDwk9Uz4r2d5g8SIRVbNOd2fj3eU+lGJXjVbEAnxezr14hwabbfwW2ptjcFzqkhG7OmQ==

‎infrastructure/application/Pulumi.staging.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ config:
3535
secure: AAABAK8LsYKKNgIS4fepW5Sh6+WKxNopxsos51eBttT7O8E8K0HYOswgrIWuYJ0R1eJHDLKRHqQ=
3636
application:file-api-key-tewkesbury:
3737
secure: AAABALlStpxyNG5SRQFVYJMGmCyteUkoU9XBTBJn2kcf6APdqO1JwxU4jiU9Qo6a6aZQXK60an7xbkuD2hla/UvjR7Wu7cXY
38+
application:skip-rate-limit-secret:
39+
secure: AAABANwjBmGN73bwF6nqQP5i8tKMe3mi0NMBdae4uaLasG2VfPvC1D4mk7QWIFqVnzsD5jCN20Eqos/r2B+EAo2rAws3JeD/
3840
application:google-client-id: 987324067365-vpsk3kgeq5n32ihjn760ihf8l7m5rhh8.apps.googleusercontent.com
3941
application:google-client-secret:
4042
secure: AAABAGQuqQDU4S+vR+cQaFoa6xAeWU9clVaNonQ/dq0R8Dke+o0y7ALOmYMy4fOX4Pa6HiZl85npU/cbwy8HdMYaiA==

‎infrastructure/application/index.ts

+4
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,10 @@ export = async () => {
392392
name: "FILE_API_KEY_TEWKESBURY",
393393
value: config.requireSecret("file-api-key-tewkesbury"),
394394
},
395+
{
396+
name: "SKIP_RATE_LIMIT_SECRET",
397+
value: config.requireSecret("skip-rate-limit-secret"),
398+
},
395399
{
396400
name: "GOOGLE_CLIENT_ID",
397401
value: config.require("google-client-id"),

0 commit comments

Comments
 (0)