Sign k6 requests with HMAC to enable WAF bypass

packages/js/k6/package.json

@@ -11,8 +11,11 @@
   "author": "Openverse Contributors <[email protected]>",
   "license": "MIT",
   "devDependencies": {
+    "@rollup/plugin-commonjs": "^26.0.1",
+    "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-typescript": "^11.1.6",
     "@types/k6": "^0.53.1",
+    "core-js": "^3.37.1",
     "glob": "^11.0.0",
     "rollup": "^4.21.1",
     "typescript": "^5.6.2"

packages/js/k6/rollup.config.ts

@@ -7,6 +7,8 @@ import { defineConfig } from "rollup"
 import { glob } from "glob"
 
 import typescript from "@rollup/plugin-typescript"
+import { nodeResolve } from "@rollup/plugin-node-resolve"
+import commonjs from "@rollup/plugin-commonjs"
 
 function getConfig(testFile: string) {
   return defineConfig({
@@ -18,7 +20,7 @@ function getConfig(testFile: string) {
       preserveModules: true,
       preserveModulesRoot: "src",
     },
-    plugins: [typescript()],
+    plugins: [typescript(), nodeResolve(), commonjs()],
   })
 }
 

packages/js/k6/src/crypto.ts

@@ -0,0 +1,15 @@
+// Courtesy of @mbforbes via https://gist.github.com/robingustafsson/7dd6463d85efdddbb0e4bcd3ecc706e1?permalink_comment_id=4884925#gistcomment-4884925
+
+import { createHMAC } from "k6/crypto"
+
+import type { Algorithm } from "k6/crypto"
+
+export function sign(
+  data: string | ArrayBuffer,
+  hashAlg: Algorithm,
+  secret: string | ArrayBuffer
+) {
+  const hasher = createHMAC(hashAlg, secret)
+  hasher.update(data)
+  return hasher.digest("base64rawurl")
+}

packages/js/k6/src/frontend/constants.ts

@@ -1,2 +1,3 @@
 export const PROJECT_ID = 3713375
-export const FRONTEND_URL = __ENV.FRONTEND_URL || "https://openverse.org/"
+// Default to location of `ov j p frontend prod`
+export const FRONTEND_URL = __ENV.FRONTEND_URL || "http://127.0.0.1:8443/"

packages/js/k6/src/frontend/scenarios.ts

@@ -1,74 +1,75 @@
-import { group } from "k6"
-import exec from "k6/execution"
-import http from "k6/http"
+import { check } from "k6"
 
-import { getRandomWord, makeResponseFailedCheck } from "../utils.js"
+import { getRandomWord } from "../utils.js"
+import { http } from "../http.js"
 
 import { FRONTEND_URL, PROJECT_ID } from "./constants.js"
 
 import type { Options, Scenario } from "k6/options"
 
 const STATIC_PAGES = ["about", "sources", "privacy", "sensitive-content"]
 const TEST_LOCALES = ["en", "ru", "es", "fa"]
-const TEST_PARAMS = "&license=by&extension=jpg,mp3&source=flickr,jamendo"
+const TEST_PARAMS = "license=by&extension=jpg,mp3&source=flickr,jamendo"
 
 const localePrefix = (locale: string) => {
   return locale === "en" ? "" : locale + "/"
 }
 
-const visitUrl = (url: string, action: Action) => {
-  // eslint-disable-next-line import/no-named-as-default-member
-  const response = http.get(url, {
-    headers: { "User-Agent": "OpenverseLoadTesting" },
-  })
-  const checkResponseFailed = makeResponseFailedCheck("", url)
-  if (checkResponseFailed(response, action)) {
-    console.error(`Failed URL: ${url}`)
-    return 0
-  }
-  return 1
-}
-
 const parseEnvLocales = (locales: string) => {
   return locales ? locales.split(",") : ["en"]
 }
 
 export function visitStaticPages() {
   const locales = parseEnvLocales(__ENV.LOCALES)
-  console.log(
-    `VU: ${exec.vu.idInTest}  -  ITER: ${exec.vu.iterationInInstance}`
-  )
+  const ovGroup = `visit static pages for locales ${locales}`
+
   for (const locale of locales) {
-    group(`visit static pages for locale ${locale}`, () => {
-      for (const page of STATIC_PAGES) {
-        visitUrl(
-          `${FRONTEND_URL}${localePrefix(locale)}${page}`,
-          "visitStaticPages"
+    for (const page of STATIC_PAGES) {
+      const url = new URL(`${localePrefix(locale)}${page}`, FRONTEND_URL)
+      const response = http.get(url.toString(), { tags: { ovGroup } })
+      const result = check(
+        response,
+        { "status was 200": (r) => r.status === 200 },
+        { ovGroup }
+      )
+
+      if (!result) {
+        console.error(
+          `Request failed ⨯ ${url}: ${response.status}\n${response.body}`
         )
       }
-    })
+    }
   }
 }
 
 export function visitSearchPages() {
   const locales = parseEnvLocales(__ENV.LOCALES)
-  const params = __ENV.PARAMS
-  const paramsString = params ? ` with params ${params}` : ""
-  console.log(
-    `VU: ${exec.vu.idInTest}  -  ITER: ${exec.vu.iterationInInstance}`
-  )
-  group(`search for random word on locales ${locales}${paramsString}`, () => {
-    for (const MEDIA_TYPE of ["image", "audio"]) {
-      for (const locale of locales) {
-        const q = getRandomWord()
-        return visitUrl(
-          `${FRONTEND_URL}${localePrefix(locale)}search/${MEDIA_TYPE}?q=${q}${params}`,
-          "visitSearchPages"
+  const ovGroup = `search for random word on locales ${locales}`
+
+  for (const MEDIA_TYPE of ["image", "audio"]) {
+    for (const locale of locales) {
+      const url = new URL(
+        `${localePrefix(locale)}search/${MEDIA_TYPE}`,
+        FRONTEND_URL
+      )
+      const params = new URLSearchParams(__ENV.PARAMS)
+      params.append("q", getRandomWord())
+      url.search = params.toString()
+
+      const response = http.get(url.toString(), { tags: { ovGroup } })
+      const result = check(
+        response,
+        { "status was 200": (r) => r.status === 200 },
+        { ovGroup }
+      )
+
+      if (!result) {
+        console.error(
+          `Request failed ⨯ ${url}: ${response.status}\n${response.body}`
         )
       }
     }
-    return undefined
-  })
+  }
 }
 
 const actions = {
@@ -95,7 +96,7 @@ const createScenario = (
 }
 
 export const SCENARIOS = {
-  staticPages: createScenario({ LOCALES: "en" }, "visitStaticPages"),
+  englishStaticPages: createScenario({ LOCALES: "en" }, "visitStaticPages"),
   localeStaticPages: createScenario(
     { LOCALES: TEST_LOCALES.join(",") },
     "visitStaticPages"
@@ -129,14 +130,14 @@ function getScenarios(
 
 export const SCENARIO_GROUPS = {
   all: getScenarios([
-    "staticPages",
+    "englishStaticPages",
     "localeStaticPages",
     "englishSearchPages",
     "localesSearchPages",
     "englishSearchPagesWithFilters",
     "localesSearchPagesWithFilters",
   ]),
-  "static-en": getScenarios(["staticPages"]),
+  "static-en": getScenarios(["englishStaticPages"]),
   "static-locales": getScenarios(["localeStaticPages"]),
   "search-en": getScenarios([
     "englishSearchPages",

packages/js/k6/src/http.ts

@@ -0,0 +1,83 @@
+/**
+ * Wrap k6/http functions to sign requests to allow firewall bypass.
+ */
+
+import {
+  get,
+  post,
+  del,
+  options,
+  head,
+  patch,
+  put,
+  type RequestBody,
+  type RefinedParams,
+  type ResponseType,
+} from "k6/http"
+import { HttpURL } from "k6/experimental/tracing"
+
+// Polyfills URL
+import "core-js/web/url"
+import "core-js/web/url-search-params"
+
+import { sign } from "./crypto"
+
+const signingSecret = __ENV.signing_secret
+
+function getSignedParams<
+  RT extends ResponseType | undefined,
+  P extends RefinedParams<RT>,
+>(url: string | HttpURL, params: P): P {
+  if (!signingSecret) {
+    return params
+  }
+
+  const _url = new URL(url.toString())
+
+  const timestamp = Math.floor(Date.now() / 1000)
+  const resource = `${_url.pathname}${_url.search}${timestamp}`
+  const mac = sign(resource, "sha256", signingSecret)
+
+  return {
+    ...params,
+    headers: {
+      ...(params.headers ?? {}),
+      "x-ov-cf-mac": mac,
+      "x-ov-cf-timestamp": timestamp.toString(),
+    },
+  }
+}
+
+type BodylessHttpFn = typeof get
+type BodiedHttpFn = typeof post
+
+function wrapHttpFunction<F extends BodylessHttpFn | BodiedHttpFn>(fn: F): F {
+  // url is always the first argument, params is always the last argument
+  const urlIdx = 0
+  const paramsIdx = fn.length - 1
+
+  return (<RT extends ResponseType | undefined>(...args: Parameters<F>) => {
+    const signedParams = getSignedParams(
+      args[urlIdx],
+      (args[paramsIdx] as RefinedParams<RT>) ?? {}
+    )
+
+    return paramsIdx === 1
+      ? (fn as BodylessHttpFn)(args[urlIdx], signedParams)
+      : (fn as BodiedHttpFn)(args[urlIdx], args[1] as RequestBody, signedParams)
+  }) as F
+}
+
+/**
+ * Wrapper around k6/http that signs requests with Openverse's
+ * HMAC shared secret, enabling firewall bypass for load testing.
+ */
+export const http = {
+  get: wrapHttpFunction(get),
+  post: wrapHttpFunction(post),
+  del: wrapHttpFunction(del),
+  options: wrapHttpFunction(options),
+  head: wrapHttpFunction(head),
+  put: wrapHttpFunction(put),
+  patch: wrapHttpFunction(patch),
+}

packages/js/k6/src/utils.ts

@@ -1,6 +1,3 @@
-import { check } from "k6"
-import { type Response } from "k6/http"
-
 // @ts-expect-error https://github.com/grafana/k6-template-typescript/issues/16
 // eslint-disable-next-line import/extensions, import/no-unresolved
 import { randomItem } from "https://jslib.k6.io/k6-utils/1.2.0/index.js"
@@ -13,18 +10,3 @@ const WORDS = open("/usr/share/dict/words")
   .filter((w) => !w.endsWith("'s"))
 
 export const getRandomWord = () => randomItem(WORDS)
-
-export const makeResponseFailedCheck = (param: string, page: string) => {
-  return (response: Response, action: string) => {
-    const requestDetail = `${param ? `for param "${param} "` : ""}at page ${page} for ${action}`
-    if (check(response, { "status was 200": (r) => r.status === 200 })) {
-      console.log(`Checked status 200 ✓ ${requestDetail}.`)
-      return false
-    } else {
-      console.error(
-        `Request failed ⨯ ${requestDetail}: ${response.status}\n${response.body}`
-      )
-      return true
-    }
-  }
-}