diff --git a/test/load-test/k6/.gitignore b/test/load-test/k6/.gitignore new file mode 100644 index 0000000000..df9d16b899 --- /dev/null +++ b/test/load-test/k6/.gitignore @@ -0,0 +1 @@ +_results/ diff --git a/test/load-test/k6/README.md b/test/load-test/k6/README.md new file mode 100644 index 0000000000..e9bcbfe493 --- /dev/null +++ b/test/load-test/k6/README.md @@ -0,0 +1,45 @@ +# k6 Load Tests + +This directory contains a [k6](https://grafana.com/docs/k6/latest/) script for +load testing ocaml.org endpoints. + +- The k6 test script is defined in [./script.js](./script.js). +- The runner script [./run.sh](./run.sh) will run `./script.js` with the k6 + docker image. + + +## Running load tests + +### Prerequisites + +- docker +- k6parser +- xdg-open (MacOS users may need to adjust the runner script) + +### Usage + +``` sh +./run.sh script.js +``` + +will run a load test against all defined endpoints using 10 virtual users, and +running for 30 seconds. + +You can specify different values for the concurrent users and duration using the +`--vus` and `--duration` flags, respectively. E.g., to test with 100 virtual +users for one minute: + +``` sh +./run.sh --vus 100 --duration 1m script.js +``` + +## Reviewing the results + +k6 will print out the a summary of results when the test is finished. See + for +documentation on how to interpret the results. + +- Detailed results are written to a gzipped JOSN file + `_results/{day}T{time}-results.gz`. +- A browser will be opened with a visualization plotting general responsiveness, + loaded with the file `_results/{day}T{time}-results.report`. diff --git a/test/load-test/k6/run.sh b/test/load-test/k6/run.sh new file mode 100755 index 0000000000..6842ae3942 --- /dev/null +++ b/test/load-test/k6/run.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env sh + +# Ignore posix compliance errors in shellcheck +#shellcheck disable=SC3000-SC3061 + +set -eu +set -o pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +RESULTS_DIR=_results +USER_ID=$(id -u) # ensure we can write to the mounted dir +TIME=$(date +%FT%T) +RESULTS="${RESULTS_DIR}/${TIME}-results.gz" +GRAPH="${RESULTS_DIR}/${TIME}-report.html" + + +mkdir "$RESULTS_DIR" + +docker run --rm \ + -u "$USER_ID" \ + -v "$SCRIPT_DIR":/home/k6 grafana/k6 \ + --out json="$RESULTS" \ + "$@" + +k6parser "$RESULTS" --output "$GRAPH" + +xdg-open "$GRAPH" diff --git a/test/load-test/k6/script.js b/test/load-test/k6/script.js new file mode 100644 index 0000000000..b945b6470b --- /dev/null +++ b/test/load-test/k6/script.js @@ -0,0 +1,89 @@ +import http from "k6/http"; +import { sleep } from "k6"; + +export const options = { + // A number specifying the number of VUs to run concurrently. + vus: 10, + // A string specifying the total duration of the test run. + duration: "30s", + + // The following section contains configuration options for execution of this + // test script in Grafana Cloud. + // + // See https://grafana.com/docs/grafana-cloud/k6/get-started/run-cloud-tests-from-the-cli/ + // to learn about authoring and running k6 test scripts in Grafana k6 Cloud. + // + // cloud: { + // // The ID of the project to which the test is assigned in the k6 Cloud UI. + // // By default tests are executed in default project. + // projectID: "", + // // The name of the test in the k6 Cloud UI. + // // Test runs with the same name will be grouped. + // name: "script.js" + // }, + + // Uncomment this section to enable the use of Browser API in your tests. + // + // See https://grafana.com/docs/k6/latest/using-k6-browser/running-browser-tests/ to learn more + // about using Browser API in your test scripts. + // + // scenarios: { + // // The scenario name appears in the result summary, tags, and so on. + // // You can give the scenario any name, as long as each name in the script is unique. + // ui: { + // // Executor is a mandatory parameter for browser-based tests. + // // Shared iterations in this case tells k6 to reuse VUs to execute iterations. + // // + // // See https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/ for other executor types. + // executor: 'shared-iterations', + // options: { + // browser: { + // // This is a mandatory parameter that instructs k6 to launch and + // // connect to a chromium-based browser, and use it to run UI-based + // // tests. + // type: 'chromium', + // }, + // }, + // }, + // } +}; + +// The function that defines VU logic. +// +// See https://grafana.com/docs/k6/latest/examples/get-started-with-k6/ to learn more +// about authoring k6 scripts. + +const base_url = "https://staging.ocaml.org"; +// base_url = "https://ocaml.com" + +function endpoint(u) { + return base_url + u; +} + +export default function () { + // Landing page + http.get(endpoint("/")); + + // Core doc page + http.get(endpoint("/p/core/latest/doc/index.html")); + + // Top level pages + http.get(endpoint("/install")); + http.get(endpoint("/docs/tour-of-ocaml")); + http.get(endpoint("/docs")); + http.get(endpoint("/platform")); + http.get(endpoint("/packages")); + http.get(endpoint("/community")); + http.get(endpoint("/changelog")); + http.get(endpoint("/play")); + http.get(endpoint("/industrial-users")); + http.get(endpoint("/academic-users")); + + // some package searches + // Grouping the urls, see https://grafana.com/docs/k6/latest/using-k6/http-requests/#url-grouping + const package_search_tag = { tags: { name: "PacakageSearch" } }; + ["http", "server", "cli", "core", "eio", "graph"].forEach((q) => { + http.get(endpoint(`/packages/autocomplete?q=${q}`), package_search_tag); + http.get(endpoint(`/packages/search?q=${q}`), package_search_tag); + }); +} diff --git a/test/load-test/locust/.gitignore b/test/load-test/locust/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/test/load-test/locust/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/test/load-test/locust/README.md b/test/load-test/locust/README.md new file mode 100644 index 0000000000..7c3418f0c9 --- /dev/null +++ b/test/load-test/locust/README.md @@ -0,0 +1,40 @@ +# Locust Load Tests + +This directory contains a [locust](https://locust.io/) script for load testing +ocaml.org endpoints. + +## Running simple load tests + +1. Start the test framework running with + + ``` sh + ./main.sh + ``` + +2. Navigate to http://0.0.0.0:8089 + +3. Configure + - the max number of users to simulate + - the number of new users to add to the simulation every second + - the host (`https://staging.ocaml.org`, `https://ocaml.org`, etc.) + +## Running load tests with multiple cores + +``` sh +./main.sh n +``` + +where `n` is the number of processes to run concurrently. + +## Reviewing the load tests + +- Click "Stop" when you are finished running your test. +- Review the various tabs, or click "Download data" for options to download the + test results. +- You can also review prometheus metrics about the staging and prod servers at + https://status.ocaml.ci.dev/d/be358r0z9ai9sf/ocaml-org + +## Adding new routines + +Tests are defined as "tasks" (sequences of site traversal) in +[./locustfile.py](./locustfile.py). diff --git a/test/load-test/locust/locustfile.py b/test/load-test/locust/locustfile.py new file mode 100644 index 0000000000..7829c68508 --- /dev/null +++ b/test/load-test/locust/locustfile.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +import random +from locust import FastHttpUser, task, between + +# obtained via https://random-word-api.herokuapp.com/word?number=200 +words = ["sacraria","seedcake","philtered","leadiest","cloverleafs","snaffle","lyrisms","ankhs","bedtimes","carrier","restabling","playlets","occupation","overreport","printers","scintigraphic","spa","corsages","enwound","gossipmonger","hydragog","waterlily","avulsions","sonneteerings","spilt","hemocytes","tamandus","rais","minaret","coliseums","ultramicrotome","attribute","phosphite","scincoids","scooting","frightfulnesses","carbamides","sculpture","irresponsive","overtasks","expertise","weet","consociations","tulles","hared","pigginesses","oversimple","theologs","adverb","inamoratas","teeny","rapacities","assonant","metestrus","cyanohydrin","smiting","polychete","merest","tautological","phyllopod","petahertz","plainspokenness","pavior","penitently","omikrons","cigarlike","foetal","diebacks","downlight","kinship","warmish","titleholders","suppositions","resuscitations","tiffing","outsung","homed","alternated","cranks","piaster","allotters","nonirradiated","protohistories","finned","decouple","shahs","foeman","perfidious","soarers","thoroughpins","gastrulating","thrivers","convention","roughened","uncircumcised","clabbering","leadscrew","panfuls","nilgais","evolver","overvoting","furrower","ichthyosaurs","internalizes","borschts","regrouping","lordlier","roguish","microseismicity","besmiling","mattoids","cholerically","fibrosarcomas","farinha","curricles","triradiate","beringed","electrolysis","kashmirs","dirdums","ignorami","otalgic","lusciously","blotty","pizzaz","educe","pendant","disposable","autolyzes","outjutting","interfused","operagoers","fustian","theretofore","dean","unsullied","goitrogenic","ultrasafe","potboil","geochemistries","outdesigned","ephedras","woodlore","illuminatingly","guardrooms","sheldrakes","leachable","theistically","reconception","beachboys","recriminates","almuds","changeabilities","flareups","machinate","verbalizations","dendrologist","unkept","copulatives","restyles","parceled","caecilians","mortgager","thunderstones","labarums","wiliness","hydroplane","already","unlatch","swineherds","alternate","whodunit","hoodiest","sainted","detract","inspiring","fantastically","macaws","adsorbs","thickets","blogs","greenfields","ariettes","camphor","hornpipe","uninventive","boatyard","boomiest","lollingly","congresspersons","painter","radiocarbons","impiously","unfeigned","matchlocks","screwballs","stickies","muddlers","resentful","meats"] + +class OcamlOrgUsere(FastHttpUser): + wait_time = between(1, 5) # range of seconds a user waits between clicks + + @task + def landing(self): + self.client.get("/") + self.client.get("/install") + self.client.get("/p/core/latest/doc/index.html") + self.client.get("/docs/tour-of-ocaml") + + @task + def top_level_pages(self): + self.client.get("/docs") + self.client.get("/platform") + self.client.get("/packages") + self.client.get("/community") + self.client.get("/changelog") + self.client.get("/play") + self.client.get("/industrial-users") + self.client.get("/academic-users") + + @task + def package_searches(self): + query = random.choice(words) + self.client.get(f"/packages/autocomplete?q={query}") + self.client.get(f"/packages/search?q={query}") diff --git a/test/load-test/locust/main.sh b/test/load-test/locust/main.sh new file mode 100755 index 0000000000..4ea0b3dd4c --- /dev/null +++ b/test/load-test/locust/main.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# Ignore posix compliance errors in shellcheck +#shellcheck disable=SC3000-SC3061 + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# see https://docs.locust.io/en/stable/running-distributed.html#distributed-load-generation +n_procs="$1" + +docker run -p 8089:8089 -v "$SCRIPT_DIR":/mnt/locust locustio/locust \ + --locustfile /mnt/locust/locustfile.py \ + ${n_procs:+"--processes=${n_procs}"} # Build the --processes flag if n_procs is not nil