Update the % for ElasticSearch 6.7 AA test

The AA test is currently deployed at a 50/50 variant split. This PR updates the AA test to throttle it the split at 0.5%. The C variant has been added to make the difference between the CDN traffic allocation and anything that falls into the Z variant (handled by the rendering app) more explicit.
alphagov · Sep 4, 2023 · fdd38d7 · fdd38d7
1 parent a97b36b
commit fdd38d7
Show file tree

Hide file tree

Showing 10 changed files with 108 additions and 10 deletions.
diff --git a/ab_tests/ab_tests.yaml b/ab_tests/ab_tests.yaml
@@ -10,3 +10,4 @@
 - EsSixPointSeven:
   - A
   - B
+  - C
diff --git a/configs/dictionaries/essixpointseven_percentages.yaml b/configs/dictionaries/essixpointseven_percentages.yaml
@@ -1,3 +1,4 @@
 ---
-A: 50
-B: 50
+A: 0.5
+B: 0.5
+C: 99
diff --git a/spec/test-outputs/www-eks-integration.out.vcl b/spec/test-outputs/www-eks-integration.out.vcl
@@ -243,6 +243,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -257,11 +261,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-eks-production.out.vcl b/spec/test-outputs/www-eks-production.out.vcl
@@ -417,6 +417,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -431,11 +435,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-eks-staging.out.vcl b/spec/test-outputs/www-eks-staging.out.vcl
@@ -419,6 +419,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -433,11 +437,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-eks-test.out.vcl b/spec/test-outputs/www-eks-test.out.vcl
@@ -243,6 +243,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -257,11 +261,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-integration.out.vcl b/spec/test-outputs/www-integration.out.vcl
@@ -243,6 +243,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -257,11 +261,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-production.out.vcl b/spec/test-outputs/www-production.out.vcl
@@ -409,6 +409,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -423,11 +427,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-staging.out.vcl b/spec/test-outputs/www-staging.out.vcl
@@ -419,6 +419,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -433,11 +437,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }

diff --git a/spec/test-outputs/www-test.out.vcl b/spec/test-outputs/www-test.out.vcl
@@ -243,6 +243,10 @@ sub vcl_recv {
         # Some users, such as remote testers, will be given a URL with a query string
         # to place them into a specific bucket.
         set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+      } else if (req.url ~ "[\?\&]ABTest-EsSixPointSeven=C(&|$)") {
+        # Some users, such as remote testers, will be given a URL with a query string
+        # to place them into a specific bucket.
+        set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
       } else if (req.http.Cookie ~ "ABTest-EsSixPointSeven") {
         # Set the value of the header to whatever decision was previously made
         set req.http.GOVUK-ABTest-EsSixPointSeven = req.http.Cookie:ABTest-EsSixPointSeven;
@@ -257,11 +261,19 @@ sub vcl_recv {
         declare local var.nominator_EsSixPointSeven_B INTEGER;
         set var.nominator_EsSixPointSeven_B = std.atoi(table.lookup(essixpointseven_percentages, "B"));
         set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_B;
+        declare local var.denominator_EsSixPointSeven_C INTEGER;
+        declare local var.nominator_EsSixPointSeven_C INTEGER;
+        set var.nominator_EsSixPointSeven_C = std.atoi(table.lookup(essixpointseven_percentages, "C"));
+        set var.denominator_EsSixPointSeven += var.nominator_EsSixPointSeven_C;
         set var.denominator_EsSixPointSeven_A = var.denominator_EsSixPointSeven;
+        set var.denominator_EsSixPointSeven_B = var.denominator_EsSixPointSeven_A;
+        set var.denominator_EsSixPointSeven_B -= var.nominator_EsSixPointSeven_A;
         if (randombool(var.nominator_EsSixPointSeven_A, var.denominator_EsSixPointSeven_A)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "A";
-        } else {
+        } else if (randombool(var.nominator_EsSixPointSeven_B, var.denominator_EsSixPointSeven_B)) {
           set req.http.GOVUK-ABTest-EsSixPointSeven = "B";
+        } else {
+          set req.http.GOVUK-ABTest-EsSixPointSeven = "C";
         }
       }
     }