From 13e467332a9df60a68af3f5506856061c32e3ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 15 Oct 2024 14:37:29 +0300 Subject: [PATCH 1/2] terraform: fix devshell --- terraform-iam/.envrc | 2 +- terraform/flake-module.nix | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/terraform-iam/.envrc b/terraform-iam/.envrc index e1ef6425..11f4c17d 100644 --- a/terraform-iam/.envrc +++ b/terraform-iam/.envrc @@ -1,4 +1,4 @@ -use flake .#terraform +use flake .#terraform-iam export AWS_CONFIG_FILE=$PWD/aws-config export AWS_PROFILE=nixos-prod diff --git a/terraform/flake-module.nix b/terraform/flake-module.nix index 36c4eaba..9d27966c 100644 --- a/terraform/flake-module.nix +++ b/terraform/flake-module.nix @@ -12,6 +12,20 @@ in { pkgs, ... }: { devShells.terraform = pkgs.mkShellNoCC { + packages = [ + pkgs.awscli2 + # TODO: migrate registry for opentofu as well. + (pkgs.opentofu.withPlugins (p: [ + p.aws + p.fastly + p.netlify + p.secret + ])) + ]; + }; + + # get rid of this, once we fix the migration above. + devShells.terraform-iam = pkgs.mkShellNoCC { packages = [ pkgs.awscli2 (pkgs.opentofu.withPlugins ( From 0bf990234d5eca926a3219f40019dbd3ade06ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 15 Oct 2024 14:39:39 +0300 Subject: [PATCH 2/2] introduce staging binary cache to migration This introduces two buckets. One will be the old cache that we want to move to glacier eventually. The other bucket is the new cache that we want to use in the future. --- terraform/cache-bucket/main.tf | 132 ++++++++ terraform/cache-bucket/providers.tf | 7 + terraform/cache-staging.tf | 333 ++++++++++++++++++++ terraform/cache-staging/diagnostic.sh | 54 ++++ terraform/cache-staging/index.html | 60 ++++ terraform/cache-staging/new-cache-test-file | 1 + terraform/cache-staging/nix-cache-info | 3 + terraform/cache-staging/old-cache-test-file | 1 + terraform/cache-staging/s3-authn.vcl | 64 ++++ terraform/dns.tf | 5 + 10 files changed, 660 insertions(+) create mode 100644 terraform/cache-bucket/main.tf create mode 100644 terraform/cache-bucket/providers.tf create mode 100644 terraform/cache-staging.tf create mode 100755 terraform/cache-staging/diagnostic.sh create mode 100644 terraform/cache-staging/index.html create mode 100644 terraform/cache-staging/new-cache-test-file create mode 100644 terraform/cache-staging/nix-cache-info create mode 100644 terraform/cache-staging/old-cache-test-file create mode 100644 terraform/cache-staging/s3-authn.vcl diff --git a/terraform/cache-bucket/main.tf b/terraform/cache-bucket/main.tf new file mode 100644 index 00000000..ddf31952 --- /dev/null +++ b/terraform/cache-bucket/main.tf @@ -0,0 +1,132 @@ +variable "bucket_name" { + type = string +} + +resource "aws_s3_bucket" "cache" { + provider = aws + bucket = var.bucket_name + + lifecycle_rule { + enabled = true + + transition { + days = 365 + storage_class = "STANDARD_IA" + } + } + + cors_rule { + allowed_headers = ["Authorization"] + allowed_methods = ["GET"] + allowed_origins = ["*"] + max_age_seconds = 3000 + } +} + +resource "aws_s3_bucket_public_access_block" "cache" { + bucket = aws_s3_bucket.cache.bucket + + block_public_acls = false + block_public_policy = false +} + +resource "aws_s3_bucket_object" "cache-nix-cache-info" { + provider = aws + depends_on = [aws_s3_bucket_public_access_block.cache] + + bucket = aws_s3_bucket.cache.bucket + content_type = "text/x-nix-cache-info" + etag = filemd5("${path.module}/../cache-staging/nix-cache-info") + key = "nix-cache-info" + source = "${path.module}/../cache-staging/nix-cache-info" +} + +resource "aws_s3_bucket_object" "cache-index-html" { + provider = aws + depends_on = [aws_s3_bucket_public_access_block.cache] + + bucket = aws_s3_bucket.cache.bucket + content_type = "text/html" + etag = filemd5("${path.module}/../cache-staging/index.html") + key = "index.html" + source = "${path.module}/../cache-staging/index.html" +} + +resource "aws_s3_bucket_policy" "cache" { + provider = aws + bucket = aws_s3_bucket.cache.id + depends_on = [aws_s3_bucket_public_access_block.cache] + + # imported from existing + policy = <S3 requests. See Fastly documentation: + # https://docs.fastly.com/en/guides/amazon-s3#using-an-amazon-s3-private-bucket + snippet { + name = "Authenticate S3 requests for new bucket" + type = "miss" + priority = 100 + content = templatefile("${path.module}/cache-staging/s3-authn.vcl", { + backend_name = "F_new_bucket" + aws_region = module.cache-staging-202410.region + bucket = module.cache-staging-202410.bucket + backend_domain = module.cache-staging-202410.bucket_domain_name + access_key = local.cache-iam.key + secret_key = local.cache-iam.secret + }) + } + + snippet { + name = "Authenticate S3 requests for old bucket" + type = "miss" + priority = 100 + content = templatefile("${path.module}/cache-staging/s3-authn.vcl", { + backend_name = "F_old_bucket" + aws_region = module.cache-staging-202010.region + bucket = module.cache-staging-202010.bucket + backend_domain = module.cache-staging-202010.bucket_domain_name + access_key = local.cache-iam.key + secret_key = local.cache-iam.secret + }) + } + + snippet { + content = "set req.url = querystring.remove(req.url);" + name = "Remove all query strings" + priority = 50 + type = "recv" + } + + + # Work around the 2GB size limit for large files + # + # See https://docs.fastly.com/en/guides/segmented-caching + snippet { + content = <<-EOT + if (req.url.path ~ "^/nar/") { + set req.enable_segmented_caching = true; + } + EOT + name = "Enable segment caching for NAR files" + priority = 60 + type = "recv" + } + + snippet { + name = "Fallback to old bucket on 403 or return 404" + type = "fetch" + priority = 90 + content = <<-EOT + if (beresp.status == 403) { + if (req.backend == F_new_bucket) { + restart; + } else { + set beresp.status = 404; + } + } + EOT + } + + # We will switch to this snipped once we retire the old bucket instead of the fallback above + #snippet { + # name = "Return 404 on 403" + # type = "fetch" + # priority = 90 + # content = <<-EOT + # if (beresp.status == 403) { + # set beresp.status = 404; + # } + # EOT + #} + + # Add a snippet to set a custom header based on the backend used + snippet { + name = "Set-Backend-Header" + type = "deliver" + priority = 70 + content = <<-EOT + if (req.backend == F_old_bucket) { + set resp.http.X-Bucket = "${module.cache-staging-202010.bucket}"; + } else if (req.backend == F_new_bucket) { + set resp.http.X-Bucket = "${module.cache-staging-202410.bucket}"; + } + EOT + } + + logging_s3 { + name = "${local.cache_staging_domain}-to-s3" + bucket_name = local.fastlylogs["bucket_name"] + compression_codec = "zstd" + domain = local.fastlylogs["s3_domain"] + format = local.fastlylogs["format"] + format_version = 2 + path = "${local.cache_staging_domain}/" + period = local.fastlylogs["period"] + message_type = "blank" + s3_iam_role = local.fastlylogs["iam_role_arn"] + } +} + +resource "fastly_tls_subscription" "cache-staging" { + domains = [for domain in fastly_service_vcl.cache-staging.domain : domain.name] + configuration_id = local.fastly_tls12_sni_configuration_id + certificate_authority = "globalsign" +} diff --git a/terraform/cache-staging/diagnostic.sh b/terraform/cache-staging/diagnostic.sh new file mode 100755 index 00000000..0b713942 --- /dev/null +++ b/terraform/cache-staging/diagnostic.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bind.dnsutils -p mtr -p curl +# shellcheck shell=bash +# impure: needs ping +# +# Run this script if you are having issues with cache.nixos.org and paste the +# output URL in a new issue in the same repo. +# + +domain=${1:-cache-staging.nixos.org} + +run() { + echo "> $*" + "$@" |& sed -e "s/^/ /" + printf "Exit: %s\n\n\n" "$?" +} + +curl_w=" +time_namelookup: %{time_namelookup} +time_connect: %{time_connect} +time_appconnect: %{time_appconnect} +time_pretransfer: %{time_pretransfer} +time_redirect: %{time_redirect} +time_starttransfer: %{time_starttransfer} +time_total: %{time_total} +" + +curl_test() { + curl -w "$curl_w" -v -o /dev/null "$@" +} + +ix() { + url=$(cat | curl -F 'f:1=<-' ix.io 2>/dev/null) + echo "Pasted at: $url" +} + +( + echo "domain=$domain" + run dig -t A "$domain" + run ping -c1 "$domain" + run ping -4 -c1 "$domain" + run ping -6 -c1 "$domain" + run mtr -c 20 -w -r "$domain" + run curl_test -4 "http://$domain/" + run curl_test -6 "http://$domain/" + run curl_test -4 "https://$domain/" + run curl_test -6 "https://$domain/" + run curl -I -4 "https://$domain/" + run curl -I -4 "https://$domain/" + run curl -I -4 "https://$domain/" + run curl -I -6 "https://$domain/" + run curl -I -6 "https://$domain/" + run curl -I -6 "https://$domain/" +) | tee /dev/stderr | ix diff --git a/terraform/cache-staging/index.html b/terraform/cache-staging/index.html new file mode 100644 index 00000000..0cc5df27 --- /dev/null +++ b/terraform/cache-staging/index.html @@ -0,0 +1,60 @@ + + + + cache-staging.nixos.org is up + + + + + + + + +
+
+

+ + logo + +

+ +

+ https://cache.nixos.org/ provides + prebuilt binaries for Nixpkgs and NixOS. It is + used automatically by the Nix package manager to + speed up builds. +

+

+
+
+
+

+ If you are having trouble, please reach out through one of the + support channels + with the results of + this diagnostics script + which will help us figure out where the issue lies. +

+

+ For questions, or support, + the support page from the NixOS website describes how to get in touch. +

+
+
+ + diff --git a/terraform/cache-staging/new-cache-test-file b/terraform/cache-staging/new-cache-test-file new file mode 100644 index 00000000..3e757656 --- /dev/null +++ b/terraform/cache-staging/new-cache-test-file @@ -0,0 +1 @@ +new diff --git a/terraform/cache-staging/nix-cache-info b/terraform/cache-staging/nix-cache-info new file mode 100644 index 00000000..7d239de8 --- /dev/null +++ b/terraform/cache-staging/nix-cache-info @@ -0,0 +1,3 @@ +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 40 diff --git a/terraform/cache-staging/old-cache-test-file b/terraform/cache-staging/old-cache-test-file new file mode 100644 index 00000000..3367afdb --- /dev/null +++ b/terraform/cache-staging/old-cache-test-file @@ -0,0 +1 @@ +old diff --git a/terraform/cache-staging/s3-authn.vcl b/terraform/cache-staging/s3-authn.vcl new file mode 100644 index 00000000..f58555b5 --- /dev/null +++ b/terraform/cache-staging/s3-authn.vcl @@ -0,0 +1,64 @@ +# VCL snippet to authenticate Fastly<->S3 requests. +# +# https://docs.fastly.com/en/guides/amazon-s3#using-an-amazon-s3-private-bucket + +if (req.method == "GET" && !req.backend.is_shield && req.backend == ${backend_name}) { + set var.awsAccessKey = "${access_key}"; + set var.awsSecretKey = "${secret_key}"; + set var.awsS3Bucket = "${bucket}"; + set var.awsRegion = "${aws_region}"; # Change this value to your own data + set var.awsS3Host = var.awsS3Bucket ".s3." var.awsRegion ".amazonaws.com"; + + set bereq.http.x-amz-content-sha256 = digest.hash_sha256(""); + set bereq.http.x-amz-date = strftime({"%Y%m%dT%H%M%SZ"}, now); + set bereq.http.x-amz-request-payer = "requester"; + set bereq.http.host = var.awsS3Host; + + set bereq.url = querystring.remove(bereq.url); + set bereq.url = regsuball(urlencode(urldecode(bereq.url.path)), {"%2F"}, "/"); + set var.dateStamp = strftime({"%Y%m%d"}, now); + set var.canonicalHeaders = "" + "host:" bereq.http.host LF + "x-amz-content-sha256:" bereq.http.x-amz-content-sha256 LF + "x-amz-date:" bereq.http.x-amz-date LF + "x-amz-request-payer:" bereq.http.x-amz-request-payer LF + ; + set var.canonicalQuery = ""; + set var.signedHeaders = "host;x-amz-content-sha256;x-amz-date;x-amz-request-payer"; + set var.canonicalRequest = "" + "GET" LF + bereq.url.path LF + var.canonicalQuery LF + var.canonicalHeaders LF + var.signedHeaders LF + digest.hash_sha256("") + ; + + set var.scope = var.dateStamp "/" var.awsRegion "/s3/aws4_request"; + + set var.stringToSign = "" + "AWS4-HMAC-SHA256" LF + bereq.http.x-amz-date LF + var.scope LF + regsub(digest.hash_sha256(var.canonicalRequest),"^0x", "") + ; + + set var.signature = digest.awsv4_hmac( + var.awsSecretKey, + var.dateStamp, + var.awsRegion, + "s3", + var.stringToSign + ); + + set bereq.http.Authorization = "AWS4-HMAC-SHA256 " + "Credential=${access_key}/" var.scope ", " + "SignedHeaders=" var.signedHeaders ", " + "Signature=" + regsub(var.signature,"^0x", "") + ; + + unset bereq.http.Accept; + unset bereq.http.Accept-Language; + unset bereq.http.User-Agent; + unset bereq.http.Fastly-Client-IP; +} diff --git a/terraform/dns.tf b/terraform/dns.tf index aa63051e..f47aefdb 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -131,6 +131,11 @@ locals { type = "CNAME" value = "dualstack.v2.shared.global.fastly.net" }, + { + hostname = "cache-staging.nixos.org" + type = "CNAME" + value = "dualstack.v2.shared.global.fastly.net" + }, { hostname = "channels.nixos.org" type = "CNAME"