chore: use brotli compression in the app

neherlab · Aug 25, 2022 · b77a098 · b77a098
1 parent 1df6e4b
commit b77a098
Show file tree

Hide file tree

Showing 5 changed files with 66 additions and 89 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -55,8 +55,8 @@ jobs:
 
       - name: Install system dependencies
         run: |
-          sudo apt-get update -qq --yes
-          sudo apt-get install -qq --yes --verbose-versions pigz rename
+          sudo apt-get update -qq --yes >/dev/null
+          sudo apt-get install -qq --yes brotli pigz rename >/dev/null
 
       - name: Install NPM dependencies
         run: |
@@ -88,74 +88,4 @@ jobs:
       - name: Precompress files
         if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
         run: |
-          pigz -dfrq public/ || true
-          pigz -kfrq public/
-
-      - name: Upload non-compressed non-HTML files
-        if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
-        run: |
-          aws s3 sync \
-            --delete \
-            --only-show-errors \
-            --cache-control "max-age=2592000, public" \
-            --metadata-directive REPLACE \
-            --exclude "*.git*" \
-            --exclude "*.html" \
-            --exclude "*.gz" \
-            "public/" "s3://${AWS_S3_BUCKET}/"
-
-      - name: Upload compressed non-HTML files
-        if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
-        run: |
-          aws s3 sync \
-            --delete \
-            --only-show-errors \
-            --cache-control "max-age=2592000, public" \
-            --content-encoding=gzip \
-            --metadata-directive REPLACE \
-            --exclude "*" \
-            --include "*.gz" \
-            --exclude "*.git*" \
-            --exclude "*.html.gz" \
-            "public/" "s3://${AWS_S3_BUCKET}/"
-
-      - name: Remove non-HTML files
-        if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
-        run: |
-          find public/ -type f \( ! -iname "*.html" -a ! -iname "*.html.gz" \) -exec rm {} \+
-          rename --filename 's/\.html//' public/*.html || true
-          rename --filename 's/\.html//' public/*.html.gz || true
-
-      - name: Upload non-compressed HTML files
-        if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
-        run: |
-          aws s3 sync \
-            --delete \
-            --only-show-errors \
-            --content-type "text/html" \
-            --cache-control "max-age=86400, public" \
-            --metadata-directive REPLACE \
-            --exclude "*.*" \
-            "public/" "s3://${AWS_S3_BUCKET}/"
-
-      - name: Upload compressed HTML files
-        if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
-        run: |
-          aws s3 sync \
-            --delete \
-            --only-show-errors \
-            --content-type "text/html" \
-            --cache-control "max-age=86400, public" \
-            --content-encoding=gzip \
-            --metadata-directive REPLACE \
-            --exclude "*" \
-            --include "*.gz" \
-            --exclude "*.*.gz" \
-            "public/" "s3://${AWS_S3_BUCKET}/"
-
-      - name: Invalidate AWS Cloudfront cache
-        if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release')
-        run: |
-          aws cloudfront create-invalidation \
-            --distribution-id ${AWS_CLOUDFRONT_DISTRIBUTION_ID} \
-            --paths "/*"
+          ./scripts/deploy_app.sh
diff --git a/infra/iam-role-trust-for-lambda-at-edge.json b/infra/iam-role-trust-for-lambda-at-edge.json
@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": [
+          "lambda.amazonaws.com",
+          "edgelambda.amazonaws.com"
+        ]
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
diff --git a/infra/web/lambda-at-edge/OriginRequest.lambda.js b/infra/web/lambda-at-edge/OriginRequest.lambda.js
@@ -1,8 +1,9 @@
-// Implements rewrite of non-gz to gz URLs using AWS Lambda@Edge. This is
-// useful if you have precompressed your files.
+// Implements rewrite of non-compressed to .gz or .br URLs using AWS
+// Lambda@Edge. This is useful if you have precompressed your files.
 //
-// Usage: Create an AWS Lambda function and attach it to "Origin Request" event
-// of a Cloudfront distribution
+// Usage:
+// Create an AWS Lambda function and attach it to "Origin Request" event of a
+// Cloudfront distribution
 
 const ARCHIVE_EXTS = [
   '.7z',
@@ -38,7 +39,10 @@ function handler(event, context, callback) {
   // If not an archive file (which are not precompressed), rewrite the URL to
   // get the corresponding .gz file
   if(!ARCHIVE_EXTS.every(ext => request.uri.endsWith(ext))) {
-    if(acceptsEncoding(headers, 'gzip')) {
+    if(acceptsEncoding(headers, 'br')) {
+      request.uri += '.br'
+    }
+    else if(acceptsEncoding(headers, 'gzip')) {
       request.uri += '.gz'
     }
   }
@@ -47,4 +51,3 @@ function handler(event, context, callback) {
 }
 
 exports.handler = handler
-
diff --git a/package.json b/package.json
@@ -4,7 +4,7 @@
   "description": "pan-genome-visualization",
   "private": true,
   "scripts": {
-    "clean": "rm -rf public/*.html public/*.html.gz public/dist/*",
+    "clean": "rm -rf public/*.html public/*.html.gz public/*.br public/dist/*",
     "build": "NODE_ENV=production npm run build:webpack && NODE_ENV=production npm run build:gulp && NODE_ENV=production npm run build:html",
     "build:webpack": "NODE_ENV=production webpack --env production",
     "build:gulp": "NODE_ENV=production gulp",

diff --git a/scripts/deploy_app.sh b/scripts/deploy_app.sh
@@ -1,17 +1,18 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -euxo pipefail
 trap "exit" INT
 
-source .env
+[ -f ".env" ] && source .env
 
-# Remove unused
+# Remove unused files
 find public/dataset/* -type d -exec rm -rf {} \+
 rm -rf public/download
 rm -rf public/phyloTree
 
 # Precompress files
 pigz -dfrq public/ || true
 pigz -kfrq public/
+find public/ -type f \( ! -iname "*.gz" -a ! -iname "*.br" \) | parallel brotli -kf || true
 
 ## Clear bucket
 #aws s3 rm \
@@ -28,25 +29,40 @@ aws s3 sync \
   --exclude "*.git*" \
   --exclude "*.html" \
   --exclude "*.gz" \
+  --exclude "*.br" \
   "public/" "s3://${AWS_S3_BUCKET}/"
 
-# Upload compressed non-HTML files
+# Upload gzip-compressed non-HTML files
 aws s3 sync \
   --delete \
   --only-show-errors \
   --cache-control "max-age=2592000, public" \
-  --content-encoding=gzip \
+  --content-encoding gzip \
   --metadata-directive REPLACE \
   --exclude "*" \
   --include "*.gz" \
   --exclude "*.git*" \
-  --exclude "*.html.gz" \
+  --exclude "*.html*" \
+  "public/" "s3://${AWS_S3_BUCKET}/"
+
+# Upload brotli-compressed non-HTML files
+aws s3 sync \
+  --delete \
+  --only-show-errors \
+  --cache-control "max-age=2592000, public" \
+  --content-encoding br \
+  --metadata-directive REPLACE \
+  --exclude "*" \
+  --include "*.br" \
+  --exclude "*.git*" \
+  --exclude "*.html*" \
   "public/" "s3://${AWS_S3_BUCKET}/"
 
 # Remove non-HTML files
-find public/ -type f \( ! -iname "*.html" -a ! -iname "*.html.gz" \) -exec rm {} \+
+find public/ -type f \( ! -iname "*.html" -a ! -iname "*.html.gz" -a ! -iname "*.html.br" \) -exec rm {} \+
 rename --filename 's/\.html//' public/*.html || true
 rename --filename 's/\.html//' public/*.html.gz || true
+rename --filename 's/\.html//' public/*.html.br || true
 
 # Upload non-compressed HTML files
 aws s3 sync \
@@ -58,19 +74,32 @@ aws s3 sync \
   --exclude "*.*" \
   "public/" "s3://${AWS_S3_BUCKET}/"
 
-# Upload compressed HTML files
+# Upload gzip-compressed HTML files
 aws s3 sync \
   --delete \
   --only-show-errors \
   --content-type "text/html" \
   --cache-control "max-age=86400, public" \
-  --content-encoding=gzip \
+  --content-encoding gzip \
   --metadata-directive REPLACE \
   --exclude "*" \
   --include "*.gz" \
   --exclude "*.*.gz" \
   "public/" "s3://${AWS_S3_BUCKET}/"
 
+# Upload brotli-compressed HTML files
+aws s3 sync \
+  --delete \
+  --only-show-errors \
+  --content-type "text/html" \
+  --cache-control "max-age=86400, public" \
+  --content-encoding br \
+  --metadata-directive REPLACE \
+  --exclude "*" \
+  --include "*.br" \
+  --exclude "*.*.br" \
+  "public/" "s3://${AWS_S3_BUCKET}/"
+
 # Update Cloudfront cache
 aws cloudfront create-invalidation \
   --no-paginate \