From b77a098f9370f10c67fbdbbb5e15c04dc53225bb Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 25 Aug 2022 04:05:50 +0200 Subject: [PATCH] chore: use brotli compression in the app --- .github/workflows/ci.yml | 76 +------------------ infra/iam-role-trust-for-lambda-at-edge.json | 15 ++++ .../lambda-at-edge/OriginRequest.lambda.js | 15 ++-- package.json | 2 +- scripts/deploy_app.sh | 47 +++++++++--- 5 files changed, 66 insertions(+), 89 deletions(-) create mode 100644 infra/iam-role-trust-for-lambda-at-edge.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 940b956..d7535be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,8 +55,8 @@ jobs: - name: Install system dependencies run: | - sudo apt-get update -qq --yes - sudo apt-get install -qq --yes --verbose-versions pigz rename + sudo apt-get update -qq --yes >/dev/null + sudo apt-get install -qq --yes brotli pigz rename >/dev/null - name: Install NPM dependencies run: | @@ -88,74 +88,4 @@ jobs: - name: Precompress files if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') run: | - pigz -dfrq public/ || true - pigz -kfrq public/ - - - name: Upload non-compressed non-HTML files - if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') - run: | - aws s3 sync \ - --delete \ - --only-show-errors \ - --cache-control "max-age=2592000, public" \ - --metadata-directive REPLACE \ - --exclude "*.git*" \ - --exclude "*.html" \ - --exclude "*.gz" \ - "public/" "s3://${AWS_S3_BUCKET}/" - - - name: Upload compressed non-HTML files - if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') - run: | - aws s3 sync \ - --delete \ - --only-show-errors \ - --cache-control "max-age=2592000, public" \ - --content-encoding=gzip \ - --metadata-directive REPLACE \ - --exclude "*" \ - --include "*.gz" \ - --exclude "*.git*" \ - --exclude "*.html.gz" \ - "public/" "s3://${AWS_S3_BUCKET}/" - - - name: Remove non-HTML files - if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') - run: | - find public/ -type f \( ! -iname "*.html" -a ! -iname "*.html.gz" \) -exec rm {} \+ - rename --filename 's/\.html//' public/*.html || true - rename --filename 's/\.html//' public/*.html.gz || true - - - name: Upload non-compressed HTML files - if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') - run: | - aws s3 sync \ - --delete \ - --only-show-errors \ - --content-type "text/html" \ - --cache-control "max-age=86400, public" \ - --metadata-directive REPLACE \ - --exclude "*.*" \ - "public/" "s3://${AWS_S3_BUCKET}/" - - - name: Upload compressed HTML files - if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') - run: | - aws s3 sync \ - --delete \ - --only-show-errors \ - --content-type "text/html" \ - --cache-control "max-age=86400, public" \ - --content-encoding=gzip \ - --metadata-directive REPLACE \ - --exclude "*" \ - --include "*.gz" \ - --exclude "*.*.gz" \ - "public/" "s3://${AWS_S3_BUCKET}/" - - - name: Invalidate AWS Cloudfront cache - if: endsWith(github.ref, '/static-app') || endsWith(github.ref, '/release') - run: | - aws cloudfront create-invalidation \ - --distribution-id ${AWS_CLOUDFRONT_DISTRIBUTION_ID} \ - --paths "/*" + ./scripts/deploy_app.sh diff --git a/infra/iam-role-trust-for-lambda-at-edge.json b/infra/iam-role-trust-for-lambda-at-edge.json new file mode 100644 index 0000000..e39c731 --- /dev/null +++ b/infra/iam-role-trust-for-lambda-at-edge.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com", + "edgelambda.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/infra/web/lambda-at-edge/OriginRequest.lambda.js b/infra/web/lambda-at-edge/OriginRequest.lambda.js index 3067908..4265c28 100644 --- a/infra/web/lambda-at-edge/OriginRequest.lambda.js +++ b/infra/web/lambda-at-edge/OriginRequest.lambda.js @@ -1,8 +1,9 @@ -// Implements rewrite of non-gz to gz URLs using AWS Lambda@Edge. This is -// useful if you have precompressed your files. +// Implements rewrite of non-compressed to .gz or .br URLs using AWS +// Lambda@Edge. This is useful if you have precompressed your files. // -// Usage: Create an AWS Lambda function and attach it to "Origin Request" event -// of a Cloudfront distribution +// Usage: +// Create an AWS Lambda function and attach it to "Origin Request" event of a +// Cloudfront distribution const ARCHIVE_EXTS = [ '.7z', @@ -38,7 +39,10 @@ function handler(event, context, callback) { // If not an archive file (which are not precompressed), rewrite the URL to // get the corresponding .gz file if(!ARCHIVE_EXTS.every(ext => request.uri.endsWith(ext))) { - if(acceptsEncoding(headers, 'gzip')) { + if(acceptsEncoding(headers, 'br')) { + request.uri += '.br' + } + else if(acceptsEncoding(headers, 'gzip')) { request.uri += '.gz' } } @@ -47,4 +51,3 @@ function handler(event, context, callback) { } exports.handler = handler - diff --git a/package.json b/package.json index ce11ee4..f37be9d 100755 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "pan-genome-visualization", "private": true, "scripts": { - "clean": "rm -rf public/*.html public/*.html.gz public/dist/*", + "clean": "rm -rf public/*.html public/*.html.gz public/*.br public/dist/*", "build": "NODE_ENV=production npm run build:webpack && NODE_ENV=production npm run build:gulp && NODE_ENV=production npm run build:html", "build:webpack": "NODE_ENV=production webpack --env production", "build:gulp": "NODE_ENV=production gulp", diff --git a/scripts/deploy_app.sh b/scripts/deploy_app.sh index 76f8e95..407ab3a 100755 --- a/scripts/deploy_app.sh +++ b/scripts/deploy_app.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -set -euo pipefail +set -euxo pipefail trap "exit" INT -source .env +[ -f ".env" ] && source .env -# Remove unused +# Remove unused files find public/dataset/* -type d -exec rm -rf {} \+ rm -rf public/download rm -rf public/phyloTree @@ -12,6 +12,7 @@ rm -rf public/phyloTree # Precompress files pigz -dfrq public/ || true pigz -kfrq public/ +find public/ -type f \( ! -iname "*.gz" -a ! -iname "*.br" \) | parallel brotli -kf || true ## Clear bucket #aws s3 rm \ @@ -28,25 +29,40 @@ aws s3 sync \ --exclude "*.git*" \ --exclude "*.html" \ --exclude "*.gz" \ + --exclude "*.br" \ "public/" "s3://${AWS_S3_BUCKET}/" -# Upload compressed non-HTML files +# Upload gzip-compressed non-HTML files aws s3 sync \ --delete \ --only-show-errors \ --cache-control "max-age=2592000, public" \ - --content-encoding=gzip \ + --content-encoding gzip \ --metadata-directive REPLACE \ --exclude "*" \ --include "*.gz" \ --exclude "*.git*" \ - --exclude "*.html.gz" \ + --exclude "*.html*" \ + "public/" "s3://${AWS_S3_BUCKET}/" + +# Upload brotli-compressed non-HTML files +aws s3 sync \ + --delete \ + --only-show-errors \ + --cache-control "max-age=2592000, public" \ + --content-encoding br \ + --metadata-directive REPLACE \ + --exclude "*" \ + --include "*.br" \ + --exclude "*.git*" \ + --exclude "*.html*" \ "public/" "s3://${AWS_S3_BUCKET}/" # Remove non-HTML files -find public/ -type f \( ! -iname "*.html" -a ! -iname "*.html.gz" \) -exec rm {} \+ +find public/ -type f \( ! -iname "*.html" -a ! -iname "*.html.gz" -a ! -iname "*.html.br" \) -exec rm {} \+ rename --filename 's/\.html//' public/*.html || true rename --filename 's/\.html//' public/*.html.gz || true +rename --filename 's/\.html//' public/*.html.br || true # Upload non-compressed HTML files aws s3 sync \ @@ -58,19 +74,32 @@ aws s3 sync \ --exclude "*.*" \ "public/" "s3://${AWS_S3_BUCKET}/" -# Upload compressed HTML files +# Upload gzip-compressed HTML files aws s3 sync \ --delete \ --only-show-errors \ --content-type "text/html" \ --cache-control "max-age=86400, public" \ - --content-encoding=gzip \ + --content-encoding gzip \ --metadata-directive REPLACE \ --exclude "*" \ --include "*.gz" \ --exclude "*.*.gz" \ "public/" "s3://${AWS_S3_BUCKET}/" +# Upload brotli-compressed HTML files +aws s3 sync \ + --delete \ + --only-show-errors \ + --content-type "text/html" \ + --cache-control "max-age=86400, public" \ + --content-encoding br \ + --metadata-directive REPLACE \ + --exclude "*" \ + --include "*.br" \ + --exclude "*.*.br" \ + "public/" "s3://${AWS_S3_BUCKET}/" + # Update Cloudfront cache aws cloudfront create-invalidation \ --no-paginate \