diff --git a/.gitignore b/.gitignore index 156e19e..b40b30b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ v10.x/layer v10.x/test/lambda.zip v12.x/layer v12.x/test/lambda.zip +v14.x/layer +v14.x/test/**/lambda.zip \ No newline at end of file diff --git a/v14.x/.dockerignore b/v14.x/.dockerignore new file mode 100644 index 0000000..b65f618 --- /dev/null +++ b/v14.x/.dockerignore @@ -0,0 +1,3 @@ +layer.zip +layer +test diff --git a/v14.x/Dockerfile b/v14.x/Dockerfile new file mode 100644 index 0000000..674906c --- /dev/null +++ b/v14.x/Dockerfile @@ -0,0 +1,15 @@ +FROM lambci/lambda-base:build + +COPY bootstrap.c bootstrap.js package.json esm-loader-hook.mjs /opt/ + +ARG NODE_VERSION + +RUN curl -sSL https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz | \ + tar -xJ -C /opt --strip-components 1 -- node-v${NODE_VERSION}-linux-x64/bin/node && \ + strip /opt/bin/node + +RUN cd /opt && \ + export NODE_MAJOR=$(echo $NODE_VERSION | awk -F. '{print "\""$1"\""}') && \ + clang -Wall -Werror -s -O2 -D NODE_MAJOR="$NODE_MAJOR" -o bootstrap bootstrap.c && \ + rm bootstrap.c && \ + zip -yr /tmp/layer.zip . diff --git a/v14.x/bootstrap.c b/v14.x/bootstrap.c new file mode 100644 index 0000000..c6bdf63 --- /dev/null +++ b/v14.x/bootstrap.c @@ -0,0 +1,43 @@ +#include +#include +#include +#include + +#ifndef NODE_MAJOR +#error Must pass NODE_MAJOR to the compiler (eg "10") +#define NODE_MAJOR "" +#endif + +#define AWS_EXECUTION_ENV "AWS_Lambda_nodejs" NODE_MAJOR "_lambci" +#define NODE_PATH "/opt/nodejs/node" NODE_MAJOR "/node_modules:" \ + "/opt/nodejs/node_modules:" \ + "/var/runtime/node_modules:" \ + "/var/runtime:" \ + "/var/task" +#define MIN_MEM_SIZE 128 +#define ARG_BUF_SIZE 32 + +int main(void) { + setenv("AWS_EXECUTION_ENV", AWS_EXECUTION_ENV, true); + setenv("NODE_PATH", NODE_PATH, true); + + const char *mem_size_str = getenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE"); + int mem_size = mem_size_str != NULL ? atoi(mem_size_str) : MIN_MEM_SIZE; + + char max_semi_space_size[ARG_BUF_SIZE]; + snprintf(max_semi_space_size, ARG_BUF_SIZE, "--max-semi-space-size=%d", mem_size * 5 / 100); + + char max_old_space_size[ARG_BUF_SIZE]; + snprintf(max_old_space_size, ARG_BUF_SIZE, "--max-old-space-size=%d", mem_size * 90 / 100); + + execv("/opt/bin/node", (char *[]){ + "node", + "--experimental-loader=/opt/esm-loader-hook.mjs", + "--expose-gc", + max_semi_space_size, + max_old_space_size, + "/opt/bootstrap.js", + NULL}); + perror("Could not execv"); + return EXIT_FAILURE; +} diff --git a/v14.x/bootstrap.js b/v14.x/bootstrap.js new file mode 100644 index 0000000..c6e8b2a --- /dev/null +++ b/v14.x/bootstrap.js @@ -0,0 +1,269 @@ +import http from 'http' +import { createRequire } from 'module' +import path from 'path' +import { stat, readFile } from "fs/promises" + +const RUNTIME_PATH = '/2018-06-01/runtime' + +const CALLBACK_USED = Symbol('CALLBACK_USED') + +const { + AWS_LAMBDA_FUNCTION_NAME, + AWS_LAMBDA_FUNCTION_VERSION, + AWS_LAMBDA_FUNCTION_MEMORY_SIZE, + AWS_LAMBDA_LOG_GROUP_NAME, + AWS_LAMBDA_LOG_STREAM_NAME, + LAMBDA_TASK_ROOT, + _HANDLER, + AWS_LAMBDA_RUNTIME_API, +} = process.env + +const [HOST, PORT] = AWS_LAMBDA_RUNTIME_API.split(':') + +start() + +async function start() { + let handler + try { + handler = await getHandler() + } catch (e) { + await initError(e) + return process.exit(1) + } + tryProcessEvents(handler) +} + +async function tryProcessEvents(handler) { + try { + await processEvents(handler) + } catch (e) { + console.error(e) + return process.exit(1) + } +} + +async function processEvents(handler) { + while (true) { + const { event, context } = await nextInvocation() + + let result + try { + result = await handler(event, context) + } catch (e) { + await invokeError(e, context) + continue + } + const callbackUsed = context[CALLBACK_USED] + + await invokeResponse(result, context) + + if (callbackUsed && context.callbackWaitsForEmptyEventLoop) { + return process.prependOnceListener('beforeExit', () => tryProcessEvents(handler)) + } + } +} + +function initError(err) { + return postError(`${RUNTIME_PATH}/init/error`, err) +} + +async function nextInvocation() { + const res = await request({ path: `${RUNTIME_PATH}/invocation/next` }) + + if (res.statusCode !== 200) { + throw new Error(`Unexpected /invocation/next response: ${JSON.stringify(res)}`) + } + + if (res.headers['lambda-runtime-trace-id']) { + process.env._X_AMZN_TRACE_ID = res.headers['lambda-runtime-trace-id'] + } else { + delete process.env._X_AMZN_TRACE_ID + } + + const deadlineMs = +res.headers['lambda-runtime-deadline-ms'] + + const context = { + awsRequestId: res.headers['lambda-runtime-aws-request-id'], + invokedFunctionArn: res.headers['lambda-runtime-invoked-function-arn'], + logGroupName: AWS_LAMBDA_LOG_GROUP_NAME, + logStreamName: AWS_LAMBDA_LOG_STREAM_NAME, + functionName: AWS_LAMBDA_FUNCTION_NAME, + functionVersion: AWS_LAMBDA_FUNCTION_VERSION, + memoryLimitInMB: AWS_LAMBDA_FUNCTION_MEMORY_SIZE, + getRemainingTimeInMillis: () => deadlineMs - Date.now(), + callbackWaitsForEmptyEventLoop: true, + } + + if (res.headers['lambda-runtime-client-context']) { + context.clientContext = JSON.parse(res.headers['lambda-runtime-client-context']) + } + + if (res.headers['lambda-runtime-cognito-identity']) { + context.identity = JSON.parse(res.headers['lambda-runtime-cognito-identity']) + } + + const event = JSON.parse(res.body) + + return { event, context } +} + +async function invokeResponse(result, context) { + const res = await request({ + method: 'POST', + path: `${RUNTIME_PATH}/invocation/${context.awsRequestId}/response`, + body: JSON.stringify(result === undefined ? null : result), + }) + if (res.statusCode !== 202) { + throw new Error(`Unexpected /invocation/response response: ${JSON.stringify(res)}`) + } +} + +function invokeError(err, context) { + return postError(`${RUNTIME_PATH}/invocation/${context.awsRequestId}/error`, err) +} + +async function postError(path, err) { + const lambdaErr = toLambdaErr(err) + const res = await request({ + method: 'POST', + path, + headers: { + 'Content-Type': 'application/json', + 'Lambda-Runtime-Function-Error-Type': lambdaErr.errorType, + }, + body: JSON.stringify(lambdaErr), + }) + if (res.statusCode !== 202) { + throw new Error(`Unexpected ${path} response: ${JSON.stringify(res)}`) + } +} + +async function getHandler() { + const moduleParts = _HANDLER.split('.') + if (moduleParts.length !== 2) { + throw new Error(`Bad handler ${_HANDLER}`) + } + + const [modulePath, handlerName] = moduleParts + const {type: moduleLoaderType, ext} = await getModuleLoaderType(`${LAMBDA_TASK_ROOT}/${modulePath}`) + + // Let any errors here be thrown as-is to aid debugging + const importPath = `${LAMBDA_TASK_ROOT}/${modulePath}.${ext}` + const module = moduleLoaderType === 'module' ? await import(importPath) : createRequire(import.meta.url)(importPath) + + const userHandler = module[handlerName] + + if (userHandler === undefined) { + throw new Error(`Handler '${handlerName}' missing on module '${modulePath}'`) + } else if (typeof userHandler !== 'function') { + throw new Error(`Handler '${handlerName}' from '${modulePath}' is not a function`) + } + + return (event, context) => new Promise((resolve, reject) => { + const callback = (err, data) => { + context[CALLBACK_USED] = true + if(err) { + reject(err) + } else { + resolve(data) + } + } + + let result + try { + result = userHandler(event, context, callback) + } catch (e) { + return reject(e) + } + if (typeof result === 'object' && result != null && typeof result.then === 'function') { + result.then(resolve, reject) + } + }) +} + +/** + * @param {string} modulePath path to executeable with no file extention + * @returns {Promise<{ + * type: 'commonjs' | 'module', + * ext: 'mjs' | 'cjs' | 'js' + * }>} loader type and extention for loading module + */ +async function getModuleLoaderType(modulePath) { + //do all promises async so they dont have to wait on eachother + const [typ, mjsExist, cjsExist] = await Promise.all([ + getPackageJsonType(modulePath), + fileExists(modulePath + '.mjs'), + fileExists(modulePath + '.cjs') + ]) + + //priority here is basically cjs -> mjs -> js + //pjson.type defaults to commonjs so always check if 'module' first + if(mjsExist && cjsExist) { + if(typ === 'module') { return {type: 'module', ext: 'mjs'} } + return {type: 'commonjs', ext: 'cjs'} + } + //only one of these exist if any + if(mjsExist) { return {type: 'module', ext: 'mjs'} } + if(cjsExist) { return {type: 'commonjs', ext: 'cjs'} } + //js is the only file, determine type based on pjson + if(typ === 'module') { return {type: 'module', ext: 'js'} } + return {type: 'commonjs', ext: 'js'} +} + +async function fileExists(fullPath) { + try { + await stat(fullPath) + return true + } catch { + return false + } +} + +/** + * @param {string} modulePath path to executeable with no file extention + * @returns {Promise<'module' | 'commonjs'>} + */ +async function getPackageJsonType(modulePath) { + //try reading pjson until we reach root. i.e. '/' !== path.dirname('/') + //there is probably a way to make it search in parallel, returning the first match in the hierarchy, but it seems more trouble than its worth + for(let dir = path.dirname(modulePath); dir !== path.dirname(dir); dir = path.dirname(dir)) { + try { + const {type} = JSON.parse(await readFile(dir + path.sep + 'package.json', 'utf-8')) + return type || 'commonjs' + } catch { + //do nothing + } + } + + //if we reach root, return empty pjson + return 'commonjs' +} + +function request(options) { + options.host = HOST + options.port = PORT + + return new Promise((resolve, reject) => { + const req = http.request(options, res => { + const bufs = [] + res.on('data', data => bufs.push(data)) + res.on('end', () => resolve({ + statusCode: res.statusCode, + headers: res.headers, + body: Buffer.concat(bufs).toString(), + })) + res.on('error', reject) + }) + req.on('error', reject) + req.end(options.body) + }) +} + +function toLambdaErr(err) { + const { name, message, stack } = err + return { + errorType: name || typeof err, + errorMessage: message || ('' + err), + stackTrace: (stack || '').split('\n').slice(1), + } +} diff --git a/v14.x/build.sh b/v14.x/build.sh new file mode 100755 index 0000000..c1be7dc --- /dev/null +++ b/v14.x/build.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +. ./config.sh + +docker build --build-arg NODE_VERSION -t node-provided-lambda-v14.x . +docker run --rm -v "$PWD":/app node-provided-lambda-v14.x cp /tmp/layer.zip /app/ diff --git a/v14.x/check.sh b/v14.x/check.sh new file mode 100755 index 0000000..4d27083 --- /dev/null +++ b/v14.x/check.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +. ./config.sh + +REGIONS="$(aws ssm get-parameters-by-path --path /aws/service/global-infrastructure/services/lambda/regions \ + --query 'Parameters[].Value' --output text | tr '[:blank:]' '\n' | grep -v -e ^cn- -e ^us-gov- | sort -r)" + +for region in $REGIONS; do + aws lambda list-layer-versions --region $region --layer-name $LAYER_NAME \ + --query 'LayerVersions[*].[LayerVersionArn]' --output text +done diff --git a/v14.x/config.sh b/v14.x/config.sh new file mode 100644 index 0000000..96f7eb3 --- /dev/null +++ b/v14.x/config.sh @@ -0,0 +1,2 @@ +export LAYER_NAME=nodejs14 +export NODE_VERSION=14.15.1 diff --git a/v14.x/esm-loader-hook.mjs b/v14.x/esm-loader-hook.mjs new file mode 100644 index 0000000..5d72064 --- /dev/null +++ b/v14.x/esm-loader-hook.mjs @@ -0,0 +1,18 @@ +import {pathToFileURL} from "url" +import path from "path" + +const searchPaths = process.env.NODE_PATH.split(path.delimiter).map(path => pathToFileURL(path).href) + +export async function resolve(specifier, context, defaultResolve) { + try { + return defaultResolve(specifier, context, defaultResolve) + } catch {} + + for (const parentURL of searchPaths) { + try { + return defaultResolve(specifier, {...context, parentURL}, defaultResolve) + } catch {} + } + + throw new Error(`Cannot find package '${specifier}': attempted to import from paths [${[context.parentURL, ...searchPaths].join(', ')}]`) +} diff --git a/v14.x/layer.zip b/v14.x/layer.zip new file mode 100644 index 0000000..743c795 Binary files /dev/null and b/v14.x/layer.zip differ diff --git a/v14.x/package.json b/v14.x/package.json new file mode 100644 index 0000000..b4b69a9 --- /dev/null +++ b/v14.x/package.json @@ -0,0 +1,5 @@ +{ + "name": "node-custom-lambda-v14.x", + "version": "1.0.0", + "type": "module" +} diff --git a/v14.x/publish.sh b/v14.x/publish.sh new file mode 100755 index 0000000..58f3c85 --- /dev/null +++ b/v14.x/publish.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +. ./config.sh + +DESCRIPTION="Node.js v${NODE_VERSION} custom runtime" +FILENAME=${LAYER_NAME}-${NODE_VERSION}.zip + +REGIONS="$(aws ssm get-parameters-by-path --path /aws/service/global-infrastructure/services/lambda/regions \ + --query 'Parameters[].Value' --output text | tr '[:blank:]' '\n' | grep -v -e ^cn- -e ^us-gov- -e ^ap-northeast-3 | sort -r)" + +aws s3api put-object --bucket lambci --key layers/${FILENAME} --body layer.zip + +for region in $REGIONS; do + aws s3api copy-object --region $region --copy-source lambci/layers/${FILENAME} \ + --bucket lambci-${region} --key layers/${FILENAME} && \ + aws lambda add-layer-version-permission --region $region --layer-name $LAYER_NAME \ + --statement-id sid1 --action lambda:GetLayerVersion --principal '*' \ + --version-number $(aws lambda publish-layer-version --region $region --layer-name $LAYER_NAME \ + --content S3Bucket=lambci-${region},S3Key=layers/${FILENAME} \ + --description "$DESCRIPTION" --query Version --output text) & +done + +for job in $(jobs -p); do + wait $job +done diff --git a/v14.x/test.sh b/v14.x/test.sh new file mode 100755 index 0000000..97922b7 --- /dev/null +++ b/v14.x/test.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +rm -rf layer && unzip layer.zip -d layer + +LOADER_TYPE="commonjs" +case "$1" in + module) LOADER_TYPE=$1 ;; + commonjs) LOADER_TYPE=$1 ;; +esac + +echo Testing test/$LOADER_TYPE +cd test/$LOADER_TYPE + +npm ci + +# Create zipfile for uploading to Lambda – we don't use this here +rm -f lambda.zip && zip -qyr lambda.zip index.js data.json node_modules + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler2 + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler3 + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler4 + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler5 + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler6 + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=$LOADER_TYPE lambci/lambda:provided index.handler7 + +docker run --rm -v "$PWD":/var/task -v "$PWD"/../../layer:/opt -e NODE_MODULE_LOADER_TYPE=invalid lambci/lambda:provided index.handler7 diff --git a/v14.x/test/commonjs/data.json b/v14.x/test/commonjs/data.json new file mode 100644 index 0000000..32dc9e7 --- /dev/null +++ b/v14.x/test/commonjs/data.json @@ -0,0 +1,3 @@ +{ + "message": "complete" +} \ No newline at end of file diff --git a/v14.x/test/commonjs/index.js b/v14.x/test/commonjs/index.js new file mode 100644 index 0000000..7d51a96 --- /dev/null +++ b/v14.x/test/commonjs/index.js @@ -0,0 +1,45 @@ +// Test that global requires work +const aws4 = require('aws4') + +const interval = setInterval(console.log, 100, 'ping') + +exports.handler = async(event, context) => { + console.log(process.version) + console.log(process.execPath) + console.log(process.execArgv) + console.log(process.argv) + console.log(process.cwd()) + console.log(process.env) + console.log(event) + console.log(context) + console.log(context.getRemainingTimeInMillis()) + console.log(aws4) + return { some: 'obj!' } +} + +exports.handler2 = (event, context) => { + setTimeout(context.done, 100, null, { some: 'obj!' }) +} + +exports.handler3 = (event, context) => { + setTimeout(context.succeed, 100, { some: 'obj!' }) +} + +exports.handler4 = (event, context) => { + setTimeout(context.fail, 100, new Error('This error should be logged')) +} + +exports.handler5 = (event, context, cb) => { + setTimeout(cb, 100, null, { some: 'obj!' }) + setTimeout(clearInterval, 100, interval) +} + +exports.handler6 = (event, context, cb) => { + context.callbackWaitsForEmptyEventLoop = false + setTimeout(cb, 100, null, { some: 'obj!' }) +} + +exports.handler7 = async (event, context) => { + const data = require(`${__dirname}/data.json`) + return { some: 'obj!', data } +} diff --git a/v14.x/test/commonjs/package-lock.json b/v14.x/test/commonjs/package-lock.json new file mode 100644 index 0000000..5fdc74c --- /dev/null +++ b/v14.x/test/commonjs/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "test", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "aws4": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + } + } +} diff --git a/v14.x/test/commonjs/package.json b/v14.x/test/commonjs/package.json new file mode 100644 index 0000000..73f1e8d --- /dev/null +++ b/v14.x/test/commonjs/package.json @@ -0,0 +1,7 @@ +{ + "name": "test", + "version": "1.0.0", + "dependencies": { + "aws4": "^1.8.0" + } +} diff --git a/v14.x/test/module/data.json b/v14.x/test/module/data.json new file mode 100644 index 0000000..32dc9e7 --- /dev/null +++ b/v14.x/test/module/data.json @@ -0,0 +1,3 @@ +{ + "message": "complete" +} \ No newline at end of file diff --git a/v14.x/test/module/index.js b/v14.x/test/module/index.js new file mode 100644 index 0000000..e633989 --- /dev/null +++ b/v14.x/test/module/index.js @@ -0,0 +1,60 @@ +// Test that global requires work +import fs from 'fs' +import { dirname } from 'path' +import { fileURLToPath } from 'url' + +import aws4 from 'aws4' + +const interval = setInterval(console.log, 100, 'ping') + +const sleep = async (millisecond) => await new Promise(res => setTimeout(res, millisecond)) + +// Test top-level await works +await sleep(10) + +export const handler = async (event, context) => { + console.log(process.version) + console.log(process.execPath) + console.log(process.execArgv) + console.log(process.argv) + console.log(process.cwd()) + console.log(process.env) + console.log(event) + console.log(context) + console.log(context.getRemainingTimeInMillis()) + console.log(aws4) + return { some: 'obj!' } +} + +export const handler2 = (event, context) => { + setTimeout(context.done, 100, null, { some: 'obj!' }) +} + +export const handler3 = (event, context) => { + setTimeout(context.succeed, 100, { some: 'obj!' }) +} + +export const handler4 = (event, context) => { + setTimeout(context.fail, 100, new Error('This error should be logged')) +} + +export const handler5 = (event, context, cb) => { + setTimeout(cb, 100, null, { some: 'obj!' }) + setTimeout(clearInterval, 100, interval) +} + +export const handler6 = (event, context, cb) => { + context.callbackWaitsForEmptyEventLoop = false + setTimeout(cb, 100, null, { some: 'obj!' }) +} + +export const handler7 = async (event, context) => { + const __dirname = dirname(fileURLToPath(import.meta.url)) + const data = await new Promise((res, rej) => { + fs.readFile(`${__dirname}/data.json`, (err, str) => { + if (err) return rej(err) + else res(JSON.parse(str)) + }) + }) + return { some: 'obj!', data } +} diff --git a/v14.x/test/module/package-lock.json b/v14.x/test/module/package-lock.json new file mode 100644 index 0000000..5fdc74c --- /dev/null +++ b/v14.x/test/module/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "test", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "aws4": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + } + } +} diff --git a/v14.x/test/module/package.json b/v14.x/test/module/package.json new file mode 100644 index 0000000..6570058 --- /dev/null +++ b/v14.x/test/module/package.json @@ -0,0 +1,8 @@ +{ + "name": "test", + "version": "1.0.0", + "type": "module", + "dependencies": { + "aws4": "^1.8.0" + } +}