Merge branch 'master' into feat/postgres14.5

Author: NanoCode012

README.md

@@ -102,6 +102,36 @@ Support for this can be enabled my making your Cloudwatch Event look like this.
 If you supply `USE_IAM_AUTH` with a value of `true`, the `PGPASSWORD` var may be omitted in the CloudWatch event.
 If you still provide it, it will be ignored.
 
+#### SecretsManager-based Postgres authentication
+
+If you prefer to not send DB details/credentials in the event parameters, you can store such details in SecretsManager and just provide the SecretId, then the function will fetch your DB details/credentials from the secret value.
+
+NOTE: the execution role for the Lambda function must have access to GetSecretValue for the given secret.
+
+Support for this can be enabled by setting the SECRETS_MANAGER_SECRET_ID, so your Cloudwatch Event looks like this:
+
+```json
+
+{
+     "SECRETS_MANAGER_SECRET_ID": "my/secret/id",
+     "S3_BUCKET" : "db-backups",
+     "ROOT": "hourly-backups"
+}
+```
+
+If you supply `SECRETS_MANAGER_SECRET_ID`, you can ommit the 'PG*' keys, and they will be fetched from your SecretsManager secret value instead with the following mapping:
+
+| Secret Value  | PG-Key |
+| ------------- | ------------- |
+| username  | PGUSER  |
+| password  | PGPASSWORD  |
+| dbname  | PGDATABASE  |
+| host  | PGHOST  |
+| port  | PGPORT  |
+
+
+You can provide overrides in your event to any PG* keys as event parameters will take precedence over secret values.
+
 ## Developer
 
 #### Bundling a new `pg_dump` binary

lib/config.js

@@ -2,9 +2,10 @@ const path = require("path");
 
 // default config that is overridden by the Lambda event
 module.exports = {
-  S3_REGION: "eu-west-1",
-  PGDUMP_PATH: path.join(__dirname, "../bin/postgres-14.5"),
-  // maximum time allowed to connect to postgres before a timeout occurs
-  PGCONNECT_TIMEOUT: 15,
-  USE_IAM_AUTH: false,
-};
+    S3_REGION: 'eu-west-1',
+    PGDUMP_PATH: path.join(__dirname, "../bin/postgres-14.5"),
+    // maximum time allowed to connect to postgres before a timeout occurs
+    PGCONNECT_TIMEOUT: 15,
+    USE_IAM_AUTH: false,
+    S3_STORAGE_CLASS: 'STANDARD'
+}

lib/handler.js

@@ -2,6 +2,7 @@ const utils = require('./utils')
 const uploadS3 = require('./upload-s3')
 const pgdump = require('./pgdump')
 const decorateWithIamToken = require('./iam')
+const decorateWithSecretsManagerCredentials = require('./secrets-manager')
 const encryption = require('./encryption')
 
 const DEFAULT_CONFIG = require('./config')
@@ -35,7 +36,18 @@ async function backup(config) {
 
 async function handler(event) {
     const baseConfig = { ...DEFAULT_CONFIG, ...event }
-    const config = event.USE_IAM_AUTH === true ? decorateWithIamToken(baseConfig) : baseConfig
+    let config
+
+    if (event.USE_IAM_AUTH === true) {
+        config = decorateWithIamToken(baseConfig)
+    }
+    else if (event.SECRETS_MANAGER_SECRET_ID) {
+        config = await decorateWithSecretsManagerCredentials(baseConfig)
+    }
+    else {
+        config = baseConfig
+    }
+
     try {
         return await backup(config)
     }

lib/secrets-manager.js

@@ -0,0 +1,53 @@
+/* eslint-disable brace-style */
+const AWS = require('aws-sdk')
+
+// configure AWS to log to stdout
+AWS.config.update({
+    logger: process.stdout
+})
+
+async function getDbCredentials(config) {
+    const secretsManager = new AWS.SecretsManager({
+        region: config.S3_REGION
+    })
+
+    const params = {
+        SecretId: config.SECRETS_MANAGER_SECRET_ID
+    }
+
+    return new Promise((resolve, reject) => {
+        secretsManager.getSecretValue(params, (err, data) => {
+            if (err) {
+                console.log('Error while getting secret value:', err)
+                reject(err)
+            } else {
+                const credentials = JSON.parse(data.SecretString)
+                resolve(credentials)
+            }
+        })
+    })
+}
+
+async function decorateWithSecretsManagerCredentials(baseConfig) {
+    try {
+        const credentials = await getDbCredentials(baseConfig)
+
+        const credsFromSecret = {}
+
+        if (credentials.username) credsFromSecret.PGUSER = credentials.username
+        if (credentials.password) credsFromSecret.PGPASSWORD = credentials.password
+        if (credentials.dbname) credsFromSecret.PGDATABASE = credentials.dbname
+        if (credentials.host) credsFromSecret.PGHOST = credentials.host
+        if (credentials.port) credsFromSecret.PGPORT = credentials.port
+
+        return {
+            ...credsFromSecret,
+            ...baseConfig
+        }
+    } catch (error) {
+        console.log(error)
+        return baseConfig
+    }
+}
+
+module.exports = decorateWithSecretsManagerCredentials

lib/upload-s3.js

@@ -12,7 +12,8 @@ async function uploadS3(stream, config, key) {
     const result = await s3.upload({
         Key: key,
         Bucket: config.S3_BUCKET,
-        Body: stream
+        Body: stream,
+        StorageClass: config.S3_STORAGE_CLASS
     }).promise()
 
     console.log('Uploaded to', result.Location)

test/handler.js

@@ -89,6 +89,34 @@ describe('Handler', () => {
         AWSMOCK.restore('RDS.Signer')
     })
 
+    it('should be able to authenticate via SecretsManager', async () => {
+        const { s3Spy, pgSpy } = makeMockHandler()
+
+        const secretsManagerMockEvent = { ...mockEvent, SECRETS_MANAGER_SECRET_ID: 'my-secret-id' }
+        const username = 'myuser'
+        const password = 'mypassword'
+        const secretValue = {
+            SecretString: JSON.stringify({ username, password })
+        }
+
+        AWSMOCK.mock('SecretsManager', 'getSecretValue', (params, callback) => {
+            expect(params.SecretId).to.eql(secretsManagerMockEvent.SECRETS_MANAGER_SECRET_ID)
+            callback(null, secretValue)
+        })
+
+        await handler(secretsManagerMockEvent)
+        // handler should have called pgSpy with correct arguments
+        expect(pgSpy.calledOnce).to.be.true
+        expect(s3Spy.calledOnce).to.be.true
+        expect(s3Spy.firstCall.args).to.have.length(3)
+        const config = s3Spy.firstCall.args[1]
+        // production code is synchronous, so this is annoying
+        expect(config.PGUSER).to.equal(username)
+        expect(config.PGPASSWORD).to.equal(password)
+
+        AWSMOCK.restore('SecretsManager')
+    })
+
     it('should upload the backup file and an iv file', async () => {
         const { s3Spy } = makeMockHandler()
 

test/secrets-manager.js

@@ -0,0 +1,90 @@
+/* eslint no-underscore-dangle: 0 */
+const { expect } = require('chai')
+const rewire = require('rewire')
+const chai = require('chai')
+const chaiAsPromised = require('chai-as-promised')
+const AWSMOCK = require('aws-sdk-mock')
+const AWS = require('aws-sdk')
+
+chai.should()
+chai.use(chaiAsPromised)
+
+AWSMOCK.setSDKInstance(AWS)
+
+const decorateWithSecretsManagerCredentials = rewire('../lib/secrets-manager')
+
+describe('secrets-manager-based auth', () => {
+    const parsedSecretValue = {
+        dbname: 'somedatabase',
+        username: 'someuser',
+        password: 'somepassword',
+        host: 'somehost',
+        port: '2345'
+    }
+    const secretValue = {
+        SecretString: JSON.stringify(parsedSecretValue)
+    }
+
+    const keyMappings = [
+        { secretKey: 'username', pgKey: 'PGUSER' },
+        { secretKey: 'password', pgKey: 'PGPASSWORD' },
+        { secretKey: 'dbname', pgKey: 'PGDATABASE' },
+        { secretKey: 'host', pgKey: 'PGHOST' },
+        { secretKey: 'port', pgKey: 'PGPORT' }
+    ]
+
+    keyMappings.forEach((map) => {
+        it(`should set ${map.pgKey} from the secret ${map.secretKey}`, async () => {
+            const mockEvent = { SECRETS_MANAGER_SECRET_ID: 'my-secret-id' }
+
+            AWSMOCK.mock('SecretsManager', 'getSecretValue', (params, callback) => {
+                expect(params.SecretId).to.eql(mockEvent.SECRETS_MANAGER_SECRET_ID)
+
+                callback(null, secretValue)
+            })
+
+            const config = await decorateWithSecretsManagerCredentials(mockEvent)
+
+            expect(config[map.pgKey]).to.eql(parsedSecretValue[map.secretKey])
+        })
+
+        context(`when the event contains an override for ${map.pgKey}`, () => {
+            it(`should set ${map.pgKey} from the event params`, async () => {
+                const mockEvent = {
+                    SECRETS_MANAGER_SECRET_ID: 'my-secret-id',
+                    [map.pgKey]: 'some-value-override'
+                }
+
+                AWSMOCK.mock('SecretsManager', 'getSecretValue', (params, callback) => {
+                    expect(params.SecretId).to.eql(mockEvent.SECRETS_MANAGER_SECRET_ID)
+
+                    callback(null, secretValue)
+                })
+
+                const config = await decorateWithSecretsManagerCredentials(mockEvent)
+
+                expect(config[map.pgKey]).to.eql(mockEvent[map.pgKey])
+            })
+        })
+    })
+
+    context('when there is an error getting the secret value', () => {
+        it('should return the given base config', async () => {
+            const mockEvent = { SECRETS_MANAGER_SECRET_ID: 'my-secret-id' }
+
+            AWSMOCK.mock('SecretsManager', 'getSecretValue', (params, callback) => {
+                expect(params.SecretId).to.eql(mockEvent.SECRETS_MANAGER_SECRET_ID)
+
+                callback('some error', secretValue)
+            })
+
+            const config = await decorateWithSecretsManagerCredentials(mockEvent)
+
+            expect(config).to.eql(mockEvent)
+        })
+    })
+
+    afterEach(() => {
+        AWSMOCK.restore('SecretsManager')
+    })
+})