Feature/allow backup cronjobs to be run by a k8s cronjob ams

CHANGELOG.md

@@ -26,6 +26,10 @@ Update version number
 * Do not wait for cms when updating site config
 * Move docker builds from travis to github actions
 
+## UNRELEASED
+* Change mongoDB S3 backup to read the database dump in chunks and upload it to the S3 bucket using a MultiPart upload setup.
+* Allow s3 backups to be disabled from the cronjobs, but instead be run through a different entrypoint (backup.js) to allow kubernetes cronjobs to be used.
+
 ## v0.20.0 (2021-12-20)
 * Fix responding to not existing routes with to much information
 * Make submissions listable & viewable, and allow them to be filtered by formId

backup.js

@@ -0,0 +1,27 @@
+const backupMapping = {
+  'mongo': 'mongodb_s3_backups',
+  'mysql': 'mysql_s3_backups'
+};
+
+if (!!process.env.BACKUP_TYPE === false) {
+  console.error ('No backup type given in ENV variables');
+  process.exit(1);
+}
+
+if (Object.keys(backupMapping).indexOf(process.env.BACKUP_TYPE) === -1) {
+  console.error ('Backup type not supported');
+  process.exit(1);
+}
+
+const backup = require(`./src/cron/${backupMapping[process.env.BACKUP_TYPE]}`);
+
+async function run() {
+  try {
+    console.log(`${backupMapping[process.env.BACKUP_TYPE]}`);
+    await backup.onTick();
+  } catch (err) {
+    console.error('Backup went wrong', err);
+  }
+}
+
+run();

src/cron/mongodb_backups.js

@@ -10,6 +10,10 @@ module.exports = {
 	cronTime: '0 0 1 * * *',
 	runOnInit: true,
 	onTick: function() {
+    if (!!process.env.PREVENT_BACKUP_CRONJOBS === true || process.env.S3_MONGO_BACKUPS === 'ON') {
+      return;
+    }
+
     const objectStoreUrl = process.env.OBJECT_STORE_URL;
     const objectStoreUser = process.env.OBJECT_STORE_USER;
     const objectStorePass = process.env.OBJECT_STORE_PASS;

src/cron/mongodb_s3_backups.js

@@ -1,76 +1,60 @@
-const AWS = require('aws-sdk');
-const fs = require('fs'); // Needed for example below
+const fs     = require('fs');
 const moment = require('moment')
-const os = require('os');
-//const BACKUP_PATH = (ZIP_NAME) => path.resolve(os.tmpdir(), ZIP_NAME);
-const { exec } = require('child_process');
-const log = require('debug')('app:cron');
-const db = require('../db');
-
-// Purpose
-// -------
-// Auto-close ideas that passed the deadline.
-//          accessKeyId: process.env.S3_KEY,
-          secretAccessKey: process.env.S3_SECRET
-// Runs every night at 1:00.
-//
-function currentTime(timezoneOffset) {
-    if (timezoneOffset) {
-        return moment(moment(moment.now()).utcOffset(timezoneOffset, true).toDate()).format("YYYY-MM-DDTHH-mm-ss");
-    } else {
-        return moment
-            .utc()
-            .format('YYYY-MM-DDTHH-mm-ss');
-    }
-}
+const {exec} = require('child_process');
+const s3     = require('../services/awsS3');
+const util   = require('util');
 
 const backupMongoDBToS3 = async () => {
-    if (process.env.S3_MONGO_BACKUPS === 'ON') {
-      const host = process.env.MONGO_DB_HOST || 'localhost';
-      const port = process.env.MONGO_DB_PORT || 27017;
-      const tmpDbFile = 'db_mongo'
-
-    //  let DB_BACKUP_NAME = `mongodb_${currentTime()}.gz`;
-
-      // Default command, does not considers username or password
-      let command = `mongodump -h ${host} --port=${port} --archive=${tmpDbFile}`;
-
-      // When Username and password is provided
-      // /
-      //if (username && password) {
-      //    command = `mongodump -h ${host} --port=${port} -d ${database} -p ${password} -u ${username} --quiet --gzip --archive=${BACKUP_PATH(DB_BACKUP_NAME)}`;
-      //}
-
-      exec(command, (err, stdout, stderr) => {
-          if (err) {
-              // Most likely, mongodump isn't installed or isn't accessible
-            console.log('errere', err);
-          } else {
-            const spacesEndpoint = new AWS.Endpoint(process.env.S3_ENDPOINT);
-
-            const created = moment().format('YYYY-MM-DD hh:mm:ss')
-            const fileContent = fs.readFileSync(tmpDbFile);
-
-            const s3 = new AWS.S3({
-                endpoint: spacesEndpoint,
-                accessKeyId: process.env.S3_KEY,
-                secretAccessKey: process.env.S3_SECRET
-            });
-
-            var params = {
-                Bucket: process.env.S3_BUCKET,
-                Key: "mongodb/mongo_" + created,
-                Body: fileContent,
-                ACL: "private"
-            };
+  if (!!process.env.PREVENT_BACKUP_CRONJOBS === true) {
+    return;
+  }
+
+  console.log('backing up to mongodb', process.env.S3_MONGO_BACKUPS);
+
+  if (process.env.S3_MONGO_BACKUPS === 'ON') {
+    const host            = process.env.MONGO_DB_HOST || 'localhost';
+    const port            = process.env.MONGO_DB_PORT || 27017;
+    const tempFile        = 'db_mongo';
+    const isOnK8s         = !!process.env.KUBERNETES_NAMESPACE;
+    const namespace       = process.env.KUBERNETES_NAMESPACE;
+    const created = moment().format('YYYY-MM-DD hh:mm:ss')
+    const fileNameInS3 = isOnK8s ? `mongodb/${namespace}/mongo_${created}` : `mongodb/mongo_${created}`;
+    const deleteTempFile = () => {
+      try {
+        console.log ('removing temp file', tempFile);
+        fs.unlinkSync(tempFile);
+      } catch (e) {
+        console.error('error removing file', tempFile, e);
+      }
+    };
+    
+    // Default command, does not considers username or password
+    let command = `mongodump -h ${host} --port=${port} --archive=${tempFile}`;
+    
+    const promiseExec = util.promisify(exec);
+
+    return promiseExec(command, async (err, stdout, stderr) => {
+      if (err) {
+        // Most likely, mongodump isn't installed or isn't accessible
+        console.error(`mongodump command error: ${err}`);
+        deleteTempFile();
+        return;
+      }
+
+      const statsFile = fs.statSync(tempFile);
+      console.info(`file size: ${Math.round(statsFile.size / 1024 / 1024)}MB`);
 
-            s3.putObject(params, function(err, data) {
-                if (err) console.log(err, err.stack);
-                else     console.log(data);
-            });
-          }
-      });
-    }
+      try {
+        await s3.uploadFile(tempFile, fileNameInS3);
+        deleteTempFile();
+        console.log('successfully uploaded to s3');
+      } catch (e) {
+        deleteTempFile();
+        throw e;
+      }
+
+    });
+  }
 }
 
 
@@ -89,6 +73,6 @@ module.exports = {
 	cronTime: '0 0 1 * * *',
 	runOnInit: true,
 	onTick: async function() {
-    backupMongoDBToS3();
+    return backupMongoDBToS3();
 	}
 };

src/cron/mysql_backups.js

@@ -10,6 +10,11 @@ module.exports = {
 	cronTime: '0 0 1 * * *',
 	runOnInit: true,
 	onTick: function() {
+    // Do not run this cronjob if we have PREVENT_BACKUP_CRONJOBS set to true, or if the S3_DBS_TO_BACKUP is not empty (which means we want to backup to S3).
+    if (!!process.env.PREVENT_BACKUP_CRONJOBS === true || !!process.env.S3_DBS_TO_BACKUP === false) {
+      return;
+    }
+
     const mysqlRootPw = process.env.MYSQL_ROOT_PASS;
     const objectStoreUrl = process.env.OBJECT_STORE_URL;
     const objectStoreUser = process.env.OBJECT_STORE_USER;

src/cron/mysql_s3_backups.js

@@ -1,18 +1,21 @@
-const AWS = require('aws-sdk');
-const fs = require('fs'); // Needed for example below
 const mysqldump = require('mysqldump');
 const moment = require('moment')
-const log = require('debug')('app:cron');
-const db = require('../db');
+const s3 = require('../services/awsS3');
 
 // Purpose
 // -------
 // Auto-close ideas that passed the deadline.
 //
 // Runs every night at 1:00.
 const backupMysqlToS3 = async () => {
+  if (!!process.env.PREVENT_BACKUP_CRONJOBS === true) {
+    return;
+  }
+
   const dbsToBackup = process.env.S3_DBS_TO_BACKUP ? process.env.S3_DBS_TO_BACKUP.split(',') : false;
-
+  const isOnK8s = !!process.env.KUBERNETES_NAMESPACE;
+  const namespace = process.env.KUBERNETES_NAMESPACE;
+
   if (dbsToBackup) {
     dbsToBackup.forEach(async function(dbName) {
       // return the dump from the function and not to a file
@@ -32,24 +35,20 @@ const backupMysqlToS3 = async () => {
           }
       });
 
-      const spacesEndpoint = new AWS.Endpoint(process.env.S3_ENDPOINT);
-
       const created = moment().format('YYYY-MM-DD hh:mm:ss')
 
-      const s3 = new AWS.S3({
-          endpoint: spacesEndpoint,
-          accessKeyId: process.env.S3_KEY,
-          secretAccessKey: process.env.S3_SECRET
-      });
+      const key = isOnK8s ? `mysql/${namespace}/${dbName}_${created}sql` : `mysql/${dbName}_${created}sql`;
 
       var params = {
           Bucket: process.env.S3_BUCKET,
-          Key: 'mysql/' + dbName + '_' + created + ".sql",
+          Key: key,
           Body: result.dump.data,
           ACL: "private"
       };
+
+      const client = s3.getClient();
 
-      s3.putObject(params, function(err, data) {
+      client.putObject(params, function(err, data) {
           if (err) console.log(err, err.stack);
           else     console.log(data);
       });
@@ -76,6 +75,6 @@ module.exports = {
 	cronTime: '0 0 1 * * *',
 	runOnInit: false,
 	onTick: async function() {
-    backupMysqlToS3();
+    return backupMysqlToS3();
 	}
 };

src/models/Submission.js

@@ -14,6 +14,7 @@ module.exports = function (db, sequelize, DataTypes) {
       type: DataTypes.INTEGER,
       allowNull: true
     },
+
     formName: {
       type: DataTypes.STRING,
       allowNull: true
@@ -78,28 +79,31 @@ module.exports = function (db, sequelize, DataTypes) {
 					attributes : ['role', 'displayName', 'nickName', 'firstName', 'lastName', 'email']
 				}]
 			},
-            forSiteId: function (siteId) {
-                return {
-                    where: {
-                        siteId: siteId,
-                    }
-                };
-            },
-            filter:    function (filtersInclude, filtersExclude) {
-                const filterKeys = [
-                    {
-                        'key': 'id'
-                    },
-                    {
-                        'key': 'status'
-                    },
-                    {
-                        'key': 'formId'
-                    },
-                ];
+			forSiteId: function (siteId) {
+					return {
+							where: {
+									siteId: siteId,
+							}
+					};
+			},
+			filter:    function (filtersInclude, filtersExclude) {
+					const filterKeys = [
+							{
+									'key': 'id'
+							},
+							{
+									'key': 'status'
+							},
+							{
+									'key': 'formName'
+							},
+              {
+                  'key': 'formId'
+              },
+					];
 
-                return getSequelizeConditionsForFilters(filterKeys, filtersInclude, sequelize, filtersExclude);
-            },
+					return getSequelizeConditionsForFilters(filterKeys, filtersInclude, sequelize, filtersExclude);
+			},
 		};
 	}