Databases: Support postgres schema in dev environment.

Fixes boxyhq#1818.
This commit adds postgres schema support to the app logic.
The dev environment uses synchronize function to create tables,
and does not run the explicit migrations. We will add schema support
for production in the next commit.

.env.example

@@ -20,6 +20,9 @@ DB_PAGE_LIMIT=50
 DB_ENCRYPTION_KEY=
 # Uncomment below if you wish to run DB migrations manually.
 #DB_MANUAL_MIGRATION=true
+# Specify postgres schema to use. In production, you will need to create the schema first
+# to use this option. Jackson will not create it for you. 
+POSTGRES_SCHEMA=
 
 # Admin Portal settings
 # SMTP details for Magic Links

lib/env.ts

@@ -50,6 +50,9 @@ const db: DatabaseOption = {
     writeCapacityUnits: process.env.DB_DYNAMODB_RCUS ? Number(process.env.DB_DYNAMODB_WCUS) : undefined,
   },
   manualMigration: process.env.DB_MANUAL_MIGRATION === 'true',
+  postgres: {
+    schema: process.env.POSTGRES_SCHEMA,
+  },
 };
 
 /** Indicates if the Jackson instance is hosted (i.e. not self-hosted) */

npm/src/db/constants.ts

@@ -0,0 +1 @@
+export const DEFAULT_POSTGRES_SCHEMA = 'public';

npm/src/db/defaultDb.ts

@@ -1,4 +1,5 @@
 import { JacksonOption } from '../typings';
+import { DEFAULT_POSTGRES_SCHEMA } from './constants';
 
 export default function defaultDb(opts: JacksonOption) {
   opts.db = opts.db || {};
@@ -12,6 +13,8 @@ export default function defaultDb(opts: JacksonOption) {
   opts.db.dynamodb.readCapacityUnits = opts.db.dynamodb.readCapacityUnits || 5;
   opts.db.dynamodb.writeCapacityUnits = opts.db.dynamodb.writeCapacityUnits || 5;
   opts.db.manualMigration = opts.db.manualMigration || false;
+  opts.db.postgres = opts.db.postgres || {};
+  opts.db.postgres.schema = opts.db.postgres.schema || DEFAULT_POSTGRES_SCHEMA;
 
   return opts;
 }

npm/src/db/sql/sql.ts

@@ -6,8 +6,9 @@ import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records, SortOrder }
 import { DataSource, DataSourceOptions, In, IsNull } from 'typeorm';
 import * as dbutils from '../utils';
 import * as mssql from './mssql';
+import { DEFAULT_POSTGRES_SCHEMA } from '../constants';
 
-class Sql implements DatabaseDriver {
+export class Sql implements DatabaseDriver {
   private options: DatabaseOption;
   private dataSource!: DataSource;
   private storeRepository;
@@ -26,6 +27,7 @@ class Sql implements DatabaseDriver {
 
   async init({ JacksonStore, JacksonIndex, JacksonTTL }): Promise<Sql> {
     const sqlType = this.options.engine === 'planetscale' ? 'mysql' : this.options.type!;
+    const postgresSchema = this.options.postgres?.schema || DEFAULT_POSTGRES_SCHEMA;
     // Synchronize by default for non-planetscale engines only if migrations are not set to run
     let synchronize = !this.options.manualMigration;
     if (this.options.engine === 'planetscale') {
@@ -53,14 +55,30 @@ class Sql implements DatabaseDriver {
             ...baseOpts,
           });
         } else {
-          this.dataSource = new DataSource(<DataSourceOptions>{
+          const dataSourceOptions = {
             url: this.options.url,
             ssl: this.options.ssl,
             ...baseOpts,
-          });
+          };
+
+          if (sqlType === 'postgres') {
+            dataSourceOptions['synchronize'] = false;
+            dataSourceOptions['schema'] = postgresSchema;
+          }
+          this.dataSource = new DataSource(<DataSourceOptions>dataSourceOptions);
         }
+
         await this.dataSource.initialize();
 
+        if (sqlType === 'postgres' && synchronize) {
+          // We skip synchronization for postgres databases because TypeORM
+          // does not create schemas if they don't exist, we manually run
+          // synchronize here if it is set to true.
+          const queryRunner = this.dataSource.createQueryRunner();
+          await queryRunner.query(`CREATE SCHEMA IF NOT EXISTS ${postgresSchema}`);
+          this.dataSource.synchronize();
+        }
+
         break;
       } catch (err) {
         console.error(`error connecting to engine: ${this.options.engine}, type: ${sqlType} db: ${err}`);

npm/src/typings.ts

@@ -418,6 +418,9 @@ export interface DatabaseOption {
     writeCapacityUnits?: number;
   };
   manualMigration?: boolean;
+  postgres?: {
+    schema?: string;
+  };
 }
 
 export interface JacksonOption {

npm/test/db/db.test.ts

@@ -9,6 +9,7 @@ const dbObjs: { [key: string]: DatabaseDriver } = {};
 const connectionStores: Storable[] = [];
 const ttlStores: Storable[] = [];
 const ttl = 2;
+const non_default_schema = 'non_default';
 
 const record1 = {
   id: '1',
@@ -130,6 +131,12 @@ const dbs = [
     ...postgresDbConfig,
     encryptionKey,
   },
+  {
+    ...postgresDbConfig,
+    postgres: {
+      schema: non_default_schema,
+    },
+  },
   {
     ...mongoDbConfig,
   },
@@ -188,7 +195,11 @@ tap.before(async () => {
   for (const idx in dbs) {
     const opts = dbs[idx];
     const db = await DB.new(opts, true);
-    dbObjs[opts.engine! + (opts.type ? opts.type : '')] = db;
+    if (opts.type === 'postgres' && opts['schema'] === non_default_schema) {
+      dbObjs[opts['schema'] + opts.engine! + (opts.type ? opts.type : '')] = db;
+    } else {
+      dbObjs[opts.engine! + (opts.type ? opts.type : '')] = db;
+    }
 
     const randomSession = Date.now();
     connectionStores.push(db.store('saml:config:' + randomSession + randomBytes(4).toString('hex')));
@@ -201,15 +212,32 @@ tap.teardown(async () => {
 });
 
 tap.test('dbs', async () => {
+  // We need this to ensure that the test runs atleast once.
+  // It is quite easy to skip the test by mistake in the future
+  // if one of the conditions change and it goes unnoticed.
+  let has_non_default_postgres_schema_test_ran = false;
   for (const idx in connectionStores) {
     const connectionStore = connectionStores[idx];
     const ttlStore = ttlStores[idx];
     const dbEngine = dbs[idx].engine!;
-    let dbType = dbEngine;
+    let dbType = dbEngine.toString();
     if (dbs[idx].type) {
       dbType += ': ' + dbs[idx].type;
     }
 
+    tap.test('Test non default postgres schema', (t) => {
+      if (dbType === 'sql: postgres' && dbs[idx].postgres?.schema === non_default_schema) {
+        t.same(
+          connectionStore['db']['db']['dataSource']['createQueryBuilder']()['connection']['options'][
+            'schema'
+          ],
+          non_default_schema
+        );
+      }
+      has_non_default_postgres_schema_test_ran = true;
+      t.end();
+    });
+
     tap.test('put(): ' + dbType, async () => {
       await connectionStore.put(
         record1.id,
@@ -527,4 +555,9 @@ tap.test('dbs', async () => {
       await value.close();
     }
   });
+
+  tap.test('Ensure that the test for non default postgres schema has ran atleast once', (t) => {
+    t.same(has_non_default_postgres_schema_test_ran, true);
+    t.end();
+  });
 });