diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index dbdce51c50b..d4e6e9a1ec3 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -1,6 +1,5 @@ export * as configs from "./configs" export * as events from "./events" -export * as migrations from "./migrations" export * as users from "./users" export * as userUtils from "./users/utils" export * as roles from "./security/roles" diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts deleted file mode 100644 index 0dd57fe639f..00000000000 --- a/packages/backend-core/src/migrations/definitions.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - MigrationType, - MigrationName, - MigrationDefinition, -} from "@budibase/types" - -export const DEFINITIONS: MigrationDefinition[] = [ - { - type: MigrationType.GLOBAL, - name: MigrationName.USER_EMAIL_VIEW_CASING, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.SYNC_QUOTAS, - }, - { - type: MigrationType.APP, - name: MigrationName.APP_URLS, - }, - { - type: MigrationType.APP, - name: MigrationName.EVENT_APP_BACKFILL, - }, - { - type: MigrationType.APP, - name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.EVENT_GLOBAL_BACKFILL, - }, - { - type: MigrationType.INSTALLATION, - name: MigrationName.EVENT_INSTALLATION_BACKFILL, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.GLOBAL_INFO_SYNC_USERS, - }, -] diff --git a/packages/backend-core/src/migrations/index.ts b/packages/backend-core/src/migrations/index.ts deleted file mode 100644 index bce0cfc75c3..00000000000 --- a/packages/backend-core/src/migrations/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./migrations" -export * from "./definitions" diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts deleted file mode 100644 index c8320b57246..00000000000 --- a/packages/backend-core/src/migrations/migrations.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { DEFAULT_TENANT_ID } from "../constants" -import { - DocumentType, - StaticDatabases, - getAllApps, - getGlobalDBName, - getDB, -} from "../db" -import environment from "../environment" -import * as platform from "../platform" -import * as context from "../context" -import { DEFINITIONS } from "." -import { - Migration, - MigrationOptions, - MigrationType, - MigrationNoOpOptions, - App, -} from "@budibase/types" - -export const getMigrationsDoc = async (db: any) => { - // get the migrations doc - try { - return await db.get(DocumentType.MIGRATIONS) - } catch (err: any) { - if (err.status && err.status === 404) { - return { _id: DocumentType.MIGRATIONS } - } else { - throw err - } - } -} - -export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => { - // filter migrations to the type and populate a no-op migration - const migrations: Migration[] = DEFINITIONS.filter( - def => def.type === opts.type - ).map(d => ({ ...d, fn: async () => {} })) - await runMigrations(migrations, { noOp: opts }) -} - -export const runMigration = async ( - migration: Migration, - options: MigrationOptions = {} -) => { - const migrationType = migration.type - const migrationName = migration.name - const silent = migration.silent - - const log = (message: string) => { - if (!silent) { - console.log(message) - } - } - - // get the db to store the migration in - let dbNames: string[] - if (migrationType === MigrationType.GLOBAL) { - dbNames = [getGlobalDBName()] - } else if (migrationType === MigrationType.APP) { - if (options.noOp) { - if (!options.noOp.appId) { - throw new Error("appId is required for noOp app migration") - } - dbNames = [options.noOp.appId] - } else { - const apps = (await getAllApps(migration.appOpts)) as App[] - dbNames = apps.map(app => app.appId) - } - } else if (migrationType === MigrationType.INSTALLATION) { - dbNames = [StaticDatabases.PLATFORM_INFO.name] - } else { - throw new Error(`Unrecognised migration type [${migrationType}]`) - } - - const length = dbNames.length - let count = 0 - - // run the migration against each db - for (const dbName of dbNames) { - count++ - const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - - const db = getDB(dbName) - - try { - const doc = await getMigrationsDoc(db) - - // the migration has already been run - if (doc[migrationName]) { - // check for force - if ( - options.force && - options.force[migrationType] && - options.force[migrationType].includes(migrationName) - ) { - log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`) - } else { - // no force, exit - return - } - } - - // check if the migration is not a no-op - if (!options.noOp) { - log( - `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` - ) - - if (migration.preventRetry) { - // eagerly set the completion date - // so that we never run this migration twice even upon failure - doc[migrationName] = Date.now() - const response = await db.put(doc) - doc._rev = response.rev - } - - // run the migration - if (migrationType === MigrationType.APP) { - await context.doInAppContext(db.name, async () => { - await migration.fn(db) - }) - } else { - await migration.fn(db) - } - - log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`) - } - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Migration: ${migrationName}] [DB: ${dbName}] Error: `, - err - ) - throw err - } - } -} - -export const runMigrations = async ( - migrations: Migration[], - options: MigrationOptions = {} -) => { - let tenantIds - - if (environment.MULTI_TENANCY) { - if (options.noOp) { - tenantIds = [options.noOp.tenantId] - } else if (!options.tenantIds || !options.tenantIds.length) { - // run for all tenants - tenantIds = await platform.tenants.getTenantIds() - } else { - tenantIds = options.tenantIds - } - } else { - // single tenancy - tenantIds = [DEFAULT_TENANT_ID] - } - - if (tenantIds.length > 1) { - console.log(`Checking migrations for ${tenantIds.length} tenants`) - } else { - console.log("Checking migrations") - } - - let count = 0 - // for all tenants - for (const tenantId of tenantIds) { - count++ - if (tenantIds.length > 1) { - console.log(`Progress [${count}/${tenantIds.length}]`) - } - // for all migrations - for (const migration of migrations) { - // run the migration - await context.doInTenant( - tenantId, - async () => await runMigration(migration, options) - ) - } - } - console.log("Migrations complete") -} diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap b/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap deleted file mode 100644 index 377900b5d5c..00000000000 --- a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap +++ /dev/null @@ -1,11 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`migrations should match snapshot 1`] = ` -{ - "_id": "migrations", - "_rev": "1-2f64479842a0513aa8b97f356b0b9127", - "createdAt": "2020-01-01T00:00:00.000Z", - "test": 1577836800000, - "updatedAt": "2020-01-01T00:00:00.000Z", -} -`; diff --git a/packages/backend-core/src/migrations/tests/migrations.spec.ts b/packages/backend-core/src/migrations/tests/migrations.spec.ts deleted file mode 100644 index af2eb33cf58..00000000000 --- a/packages/backend-core/src/migrations/tests/migrations.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { testEnv, DBTestConfiguration } from "../../../tests/extra" -import * as migrations from "../index" -import * as context from "../../context" -import { MigrationType } from "@budibase/types" - -testEnv.multiTenant() - -describe("migrations", () => { - const config = new DBTestConfiguration() - - const migrationFunction = jest.fn() - - const MIGRATIONS = [ - { - type: MigrationType.GLOBAL, - name: "test" as any, - fn: migrationFunction, - }, - ] - - beforeEach(() => { - config.newTenant() - }) - - afterEach(async () => { - jest.clearAllMocks() - }) - - const migrate = () => { - return migrations.runMigrations(MIGRATIONS, { - tenantIds: [config.tenantId], - }) - } - - it("should run a new migration", async () => { - await config.doInTenant(async () => { - await migrate() - expect(migrationFunction).toHaveBeenCalled() - const db = context.getGlobalDB() - const doc = await migrations.getMigrationsDoc(db) - expect(doc.test).toBeDefined() - }) - }) - - it("should match snapshot", async () => { - await config.doInTenant(async () => { - await migrate() - const doc = await migrations.getMigrationsDoc(context.getGlobalDB()) - expect(doc).toMatchSnapshot() - }) - }) - - it("should skip a previously run migration", async () => { - await config.doInTenant(async () => { - const db = context.getGlobalDB() - await migrate() - const previousDoc = await migrations.getMigrationsDoc(db) - await migrate() - const currentDoc = await migrations.getMigrationsDoc(db) - expect(migrationFunction).toHaveBeenCalledTimes(1) - expect(currentDoc.test).toBe(previousDoc.test) - }) - }) -}) diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.ts similarity index 67% rename from packages/backend-core/tests/core/users/users.spec.js rename to packages/backend-core/tests/core/users/users.spec.ts index dde0d87fb7b..b14f5532665 100644 --- a/packages/backend-core/tests/core/users/users.spec.js +++ b/packages/backend-core/tests/core/users/users.spec.ts @@ -1,17 +1,17 @@ -const _ = require("lodash/fp") -const { structures } = require("../../../tests") +import { range } from "lodash/fp" +import { structures } from "../.." jest.mock("../../../src/context") jest.mock("../../../src/db") -const context = require("../../../src/context") -const db = require("../../../src/db") +import * as context from "../../../src/context" +import * as db from "../../../src/db" -const { getCreatorCount } = require("../../../src/users/users") +import { getCreatorCount } from "../../../src/users/users" describe("Users", () => { - let getGlobalDBMock - let paginationMock + let getGlobalDBMock: jest.SpyInstance + let paginationMock: jest.SpyInstance beforeEach(() => { jest.resetAllMocks() @@ -22,11 +22,10 @@ describe("Users", () => { jest.spyOn(db, "getGlobalUserParams") }) - it("Retrieves the number of creators", async () => { - const getUsers = (offset, limit, creators = false) => { - const range = _.range(offset, limit) + it("retrieves the number of creators", async () => { + const getUsers = (offset: number, limit: number, creators = false) => { const opts = creators ? { builder: { global: true } } : undefined - return range.map(() => structures.users.user(opts)) + return range(offset, limit).map(() => structures.users.user(opts)) } const page1Data = getUsers(0, 8) const page2Data = getUsers(8, 12, true) diff --git a/packages/frontend-core/src/api/migrations.ts b/packages/frontend-core/src/api/migrations.ts index 8213691205b..35d2f95dbc4 100644 --- a/packages/frontend-core/src/api/migrations.ts +++ b/packages/frontend-core/src/api/migrations.ts @@ -1,8 +1,8 @@ -import { GetOldMigrationStatus } from "@budibase/types" +import { GetMigrationStatus } from "@budibase/types" import { BaseAPIClient } from "./types" export interface MigrationEndpoints { - getMigrationStatus: () => Promise + getMigrationStatus: () => Promise } export const buildMigrationEndpoints = ( diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 0794c535061..84e1abea69e 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -43,7 +43,6 @@ async function init() { BB_ADMIN_USER_EMAIL: "", BB_ADMIN_USER_PASSWORD: "", PLUGINS_DIR: "", - HTTP_MIGRATIONS: "0", HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 4169087a63a..b62d022cbda 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -27,7 +27,6 @@ import { env as envCore, ErrorCode, events, - migrations, objectStore, roles, tenancy, @@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro" import { App, Layout, - MigrationType, PlanType, Screen, UserCtx, @@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest, app: App) { } async function appPostCreate(ctx: UserCtx, app: App) { - const tenantId = tenancy.getTenantId() - await migrations.backPopulateMigrations({ - type: MigrationType.APP, - tenantId, - appId: app.appId, - }) - await creationEvents(ctx.request, app) // app import, template creation and duplication diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index edf4ec6f515..fc3a1a15482 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,35 +1,11 @@ import { context } from "@budibase/backend-core" -import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { - Ctx, - FetchOldMigrationResponse, - GetOldMigrationStatus, - RuneOldMigrationResponse, - RunOldMigrationRequest, -} from "@budibase/types" +import { Ctx, GetMigrationStatus } from "@budibase/types" import { getAppMigrationVersion, getLatestEnabledMigrationId, } from "../../appMigrations" -export async function migrate( - ctx: Ctx -) { - const options = ctx.request.body - // don't await as can take a while, just return - migrationImpl(options) - ctx.body = { message: "Migration started." } -} - -export async function fetchDefinitions( - ctx: Ctx -) { - ctx.body = MIGRATIONS -} - -export async function getMigrationStatus( - ctx: Ctx -) { +export async function getMigrationStatus(ctx: Ctx) { const appId = context.getAppId() if (!appId) { diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index 918b197de29..0ffc3345518 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -1,16 +1,8 @@ import Router from "@koa/router" import * as migrationsController from "../controllers/migrations" -import { auth } from "@budibase/backend-core" const router: Router = new Router() -router - .post("/api/migrations/run", auth.internalApi, migrationsController.migrate) - .get( - "/api/migrations/definitions", - auth.internalApi, - migrationsController.fetchDefinitions - ) - .get("/api/migrations/status", migrationsController.getMigrationStatus) +router.get("/api/migrations/status", migrationsController.getMigrationStatus) export default router diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 45d675ec3f2..a9867b1231c 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -54,7 +54,6 @@ const environment = { REDIS_URL: process.env.REDIS_URL, REDIS_PASSWORD: process.env.REDIS_PASSWORD, REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, - HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, CLUSTER_MODE: process.env.CLUSTER_MODE, API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts deleted file mode 100644 index be03d3c81ee..00000000000 --- a/packages/server/src/migrations/functions/appUrls.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import sdk from "../../sdk" - -/** - * Date: - * January 2022 - * - * Description: - * Add the url to the app metadata if it doesn't exist - */ -export const run = async (appDb: any) => { - let metadata - try { - metadata = await appDb.get(dbCore.DocumentType.APP_METADATA) - } catch (e) { - // sometimes the metadata document doesn't exist - // exit early instead of failing the migration - console.error("Error retrieving app metadata. Skipping", e) - return - } - - if (!metadata.url) { - metadata.url = sdk.applications.getAppUrl({ name: metadata.name }) - console.log(`Adding url to app: ${metadata.url}`) - await appDb.put(metadata) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts deleted file mode 100644 index 51a37108b0f..00000000000 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ /dev/null @@ -1,149 +0,0 @@ -import * as automations from "./app/automations" -import * as datasources from "./app/datasources" -import * as layouts from "./app/layouts" -import * as queries from "./app/queries" -import * as roles from "./app/roles" -import * as tables from "./app/tables" -import * as screens from "./app/screens" -import * as global from "./global" -import { App, AppBackfillSucceededEvent, Event } from "@budibase/types" -import { db as dbUtils, events } from "@budibase/backend-core" -import env from "../../../environment" -import { DEFAULT_TIMESTAMP } from "." - -const failGraceful = env.SELF_HOSTED && !env.isDev() - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - console.trace(e) - throw e -} - -const EVENTS = [ - Event.AUTOMATION_CREATED, - Event.AUTOMATION_STEP_CREATED, - Event.DATASOURCE_CREATED, - Event.LAYOUT_CREATED, - Event.QUERY_CREATED, - Event.ROLE_CREATED, - Event.SCREEN_CREATED, - Event.TABLE_CREATED, - Event.VIEW_CREATED, - Event.VIEW_CALCULATION_CREATED, - Event.VIEW_FILTER_CREATED, - Event.APP_PUBLISHED, - Event.APP_CREATED, -] - -/** - * Date: - * May 2022 - * - * Description: - * Backfill app events. - */ - -export const run = async (appDb: any) => { - try { - if (await global.isComplete()) { - // make sure new apps aren't backfilled - // return if the global migration for this tenant is complete - // which runs after the app migrations - return - } - - // tell the event pipeline to start caching - // events for this tenant - await events.backfillCache.start(EVENTS) - - let timestamp: string | number = DEFAULT_TIMESTAMP - const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA) - if (app.createdAt) { - timestamp = app.createdAt as string - } - - if (dbUtils.isProdAppID(app.appId)) { - await events.app.published(app, timestamp) - } - - const totals: any = {} - const errors: any = [] - - if (dbUtils.isDevAppID(app.appId)) { - await events.app.created(app, timestamp) - try { - totals.automations = await automations.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.datasources = await datasources.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.layouts = await layouts.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.queries = await queries.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.roles = await roles.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.screens = await screens.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.tables = await tables.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - } - - const properties: AppBackfillSucceededEvent = { - appId: app.appId, - automations: totals.automations, - datasources: totals.datasources, - layouts: totals.layouts, - queries: totals.queries, - roles: totals.roles, - tables: totals.tables, - screens: totals.screens, - } - - if (errors.length) { - properties.errors = errors.map((e: any) => - JSON.stringify(e, Object.getOwnPropertyNames(e)) - ) - properties.errorCount = errors.length - } else { - properties.errorCount = 0 - } - - await events.backfill.appSucceeded(properties) - // tell the event pipeline to stop caching events for this tenant - await events.backfillCache.end() - } catch (e) { - handleError(e) - await events.backfill.appFailed(e) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts deleted file mode 100644 index 20da8fd3c0e..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/automations.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getAutomationParams } from "../../../../db/utils" -import { Automation } from "@budibase/types" - -const getAutomations = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getAutomationParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const automations = await getAutomations(appDb) - - for (const automation of automations) { - await events.automation.created(automation, timestamp) - - for (const step of automation.definition.steps) { - await events.automation.stepCreated(automation, step, timestamp) - } - } - - return automations.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts deleted file mode 100644 index 5d7e1ad8661..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/datasources.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getDatasourceParams } from "../../../../db/utils" -import { Datasource } from "@budibase/types" - -const getDatasources = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getDatasourceParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const datasources: Datasource[] = await getDatasources(appDb) - - for (const datasource of datasources) { - await events.datasource.created(datasource, timestamp) - } - - return datasources.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts deleted file mode 100644 index ee5806459bf..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/layouts.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getLayoutParams } from "../../../../db/utils" -import { Layout } from "@budibase/types" - -const getLayouts = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getLayoutParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const layouts: Layout[] = await getLayouts(appDb) - - for (const layout of layouts) { - // exclude default layouts - if ( - layout._id === "layout_private_master" || - layout._id === "layout_public_master" - ) { - continue - } - await events.layout.created(layout, timestamp) - } - - return layouts.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts deleted file mode 100644 index e028721bce4..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getQueryParams } from "../../../../db/utils" -import { Query, Datasource, SourceName } from "@budibase/types" - -const getQueries = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getQueryParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -const getDatasource = async ( - appDb: any, - datasourceId: string -): Promise => { - return appDb.get(datasourceId) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const queries: Query[] = await getQueries(appDb) - - for (const query of queries) { - let datasource: Datasource - - try { - datasource = await getDatasource(appDb, query.datasourceId) - } catch (e: any) { - // handle known bug where a datasource has been deleted - // and the query has not - if (e.status === 404) { - datasource = { - type: "unknown", - _id: query.datasourceId, - source: "unknown" as SourceName, - } - } else { - throw e - } - } - - await events.query.created(datasource, query, timestamp) - } - - return queries.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts deleted file mode 100644 index 494b6f69239..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/roles.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getRoleParams } from "../../../../db/utils" -import { Role } from "@budibase/types" - -const getRoles = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const roles = await getRoles(appDb) - - for (const role of roles) { - await events.role.created(role, timestamp) - } - - return roles.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts deleted file mode 100644 index ab3b4b9d3c3..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/screens.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getScreenParams } from "../../../../db/utils" -import { Screen } from "@budibase/types" - -const getScreens = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getScreenParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const screens = await getScreens(appDb) - - for (const screen of screens) { - await events.screen.created(screen, timestamp) - } - - return screens.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts deleted file mode 100644 index e8437bd5298..00000000000 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { events } from "@budibase/backend-core" -import { Database } from "@budibase/types" -import sdk from "../../../../sdk" - -export const backfill = async (appDb: Database, timestamp: string | number) => { - const tables = await sdk.tables.getAllInternalTables(appDb) - - for (const table of tables) { - await events.table.created(table, timestamp) - } - - return tables.length -} diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts deleted file mode 100644 index 7f718cee2ff..00000000000 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ /dev/null @@ -1,214 +0,0 @@ -import * as users from "./global/users" -import * as configs from "./global/configs" -import * as quotas from "./global/quotas" -import { - tenancy, - events, - migrations, - accounts, - db as dbUtils, -} from "@budibase/backend-core" -import { - App, - CloudAccount, - Event, - Hosting, - QuotaUsage, - TenantBackfillSucceededEvent, - User, -} from "@budibase/types" -import env from "../../../environment" -import { DEFAULT_TIMESTAMP } from "." - -const failGraceful = env.SELF_HOSTED && !env.isDev() - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - throw e -} - -const formatUsage = (usage: QuotaUsage) => { - let maxAutomations = 0 - let maxQueries = 0 - let rows = 0 - - if (usage) { - if (usage.usageQuota) { - rows = usage.usageQuota.rows - } - - if (usage.monthly) { - for (const value of Object.values(usage.monthly)) { - if (value.automations > maxAutomations) { - maxAutomations = value.automations - } - if (value.queries > maxQueries) { - maxQueries = value.queries - } - } - } - } - - return { - maxAutomations, - maxQueries, - rows, - } -} - -const EVENTS = [ - Event.EMAIL_SMTP_CREATED, - Event.AUTH_SSO_CREATED, - Event.AUTH_SSO_ACTIVATED, - Event.ORG_NAME_UPDATED, - Event.ORG_LOGO_UPDATED, - Event.ORG_PLATFORM_URL_UPDATED, - Event.USER_CREATED, - Event.USER_PERMISSION_ADMIN_ASSIGNED, - Event.USER_PERMISSION_BUILDER_ASSIGNED, - Event.ROLE_ASSIGNED, - Event.ROWS_CREATED, - Event.QUERIES_RUN, - Event.AUTOMATIONS_RUN, -] - -/** - * Date: - * May 2022 - * - * Description: - * Backfill global events. - */ - -export const run = async (db: any) => { - try { - const tenantId = tenancy.getTenantId() - let timestamp: string | number = DEFAULT_TIMESTAMP - - const totals: any = {} - const errors: any = [] - - let allUsers: User[] = [] - try { - allUsers = await users.getUsers(db) - } catch (e: any) { - handleError(e, errors) - } - - if (!allUsers || allUsers.length === 0) { - // first time startup - we don't need to backfill anything - // tenant will be identified when admin user is created - if (env.SELF_HOSTED) { - await events.installation.firstStartup() - } - return - } - - try { - const installTimestamp = await getInstallTimestamp(db, allUsers) - if (installTimestamp) { - timestamp = installTimestamp - } - } catch (e) { - handleError(e, errors) - } - - let account: CloudAccount | undefined - if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - account = await accounts.getAccountByTenantId(tenantId) - } - - try { - await events.identification.identifyTenantGroup( - tenantId, - env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD, - timestamp - ) - } catch (e) { - handleError(e, errors) - } - - // tell the event pipeline to start caching - // events for this tenant - await events.backfillCache.start(EVENTS) - - try { - await configs.backfill(db, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.users = await users.backfill(db, account) - } catch (e) { - handleError(e, errors) - } - - try { - const allApps = (await dbUtils.getAllApps({ dev: true })) as App[] - totals.apps = allApps.length - - totals.usage = await quotas.backfill(allApps) - } catch (e) { - handleError(e, errors) - } - - const properties: TenantBackfillSucceededEvent = { - apps: totals.apps, - users: totals.users, - ...formatUsage(totals.usage), - usage: totals.usage, - } - - if (errors.length) { - properties.errors = errors.map((e: any) => - JSON.stringify(e, Object.getOwnPropertyNames(e)) - ) - properties.errorCount = errors.length - } else { - properties.errorCount = 0 - } - - await events.backfill.tenantSucceeded(properties) - // tell the event pipeline to stop caching events for this tenant - await events.backfillCache.end() - } catch (e) { - handleError(e) - await events.backfill.tenantFailed(e) - } -} - -export const isComplete = async (): Promise => { - const globalDb = tenancy.getGlobalDB() - const migrationsDoc = await migrations.getMigrationsDoc(globalDb) - return !!migrationsDoc.event_global_backfill -} - -export const getInstallTimestamp = async ( - globalDb: any, - allUsers?: User[] -): Promise => { - if (!allUsers) { - allUsers = await users.getUsers(globalDb) - } - - // get the oldest user timestamp - if (allUsers) { - const timestamps = allUsers - .map(user => user.createdAt) - .filter(timestamp => !!timestamp) - .sort( - (a, b) => - new Date(a as number).getTime() - new Date(b as number).getTime() - ) - - if (timestamps.length) { - return timestamps[0] - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts deleted file mode 100644 index 04eb9caff2f..00000000000 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - events, - DocumentType, - SEPARATOR, - UNICODE_MAX, -} from "@budibase/backend-core" -import { - Config, - isSMTPConfig, - isGoogleConfig, - isOIDCConfig, - isSettingsConfig, - ConfigType, - DatabaseQueryOpts, -} from "@budibase/types" -import env from "./../../../../environment" - -export function getConfigParams(): DatabaseQueryOpts { - return { - include_docs: true, - startkey: `${DocumentType.CONFIG}${SEPARATOR}`, - endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`, - } -} - -const getConfigs = async (globalDb: any): Promise => { - const response = await globalDb.allDocs(getConfigParams()) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async ( - globalDb: any, - timestamp: string | number | undefined -) => { - const configs = await getConfigs(globalDb) - - for (const config of configs) { - if (isSMTPConfig(config)) { - await events.email.SMTPCreated(timestamp) - } - if (isGoogleConfig(config)) { - await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp) - if (config.config.activated) { - await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp) - } - } - if (isOIDCConfig(config)) { - await events.auth.SSOCreated(ConfigType.OIDC, timestamp) - if (config.config.configs[0].activated) { - await events.auth.SSOActivated(ConfigType.OIDC, timestamp) - } - } - if (isSettingsConfig(config)) { - const company = config.config.company - if (company && company !== "Budibase") { - await events.org.nameUpdated(timestamp) - } - - const logoUrl = config.config.logoUrl - if (logoUrl) { - await events.org.logoUpdated(timestamp) - } - - const platformUrl = config.config.platformUrl - if ( - platformUrl && - platformUrl !== "http://localhost:10000" && - env.SELF_HOSTED - ) { - await events.org.platformURLUpdated(timestamp) - } - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/quotas.ts b/packages/server/src/migrations/functions/backfill/global/quotas.ts deleted file mode 100644 index 505274a318a..00000000000 --- a/packages/server/src/migrations/functions/backfill/global/quotas.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { DEFAULT_TIMESTAMP } from "./../index" -import { events } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { App } from "@budibase/types" - -const getOldestCreatedAt = (allApps: App[]): string | undefined => { - const timestamps = allApps - .filter(app => !!app.createdAt) - .map(app => app.createdAt as string) - .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) - - if (timestamps.length) { - return timestamps[0] - } -} - -const getMonthTimestamp = (monthString: string): number => { - const parts = monthString.split("-") - const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation - const year = parseInt(parts[1]) - - // using 0 as the day in next month gives us last day in previous month - const date = new Date(year, month + 1, 0).getTime() - const now = new Date().getTime() - - if (date > now) { - return now - } else { - return date - } -} - -export const backfill = async (allApps: App[]) => { - const usage = await quotas.getQuotaUsage() - - const rows = usage.usageQuota.rows - let timestamp: string | number = DEFAULT_TIMESTAMP - - const oldestAppTimestamp = getOldestCreatedAt(allApps) - if (oldestAppTimestamp) { - timestamp = oldestAppTimestamp - } - - await events.rows.created(rows, timestamp) - - for (const [monthString, quotas] of Object.entries(usage.monthly)) { - if (monthString === "current") { - continue - } - const monthTimestamp = getMonthTimestamp(monthString) - - const queries = quotas.queries - await events.query.run(queries, monthTimestamp) - - const automations = quotas.automations - await events.automation.run(automations, monthTimestamp) - } - - return usage -} diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts deleted file mode 100644 index b3dae822d7a..00000000000 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - events, - db as dbUtils, - users as usersCore, -} from "@budibase/backend-core" -import { User, CloudAccount } from "@budibase/types" -import { DEFAULT_TIMESTAMP } from ".." - -// manually define user doc params - normally server doesn't read users from the db -const getUserParams = (props: any) => { - return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props) -} - -export const getUsers = async (globalDb: any): Promise => { - const response = await globalDb.allDocs( - getUserParams({ - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async ( - globalDb: any, - account: CloudAccount | undefined -) => { - const users = await getUsers(globalDb) - - for (const user of users) { - let timestamp: string | number = DEFAULT_TIMESTAMP - if (user.createdAt) { - timestamp = user.createdAt - } - await events.identification.identifyUser(user, account, timestamp) - await events.user.created(user, timestamp) - - if (usersCore.hasAdminPermissions(user)) { - await events.user.permissionAdminAssigned(user, timestamp) - } - - if (usersCore.hasBuilderPermissions(user)) { - await events.user.permissionBuilderAssigned(user, timestamp) - } - - if (user.roles) { - for (const [, role] of Object.entries(user.roles)) { - await events.role.assigned(user, role, timestamp) - } - } - } - - return users.length -} diff --git a/packages/server/src/migrations/functions/backfill/index.ts b/packages/server/src/migrations/functions/backfill/index.ts deleted file mode 100644 index 00c04722b4c..00000000000 --- a/packages/server/src/migrations/functions/backfill/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * as app from "./app" -export * as global from "./global" -export * as installation from "./installation" - -// historical events are free in posthog - make sure we default to a -// historical time if no other can be found -export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime() diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts deleted file mode 100644 index 3c2b8bd3fc2..00000000000 --- a/packages/server/src/migrations/functions/backfill/installation.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { DEFAULT_TIMESTAMP } from "./index" -import { events, tenancy, installation } from "@budibase/backend-core" -import { Installation } from "@budibase/types" -import * as global from "./global" -import env from "../../../environment" - -const failGraceful = env.SELF_HOSTED - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - throw e -} - -/** - * Date: - * May 2022 - * - * Description: - * Backfill installation events. - */ - -export const run = async () => { - try { - // need to use the default tenant to try to get the installation time - await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { - const db = tenancy.getGlobalDB() - let timestamp: string | number = DEFAULT_TIMESTAMP - - const installTimestamp = await global.getInstallTimestamp(db) - if (installTimestamp) { - timestamp = installTimestamp - } - - const install: Installation = await installation.getInstall() - await events.identification.identifyInstallationGroup( - install.installId, - timestamp - ) - }) - await events.backfill.installationSucceeded() - } catch (e) { - handleError(e) - await events.backfill.installationFailed(e) - } -} diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts deleted file mode 100644 index 83a7670e789..00000000000 --- a/packages/server/src/migrations/functions/syncQuotas.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { runQuotaMigration } from "./usageQuotas" -import * as syncApps from "./usageQuotas/syncApps" -import * as syncRows from "./usageQuotas/syncRows" -import * as syncPlugins from "./usageQuotas/syncPlugins" -import * as syncUsers from "./usageQuotas/syncUsers" -import * as syncCreators from "./usageQuotas/syncCreators" - -/** - * Synchronise quotas to the state of the db. - */ -export const run = async () => { - await runQuotaMigration(async () => { - await syncApps.run() - await syncRows.run() - await syncPlugins.run() - await syncUsers.run() - await syncCreators.run() - }) -} diff --git a/packages/server/src/migrations/functions/tableSettings.ts b/packages/server/src/migrations/functions/tableSettings.ts deleted file mode 100644 index 2db3df0d0f2..00000000000 --- a/packages/server/src/migrations/functions/tableSettings.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { getScreenParams } from "../../db/utils" -import { Screen } from "@budibase/types" -import { makePropSafe as safe } from "@budibase/string-templates" -/** - * Date: - * November 2022 - * - * Description: - * Update table settings to use actions instead of links. We do not remove the - * legacy values here as we cannot guarantee that their apps are up-t-date. - * It is safe to simply save both the new and old structure in the definition. - * - * Migration 1: - * Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables - * and table blocks are migrated into a "Navigate To" action under the new - * "onClick" setting. - * - * Migration 2: - * Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are - * migrated into a "Navigate To" action under the new "onClickTitleButton" - * setting. - */ -export const run = async (appDb: any) => { - // Get all app screens - let screens: Screen[] - try { - screens = ( - await appDb.allDocs( - getScreenParams(null, { - include_docs: true, - }) - ) - ).rows.map((row: any) => row.doc) - } catch (e) { - // sometimes the metadata document doesn't exist - // exit early instead of failing the migration - console.error("Error retrieving app metadata. Skipping", e) - return - } - - // Recursively update any relevant components and mutate the screen docs - for (let screen of screens) { - const changed = migrateTableSettings(screen.props) - - // Save screen if we updated it - if (changed) { - await appDb.put(screen) - console.log( - `Screen ${screen.routing?.route} contained table settings which were migrated` - ) - } - } -} - -// Recursively searches and mutates a screen doc to migrate table component -// and table block settings -const migrateTableSettings = (component: any) => { - let changed = false - if (!component) { - return changed - } - - // Migration 1: migrate table row click settings - if ( - component._component.endsWith("/table") || - component._component.endsWith("/tableblock") - ) { - const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component - if (linkRows && !onClick) { - const column = linkColumn || "_id" - const action = convertLinkSettingToAction(linkURL, !!linkPeek, column) - if (action) { - changed = true - component.onClick = action - if (component._component.endsWith("/tableblock")) { - component.clickBehaviour = "actions" - } - } - } - } - - // Migration 2: migrate table block title button settings - if (component._component.endsWith("/tableblock")) { - const { - showTitleButton, - titleButtonURL, - titleButtonPeek, - onClickTitleButton, - } = component - if (showTitleButton && !onClickTitleButton) { - const action = convertLinkSettingToAction( - titleButtonURL, - !!titleButtonPeek - ) - if (action) { - changed = true - component.onClickTitleButton = action - component.titleButtonClickBehaviour = "actions" - } - } - } - - // Recurse down the tree as needed - component._children?.forEach((child: any) => { - const childChanged = migrateTableSettings(child) - changed = changed || childChanged - }) - return changed -} - -// Util ti convert the legacy settings into a navigation action structure -const convertLinkSettingToAction = ( - linkURL: string, - linkPeek: boolean, - linkColumn?: string -) => { - // Sanity check we have a URL - if (!linkURL) { - return null - } - - // Default URL to the old URL setting - let url = linkURL - - // If we enriched the old URL with a column, update the url - if (linkColumn && linkURL.includes("/:")) { - // Convert old link URL setting, which is a screen URL, into a valid - // binding using the new clicked row binding - const split = linkURL.split("/:") - const col = linkColumn || "_id" - const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}` - url = `${split[0]}/${binding}` - } - - // Create action structure - return [ - { - "##eventHandlerType": "Navigate To", - parameters: { - url, - peek: linkPeek, - }, - }, - ] -} diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js deleted file mode 100644 index 7a2f3245527..00000000000 --- a/packages/server/src/migrations/functions/tests/appUrls.spec.js +++ /dev/null @@ -1,26 +0,0 @@ -const { db: dbCore } = require("@budibase/backend-core") -const TestConfig = require("../../../tests/utilities/TestConfiguration") - -const migration = require("../appUrls") - -describe("run", () => { - let config = new TestConfig(false) - - beforeAll(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - const app = await config.createApp("testApp") - const metadata = await dbCore.doWithDB(app.appId, async db => { - const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA) - delete metadataDoc.url - await db.put(metadataDoc) - await migration.run(db) - return await db.get(dbCore.DocumentType.APP_METADATA) - }) - expect(metadata.url).toEqual("/testapp") - }) -}) diff --git a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts b/packages/server/src/migrations/functions/tests/tableSettings.spec.ts deleted file mode 100644 index 8d28a433225..00000000000 --- a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { App, Screen } from "@budibase/types" - -import { db as dbCore } from "@budibase/backend-core" -import TestConfig from "../../../tests/utilities/TestConfiguration" -import { run as runMigration } from "../tableSettings" - -describe("run", () => { - const config = new TestConfig(false) - let app: App - let screen: Screen - - beforeAll(async () => { - await config.init() - app = await config.createApp("testApp") - screen = await config.createScreen() - }) - - afterAll(config.end) - - it("migrates table block row on click settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const onClick = screen.props._children?.[0].onClick - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe( - `/rows/{{ [eventContext].[row].[name] }}` - ) - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("migrates table row on click settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table", - _styles: {}, - _component: "@budibase/standard-components/table", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const onClick = screen.props._children?.[0].onClick - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe( - `/rows/{{ [eventContext].[row].[name] }}` - ) - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("migrates table block title button settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - showTitleButton: true, - titleButtonURL: "/url", - titleButtonPeek: true, - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClickTitleButton" setting - const onClick = screen.props._children?.[0].onClickTitleButton - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe("/url") - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("ignores components that have already been migrated", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - onClick: "foo", - }, - ] - const initialDefinition = JSON.stringify(screen.props._children?.[0]) - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const newDefinition = JSON.stringify(screen.props._children?.[0]) - expect(initialDefinition).toEqual(newDefinition) - }) -}) diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js deleted file mode 100644 index f0da25893c6..00000000000 --- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js +++ /dev/null @@ -1,34 +0,0 @@ -jest.mock("@budibase/backend-core", () => { - const core = jest.requireActual("@budibase/backend-core") - return { - ...core, - db: { - ...core.db, - createNewUserEmailView: jest.fn(), - }, - } -}) -const { context, db: dbCore } = require("@budibase/backend-core") -const TestConfig = require("../../../tests/utilities/TestConfiguration") - -// mock email view creation - -const migration = require("../userEmailViewCasing") - -describe("run", () => { - let config = new TestConfig(false) - - beforeAll(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - await config.doInTenant(async () => { - const globalDb = context.getGlobalDB() - await migration.run(globalDb) - expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts deleted file mode 100644 index e94e993b21c..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const runQuotaMigration = async (migration: () => Promise) => { - await migration() -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts deleted file mode 100644 index 80ed1953d37..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - // get app count - const devApps = await dbCore.getAllApps({ dev: true }) - const appCount = devApps ? devApps.length : 0 - - // sync app count - console.log(`Syncing app count: ${appCount}`) - await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts deleted file mode 100644 index ce53be925a5..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { users } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - const creatorCount = await users.getCreatorCount() - console.log(`Syncing creator count: ${creatorCount}`) - await quotas.setUsage( - creatorCount, - StaticQuotaName.CREATORS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts b/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts deleted file mode 100644 index b00970aea27..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { logging } from "@budibase/backend-core" -import { plugins } from "@budibase/pro" - -export const run = async () => { - try { - await plugins.checkPluginQuotas() - } catch (err) { - logging.logAlert("Failed to update plugin quotas", err) - } -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts deleted file mode 100644 index 506218a41f3..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import { getUniqueRows } from "../../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" -import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types" - -export const run = async () => { - // get all rows in all apps - const allApps = (await dbCore.getAllApps({ all: true })) as App[] - const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] - const { appRows } = await getUniqueRows(appIds) - - // get the counts per app - const counts: { [key: string]: number } = {} - let rowCount = 0 - Object.entries(appRows).forEach(([appId, rows]) => { - counts[appId] = rows.length - rowCount += rows.length - }) - - // sync row count - console.log(`Syncing row count: ${rowCount}`) - await quotas.setUsagePerApp( - counts, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts b/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts deleted file mode 100644 index c9913dced82..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { users } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - const userCount = await users.getUserCount() - console.log(`Syncing user count: ${userCount}`) - await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts deleted file mode 100644 index 1d4d4d0f711..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts +++ /dev/null @@ -1,35 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncApps from "../syncApps" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -describe("syncApps", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - return config.doInContext(undefined, async () => { - // create the usage quota doc and mock usages - await quotas.getQuotaUsage() - await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC) - - let usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.apps).toEqual(3) - - // create an extra app to test the migration - await config.createApp("quota-test") - - // migrate - await syncApps.run() - - // assert the migration worked - usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.apps).toEqual(2) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts deleted file mode 100644 index 93b7d4949b6..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncCreators from "../syncCreators" -import { quotas } from "@budibase/pro" - -describe("syncCreators", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("syncs creators", async () => { - return config.doInContext(undefined, async () => { - await config.createUser({ admin: { global: true } }) - - await syncCreators.run() - - const usageDoc = await quotas.getQuotaUsage() - // default + additional creator - const creatorsCount = 2 - expect(usageDoc.usageQuota.creators).toBe(creatorsCount) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts deleted file mode 100644 index 730278683cd..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncRows from "../syncRows" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" -import { db as dbCore, context } from "@budibase/backend-core" - -describe("syncRows", () => { - const config = new TestConfig() - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - return config.doInContext(undefined, async () => { - // create the usage quota doc and mock usages - await quotas.getQuotaUsage() - await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC) - - let usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.rows).toEqual(300) - - // app 1 - const app1 = config.app - await context.doInAppContext(app1!.appId, async () => { - await config.createTable() - await config.createRow() - }) - // app 2 - const app2 = await config.createApp("second-app") - await context.doInAppContext(app2.appId, async () => { - await config.createTable() - await config.createRow() - await config.createRow() - }) - - // migrate - await syncRows.run() - - // assert the migration worked - usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.rows).toEqual(3) - expect( - usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows - ).toEqual(1) - expect( - usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows - ).toEqual(2) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts deleted file mode 100644 index 2731cc041db..00000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncUsers from "../syncUsers" -import { quotas } from "@budibase/pro" - -describe("syncUsers", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("syncs users", async () => { - return config.doInContext(undefined, async () => { - await config.createUser() - - await syncUsers.run() - - const usageDoc = await quotas.getQuotaUsage() - // default + additional user - const userCount = 2 - expect(usageDoc.usageQuota.users).toBe(userCount) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts deleted file mode 100644 index 078289cddf8..00000000000 --- a/packages/server/src/migrations/functions/userEmailViewCasing.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" - -/** - * Date: - * October 2021 - * - * Description: - * Recreate the user email view to include latest changes i.e. lower casing the email address - */ - -export const run = async () => { - await dbCore.createNewUserEmailView() -} diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts deleted file mode 100644 index a66d7931420..00000000000 --- a/packages/server/src/migrations/index.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { locks, migrations } from "@budibase/backend-core" -import { - Migration, - MigrationOptions, - MigrationName, - LockType, - LockName, -} from "@budibase/types" -import env from "../environment" - -// migration functions -import * as userEmailViewCasing from "./functions/userEmailViewCasing" -import * as syncQuotas from "./functions/syncQuotas" -import * as appUrls from "./functions/appUrls" -import * as tableSettings from "./functions/tableSettings" -import * as backfill from "./functions/backfill" -/** - * Populate the migration function and additional configuration from - * the static migration definitions. - */ -export const buildMigrations = () => { - const definitions = migrations.DEFINITIONS - const serverMigrations: Migration[] = [] - - for (const definition of definitions) { - switch (definition.name) { - case MigrationName.USER_EMAIL_VIEW_CASING: { - serverMigrations.push({ - ...definition, - fn: userEmailViewCasing.run, - }) - break - } - case MigrationName.SYNC_QUOTAS: { - serverMigrations.push({ - ...definition, - fn: syncQuotas.run, - }) - break - } - case MigrationName.APP_URLS: { - serverMigrations.push({ - ...definition, - appOpts: { all: true }, - fn: appUrls.run, - }) - break - } - case MigrationName.EVENT_APP_BACKFILL: { - serverMigrations.push({ - ...definition, - appOpts: { all: true }, - fn: backfill.app.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.EVENT_GLOBAL_BACKFILL: { - serverMigrations.push({ - ...definition, - fn: backfill.global.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.EVENT_INSTALLATION_BACKFILL: { - serverMigrations.push({ - ...definition, - fn: backfill.installation.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: { - serverMigrations.push({ - ...definition, - appOpts: { dev: true }, - fn: tableSettings.run, - }) - break - } - } - } - - return serverMigrations -} - -export const MIGRATIONS = buildMigrations() - -export const migrate = async (options?: MigrationOptions) => { - if (env.SELF_HOSTED) { - // self host runs migrations on startup - // make sure only a single instance runs them - await migrateWithLock(options) - } else { - await migrations.runMigrations(MIGRATIONS, options) - } -} - -const migrateWithLock = async (options?: MigrationOptions) => { - await locks.doWithLock( - { - type: LockType.TRY_ONCE, - name: LockName.MIGRATIONS, - ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes - systemLock: true, - }, - async () => { - await migrations.runMigrations(MIGRATIONS, options) - } - ) -} diff --git a/packages/server/src/migrations/tests/helpers.ts b/packages/server/src/migrations/tests/helpers.ts deleted file mode 100644 index 35831a2fd08..00000000000 --- a/packages/server/src/migrations/tests/helpers.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Mimic configs test configuration from worker, creation configs directly in database - -import * as structures from "./structures" -import { configs } from "@budibase/backend-core" -import { Config } from "@budibase/types" - -export const saveSettingsConfig = async (globalDb: any) => { - const config = structures.settings() - await saveConfig(config, globalDb) -} - -export const saveGoogleConfig = async (globalDb: any) => { - const config = structures.google() - await saveConfig(config, globalDb) -} - -export const saveOIDCConfig = async (globalDb: any) => { - const config = structures.oidc() - await saveConfig(config, globalDb) -} - -export const saveSmtpConfig = async (globalDb: any) => { - const config = structures.smtp() - await saveConfig(config, globalDb) -} - -const saveConfig = async (config: Config, globalDb: any) => { - config._id = configs.generateConfigID(config.type) - - let response - try { - response = await globalDb.get(config._id) - config._rev = response._rev - await globalDb.put(config) - } catch (e: any) { - if (e.status === 404) { - await globalDb.put(config) - } - } -} diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts deleted file mode 100644 index 3a23d8f0119..00000000000 --- a/packages/server/src/migrations/tests/index.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { - events, - migrations, - tenancy, - DocumentType, - context, -} from "@budibase/backend-core" -import TestConfig from "../../tests/utilities/TestConfiguration" -import * as structures from "../../tests/utilities/structures" -import { MIGRATIONS } from "../" -import * as helpers from "./helpers" - -import tk from "timekeeper" -import { View } from "@budibase/types" - -const timestamp = new Date().toISOString() -tk.freeze(timestamp) - -const clearMigrations = async () => { - const dbs = [context.getDevAppDB(), context.getProdAppDB()] - for (const db of dbs) { - const doc = await db.get(DocumentType.MIGRATIONS) - const newDoc = { _id: doc._id, _rev: doc._rev } - await db.put(newDoc) - } -} - -describe("migrations", () => { - const config = new TestConfig() - - beforeAll(async () => { - await config.init() - }) - - afterAll(() => { - config.end() - }) - - describe("backfill", () => { - it("runs app db migration", async () => { - await config.doInContext(undefined, async () => { - await clearMigrations() - await config.createAutomation() - await config.createAutomation(structures.newAutomation()) - await config.createDatasource() - await config.createDatasource() - await config.createLayout() - await config.createQuery() - await config.createQuery() - await config.createRole() - await config.createRole() - await config.createTable() - await config.createLegacyView() - await config.createTable() - await config.createLegacyView( - structures.view(config.table!._id!) as View - ) - await config.createScreen() - await config.createScreen() - - jest.clearAllMocks() - const migration = MIGRATIONS.filter( - m => m.name === "event_app_backfill" - )[0] - await migrations.runMigration(migration) - - expect(events.app.created).toHaveBeenCalledTimes(1) - expect(events.app.published).toHaveBeenCalledTimes(1) - expect(events.automation.created).toHaveBeenCalledTimes(2) - expect(events.automation.stepCreated).toHaveBeenCalledTimes(1) - expect(events.datasource.created).toHaveBeenCalledTimes(2) - expect(events.layout.created).toHaveBeenCalledTimes(1) - expect(events.query.created).toHaveBeenCalledTimes(2) - expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation) - expect(events.table.created).toHaveBeenCalledTimes(3) - expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2) - - // to make sure caching is working as expected - expect( - events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(20) // Addition of of the events above - }) - }) - }) - - it("runs global db migration", async () => { - await config.doInContext(undefined, async () => { - await clearMigrations() - const appId = config.getProdAppId() - const roles = { [appId]: "role_12345" } - await config.createUser({ - builder: { global: false }, - admin: { global: true }, - roles, - }) // admin only - await config.createUser({ - builder: { global: false }, - admin: { global: false }, - roles, - }) // non admin non builder - await config.createTable() - await config.createRow() - await config.createRow() - - const db = tenancy.getGlobalDB() - await helpers.saveGoogleConfig(db) - await helpers.saveOIDCConfig(db) - await helpers.saveSettingsConfig(db) - await helpers.saveSmtpConfig(db) - - jest.clearAllMocks() - const migration = MIGRATIONS.filter( - m => m.name === "event_global_backfill" - )[0] - await migrations.runMigration(migration) - - expect(events.user.created).toHaveBeenCalledTimes(3) - expect(events.role.assigned).toHaveBeenCalledTimes(2) - expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user - expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above - expect(events.rows.created).toHaveBeenCalledTimes(1) - expect(events.rows.created).toHaveBeenCalledWith(2, timestamp) - expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1) - expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2) - expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2) - expect(events.org.logoUpdated).toHaveBeenCalledTimes(1) - expect(events.org.nameUpdated).toHaveBeenCalledTimes(1) - expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1) - expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1) - - // to make sure caching is working as expected - expect( - events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(19) - }) - }) -}) diff --git a/packages/server/src/migrations/tests/structures.ts b/packages/server/src/migrations/tests/structures.ts deleted file mode 100644 index e2113e6a7c3..00000000000 --- a/packages/server/src/migrations/tests/structures.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { utils } from "@budibase/backend-core" -import { - SMTPConfig, - OIDCConfig, - GoogleConfig, - SettingsConfig, - ConfigType, -} from "@budibase/types" - -export const oidc = (conf?: OIDCConfig): OIDCConfig => { - return { - type: ConfigType.OIDC, - config: { - configs: [ - { - configUrl: "http://someconfigurl", - clientID: "clientId", - clientSecret: "clientSecret", - logo: "Microsoft", - name: "Active Directory", - uuid: utils.newid(), - activated: true, - scopes: [], - ...conf, - }, - ], - }, - } -} - -export const google = (conf?: GoogleConfig): GoogleConfig => { - return { - type: ConfigType.GOOGLE, - config: { - clientID: "clientId", - clientSecret: "clientSecret", - activated: true, - ...conf, - }, - } -} - -export const smtp = (conf?: SMTPConfig): SMTPConfig => { - return { - type: ConfigType.SMTP, - config: { - port: 12345, - host: "smtptesthost.com", - from: "testfrom@example.com", - subject: "Hello!", - secure: false, - ...conf, - }, - } -} - -export const settings = (conf?: SettingsConfig): SettingsConfig => { - return { - type: ConfigType.SETTINGS, - config: { - platformUrl: "http://mycustomdomain.com", - logoUrl: "http://mylogourl,com", - company: "mycompany", - ...conf, - }, - } -} diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index edca64db7da..54e982ae568 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -15,7 +15,6 @@ import { watch } from "../watch" import * as automations from "../automations" import * as fileSystem from "../utilities/fileSystem" import { default as eventEmitter, init as eventInit } from "../events" -import * as migrations from "../migrations" import * as bullboard from "../automations/bullboard" import * as appMigrations from "../appMigrations/queue" import * as pro from "@budibase/pro" @@ -106,18 +105,6 @@ export async function startup( initialiseWebsockets(app, server) } - // run migrations on startup if not done via http - // not recommended in a clustered environment - if (!env.HTTP_MIGRATIONS && !env.isTest()) { - console.log("Running migrations") - try { - await migrations.migrate() - } catch (e) { - logging.logAlert("Error performing migrations. Exiting.", e) - shutdown(server) - } - } - // monitor plugin directory if required if ( env.SELF_HOSTED && diff --git a/packages/types/src/api/web/global/oldMigration.ts b/packages/types/src/api/web/global/oldMigration.ts index 812ee1e593f..f0da6c52f36 100644 --- a/packages/types/src/api/web/global/oldMigration.ts +++ b/packages/types/src/api/web/global/oldMigration.ts @@ -1,12 +1,3 @@ -import { Migration, MigrationOptions } from "../../../sdk" - -export interface RunOldMigrationRequest extends MigrationOptions {} -export interface RuneOldMigrationResponse { - message: string -} - -export type FetchOldMigrationResponse = Migration[] - -export interface GetOldMigrationStatus { +export interface GetMigrationStatus { migrated: boolean } diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts index 9b03ddd4384..18ed533e9dd 100644 --- a/packages/types/src/api/web/system/index.ts +++ b/packages/types/src/api/web/system/index.ts @@ -3,6 +3,5 @@ export * from "./status" export * from "./ops" export * from "./account" export * from "./log" -export * from "./migration" export * from "./restore" export * from "./tenant" diff --git a/packages/types/src/api/web/system/migration.ts b/packages/types/src/api/web/system/migration.ts deleted file mode 100644 index a18112744cb..00000000000 --- a/packages/types/src/api/web/system/migration.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { MigrationDefinition, MigrationOptions } from "../../../sdk" - -export interface RunGlobalMigrationRequest extends MigrationOptions {} -export interface RunGlobalMigrationResponse { - message: string -} - -export type FetchMigrationDefinitionsResponse = MigrationDefinition[] diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 86eb5b1a244..eb9e23b3d19 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -4,7 +4,6 @@ export * from "./hosting" export * from "./context" export * from "./events" export * from "./licensing" -export * from "./migrations" export * from "./datasources" export * from "./search" export * from "./koa" diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts deleted file mode 100644 index 6db0c858794..00000000000 --- a/packages/types/src/sdk/migrations.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Database } from "./db" - -export interface Migration extends MigrationDefinition { - appOpts?: object - fn: (db: Database) => Promise - silent?: boolean - preventRetry?: boolean -} - -export enum MigrationType { - // run once per tenant, recorded in global db, global db is provided as an argument - GLOBAL = "global", - // run per app, recorded in each app db, app db is provided as an argument - APP = "app", - // run once, recorded in global info db, global info db is provided as an argument - INSTALLATION = "installation", -} - -export interface MigrationNoOpOptions { - type: MigrationType - tenantId: string - appId?: string -} - -/** - * e.g. - * { - * tenantIds: ['bb'], - * force: { - * global: ['quota_1'] - * } - * } - */ -export interface MigrationOptions { - tenantIds?: string[] - force?: { - [type: string]: string[] - } - noOp?: MigrationNoOpOptions -} - -export enum MigrationName { - USER_EMAIL_VIEW_CASING = "user_email_view_casing", - APP_URLS = "app_urls", - EVENT_APP_BACKFILL = "event_app_backfill", - EVENT_GLOBAL_BACKFILL = "event_global_backfill", - EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", - GLOBAL_INFO_SYNC_USERS = "global_info_sync_users", - TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions", - // increment this number to re-activate this migration - SYNC_QUOTAS = "sync_quotas_2", -} - -export interface MigrationDefinition { - type: MigrationType - name: MigrationName -} diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 0bcdadfefcd..e36c45a3ba1 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -28,7 +28,6 @@ import { LockType, LookupAccountHolderResponse, LookupTenantUserResponse, - MigrationType, PlatformUserByEmail, SaveUserResponse, SearchUsersRequest, @@ -45,7 +44,6 @@ import { cache, ErrorCode, events, - migrations, platform, tenancy, db, @@ -187,10 +185,6 @@ export const adminUser = async ( if (env.MULTI_TENANCY) { // store the new tenant record in the platform db await platform.tenants.addTenant(tenantId) - await migrations.backPopulateMigrations({ - type: MigrationType.GLOBAL, - tenantId, - }) } await tenancy.doInTenant(tenantId, async () => { diff --git a/packages/worker/src/api/controllers/system/migrations.ts b/packages/worker/src/api/controllers/system/migrations.ts deleted file mode 100644 index fc253d839d9..00000000000 --- a/packages/worker/src/api/controllers/system/migrations.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { - FetchMigrationDefinitionsResponse, - RunGlobalMigrationRequest, - RunGlobalMigrationResponse, - UserCtx, -} from "@budibase/types" - -const { migrate, MIGRATIONS } = require("../../../migrations") - -export const runMigrations = async ( - ctx: UserCtx -) => { - const options = ctx.request.body - // don't await as can take a while, just return - migrate(options) - ctx.body = { message: "Migration started." } -} - -export const fetchDefinitions = async ( - ctx: UserCtx -) => { - ctx.body = MIGRATIONS -} diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 741026543c7..d4ddb415226 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -12,7 +12,6 @@ import tenantsRoutes from "./system/tenants" import statusRoutes from "./system/status" import selfRoutes from "./global/self" import licenseRoutes from "./global/license" -import migrationRoutes from "./system/migrations" import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" import systemLogRoutes from "./system/logs" @@ -34,7 +33,6 @@ export const routes: Router[] = [ licenseRoutes, pro.groups, pro.auditLogs, - migrationRoutes, accountRoutes, restoreRoutes, eventRoutes, diff --git a/packages/worker/src/api/routes/system/migrations.ts b/packages/worker/src/api/routes/system/migrations.ts deleted file mode 100644 index a8189b5a913..00000000000 --- a/packages/worker/src/api/routes/system/migrations.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Router from "@koa/router" -import * as migrationsController from "../../controllers/system/migrations" -import { auth } from "@budibase/backend-core" - -const router: Router = new Router() - -router - .post( - "/api/system/migrations/run", - auth.internalApi, - migrationsController.runMigrations - ) - .get( - "/api/system/migrations/definitions", - auth.internalApi, - migrationsController.fetchDefinitions - ) - -export default router diff --git a/packages/worker/src/api/routes/system/tests/migrations.spec.ts b/packages/worker/src/api/routes/system/tests/migrations.spec.ts deleted file mode 100644 index fe91a1070c8..00000000000 --- a/packages/worker/src/api/routes/system/tests/migrations.spec.ts +++ /dev/null @@ -1,63 +0,0 @@ -const migrateFn = jest.fn() - -import { TestConfiguration } from "../../../../tests" - -jest.mock("../../../../migrations", () => { - return { - ...jest.requireActual("../../../../migrations"), - migrate: migrateFn, - } -}) - -describe("/api/system/migrations", () => { - const config = new TestConfiguration() - - beforeAll(async () => { - await config.beforeAll() - }) - - afterAll(async () => { - await config.afterAll() - }) - - afterEach(() => { - jest.clearAllMocks() - }) - - describe("POST /api/system/migrations/run", () => { - it("fails with no internal api key", async () => { - const res = await config.api.migrations.runMigrations({ - headers: {}, - status: 403, - }) - expect(res.body).toEqual({ message: "Unauthorized", status: 403 }) - expect(migrateFn).toHaveBeenCalledTimes(0) - }) - - it("runs migrations", async () => { - const res = await config.api.migrations.runMigrations() - expect(res.body.message).toBeDefined() - expect(migrateFn).toHaveBeenCalledTimes(1) - }) - }) - - describe("DELETE /api/system/migrations/definitions", () => { - it("fails with no internal api key", async () => { - const res = await config.api.migrations.getMigrationDefinitions({ - headers: {}, - status: 403, - }) - expect(res.body).toEqual({ message: "Unauthorized", status: 403 }) - }) - - it("returns definitions", async () => { - const res = await config.api.migrations.getMigrationDefinitions() - expect(res.body).toEqual([ - { - name: "global_info_sync_users", - type: "global", - }, - ]) - }) - }) -}) diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts deleted file mode 100644 index 642fbeb54e2..00000000000 --- a/packages/worker/src/migrations/index.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { migrations, locks } from "@budibase/backend-core" -import { - Migration, - MigrationOptions, - MigrationName, - LockType, - LockName, -} from "@budibase/types" -import env from "../environment" - -// migration functions -import * as syncUserInfo from "./functions/globalInfoSyncUsers" - -/** - * Populate the migration function and additional configuration from - * the static migration definitions. - */ -export const buildMigrations = () => { - const definitions = migrations.DEFINITIONS - const workerMigrations: Migration[] = [] - - for (const definition of definitions) { - switch (definition.name) { - case MigrationName.GLOBAL_INFO_SYNC_USERS: { - // only needed in cloud - if (!env.SELF_HOSTED) { - workerMigrations.push({ - ...definition, - fn: syncUserInfo.run, - }) - } - break - } - } - } - - return workerMigrations -} - -export const MIGRATIONS = buildMigrations() - -export const migrate = async (options?: MigrationOptions) => { - if (env.SELF_HOSTED) { - await migrateWithLock(options) - } else { - await migrations.runMigrations(MIGRATIONS, options) - } -} - -const migrateWithLock = async (options?: MigrationOptions) => { - await locks.doWithLock( - { - type: LockType.TRY_ONCE, - name: LockName.MIGRATIONS, - ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes - systemLock: true, - }, - async () => { - await migrations.runMigrations(MIGRATIONS, options) - } - ) -} diff --git a/packages/worker/src/tests/api/index.ts b/packages/worker/src/tests/api/index.ts index fa4e54184c4..0f0334c8da9 100644 --- a/packages/worker/src/tests/api/index.ts +++ b/packages/worker/src/tests/api/index.ts @@ -6,7 +6,6 @@ import { EmailAPI } from "./email" import { SelfAPI } from "./self" import { UserAPI } from "./users" import { EnvironmentAPI } from "./environment" -import { MigrationAPI } from "./migrations" import { StatusAPI } from "./status" import { RestoreAPI } from "./restore" import { TenantAPI } from "./tenants" @@ -26,7 +25,6 @@ export default class API { self: SelfAPI users: UserAPI environment: EnvironmentAPI - migrations: MigrationAPI status: StatusAPI restore: RestoreAPI tenants: TenantAPI @@ -46,7 +44,6 @@ export default class API { this.self = new SelfAPI(config) this.users = new UserAPI(config) this.environment = new EnvironmentAPI(config) - this.migrations = new MigrationAPI(config) this.status = new StatusAPI(config) this.restore = new RestoreAPI(config) this.tenants = new TenantAPI(config) diff --git a/packages/worker/src/tests/api/migrations.ts b/packages/worker/src/tests/api/migrations.ts deleted file mode 100644 index 6d62fe994af..00000000000 --- a/packages/worker/src/tests/api/migrations.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { TestAPI, TestAPIOpts } from "./base" - -export class MigrationAPI extends TestAPI { - runMigrations = (opts?: TestAPIOpts) => { - return this.request - .post(`/api/system/migrations/run`) - .set(opts?.headers ? opts.headers : this.config.internalAPIHeaders()) - .expect(opts?.status ? opts.status : 200) - } - - getMigrationDefinitions = (opts?: TestAPIOpts) => { - return this.request - .get(`/api/system/migrations/definitions`) - .set(opts?.headers ? opts.headers : this.config.internalAPIHeaders()) - .expect(opts?.status ? opts.status : 200) - } -}