From d16cf340d223283dafbc5f8f4891fbff1d11ec48 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Wed, 22 Jan 2025 13:28:37 -0500 Subject: [PATCH] Add Jobs for Product Creation, Builds, and Publishing (#1060) * Add jobs for product creation, build, and publish * Hook into jobs * Add product creation to frontend * Delete Product from BuildEngine * Imitate correct build/publish failure output * Handle post-build artifact creation * Fix database write for artifacts * Handle special artifact types in build * Update ProductPublications in publish * Move workflow creation call to products.create Fix product validation * Update Project DateActive * Fix product form closing * Handle no available product definitions * Properly handle GOOGLE_PLAY_UPLOADED - Tighten typing of environment - Create bullmq job to get versionCode for GOOGLE_PLAY_UPLOADED, because XState won't properly handle asynchronous actions (ugh) * Add environment population functions from DWKit * Assign targets and env for build * Assign targets and env for publish * Add PWA properties to workflow definitions * Fix environment merging * remove async from workflow send why was this async in the first place??? yes, it was my fault, but still, why??? * Add missing writes to WorkflowType * Fix getWorkflowParameters with null Properties * Use transitions for Project.DateActive * Product icon in selector * Remove unneeded TODO * Investigate blank activity name at startup The activityName was sometimes showing up blank. After investigating further, I was able to narrow it down to existing just at the start of a new product. This issue was sometimes persisting when the jobs backend was incorrect. * Preserve artifact history Had been deleting previous artifacts based on logic from S1 backend. This was determined to be unneeded, as the relevant code in S1 was determined to be unreachable after talking with @chrisvire * Rename WorkflowContextBase to WorkflowInstanceContext * Fix product creation workflow options init * Create WorkflowInstance in database on create * Log artifacts in build * Fix lint errors in getWorkflowParameters * Fix null description bug * Fix check errors * Fix errors in publish job * Add close button to modal * Don't close modal when going back from store * Prevent cancel button from submitting form * Fix styling in creation form * Fix storeLanguage check in Product validation * Remove store language from UI product creation * Bump dev tsconfig for node-server to NodeNext * Remove check for 'expired' status * Switch updateProjectDateActive back Switch updateProjectDateActive back to expected functionality based on S1. I still don't fully understand under which conditions we would want to be running this. Right now I just have it to where it will execute each time a product is created, updated, or deleted. * Remove duplicate function call * Fix check errors with updateProjectDateActive * Fix background color for visibility * Add copy icon to copy project url * Add link styling to console text * Add close button to details modal * Redirect to parent project on task submission * More concise query for redirect * Remove unnecessary IDs from task display * Add background color to table header, and border to cells * Create db wrappers for workflowInstances Moved updateProjectDateActive to be handled by workflowInstances rather than products * Configure option on SortTable to handle row click Open artifact link in new tab when row is clicked. * Add noStoresAvailable message to modal * Make scope argument mandatory for get parameters * Return params and env for debugging * Rename targets and channel with "default" prefix This is for clarification when looking at the job parameters * Open console text in new tab * Include Artifacts in Synchronize Data --- scripts/DB/default_workflow.sql | 16 +- .../common/bullmq/queues.ts | 24 ++ .../common/bullmq/types.ts | 87 ++++++++ .../common/databaseProxy/Products.ts | 78 +++++-- .../common/databaseProxy/WorkflowInstances.ts | 80 +++++++ .../common/databaseProxy/index.ts | 6 +- .../common/databaseProxy/utility.ts | 2 +- .../common/public/prisma.ts | 2 + .../common/public/workflow.ts | 30 ++- .../common/workflow/index.ts | 44 ++-- .../common/workflow/startup-workflow.ts | 107 ++++++--- .../node-server/BullWorker.ts | 54 +++++ .../SIL.AppBuilder.Portal/node-server/dev.ts | 4 + .../node-server/index.ts | 4 + .../node-server/job-executors/build.ts | 208 ++++++++++++++++++ .../job-executors/common.build-publish.ts | 139 ++++++++++++ .../node-server/job-executors/index.ts | 3 + .../node-server/job-executors/product.ts | 169 ++++++++++++++ .../node-server/job-executors/publish.ts | 195 ++++++++++++++++ .../node-server/tsconfig.dev.json | 4 +- .../src/lib/components/ProductDetails.svelte | 26 ++- .../src/lib/components/SortTable.svelte | 25 ++- .../projects/[id=idNumber]/+page.server.ts | 190 +++++++++++----- .../projects/[id=idNumber]/+page.svelte | 133 ++++++++++- .../routes/(authenticated)/tasks/+page.svelte | 24 +- .../tasks/[product_id]/+page.server.ts | 13 +- .../tasks/[product_id]/+page.svelte | 59 ++--- 27 files changed, 1524 insertions(+), 202 deletions(-) create mode 100644 source/SIL.AppBuilder.Portal/common/databaseProxy/WorkflowInstances.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/build.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/common.build-publish.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/publish.ts diff --git a/scripts/DB/default_workflow.sql b/scripts/DB/default_workflow.sql index 2cd908f85..36a11eb99 100644 --- a/scripts/DB/default_workflow.sql +++ b/scripts/DB/default_workflow.sql @@ -99,8 +99,8 @@ DO UPDATE SET "Type" = excluded."Type", "ProductType" = excluded."ProductType"; -INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES -(9, 'pwa_cloud', '1', 'SIL Default Workflow for Publishing PWA to Cloud', 'SIL_Default_AppBuilders_Pwa_Cloud', 'SIL_AppBuilders_Web_Flow', 3, 1, 3) +INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES +(9, 'pwa_cloud', '1', 'SIL Default Workflow for Publishing PWA to Cloud', 'SIL_Default_AppBuilders_Pwa_Cloud', 'SIL_AppBuilders_Web_Flow', 3, 1, '{ "build:targets" : "pwa" }', 3) ON CONFLICT ("Id") DO UPDATE SET "Name" = excluded."Name", @@ -112,8 +112,8 @@ DO UPDATE SET "Type" = excluded."Type", "ProductType" = excluded."ProductType"; -INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES -(10, 'pwa_cloud_rebuild', '1', 'SIL Default Workflow for Rebuilding PWA to Cloud', 'SIL_Default_AppBuilders_Pwa_Cloud_Rebuild', 'SIL_AppBuilders_Web_Flow', 3, 2, 3) +INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES +(10, 'pwa_cloud_rebuild', '1', 'SIL Default Workflow for Rebuilding PWA to Cloud', 'SIL_Default_AppBuilders_Pwa_Cloud_Rebuild', 'SIL_AppBuilders_Web_Flow', 3, 2, '{ "build:targets" : "pwa" }', 3) ON CONFLICT ("Id") DO UPDATE SET "Name" = excluded."Name", @@ -151,8 +151,8 @@ DO UPDATE SET "Type" = excluded."Type", "ProductType" = excluded."ProductType"; -INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES -(13, 'asset_package', '1', 'SIL Default Workflow for Publishing Asset Packages', 'SIL_NoAdmin_AppBuilders_Android_S3', 'SIL_AppBuilders_AssetPackage_Flow', 2, 1, '{ "build:targets" : "asset-package" }', 2) +INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES +(13, 'asset_package', '1', 'SIL Default Workflow for Publishing Asset Packages', 'SIL_NoAdmin_AppBuilders_Android_S3', 'SIL_AppBuilders_AssetPackage_Flow', 2, 1, 2) ON CONFLICT ("Id") DO UPDATE SET "Name" = excluded."Name", @@ -165,8 +165,8 @@ DO UPDATE SET "Properties" = excluded."Properties", "ProductType" = excluded."ProductType"; -INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES -(14, 'asset_package_rebuild', '1', 'SIL Default Workflow for Rebuilding Asset Packages', 'SIL_Default_AppBuilders_Android_S3_Rebuild', 'SIL_AppBuilders_AssetPackage_Flow', 2, 2, '{ "build:targets" : "asset-package" }', 3) +INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES +(14, 'asset_package_rebuild', '1', 'SIL Default Workflow for Rebuilding Asset Packages', 'SIL_Default_AppBuilders_Android_S3_Rebuild', 'SIL_AppBuilders_AssetPackage_Flow', 2, 2, 3) ON CONFLICT ("Id") DO UPDATE SET "Name" = excluded."Name", diff --git a/source/SIL.AppBuilder.Portal/common/bullmq/queues.ts b/source/SIL.AppBuilder.Portal/common/bullmq/queues.ts index 025d42d76..13b698173 100644 --- a/source/SIL.AppBuilder.Portal/common/bullmq/queues.ts +++ b/source/SIL.AppBuilder.Portal/common/bullmq/queues.ts @@ -2,12 +2,36 @@ import { Queue } from 'bullmq'; import type { Job } from './types.js'; import { QueueName } from './types.js'; +/** Queue for Product Builds */ +export const Builds = new Queue(QueueName.Builds, { + connection: { + host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis' + } +}); /** Queue for default recurring jobs such as the BuildEngine status check */ export const DefaultRecurring = new Queue(QueueName.DefaultRecurring, { connection: { host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis' } }); +/** Queue for miscellaneous jobs such as Product and Project Creation */ +export const Miscellaneous = new Queue(QueueName.Miscellaneous, { + connection: { + host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis' + } +}); +/** Queue for Product Publishing */ +export const Publishing = new Queue(QueueName.Publishing, { + connection: { + host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis' + } +}); +/** Queue for jobs that poll BuildEngine, such as checking the status of a build */ +export const RemotePolling = new Queue(QueueName.RemotePolling, { + connection: { + host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis' + } +}); /** Queue for operations on UserTasks */ export const UserTasks = new Queue(QueueName.UserTasks, { connection: { diff --git a/source/SIL.AppBuilder.Portal/common/bullmq/types.ts b/source/SIL.AppBuilder.Portal/common/bullmq/types.ts index 0788c8af1..41e74caa4 100644 --- a/source/SIL.AppBuilder.Portal/common/bullmq/types.ts +++ b/source/SIL.AppBuilder.Portal/common/bullmq/types.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-namespace */ +import type { Channels } from '../build-engine-api/types.js'; import { RoleId } from '../public/prisma.js'; interface RetryOptions { @@ -17,18 +18,97 @@ export const Retry5e5: RetryOptions = { } }; +interface RepeatOptions { + readonly repeat: { + readonly pattern: string; + }; +} +/** Repeat a job every minute */ +export const RepeatEveryMinute: RepeatOptions = { + repeat: { + pattern: '*/1 * * * *' // every minute + } +}; + export enum QueueName { + Builds = 'Builds', DefaultRecurring = 'Default Recurring', + Miscellaneous = 'Miscellaneous', + Publishing = 'Publishing', + RemotePolling = 'Remote Polling', UserTasks = 'User Tasks' } export enum JobType { + // Build Tasks + Build_Product = 'Build Product', + Build_Check = 'Check Product Build', + // Product Tasks + Product_Create = 'Create Product', + Product_Delete = 'Delete Product', + Product_GetVersionCode = 'Get VersionCode for Uploaded Product', + // Publishing Tasks + Publish_Product = 'Publish Product', + Publish_Check = 'Check Product Publish', // System Tasks System_CheckStatuses = 'Check System Statuses', // UserTasks UserTasks_Modify = 'Modify UserTasks' } +export namespace Build { + export interface Product { + type: JobType.Build_Product; + productId: string; + defaultTargets: string; + environment: { [key: string]: string }; + } + export interface Check { + type: JobType.Build_Check; + organizationId: number; + productId: string; + jobId: number; + buildId: number; + productBuildId: number; + } +} + +export namespace Product { + export interface Create { + type: JobType.Product_Create; + productId: string; + } + export interface Delete { + type: JobType.Product_Delete; + organizationId: number; + workflowJobId: number; + } + export interface GetVersionCode { + type: JobType.Product_GetVersionCode; + productId: string; + } +} + +export namespace Publish { + export interface Product { + type: JobType.Publish_Product; + productId: string; + defaultChannel: Channels; + defaultTargets: string; + environment: { [key: string]: string }; + } + + export interface Check { + type: JobType.Publish_Check; + organizationId: number; + productId: string; + jobId: number; + buildId: number; + releaseId: number; + publicationId: number; + } +} + export namespace System { export interface CheckStatuses { type: JobType.System_CheckStatuses; @@ -75,6 +155,13 @@ export namespace UserTasks { export type Job = JobTypeMap[keyof JobTypeMap]; export type JobTypeMap = { + [JobType.Build_Product]: Build.Product; + [JobType.Build_Check]: Build.Check; + [JobType.Product_Create]: Product.Create; + [JobType.Product_Delete]: Product.Delete; + [JobType.Product_GetVersionCode]: Product.GetVersionCode; + [JobType.Publish_Product]: Publish.Product; + [JobType.Publish_Check]: Publish.Check; [JobType.System_CheckStatuses]: System.CheckStatuses; [JobType.UserTasks_Modify]: UserTasks.Modify; // Add more mappings here as needed diff --git a/source/SIL.AppBuilder.Portal/common/databaseProxy/Products.ts b/source/SIL.AppBuilder.Portal/common/databaseProxy/Products.ts index ff74e2e57..ff8a3536f 100644 --- a/source/SIL.AppBuilder.Portal/common/databaseProxy/Products.ts +++ b/source/SIL.AppBuilder.Portal/common/databaseProxy/Products.ts @@ -1,5 +1,9 @@ import type { Prisma } from '@prisma/client'; +import { Workflow } from 'sil.appbuilder.portal.common'; +import { BullMQ, Queues } from '../index.js'; import prisma from '../prisma.js'; +import { WorkflowType } from '../public/prisma.js'; +import { delete as deleteInstance } from './WorkflowInstances.js'; import type { RequirePrimitive } from './utility.js'; export async function create( @@ -9,8 +13,8 @@ export async function create( !(await validateProductBase( productData.ProjectId, productData.ProductDefinitionId, - productData.StoreId, - productData.StoreLanguageId + productData.StoreId ?? undefined, + productData.StoreLanguageId ?? undefined )) ) return false; @@ -21,6 +25,34 @@ export async function create( const res = await prisma.products.create({ data: productData }); + + if (res) { + const flowDefinition = ( + await prisma.productDefinitions.findUnique({ + where: { + Id: productData.ProductDefinitionId + }, + select: { + Workflow: { + select: { + Id: true, + Type: true, + ProductType: true, + WorkflowOptions: true + } + } + } + }) + )?.Workflow; + + if (flowDefinition?.Type === WorkflowType.Startup) { + await Workflow.create(res.Id, { + productType: flowDefinition.ProductType, + options: new Set(flowDefinition.WorkflowOptions) + }); + } + } + return res.Id; } catch (e) { return false; @@ -62,14 +94,33 @@ export async function update( return true; } -function deleteProduct(productId: string) { +async function deleteProduct(productId: string) { // Delete all userTasks for this product, and delete the product + const product = await prisma.products.findUnique({ + where: { + Id: productId + }, + select: { + Project: { + select: { + Id: true, + OrganizationId: true + } + }, + WorkflowJobId: true + } + }); + await Queues.Miscellaneous.add( + `Delete Product #${productId} from BuildEngine`, + { + type: BullMQ.JobType.Product_Delete, + organizationId: product!.Project.OrganizationId, + workflowJobId: product!.WorkflowJobId + }, + BullMQ.Retry5e5 + ); return prisma.$transaction([ - prisma.workflowInstances.deleteMany({ - where: { - ProductId: productId - } - }), + deleteInstance(productId), prisma.userTasks.deleteMany({ where: { ProductId: productId @@ -81,7 +132,6 @@ function deleteProduct(productId: string) { } }) ]); - // TODO: delete from BuildEngine } export { deleteProduct as delete }; @@ -140,7 +190,7 @@ async function validateProductBase( Id: true, // StoreLanguage must be allowed by Store, if the StoreLanguage is defined StoreLanguages: - storeLanguageId === undefined || storeLanguageId === null + storeLanguageId === undefined ? undefined : { where: { @@ -166,7 +216,6 @@ async function validateProductBase( } } }); - // 3. The store is allowed by the organization return ( (project?.Organization.OrganizationStores.length ?? 0) > 0 && @@ -175,10 +224,11 @@ async function validateProductBase( project?.Organization.OrganizationStores[0].Store.StoreType.Id && // 2. The project has a WorkflowProjectUrl // handled by query - // 4. The language is allowed by the store - (storeLanguageId ?? + // 4. The language, if specified, is allowed by the store + ((storeLanguageId && (project?.Organization.OrganizationStores[0].Store.StoreType.StoreLanguages.length ?? 0) > - 0) && + 0) || + storeLanguageId === undefined) && // 5. The product type is allowed by the organization (project?.Organization.OrganizationProductDefinitions.length ?? 0) > 0 ); diff --git a/source/SIL.AppBuilder.Portal/common/databaseProxy/WorkflowInstances.ts b/source/SIL.AppBuilder.Portal/common/databaseProxy/WorkflowInstances.ts new file mode 100644 index 000000000..bfe4509d0 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/common/databaseProxy/WorkflowInstances.ts @@ -0,0 +1,80 @@ +import type { Prisma, PrismaPromise } from '@prisma/client'; +import prisma from '../prisma.js'; +import { update as projectUpdate } from './Projects.js'; +import type { RequirePrimitive } from './utility.js'; + +export async function upsert(instanceData: { + where: Prisma.WorkflowInstancesWhereUniqueInput; + create: RequirePrimitive; + update: RequirePrimitive; +}) { + const timestamp = new Date(); + const res = await prisma.workflowInstances.upsert(instanceData); + + if (res.DateCreated && res.DateCreated > timestamp) { + const product = await prisma.products.findUniqueOrThrow({ + where: { + Id: instanceData.create.ProductId + }, + select: { + ProjectId: true + } + }); + + await projectUpdate(product.ProjectId, { DateActive: new Date() }); + } + return res; +} + +//@ts-expect-error this was complaining about it not returning a global Promise. PrismaPromise extends global Promise and is require by prisma.$transaction, which for some reason didn't like a function that otherwise returned a called function that does indeed return a PrismaPromise. +async function deleteInstance(productId: string): PrismaPromise { + const product = await prisma.products.findUniqueOrThrow({ + where: { Id: productId }, + select: { ProjectId: true } + }); + const project = await prisma.projects.findUniqueOrThrow({ + where: { + Id: product.ProjectId + }, + select: { + Products: { + where: { + Id: { not: productId } + }, + select: { + WorkflowInstance: { + select: { + Id: true + } + }, + DateUpdated: true + } + }, + DateActive: true + } + }); + + const projectDateActive = project.DateActive; + + let dateActive = new Date(0); + project.Products.forEach((product) => { + if (product.WorkflowInstance) { + if (product.DateUpdated && product.DateUpdated > dateActive) { + dateActive = product.DateUpdated; + } + } + }); + + if (dateActive > new Date(0)) { + project.DateActive = dateActive; + } else { + project.DateActive = null; + } + + if (project.DateActive != projectDateActive) { + await projectUpdate(product.ProjectId, { DateActive: project.DateActive }); + } + return prisma.workflowInstances.delete({ where: { ProductId: productId } }); +} +export { deleteInstance as delete }; + diff --git a/source/SIL.AppBuilder.Portal/common/databaseProxy/index.ts b/source/SIL.AppBuilder.Portal/common/databaseProxy/index.ts index e081ba9a9..187a36182 100644 --- a/source/SIL.AppBuilder.Portal/common/databaseProxy/index.ts +++ b/source/SIL.AppBuilder.Portal/common/databaseProxy/index.ts @@ -1,7 +1,7 @@ import { Prisma, PrismaClient } from '@prisma/client'; -import { WRITE_METHODS } from '../ReadonlyPrisma.js'; import prisma from '../prisma.js'; +import { WRITE_METHODS } from '../ReadonlyPrisma.js'; import * as groupMemberships from './GroupMemberships.js'; import * as groups from './Groups.js'; import * as organizationMemberships from './OrganizationMemberships.js'; @@ -11,6 +11,7 @@ import * as products from './Products.js'; import * as projects from './Projects.js'; import * as userRoles from './UserRoles.js'; import * as utility from './utility.js'; +import * as workflowInstances from './WorkflowInstances.js'; type RecurseRemove = { [K in keyof T]: T[K] extends V | null | undefined ? never : RecurseRemove; @@ -41,7 +42,8 @@ const handlers = { organizationProductDefinitions, organizationMemberships, userRoles, - utility + utility, + workflowInstances }; // @ts-expect-error this is in fact immediately populated const obj: DataType = {}; diff --git a/source/SIL.AppBuilder.Portal/common/databaseProxy/utility.ts b/source/SIL.AppBuilder.Portal/common/databaseProxy/utility.ts index 7a17abd74..f025336d0 100644 --- a/source/SIL.AppBuilder.Portal/common/databaseProxy/utility.ts +++ b/source/SIL.AppBuilder.Portal/common/databaseProxy/utility.ts @@ -1,7 +1,7 @@ import prisma from '../prisma.js'; export type RequirePrimitive = { - [K in keyof T]: Extract; + [K in keyof T]: Extract; }; export async function getUserIfExists(externalId: string) { diff --git a/source/SIL.AppBuilder.Portal/common/public/prisma.ts b/source/SIL.AppBuilder.Portal/common/public/prisma.ts index de7a6468d..106ce3ffa 100644 --- a/source/SIL.AppBuilder.Portal/common/public/prisma.ts +++ b/source/SIL.AppBuilder.Portal/common/public/prisma.ts @@ -18,3 +18,5 @@ export enum WorkflowType { Rebuild, Republish } + +export const WorkflowTypeString = ['', 'Startup', 'Rebuild', 'Republish']; diff --git a/source/SIL.AppBuilder.Portal/common/public/workflow.ts b/source/SIL.AppBuilder.Portal/common/public/workflow.ts index d629f9e79..4931a2b5b 100644 --- a/source/SIL.AppBuilder.Portal/common/public/workflow.ts +++ b/source/SIL.AppBuilder.Portal/common/public/workflow.ts @@ -68,7 +68,7 @@ export enum WorkflowAction { Publish_Failed = 'Publish Failed' } -export type WorkflowContextBase = { +export type WorkflowInstanceContext = { instructions: | 'asset_package_verify_and_publish' | 'app_configuration' @@ -98,11 +98,31 @@ export type WorkflowContextBase = { includeReviewers: boolean; includeArtifacts: 'apk' | 'aab' | boolean; start?: WorkflowState; - // Not sure how this is used, but will figure out when integrating into backend - environment: { [key: string]: unknown }; + environment: Environment; }; -export type WorkflowContext = WorkflowContextBase & WorkflowInput; +export type Environment = { [key: ENVKeys | string]: string }; + +export enum ENVKeys { + // Set by Workflow + PUBLISH_GOOGLE_PLAY_UPLOADED_BUILD_ID = 'PUBLISH_GOOGLE_PLAY_UPLOADED_BUILD_ID', + PUBLISH_GOOGLE_PLAY_UPLOADED_VERSION_CODE = 'PUBLISH_GOOGLE_PLAY_UPLOADED_VERSION_CODE', + GOOGLE_PLAY_EXISTING = 'GOOGLE_PLAY_EXISTING', + GOOGLE_PLAY_DRAFT = 'GOOGLE_PLAY_DRAFT', + // Before Build + UI_URL = 'UI_URL', + PRODUCT_ID = 'PRODUCT_ID', + PROJECT_ID = 'PROJECT_ID', + PROJECT_NAME = 'PROJECT_NAME', + PROJECT_DESCRIPTION = 'PROJECT_DESCRIPTION', + PROJECT_URL = 'PROJECT_URL', + PROJECT_LANGUAGE = 'PROJECT_LANGUAGE', + PROJECT_ORGANIZATION = 'PROJECT_ORGANIZATION', + PROJECT_OWNER_NAME = 'PROJECT_OWNER_NAME', + PROJECT_OWNER_EMAIL = 'PROJECT_OWNER_EMAIL' +} + +export type WorkflowContext = WorkflowInstanceContext & WorkflowInput; export type WorkflowConfig = { options: Set; @@ -243,6 +263,6 @@ export type Snapshot = { instanceId: number; definitionId: number; state: string; - context: WorkflowContextBase; + context: WorkflowInstanceContext; config: WorkflowConfig; }; diff --git a/source/SIL.AppBuilder.Portal/common/workflow/index.ts b/source/SIL.AppBuilder.Portal/common/workflow/index.ts index 690e9407d..6dd554600 100644 --- a/source/SIL.AppBuilder.Portal/common/workflow/index.ts +++ b/source/SIL.AppBuilder.Portal/common/workflow/index.ts @@ -17,8 +17,8 @@ import type { StateNode, WorkflowConfig, WorkflowContext, - WorkflowContextBase, - WorkflowEvent + WorkflowEvent, + WorkflowInstanceContext } from '../public/workflow.js'; import { ActionType, @@ -64,6 +64,7 @@ export class Workflow { }); flow.flow.start(); + await flow.createSnapshot(flow.flow.getSnapshot().context); await DatabaseWrites.productTransitions.create({ data: { ProductId: productId, @@ -119,7 +120,7 @@ export class Workflow { } /** Send a transition event to the workflow. */ - public async send(event: WorkflowEvent): Promise { + public send(event: WorkflowEvent): void { this.flow?.send(event); } @@ -134,7 +135,7 @@ export class Workflow { /** Retrieves the workflow's snapshot from the database */ public static async getSnapshot(productId: string): Promise { - const snap = await prisma.workflowInstances.findUnique({ + const instance = await prisma.workflowInstances.findUnique({ where: { ProductId: productId }, @@ -151,17 +152,17 @@ export class Workflow { } } }); - if (!snap) { + if (!instance) { return null; } return { - instanceId: snap.Id, - definitionId: snap.WorkflowDefinition.Id, - state: snap.State, - context: JSON.parse(snap.Context) as WorkflowContextBase, + instanceId: instance.Id, + definitionId: instance.WorkflowDefinition.Id, + state: instance.State, + context: JSON.parse(instance.Context) as WorkflowInstanceContext, config: { - productType: snap.WorkflowDefinition.ProductType, - options: new Set(snap.WorkflowDefinition.WorkflowOptions) + productType: instance.WorkflowDefinition.ProductType, + options: new Set(instance.WorkflowDefinition.WorkflowOptions) } }; } @@ -246,10 +247,10 @@ export class Workflow { /* PRIVATE METHODS */ private async inspect(event: InspectedSnapshotEvent): Promise { const old = this.currentState; - const snap = this.flow!.getSnapshot(); - this.currentState = StartupWorkflow.getStateNodeById(`#${StartupWorkflow.id}.${snap.value}`); + const xSnap = this.flow!.getSnapshot(); + this.currentState = StartupWorkflow.getStateNodeById(`#${StartupWorkflow.id}.${xSnap.value}`); - if (old && Workflow.stateName(old) !== snap.value) { + if (old && Workflow.stateName(old) !== xSnap.value) { await this.updateProductTransitions( event.event.userId, Workflow.stateName(old), @@ -270,14 +271,10 @@ export class Workflow { ProductId: this.productId } }); - if (snap.value in TerminalStates) { - await DatabaseWrites.workflowInstances.delete({ - where: { - ProductId: this.productId - } - }); + if (xSnap.value in TerminalStates) { + await DatabaseWrites.workflowInstances.delete(this.productId); } else { - await this.createSnapshot(snap.context); + await this.createSnapshot(xSnap.context); // This will also create the dummy entries in the ProductTransitions table await Queues.UserTasks.add(`Update UserTasks for Product #${this.productId}`, { type: BullMQ.JobType.UserTasks_Modify, @@ -300,7 +297,7 @@ export class Workflow { hasReviewers: undefined, productType: undefined, options: undefined - } as WorkflowContextBase; + } as WorkflowInstanceContext; return DatabaseWrites.workflowInstances.upsert({ where: { ProductId: this.productId @@ -479,7 +476,8 @@ export class Workflow { DestinationState: destinationState, Command: command ?? null, DateTransition: new Date(), - Comment: comment ?? null + Comment: comment ?? null, + WorkflowType: WorkflowType.Startup // TODO: Change this once we support more workflow types } }); } diff --git a/source/SIL.AppBuilder.Portal/common/workflow/startup-workflow.ts b/source/SIL.AppBuilder.Portal/common/workflow/startup-workflow.ts index b278c3a6c..972c5396c 100644 --- a/source/SIL.AppBuilder.Portal/common/workflow/startup-workflow.ts +++ b/source/SIL.AppBuilder.Portal/common/workflow/startup-workflow.ts @@ -1,4 +1,5 @@ import { assign, setup } from 'xstate'; +import { BullMQ, Queues } from '../index.js'; import { RoleId } from '../public/prisma.js'; import type { WorkflowContext, @@ -9,6 +10,7 @@ import type { } from '../public/workflow.js'; import { ActionType, + ENVKeys, ProductType, WorkflowAction, WorkflowOptions, @@ -243,9 +245,12 @@ export const StartupWorkflow = setup({ [WorkflowState.Product_Creation]: { entry: [ assign({ instructions: 'waiting' }), - () => { - // TODO: hook into build engine - console.log('Creating Product'); + ({ context }) => { + Queues.Miscellaneous.add(`Create Product #${context.productId}`, { + type: BullMQ.JobType.Product_Create, + productId: context.productId + }, + BullMQ.Retry5e5); } ], on: { @@ -293,10 +298,10 @@ export const StartupWorkflow = setup({ } }, actions: assign({ - environment: ({ context }) => { - context.environment.googlePlayExisting = true; - return context.environment; - } + environment: ({ context }) => ({ + ...context.environment, + [ENVKeys.GOOGLE_PLAY_EXISTING]: '1' + }) }), target: WorkflowState.Product_Build }, @@ -343,7 +348,11 @@ export const StartupWorkflow = setup({ [WorkflowState.Synchronize_Data]: { entry: assign({ instructions: 'synchronize_data', - includeFields: ['storeDescription', 'listingLanguageCode'] + includeFields: ['storeDescription', 'listingLanguageCode'], + includeArtifacts: true + }), + exit: assign({ + includeArtifacts: false }), on: { [WorkflowAction.Continue]: { @@ -425,9 +434,22 @@ export const StartupWorkflow = setup({ assign({ instructions: 'waiting' }), - () => { - // TODO: hook into build engine - console.log('Building Product'); + ({ context }) => { + Queues.Builds.add(`Build Product #${context.productId}`, { + type: BullMQ.JobType.Build_Product, + productId: context.productId, + defaultTargets: context.productType === ProductType.Android_S3 + ? 'apk' + : context.productType === ProductType.AssetPackage + ? 'asset-package' + : context.productType === ProductType.Web + ? 'html' + : //ProductType.Android_GooglePlay + //default + 'apk play-listing', + environment: context.environment + }, + BullMQ.Retry5e5); } ], on: { @@ -441,15 +463,15 @@ export const StartupWorkflow = setup({ }, guard: ({ context }) => context.productType === ProductType.Android_GooglePlay && - !context.environment.googlePlayUploaded, + !context.environment[ENVKeys.PUBLISH_GOOGLE_PLAY_UPLOADED_BUILD_ID], target: WorkflowState.App_Store_Preview }, { meta: { type: ActionType.Auto }, guard: ({ context }) => context.productType !== ProductType.Android_GooglePlay || - !!context.environment.googlePlayExisting || - !!context.environment.googlePlayUploaded, + context.environment[ENVKeys.GOOGLE_PLAY_EXISTING] === '1' || + !!context.environment[ENVKeys.PUBLISH_GOOGLE_PLAY_UPLOADED_BUILD_ID], target: WorkflowState.Verify_and_Publish } ], @@ -538,10 +560,10 @@ export const StartupWorkflow = setup({ instructions: 'create_app_entry', includeFields: ['storeDescription', 'listingLanguageCode'], includeArtifacts: true, - environment: ({ context }) => { - context.environment.googlePlayDraft = true; - return context.environment; - } + environment: ({ context }) => ({ + ...context.environment, + [ENVKeys.GOOGLE_PLAY_DRAFT]: '1' + }) }), exit: assign({ includeArtifacts: false }), on: { @@ -554,12 +576,13 @@ export const StartupWorkflow = setup({ options: { has: WorkflowOptions.AdminStoreAccess } } }, - actions: assign({ - environment: ({ context }) => { - context.environment.googlePlayUploaded = true; - return context.environment; - } - }), + actions: ({ context }) => { + // Given that the Set Google Play Uploaded action in S1 require DB and BuildEngine queries, this is probably the best way to do this + Queues.Miscellaneous.add(`Get VersionCode for Product #${context.productId}`, { + type: BullMQ.JobType.Product_GetVersionCode, + productId: context.productId + }); + }, target: WorkflowState.Verify_and_Publish }, { @@ -570,12 +593,12 @@ export const StartupWorkflow = setup({ options: { none: new Set([WorkflowOptions.AdminStoreAccess]) } } }, - actions: assign({ - environment: ({ context }) => { - context.environment.googlePlayUploaded = true; - return context.environment; - } - }), + actions: ({ context }) => { + Queues.Miscellaneous.add(`Get VersionCode for Product #${context.productId}`, { + type: BullMQ.JobType.Product_GetVersionCode, + productId: context.productId + }); + }, target: WorkflowState.Verify_and_Publish } ], @@ -665,9 +688,23 @@ export const StartupWorkflow = setup({ [WorkflowState.Product_Publish]: { entry: [ assign({ instructions: 'waiting' }), - () => { - // TODO: hook into build engine - console.log('Publishing Product'); + ({ context }) => { + Queues.Publishing.add(`Publish Product #${context.productId}`, { + type: BullMQ.JobType.Publish_Product, + productId: context.productId, + defaultChannel: 'production', //default unless overriden by WorkflowDefinition.Properties or ProductDefinition.Properties + defaultTargets: + context.productType === ProductType.Android_S3 + ? 's3-bucket' + : context.productType === ProductType.Web + ? 'rclone' + : //ProductType.Android_GooglePlay + //ProductType.AssetPackage + //default + 'google-play', + environment: context.environment + }, + BullMQ.Retry5e5); } ], on: { @@ -681,14 +718,14 @@ export const StartupWorkflow = setup({ }, guard: ({ context }) => context.productType === ProductType.Android_GooglePlay && - !context.environment.googlePlayExisting, + !context.environment[ENVKeys.GOOGLE_PLAY_EXISTING], target: WorkflowState.Make_It_Live }, { meta: { type: ActionType.Auto }, guard: ({ context }) => context.productType !== ProductType.Android_GooglePlay || - !!context.environment.googlePlayExisting, + context.environment[ENVKeys.GOOGLE_PLAY_EXISTING] === '1', target: WorkflowState.Published } ], diff --git a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts index 2c577a32d..8bc6b79b9 100644 --- a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts +++ b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts @@ -14,6 +14,18 @@ export abstract class BullWorker { abstract run(job: Job): Promise; } +export class Builds extends BullWorker { + constructor() { + super(BullMQ.QueueName.Builds); + } + async run(job: Job) { + switch (job.data.type) { + case BullMQ.JobType.Build_Product: + return Executor.Build.product(job as Job); + } + } +} + export class DefaultRecurring extends BullWorker { constructor() { super(BullMQ.QueueName.DefaultRecurring); @@ -38,6 +50,48 @@ export class DefaultRecurring extends BullWorker { } } +export class Miscellaneous extends BullWorker { + constructor() { + super(BullMQ.QueueName.Miscellaneous); + } + async run(job: Job) { + switch (job.data.type) { + case BullMQ.JobType.Product_Create: + return Executor.Product.create(job as Job); + case BullMQ.JobType.Product_Delete: + return Executor.Product.deleteProduct(job as Job); + case BullMQ.JobType.Product_GetVersionCode: + return Executor.Product.getVersionCode(job as Job); + } + } +} + +export class Publishing extends BullWorker { + constructor() { + super(BullMQ.QueueName.Publishing); + } + async run(job: Job) { + switch (job.data.type) { + case BullMQ.JobType.Publish_Product: + return Executor.Publish.product(job as Job); + } + } +} + +export class RemotePolling extends BullWorker { + constructor() { + super(BullMQ.QueueName.RemotePolling); + } + async run(job: Job) { + switch (job.data.type) { + case BullMQ.JobType.Build_Check: + return Executor.Build.check(job as Job); + case BullMQ.JobType.Publish_Check: + return Executor.Publish.check(job as Job); + } + } +} + export class UserTasks extends BullWorker { constructor() { super(BullMQ.QueueName.UserTasks); diff --git a/source/SIL.AppBuilder.Portal/node-server/dev.ts b/source/SIL.AppBuilder.Portal/node-server/dev.ts index 2ddc0ec66..e98c90e4c 100644 --- a/source/SIL.AppBuilder.Portal/node-server/dev.ts +++ b/source/SIL.AppBuilder.Portal/node-server/dev.ts @@ -18,5 +18,9 @@ createBullBoard({ app.use(serverAdapter.getRouter()); app.listen(3000, () => console.log('Dev server started')); +new Workers.Builds(); new Workers.DefaultRecurring(); +new Workers.Miscellaneous(); +new Workers.Publishing(); +new Workers.RemotePolling(); new Workers.UserTasks(); diff --git a/source/SIL.AppBuilder.Portal/node-server/index.ts b/source/SIL.AppBuilder.Portal/node-server/index.ts index 03b4d6689..a5b150a9b 100644 --- a/source/SIL.AppBuilder.Portal/node-server/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/index.ts @@ -75,7 +75,11 @@ app.get('/healthcheck', (req, res) => { // Running on svelte process right now. Consider putting on new thread // Fine like this if majority of job time is waiting for network requests // If there is much processing it should be moved to another thread +new Workers.Builds(); new Workers.DefaultRecurring(); +new Workers.Miscellaneous(); +new Workers.Publishing(); +new Workers.RemotePolling(); new Workers.UserTasks(); const serverAdapter = new ExpressAdapter(); diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/build.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/build.ts new file mode 100644 index 000000000..da6fd9187 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/build.ts @@ -0,0 +1,208 @@ +import { Job } from 'bullmq'; +import { + BuildEngine, + BullMQ, + DatabaseWrites, + prisma, + Queues, + Workflow +} from 'sil.appbuilder.portal.common'; +import { WorkflowAction } from 'sil.appbuilder.portal.common/workflow'; +import { + addProductPropertiesToEnvironment, + getWorkflowParameters +} from './common.build-publish.js'; + +export async function product(job: Job): Promise { + const productData = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + Project: { + select: { + OrganizationId: true + } + }, + WorkflowJobId: true, + WorkflowInstance: { + select: { + Id: true + } + } + } + }); + if (!productData) { + throw new Error(`Product #${job.data.productId} does not exist!`); + } + job.updateProgress(10); + // reset previous build + await DatabaseWrites.products.update(job.data.productId, { + WorkflowBuildId: 0 + }); + job.updateProgress(20); + const params = await getWorkflowParameters(productData.WorkflowInstance.Id, 'build'); + job.updateProgress(30); + const env = await addProductPropertiesToEnvironment(job.data.productId); + job.updateProgress(40); + const response = await BuildEngine.Requests.createBuild( + { type: 'query', organizationId: productData.Project.OrganizationId }, + productData.WorkflowJobId, + { + targets: params['targets'] ?? job.data.defaultTargets, + environment: { ...env, ...params.environment, ...job.data.environment } + } + ); + job.updateProgress(50); + if (response.responseType === 'error') { + const flow = await Workflow.restore(job.data.productId); + // TODO: Send Notification of Failure + flow.send({ type: WorkflowAction.Build_Failed, userId: null, comment: response.message }); + } else { + await DatabaseWrites.products.update(job.data.productId, { + WorkflowBuildId: response.id + }); + job.updateProgress(65); + + const productBuild = await DatabaseWrites.productBuilds.create({ + data: { + ProductId: job.data.productId, + BuildId: response.id + } + }); + + job.updateProgress(85); + + await Queues.RemotePolling.add( + `Check status of Build #${response.id}`, + { + type: BullMQ.JobType.Build_Check, + productId: job.data.productId, + organizationId: productData.Project.OrganizationId, + jobId: productData.WorkflowJobId, + buildId: response.id, + productBuildId: productBuild.Id + }, + BullMQ.RepeatEveryMinute + ); + } + job.updateProgress(100); + return { + response, + params, + env + }; +} + +export async function check(job: Job): Promise { + const response = await BuildEngine.Requests.getBuild( + { type: 'query', organizationId: job.data.organizationId }, + job.data.jobId, + job.data.buildId + ); + job.updateProgress(50); + if (response.responseType === 'error') { + throw new Error(response.message); + } else { + if (response.status === 'completed') { + await Queues.RemotePolling.removeRepeatableByKey(job.repeatJobKey); + if (response.error) { + job.log(response.error); + } + let latestArtifactDate = new Date(0); + job.log('ARTIFACTS:'); + await DatabaseWrites.productArtifacts.createMany({ + data: await Promise.all( + Object.entries(response.artifacts).map(async ([type, url]) => { + job.log(`${type}: ${url}`); + const res = await fetch(url, { method: 'HEAD' }); + const lastModified = new Date(res.headers.get('Last-Modified')); + if (lastModified > latestArtifactDate) { + latestArtifactDate = lastModified; + } + + // On version.json, update the ProductBuild.Version + if (type === 'version' && res.headers.get('Content-Type') === 'application/json') { + const version = JSON.parse(await fetch(url).then((r) => r.text())); + if (version['version']) { + await DatabaseWrites.productBuilds.update({ + where: { + Id: job.data.productBuildId + }, + data: { + Version: version['version'] + } + }); + if (response.result === 'SUCCESS') { + await DatabaseWrites.products.update(job.data.productId, { + VersionBuilt: version['version'] + }); + } + } + } + + // On play-listing-manifest.json, update the Project.DefaultLanguage + if ( + type == 'play-listing-manifest' && + res.headers.get('Content-Type') === 'application/json' + ) { + const manifest = JSON.parse(await fetch(url).then((r) => r.text())); + if (manifest['default-language']) { + const lang = await prisma.storeLanguages.findFirst({ + where: { + Name: manifest['default-language'] + }, + select: { + Id: true + } + }); + if (lang !== null) { + await DatabaseWrites.products.update(job.data.productId, { + StoreLanguageId: lang.Id + }); + } + } + } + + return { + ProductId: job.data.productId, + ProductBuildId: job.data.productBuildId, + ArtifactType: type, + Url: url, + ContentType: res.headers.get('Content-Type'), + FileSize: + res.headers.get('Content-Type') !== 'text/html' + ? parseInt(res.headers.get('Content-Length')) + : undefined + }; + }) + ) + }); + await DatabaseWrites.products.update(job.data.productId, { + DateBuilt: latestArtifactDate + }); + job.updateProgress(80); + await DatabaseWrites.productBuilds.update({ + where: { + Id: job.data.productBuildId + }, + data: { + Success: response.result === 'SUCCESS' + } + }); + job.updateProgress(90); + const flow = await Workflow.restore(job.data.productId); + if (response.result === 'SUCCESS') { + flow.send({ type: WorkflowAction.Build_Successful, userId: null }); + } else { + flow.send({ + type: WorkflowAction.Build_Failed, + userId: null, + comment: `system.build-failed,${response.artifacts['consoleText'] ?? ''}` + }); + } + } + job.updateProgress(100); + return response; + } +} diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/common.build-publish.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/common.build-publish.ts new file mode 100644 index 000000000..96828c897 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/common.build-publish.ts @@ -0,0 +1,139 @@ +import { prisma } from 'sil.appbuilder.portal.common'; +import { WorkflowTypeString } from 'sil.appbuilder.portal.common/prisma'; +import { Environment, ENVKeys } from 'sil.appbuilder.portal.common/workflow'; + +export async function addProductPropertiesToEnvironment(productId: string) { + const product = await prisma.products.findUnique({ + where: { + Id: productId + }, + select: { + Project: { + select: { + Id: true, + Name: true, + Description: true, + Language: true, + Organization: { + select: { + Name: true + } + }, + Owner: { + select: { + Name: true, + Email: true + } + } + } + }, + Properties: true + } + }); + const uiUrl = process.env.UI_URL || 'http://localhost:5173'; + const projectUrl = uiUrl + '/projects/' + product.Project.Id; + + return { + [ENVKeys.UI_URL]: uiUrl, + [ENVKeys.PRODUCT_ID]: productId, + [ENVKeys.PROJECT_ID]: '' + product.Project.Id, + [ENVKeys.PROJECT_NAME]: product.Project.Name ?? '', + [ENVKeys.PROJECT_DESCRIPTION]: product.Project.Description ?? '', + [ENVKeys.PROJECT_URL]: projectUrl, + [ENVKeys.PROJECT_LANGUAGE]: product.Project.Language ?? '', + [ENVKeys.PROJECT_ORGANIZATION]: product.Project.Organization.Name, + [ENVKeys.PROJECT_OWNER_NAME]: product.Project.Owner.Name, + [ENVKeys.PROJECT_OWNER_EMAIL]: product.Project.Owner.Email, + ...(product.Properties ? JSON.parse(product.Properties).environment ?? {} : {}) + } as Environment; +} + +export async function getWorkflowParameters(workflowInstanceId: number, scope: 'build' | 'publish') { + const instance = await prisma.workflowInstances.findUnique({ + where: { + Id: workflowInstanceId + }, + select: { + Product: { + select: { + ProductDefinition: { + select: { + Properties: true, + Name: true + } + } + } + }, + WorkflowDefinition: { + select: { + Properties: true, + Type: true + } + } + } + }); + let environment: Environment = { + WORKFLOW_TYPE: WorkflowTypeString[instance.WorkflowDefinition.Type], + WORKFLOW_PRODUCT_NAME: instance.Product.ProductDefinition.Name + }; + + const result: { [key: string]: string } = {}; + const scoped: { [key: string]: string } = {}; + Object.entries(JSON.parse(instance.WorkflowDefinition.Properties ?? '{}')).forEach(([k, v]) => { + const strValue = JSON.stringify(v); + let strKey = k; + if (strKey === 'environment') { + // merge environment + environment = { + ...environment, + ...JSON.parse(strValue) + }; + } + // Allow for scoped names so "build:targets" will become "targets" + // Scoped values should be assigned after non-scoped + else if (strKey.includes(':')) { + // Use scoped values for this scope and ignore others + if (scope && strKey.startsWith(scope + ':')) { + strKey = strKey.split(':')[1]; + scoped[strKey] = strValue; + } + } else { + result[strKey] = strValue; + } + }); + Object.entries(JSON.parse(instance.Product.ProductDefinition.Properties ?? '{}')).forEach(([k, v]) => { + const strValue = JSON.stringify(v); + let strKey = k; + if (strKey === 'environment') { + // merge environment + environment = { + ...environment, + ...JSON.parse(strValue) + }; + } + // Allow for scoped names so "build:targets" will become "targets" + // Scoped values should be assigned after non-scoped + else if (strKey.includes(':')) { + // Use scoped values for this scope and ignore others + if (scope && strKey.startsWith(scope + ':')) { + strKey = strKey.split(':')[1]; + scoped[strKey] = strValue; + } + } else { + result[strKey] = strValue; + } + }); + Object.entries(scoped).forEach(([k, v]) => { + if (k === 'environment') { + environment = { + ...environment, + ...JSON.parse(v) + } + } + else { + result[k] = v; + } + }); + + return { ...result, environment: environment }; +} diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts index f0bab14d9..058c8bdad 100644 --- a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts @@ -1,2 +1,5 @@ +export * as Build from './build.js'; +export * as Product from './product.js'; +export * as Publish from './publish.js'; export * as System from './system.js'; export * as UserTasks from './userTasks.js'; diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts new file mode 100644 index 000000000..da4054c42 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts @@ -0,0 +1,169 @@ +import { Job } from 'bullmq'; +import { + BuildEngine, + BullMQ, + DatabaseWrites, + prisma, + Workflow +} from 'sil.appbuilder.portal.common'; +import { + ENVKeys, + WorkflowAction, + WorkflowInstanceContext +} from 'sil.appbuilder.portal.common/workflow'; + +export async function create(job: Job): Promise { + const productData = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + Project: { + select: { + ApplicationType: { + select: { + Name: true + } + }, + WorkflowProjectUrl: true, + OrganizationId: true + } + }, + Store: { + select: { + Name: true + } + } + } + }); + job.updateProgress(25); + const response = await BuildEngine.Requests.createJob( + { type: 'query', organizationId: productData.Project.OrganizationId }, + { + request_id: job.data.productId, + git_url: productData.Project.WorkflowProjectUrl, + app_id: productData.Project.ApplicationType.Name, + publisher_id: productData.Store.Name + } + ); + job.updateProgress(50); + if (response.responseType === 'error') { + // TODO: What do I do here? Wait some period of time and retry? + job.log(response.message); + throw new Error(response.message); + } else { + await DatabaseWrites.products.update(job.data.productId, { + WorkflowJobId: response.id + }); + job.updateProgress(75); + const flow = await Workflow.restore(job.data.productId); + + flow.send({ type: WorkflowAction.Product_Created, userId: null }); + + job.updateProgress(100); + return response; + } +} + +export async function deleteProduct(job: Job): Promise { + const response = await BuildEngine.Requests.deleteJob( + { type: 'query', organizationId: job.data.organizationId }, + job.data.workflowJobId + ); + job.updateProgress(50); + if (response.responseType === 'error') { + job.log(response.message); + throw new Error(response.message); + } else { + job.updateProgress(100); + return response.status; + } +} + +export async function getVersionCode(job: Job): Promise { + let versionCode = 0; + const product = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + WorkflowBuildId: true, + WorkflowJobId: true, + Project: { + select: { + Organization: { + select: { + BuildEngineUrl: true, + BuildEngineApiAccessToken: true + } + } + } + } + } + }); + job.updateProgress(30); + if (product?.WorkflowBuildId && product?.WorkflowJobId) { + const productBuild = await prisma.productBuilds.findFirst({ + where: { + ProductId: job.data.productId, + BuildId: product.WorkflowBuildId + }, + select: { + Id: true + } + }); + if (!productBuild) { + return 0; + } + job.updateProgress(45); + const versionCodeArtifact = await prisma.productArtifacts.findFirst({ + where: { + ProductId: job.data.productId, + ProductBuildId: productBuild.Id, + ArtifactType: 'version' + }, + select: { + Url: true + } + }); + if (!versionCodeArtifact) { + return 0; + } + job.updateProgress(60); + const version = JSON.parse(await fetch(versionCodeArtifact.Url).then((r) => r.text())); + if (version['versionCode'] !== undefined) { + versionCode = parseInt(version['versionCode']); + } + job.updateProgress(75); + } + if (versionCode) { + const instance = await prisma.workflowInstances.findUnique({ + where: { + ProductId: job.data.productId + }, + select: { + Id: true, + Context: true + } + }); + const ctx: WorkflowInstanceContext = JSON.parse(instance.Context); + job.updateProgress(90); + await DatabaseWrites.workflowInstances.update({ + where: { + Id: instance.Id + }, + data: { + Context: JSON.stringify({ + ...ctx, + environment: { + ...ctx.environment, + [ENVKeys.PUBLISH_GOOGLE_PLAY_UPLOADED_BUILD_ID]: '' + product.WorkflowBuildId, + [ENVKeys.PUBLISH_GOOGLE_PLAY_UPLOADED_VERSION_CODE]: '' + versionCode + } + } as WorkflowInstanceContext) + } + }); + } + job.updateProgress(100); + return versionCode; +} diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/publish.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/publish.ts new file mode 100644 index 000000000..0b2a9c96b --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/publish.ts @@ -0,0 +1,195 @@ +import { Job } from 'bullmq'; +import { + BuildEngine, + BullMQ, + DatabaseWrites, + prisma, + Queues, + Workflow +} from 'sil.appbuilder.portal.common'; +import { WorkflowAction } from 'sil.appbuilder.portal.common/workflow'; +import { + addProductPropertiesToEnvironment, + getWorkflowParameters +} from './common.build-publish.js'; + +export async function product(job: Job): Promise { + const productData = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + Project: { + select: { + OrganizationId: true + } + }, + WorkflowJobId: true, + WorkflowBuildId: true, + WorkflowInstance: { + select: { + Id: true + } + } + } + }); + job.updateProgress(10); + const productBuild = await prisma.productBuilds.findFirst({ + where: { + BuildId: productData.WorkflowBuildId + }, + select: { + Id: true + } + }); + if (!productData.WorkflowBuildId || !productBuild) { + const flow = await Workflow.restore(job.data.productId); + // TODO: Send notification of failure + flow.send({ + type: WorkflowAction.Publish_Failed, + userId: null, + comment: 'Product does not have a ProductBuild available.' + }); + job.updateProgress(100); + return productData; + } + job.updateProgress(15); + await DatabaseWrites.products.update(job.data.productId, { + WorkflowPublishId: 0 + }); + job.updateProgress(20); + const params = await getWorkflowParameters(productData.WorkflowInstance.Id, 'publish'); + const channel = params['channel'] ?? job.data.defaultChannel; + job.updateProgress(30); + const env = await addProductPropertiesToEnvironment(job.data.productId); + job.updateProgress(40); + const response = await BuildEngine.Requests.createRelease( + { type: 'query', organizationId: productData.Project.OrganizationId }, + productData.WorkflowJobId, + productData.WorkflowBuildId, + { + channel: channel, + targets: params['targets'] ?? job.data.defaultTargets, + environment: { ...env, ...params.environment, ...job.data.environment } + } + ); + job.updateProgress(50); + if (response.responseType === 'error') { + const flow = await Workflow.restore(job.data.productId); + // TODO: Send notification of failure + flow.send({ type: WorkflowAction.Publish_Failed, userId: null, comment: response.message }); + } else { + await DatabaseWrites.products.update(job.data.productId, { + WorkflowPublishId: response.id + }); + job.updateProgress(65); + + const pub = await DatabaseWrites.productPublications.create({ + data: { + ProductId: job.data.productId, + ProductBuildId: productBuild.Id, + ReleaseId: response.id, + Channel: channel + } + }); + + job.updateProgress(85); + + await Queues.RemotePolling.add( + `Check status of Publish #${response.id}`, + { + type: BullMQ.JobType.Publish_Check, + productId: job.data.productId, + organizationId: productData.Project.OrganizationId, + jobId: productData.WorkflowJobId, + buildId: productData.WorkflowBuildId, + releaseId: response.id, + publicationId: pub.Id + }, + BullMQ.RepeatEveryMinute + ); + + } + job.updateProgress(100); + return { + response, + params, + env + }; +} + +export async function check(job: Job): Promise { + const response = await BuildEngine.Requests.getRelease( + { type: 'query', organizationId: job.data.organizationId }, + job.data.jobId, + job.data.buildId, + job.data.releaseId + ); + job.updateProgress(50); + if (response.responseType === 'error') { + throw new Error(response.message); + } else { + if (response.status === 'completed') { + await Queues.RemotePolling.removeRepeatableByKey(job.repeatJobKey); + if (response.error) { + job.log(response.error); + } + let packageName: string | undefined = undefined; + const flow = await Workflow.restore(job.data.productId); + if (response.result === 'SUCCESS') { + const publishUrlFile = response.artifacts['publishUrl']; + await DatabaseWrites.products.update(job.data.productId, { + DatePublished: new Date(), + PublishLink: publishUrlFile + ? (await fetch(publishUrlFile).then((r) => r.text()))?.trim() ?? undefined + : undefined + }); + flow.send({ type: WorkflowAction.Publish_Completed, userId: null }); + const packageFile = await prisma.productPublications.findUnique({ + where: { + Id: job.data.publicationId + }, + select: { + ProductBuild: { + select: { + ProductArtifacts: { + where: { + ArtifactType: 'package_name' + }, + select: { + Url: true + }, + take: 1 + } + } + } + } + }); + if (packageFile?.ProductBuild.ProductArtifacts[0]) { + packageName = await fetch(packageFile.ProductBuild.ProductArtifacts[0].Url).then((r) => + r.text() + ); + } + } else { + flow.send({ + type: WorkflowAction.Publish_Failed, + userId: null, + comment: `system.publish-failed,${response.artifacts['consoleText'] ?? ''}` + }); + } + job.updateProgress(80); + await DatabaseWrites.productPublications.update({ + where: { + Id: job.data.publicationId + }, + data: { + Success: response.result === 'SUCCESS', + LogUrl: response.console_text, + Package: packageName?.trim() + } + }); + } + job.updateProgress(100); + return response; + } +} diff --git a/source/SIL.AppBuilder.Portal/node-server/tsconfig.dev.json b/source/SIL.AppBuilder.Portal/node-server/tsconfig.dev.json index f9128699b..5dd2ecfe8 100644 --- a/source/SIL.AppBuilder.Portal/node-server/tsconfig.dev.json +++ b/source/SIL.AppBuilder.Portal/node-server/tsconfig.dev.json @@ -3,7 +3,7 @@ "compilerOptions": { "esModuleInterop": true, "skipLibCheck": true, - "module": "ES6", - "moduleResolution": "node" + "module": "NodeNext", + "moduleResolution": "NodeNext" } } diff --git a/source/SIL.AppBuilder.Portal/src/lib/components/ProductDetails.svelte b/source/SIL.AppBuilder.Portal/src/lib/components/ProductDetails.svelte index a57c8cc0b..f2e541c41 100644 --- a/source/SIL.AppBuilder.Portal/src/lib/components/ProductDetails.svelte +++ b/source/SIL.AppBuilder.Portal/src/lib/components/ProductDetails.svelte @@ -2,6 +2,7 @@ import * as m from '$lib/paraglide/messages'; import { getTimeDateString } from '$lib/timeUtils'; import { ProductTransitionType } from 'sil.appbuilder.portal.common/prisma'; + import IconContainer from './IconContainer.svelte'; export let product: { Id: string; @@ -39,11 +40,24 @@ } return ''; } + + let detailsModal: HTMLDialogElement; - +
-

{m.project_products_transitions_productDetails()}

+
+

{m.project_products_transitions_productDetails()}

+ +
@@ -99,7 +113,11 @@ {/if}
- + {m.project_products_publications_console()} {:else} @@ -124,4 +142,4 @@ :root:has(:is(.modal-open, .modal:target, .modal-toggle:checked + .modal, .modal[open])) { scrollbar-gutter: unset; } - \ No newline at end of file + diff --git a/source/SIL.AppBuilder.Portal/src/lib/components/SortTable.svelte b/source/SIL.AppBuilder.Portal/src/lib/components/SortTable.svelte index 184c203ef..800aa7600 100644 --- a/source/SIL.AppBuilder.Portal/src/lib/components/SortTable.svelte +++ b/source/SIL.AppBuilder.Portal/src/lib/components/SortTable.svelte @@ -3,10 +3,10 @@ @component -->
@@ -115,9 +127,107 @@ {m.products_definition()}
- + +
+
+
+

{m.project_products_popup_addTitle()}

+ +
+
+
+ {#each data.productsToAdd as productDef, i} + + + + {/each} +
+
+
+
+

+ {m.products_storeSelect({ + name: data.productsToAdd[selectedProduct]?.Name || '' + })} +

+ +
+
+
+ {#if availableStores.length} + {#each availableStores as store} + + + + {/each} + {:else} + {m.products_noStoresAvailable()} + {/if} +
+
+ +
+ + +
{#if !data.project?.Products.length} @@ -160,10 +270,10 @@ {m.project_productFiles()} - {#if data.session?.user.roles.find((role) => role[0] === data.project?.OrganizationId && role[1] === RoleId.OrgAdmin)} + {#if data.session?.user.roles.find((role) => role[0] === data.project?.Organization.Id && role[1] === RoleId.OrgAdmin)}
  • - + {m.project_products_popup_properties()}
    • @@ -202,6 +312,7 @@ {m.tasks_forNames({ allowedNames: product.ActiveTransition?.AllowedUserNames || m.appName(), activityName: product.ActiveTransition?.InitialState ?? '' + // activityName appears to show up blank primarily at the very startup of a new product? })} {#if product.UserTasks.slice(-1)[0]?.UserId === $page.data.session?.user.userId} @@ -209,7 +320,7 @@ {/if}
    - + {/each} {/if} @@ -283,7 +394,7 @@ {m.project_side_organization()} - {data.organizations.find((o) => data.project?.OrganizationId === o.Id)?.Name} + {data.organizations.find((o) => data.project?.Organization.Id === o.Id)?.Name}
    @@ -310,7 +421,7 @@
      @@ -318,7 +429,7 @@
    {#if task.Comment} -
    {task.Comment}
    +
    + {#if task.Comment.startsWith('system.')} + {#if task.Comment.startsWith('system.build-failed')} + + {m.system_buildFailed()} + + {:else if task.Comment.startsWith('system.publish-failed')} + + {m.system_publishFailed()} + + {/if} +
    + + {m.project_products_publications_console()} + + {:else} + {task.Comment} + {/if} +
    {/if} {/each} diff --git a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.server.ts b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.server.ts index 51d229fb1..c48ef06e5 100644 --- a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.server.ts +++ b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.server.ts @@ -57,7 +57,6 @@ export const load = (async ({ params, url, locals }) => { Reviewers: snap?.context.includeReviewers ? { select: { - Id: true, Name: true, Email: true } @@ -122,11 +121,9 @@ export const load = (async ({ params, url, locals }) => { : undefined //include all }, select: { - ProductBuildId: true, ArtifactType: true, FileSize: true, - Url: true, - Id: true + Url: true } }) : []; @@ -196,8 +193,12 @@ export const actions = { }); } - redirect(302, '/tasks'); // keep the redirect for now. It takes a bit to update the db. - //TODO: maybe switch to just a page reload to show the new instructions once we use sockets? As it is right now, the `waiting` instructions will never be shown because of this redirect. + const product = await prisma.products.findUnique({ + where: { Id: params.product_id }, + select: { ProjectId: true } + }); + + redirect(302, `/projects/${product?.ProjectId}`); } } satisfies Actions; diff --git a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.svelte b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.svelte index 22718c93c..cd51faaf4 100644 --- a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.svelte +++ b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/tasks/[product_id]/+page.svelte @@ -1,10 +1,11 @@
    @@ -30,7 +32,7 @@ {#if data.actions?.length}
    {#each data.actions as action} -
    - + + + + {/if} {#if data.fields.displayProductDescription && data.fields.appType && data.fields.projectLanguageCode} @@ -195,11 +212,6 @@ className="max-h-96" data={data.files} columns={[ - { - id: 'buildId', - header: m.tasks_files_buildId(), - data: (f) => f.ProductBuildId - }, { id: 'artifactType', header: m.project_type(), @@ -217,15 +229,12 @@ id: 'url', header: m.tasks_files_link(), data: (f) => f.Url, - render: (u) => `${u}` - }, - { - id: 'fileId', - header: m.tasks_files_fileId(), - data: (f) => f.Id, - sortable: true + render: (u) => `${u}` } ]} + onRowClick={(data) => { + window.open(data.Url, '_blank')?.focus(); + }} />
    {/if} @@ -236,12 +245,6 @@ className="max-h-96" data={data.reviewers} columns={[ - { - id: 'id', - header: m.common_id(), - data: (r) => r.Id, - sortable: true - }, { id: 'name', header: m.common_name(),