diff --git a/__tests__/example_models/minimal_model/minimal_metadata.json b/__tests__/example_models/minimal_model/minimal_metadata.json index 385437ea1..1ffe93a89 100644 --- a/__tests__/example_models/minimal_model/minimal_metadata.json +++ b/__tests__/example_models/minimal_model/minimal_metadata.json @@ -12,6 +12,7 @@ "manager": "user" }, "buildOptions": { - "exportRawModel": true + "exportRawModel": true, + "uploadType": "Code and binaries" } } diff --git a/__tests__/selenium_tests/test_upload.ts b/__tests__/selenium_tests/test_upload.ts index eb88f2add..33f5bae8e 100644 --- a/__tests__/selenium_tests/test_upload.ts +++ b/__tests__/selenium_tests/test_upload.ts @@ -86,6 +86,9 @@ describe('End to end test', () => { await driver.wait(until.urlContains('/model/')) const modelUrl = await driver.getCurrentUrl() const mName = modelUrl.match('/.*/model/(?[^/]*)')!.groups!.name + + logger.info(`model name is ${mName}`) + modelInfo.url = modelUrl modelInfo.name = mName @@ -263,6 +266,8 @@ describe('End to end test', () => { { silentErrors: true } ) + logger.info({ modelInfo }, 'the model info') + logger.info('pulling container') await runCommand(`docker pull ${imageName}`, logger.debug.bind(logger), logger.error.bind(logger), { silentErrors: true, diff --git a/pages/deployment/[uuid].tsx b/pages/deployment/[uuid].tsx index 7eadf1c1a..82b8cbfda 100644 --- a/pages/deployment/[uuid].tsx +++ b/pages/deployment/[uuid].tsx @@ -1,7 +1,9 @@ +import { ObjectId } from 'mongoose' import Info from '@mui/icons-material/Info' import DownArrow from '@mui/icons-material/KeyboardArrowDownTwoTone' import UpArrow from '@mui/icons-material/KeyboardArrowUpTwoTone' import RestartAlt from '@mui/icons-material/RestartAltTwoTone' +import Alert from '@mui/material/Alert' import Button from '@mui/material/Button' import Dialog from '@mui/material/Dialog' import DialogContent from '@mui/material/DialogContent' @@ -24,7 +26,7 @@ import Box from '@mui/system/Box' import dynamic from 'next/dynamic' import Link from 'next/link' import { useRouter } from 'next/router' -import React, { MouseEvent, useEffect, useState } from 'react' +import React, { MouseEvent, useEffect, useMemo, useState } from 'react' import { Elements } from 'react-flow-renderer' import { useGetDeployment } from '../../data/deployment' import { useGetUiConfig } from '../../data/uiConfig' @@ -39,6 +41,8 @@ import Wrapper from '../../src/Wrapper' import { createDeploymentComplianceFlow } from '../../utils/complianceFlow' import { postEndpoint } from '../../data/api' import RawModelExportList from '../../src/RawModelExportList' +import { VersionDoc } from '../../server/models/Version' +import { ModelUploadType } from '../../types/interfaces' const ComplianceFlow = dynamic(() => import('../../src/ComplianceFlow')) @@ -84,6 +88,9 @@ function CodeLine({ line }) { ) } +const isVersionDoc = (value: unknown): value is VersionDoc => + !!value && (value as VersionDoc)._id && (value as VersionDoc).version + export default function Deployment() { const router = useRouter() const { uuid, tab }: { uuid?: string; tab?: TabOptions } = router.query @@ -101,10 +108,24 @@ export default function Deployment() { const theme: any = useTheme() || lightTheme + const initialVersionRequested = useMemo(() => { + if (!deployment) return undefined + const initialVersion = deployment.versions.find( + (version) => + isVersionDoc(version) && version.version === deployment.metadata.highLevelDetails.initialVersionRequested + ) + return isVersionDoc(initialVersion) ? initialVersion : undefined + }, [deployment]) + + const hasUploadType = useMemo( + () => initialVersionRequested !== undefined && !!initialVersionRequested.metadata.buildOptions.uploadType, + [initialVersionRequested] + ) + useEffect(() => { if (deployment?.metadata?.highLevelDetails !== undefined) { - const { modelID, initialVersionRequested } = deployment.metadata.highLevelDetails - setTag(`${modelID}:${initialVersionRequested}`) + const { modelID, versionRequested } = deployment.metadata.highLevelDetails + setTag(`${modelID}:${versionRequested}`) } }, [deployment]) @@ -165,11 +186,32 @@ export default function Deployment() { return ( <> - - - + {hasUploadType || + (initialVersionRequested?.metadata.buildOptions.uploadType !== ModelUploadType.ModelCard && ( + + + + ))} + {hasUploadType && initialVersionRequested?.metadata.buildOptions.uploadType === ModelUploadType.ModelCard && ( + + + This model version was uploaded as just a model card + + + )} - + diff --git a/pages/model/[uuid].tsx b/pages/model/[uuid].tsx index 08fa4a36a..b849f85a5 100644 --- a/pages/model/[uuid].tsx +++ b/pages/model/[uuid].tsx @@ -37,7 +37,7 @@ import { Types } from 'mongoose' import dynamic from 'next/dynamic' import Link from 'next/link' import { useRouter } from 'next/router' -import React, { MouseEvent, useEffect, useState } from 'react' +import React, { MouseEvent, useEffect, useMemo, useState } from 'react' import { Elements } from 'react-flow-renderer' import UserAvatar from 'src/common/UserAvatar' import ModelOverview from 'src/ModelOverview' @@ -48,7 +48,7 @@ import ApprovalsChip from '../../src/common/ApprovalsChip' import EmptyBlob from '../../src/common/EmptyBlob' import MultipleErrorWrapper from '../../src/errors/MultipleErrorWrapper' import { lightTheme } from '../../src/theme' -import { Deployment, User, Version } from '../../types/interfaces' +import { Deployment, ModelUploadType, User, Version } from '../../types/interfaces' const ComplianceFlow = dynamic(() => import('../../src/ComplianceFlow')) @@ -82,6 +82,8 @@ function Model() { const { version, isVersionLoading, isVersionError, mutateVersion } = useGetModelVersion(uuid, selectedVersion) const { deployments, isDeploymentsLoading, isDeploymentsError } = useGetModelDeployments(uuid) + const hasUploadType = useMemo(() => version !== undefined && !!version.metadata.buildOptions.uploadType, [version]) + const onVersionChange = setTargetValue(setSelectedVersion) const theme: any = useTheme() || lightTheme @@ -170,6 +172,13 @@ function Model() { return ( + {hasUploadType && version.metadata.buildOptions.uploadType === ModelUploadType.ModelCard && ( + + + This model version was uploaded as just a model card + + + )} @@ -287,7 +296,11 @@ function Model() { > - + diff --git a/pages/model/[uuid]/new-version.tsx b/pages/model/[uuid]/new-version.tsx index dc2a10f71..ba5b002d1 100644 --- a/pages/model/[uuid]/new-version.tsx +++ b/pages/model/[uuid]/new-version.tsx @@ -7,7 +7,7 @@ import { useGetSchema } from '../../../data/schema' import LoadingBar from '../../../src/common/LoadingBar' import MultipleErrorWrapper from '../../../src/errors/MultipleErrorWrapper' import Form from '../../../src/Form/Form' -import RenderFileTab, { RenderBasicFileTab, FileTabComplete } from '../../../src/Form/RenderFileTab' +import RenderFileTab, { RenderBasicFileTab, fileTabComplete } from '../../../src/Form/RenderFileTab' import ModelExportAndSubmission from '../../../src/Form/ModelExportAndSubmission' import { RenderButtonsInterface } from '../../../src/Form/RenderButtons' import SubmissionError from '../../../src/Form/SubmissionError' @@ -66,6 +66,7 @@ function Upload() { state: { binary: undefined, code: undefined, + steps, }, schemaRef: cModel.schemaRef, @@ -75,7 +76,7 @@ function Upload() { render: RenderFileTab, renderBasic: RenderBasicFileTab, - isComplete: FileTabComplete, + isComplete: fileTabComplete, }) ) diff --git a/pages/upload.tsx b/pages/upload.tsx index 4e5394784..2eeae1af1 100644 --- a/pages/upload.tsx +++ b/pages/upload.tsx @@ -1,3 +1,6 @@ +import axios from 'axios' +import { useRouter } from 'next/router' +import { useEffect, useState } from 'react' import { useGetDefaultSchema, useGetSchemas } from '@/data/schema' import { useGetCurrentUser } from '@/data/user' import LoadingBar from '@/src/common/LoadingBar' @@ -6,7 +9,7 @@ import MultipleErrorWrapper from '@/src/errors/MultipleErrorWrapper' import Form from '@/src/Form/Form' import ModelExportAndSubmission from '@/src/Form/ModelExportAndSubmission' import { RenderButtonsInterface } from '@/src/Form/RenderButtons' -import RenderFileTab, { FileTabComplete, RenderBasicFileTab } from '@/src/Form/RenderFileTab' +import RenderFileTab, { fileTabComplete, RenderBasicFileTab } from '@/src/Form/RenderFileTab' import SchemaSelector from '@/src/Form/SchemaSelector' import SubmissionError from '@/src/Form/SubmissionError' import Wrapper from '@/src/Wrapper' @@ -15,9 +18,6 @@ import { createStep, getStepsData, getStepsFromSchema } from '@/utils/formUtils' import Box from '@mui/material/Box' import Grid from '@mui/material/Grid' import Paper from '@mui/material/Paper' -import axios from 'axios' -import { useRouter } from 'next/router' -import { useEffect, useState } from 'react' function renderSubmissionTab({ splitSchema, @@ -84,6 +84,7 @@ function Upload() { state: { binary: undefined, code: undefined, + steps, }, schemaRef: reference, @@ -93,7 +94,7 @@ function Upload() { render: RenderFileTab, renderBasic: RenderBasicFileTab, - isComplete: FileTabComplete, + isComplete: fileTabComplete, }) ) diff --git a/server/models/Version.ts b/server/models/Version.ts index ec4b9ceb6..3fbaef3da 100644 --- a/server/models/Version.ts +++ b/server/models/Version.ts @@ -4,11 +4,6 @@ import { LogStatement } from './Deployment' import { approvalStateOptions, ApprovalStates } from '../../types/interfaces' import { ModelDoc } from './Model' -interface FilePaths { - rawBinaryPath: string - rawCodePath: string -} - export interface Version { model: ModelDoc | Types.ObjectId version: string diff --git a/server/routes/v1/upload.ts b/server/routes/v1/upload.ts index 7a269f923..00424e7d4 100644 --- a/server/routes/v1/upload.ts +++ b/server/routes/v1/upload.ts @@ -10,14 +10,14 @@ import { updateDeploymentVersions } from '../../services/deployment' import { createModel, findModelByUuid } from '../../services/model' import { createVersionRequests } from '../../services/request' import { findSchemaByRef } from '../../services/schema' -import { createVersion } from '../../services/version' +import { createVersion, markVersionBuilt } from '../../services/version' import MinioStore from '../../utils/MinioStore' import { getUploadQueue } from '../../utils/queues' import { BadReq, Conflict, GenericError } from '../../utils/result' import { ensureUserRole } from '../../utils/user' import { validateSchema } from '../../utils/validateSchema' import VersionModel from '../../models/Version' -import { UploadModes } from '../../../types/interfaces' +import { ModelUploadType, UploadModes } from '../../../types/interfaces' export interface MinioFile { [fieldname: string]: Array @@ -44,15 +44,28 @@ export const postUpload = [ const mode = (req.query.mode as string) || UploadModes.NewModel const modelUuid = req.query.modelUuid as string - if (!files.binary) { + let metadata + + try { + metadata = JSON.parse(req.body.metadata) + } catch (e) { + req.log.warn({ code: 'metadata_invalid_json', metadata: req.body.metadata }, 'Metadata is not valid JSON') + return res.status(400).json({ + message: `Unable to parse schema as JSON`, + }) + } + + const uploadType = metadata.buildOptions.uploadType as ModelUploadType + + if (metadata.uploadType === ModelUploadType.Zip && !files.binary) { throw BadReq({ code: 'binary_file_not_found' }, 'Unable to find binary file') } - if (!files.code) { + if (uploadType === ModelUploadType.Zip && !files.code) { throw BadReq({ code: 'code_file_not_found' }, 'Unable to find code file') } - if (!files.binary[0].originalname.toLowerCase().endsWith('.zip')) { + if (uploadType === ModelUploadType.Zip && !files.binary[0].originalname.toLowerCase().endsWith('.zip')) { req.log.warn( { code: 'binary_wrong_file_type', filename: files.binary[0].originalname }, 'Binary is not a zip file' @@ -62,7 +75,7 @@ export const postUpload = [ }) } - if (!files.code[0].originalname.toLowerCase().endsWith('.zip')) { + if (uploadType === ModelUploadType.Zip && !files.code[0].originalname.toLowerCase().endsWith('.zip')) { req.log.warn({ code: 'code_wrong_file_type', filename: files.code[0].originalname }, 'Code is not a zip file') return res.status(400).json({ message: `Unable to process code, file not a zip.`, @@ -76,17 +89,6 @@ export const postUpload = [ ) } - let metadata - - try { - metadata = JSON.parse(req.body.metadata) - } catch (e) { - req.log.warn({ code: 'metadata_invalid_json', metadata: req.body.metadata }, 'Metadata is not valid JSON') - return res.status(400).json({ - message: `Unable to parse schema as JSON`, - }) - } - const schema = await findSchemaByRef(metadata.schemaRef) if (!schema) { req.log.warn({ code: 'schema_not_found', schemaRef: metadata.schemaRef }, 'Schema not found') @@ -184,7 +186,6 @@ export const postUpload = [ await model.save() version.model = model._id - await version.save() req.log.info({ code: 'created_model', model }, 'Created model document') @@ -196,35 +197,43 @@ export const postUpload = [ 'Successfully created requests for reviews' ) - const jobId = await ( - await getUploadQueue() - ).add({ - versionId: version._id, - userId: req.user._id, - binary: createFileRef(files.binary[0], 'binary', version), - code: createFileRef(files.code[0], 'code', version), - }) + if (uploadType === ModelUploadType.ModelCard) { + await markVersionBuilt(version._id) + await version.save() + } else { + await version.save() + } - req.log.info({ code: 'created_upload_job', jobId }, 'Successfully created job in upload queue') + if (uploadType === ModelUploadType.Zip) { + const jobId = await ( + await getUploadQueue() + ).add({ + versionId: version._id, + userId: req.user?._id, + binary: createFileRef(files.binary[0], 'binary', version), + code: createFileRef(files.code[0], 'code', version), + }) - try { - const rawBinaryPath = `model/${model._id}/version/${version._id}/raw/binary/${files.binary[0].path}` - const client = getClient() - await copyFile(`${files.binary[0].bucket}/${files.binary[0].path}`, rawBinaryPath) - await client.removeObject(files.binary[0].bucket, files.binary[0].path) - const rawCodePath = `model/${model._id}/version/${version._id}/raw/code/${files.code[0].path}` - await copyFile(`${files.code[0].bucket}/${files.code[0].path}`, rawCodePath) - await client.removeObject(files.code[0].bucket, files.code[0].path) - await VersionModel.findOneAndUpdate({ _id: version._id }, { files: { rawCodePath, rawBinaryPath } }) - req.log.info( - { code: 'adding_file_paths', rawCodePath, rawBinaryPath }, - `Adding paths for raw model exports of files to version.` - ) - } catch (e: any) { - throw GenericError({}, 'Error uploading raw code and binary to Minio', 500) + req.log.info({ code: 'created_upload_job', jobId }, 'Successfully created job in upload queue') + + try { + const rawBinaryPath = `model/${model._id}/version/${version._id}/raw/binary/${files.binary[0].path}` + const client = getClient() + await copyFile(`${files.binary[0].bucket}/${files.binary[0].path}`, rawBinaryPath) + await client.removeObject(files.binary[0].bucket, files.binary[0].path) + const rawCodePath = `model/${model._id}/version/${version._id}/raw/code/${files.code[0].path}` + await copyFile(`${files.code[0].bucket}/${files.code[0].path}`, rawCodePath) + await client.removeObject(files.code[0].bucket, files.code[0].path) + await VersionModel.findOneAndUpdate({ _id: version._id }, { files: { rawCodePath, rawBinaryPath } }) + req.log.info( + { code: 'adding_file_paths', rawCodePath, rawBinaryPath }, + `Adding paths for raw model exports of files to version.` + ) + } catch (e: any) { + throw GenericError({}, 'Error uploading raw code and binary to Minio', 500) + } } - // then return reference to user return res.json({ uuid: model.uuid, }) diff --git a/server/scripts/example_schemas/minimal_upload_schema.json b/server/scripts/example_schemas/minimal_upload_schema.json index 566e6eb5a..ea17d4052 100644 --- a/server/scripts/example_schemas/minimal_upload_schema.json +++ b/server/scripts/example_schemas/minimal_upload_schema.json @@ -105,8 +105,15 @@ "title": "Export raw models", "description": "If enabled, allow raw uploaded model files to be downloaded by deployments.", "type": "boolean" + }, + "uploadType": { + "type": "string", + "title": "Upload type", + "description": "Select the type of model upload that you want", + "enum": ["Code and binaries", "Model card only"] } - } + }, + "required": ["uploadType"] } }, "required": ["timeStamp", "highLevelDetails"] diff --git a/server/scripts/example_schemas/minimal_upload_schema_examples.json b/server/scripts/example_schemas/minimal_upload_schema_examples.json index 3ffe92e6f..c32d45840 100644 --- a/server/scripts/example_schemas/minimal_upload_schema_examples.json +++ b/server/scripts/example_schemas/minimal_upload_schema_examples.json @@ -30,7 +30,8 @@ "manager": "user" }, "buildOptions": { - "exportRawModel": true + "exportRawModel": true, + "uploadType": "Code and binaries" } }, { diff --git a/server/services/deployment.ts b/server/services/deployment.ts index 524c33db5..ad328da4f 100644 --- a/server/services/deployment.ts +++ b/server/services/deployment.ts @@ -33,9 +33,9 @@ export async function filterDeployment(user: UserDoc, unfiltered: T): Promise return Array.isArray(unfiltered) ? (filtered as unknown as T) : filtered[0] } -export async function findDeploymentByUuid(user: UserDoc, uuid: string, opts?: GetDeploymentOptions) { +export async function findDeploymentByUuid(user: UserDoc, uuid: string, _opts?: GetDeploymentOptions) { let deployment = DeploymentModel.findOne({ uuid }) - if (opts?.populate) deployment = deployment.populate('model') + deployment = deployment.populate('model', ['_id', 'uuid']).populate('versions', ['version', 'metadata']) return filterDeployment(user, await deployment) } diff --git a/src/Form/RenderFileTab.tsx b/src/Form/RenderFileTab.tsx index 527dd640e..7788ce7cc 100644 --- a/src/Form/RenderFileTab.tsx +++ b/src/Form/RenderFileTab.tsx @@ -5,7 +5,8 @@ import Grid from '@mui/material/Grid' import Stack from '@mui/material/Stack' import Typography from '@mui/material/Typography' import { styled } from '@mui/system' -import { RenderInterface, Step } from '../../types/interfaces' +import React, { useMemo } from 'react' +import { RenderInterface, Step, ModelUploadType } from '../../types/interfaces' import { setStepState } from '../../utils/formUtils' import FileInput from '../common/FileInput' @@ -20,6 +21,11 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren display: 'none', }) + const buildOptionsStep = useMemo( + () => splitSchema.steps.find((buildOptionSchemaStep) => buildOptionSchemaStep.section === 'buildOptions'), + [splitSchema.steps] + ) + const handleCodeChange = (e: any) => { setStepState(splitSchema, setSplitSchema, step, { ...state, code: e.target.files[0] }) } @@ -37,43 +43,66 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren return ( - - - - - - - - - + {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.Zip && ( + + + + + + + + + + )} + {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.ModelCard && ( + Uploading a model card without any code or binary files + )} ) } -export function FileTabComplete(step: Step) { - return step.state.binary && step.state.code +export function fileTabComplete(step: Step) { + const buildOptionsStep = step.state.steps.find( + (buildOptionSchemaStep) => buildOptionSchemaStep.section === 'buildOptions' + ) + const hasUploadType = !!buildOptionsStep.state.uploadType + if (!hasUploadType) { + return true + } + return ( + (buildOptionsStep !== undefined && + buildOptionsStep.state.uploadType === ModelUploadType.Zip && + step.state.binary && + step.state.code) || + buildOptionsStep.state.uploadType === ModelUploadType.ModelCard + ) } export function RenderBasicFileTab({ step, splitSchema, setSplitSchema }: RenderInterface) { const { state } = step const { binary, code } = state + const buildOptionsStep = useMemo( + () => step.state.steps.find((buildOptionSchemaStep) => buildOptionSchemaStep.section === 'buildOptions'), + [step] + ) + const handleCodeChange = (e: any) => { setStepState(splitSchema, setSplitSchema, step, { ...state, code: e.target.files[0] }) } @@ -82,16 +111,20 @@ export function RenderBasicFileTab({ step, splitSchema, setSplitSchema }: Render setStepState(splitSchema, setSplitSchema, step, { ...state, binary: e.target.files[0] }) } + const hasUploadType = useMemo(() => !!buildOptionsStep.state.uploadType, [buildOptionsStep]) + return ( - - - - + {(!hasUploadType || + (buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.Zip)) && ( + + + + + )} + {hasUploadType && buildOptionsStep.state.uploadType === ModelUploadType.ModelCard && ( + Uploading a model card without any code or binary files + )} ) } - -export function BasicFileTabComplete(step: Step) { - return step.state.binary && step.state.code -} diff --git a/src/RawModelExportList.tsx b/src/RawModelExportList.tsx index 9429b570c..576314b08 100644 --- a/src/RawModelExportList.tsx +++ b/src/RawModelExportList.tsx @@ -9,9 +9,11 @@ import Button from '@mui/material/Button' import { useGetModelById, useGetModelVersions } from '../data/model' import { Deployment } from '../types/interfaces' import EmptyBlob from './common/EmptyBlob' +import { ModelDoc } from '../server/models/Model' function RawModelExportList({ deployment }: { deployment: Deployment }) { - const { model } = useGetModelById(deployment.model.toString()) + const modelFromDeployment: ModelDoc = deployment.model as ModelDoc + const { model } = useGetModelById(modelFromDeployment._id.toString()) const { versions } = useGetModelVersions(model?.uuid) return ( diff --git a/types/interfaces.ts b/types/interfaces.ts index 06d2ebf74..906fff356 100644 --- a/types/interfaces.ts +++ b/types/interfaces.ts @@ -172,6 +172,12 @@ export type DocFileOrHeading = DocHeading | DocFile export type DocsMenuContent = DocFileOrHeading[] +export enum ModelUploadType { + Zip = 'Code and binaries', + ModelCard = 'Model card only', + Docker = 'Upload an exported Docker container', +} + export enum UploadModes { NewModel = 'newModel', NewVersion = 'newVersion',