From 9fc86635b3b2e4ed650b7831bf09fd53b16d4e17 Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Fri, 1 Dec 2023 07:13:14 +0500 Subject: [PATCH] error handling for malformed file uploads --- .../agenta_backend/routers/testset_router.py | 6 ++- .../components/TestSetTable/TestsetTable.tsx | 2 +- agenta-web/src/lib/helpers/evaluate.ts | 2 +- .../src/lib/helpers/fileManipulations.ts | 46 +++++++++++++++++++ agenta-web/src/lib/helpers/utils.ts | 19 -------- .../[app_id]/testsets/new/upload/index.tsx | 33 ++++++++++--- 6 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 agenta-web/src/lib/helpers/fileManipulations.ts diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 21e4ff37c..9be1a8c71 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -9,6 +9,7 @@ from fastapi import HTTPException, APIRouter, UploadFile, File, Form, Request from fastapi.responses import JSONResponse +from pydantic import ValidationError from agenta_backend.models.api.testset_model import ( TestSetSimpleResponse, @@ -99,7 +100,10 @@ async def upload_file( document["csvdata"].append(row) user = await get_user(user_uid=user_org_data["uid"]) - testset_instance = TestSetDB(**document, user=user) + try: + testset_instance = TestSetDB(**document, user=user) + except ValidationError as e: + raise HTTPException(status_code=403, detail=e.errors()) result = await engine.save(testset_instance) if isinstance(result.id, ObjectId): diff --git a/agenta-web/src/components/TestSetTable/TestsetTable.tsx b/agenta-web/src/components/TestSetTable/TestsetTable.tsx index db9220695..5deff0c94 100644 --- a/agenta-web/src/components/TestSetTable/TestsetTable.tsx +++ b/agenta-web/src/components/TestSetTable/TestsetTable.tsx @@ -15,7 +15,7 @@ import useStateCallback from "@/hooks/useStateCallback" import {AxiosResponse} from "axios" import EditRowModal from "./EditRowModal" import {getVariantInputParameters} from "@/lib/helpers/variantHelper" -import {convertToCsv, downloadCsv} from "@/lib/helpers/utils" +import {convertToCsv, downloadCsv} from "@/lib/helpers/fileManipulations" import {NoticeType} from "antd/es/message/interface" import {GenericObject, KeyValuePair} from "@/lib/Types" diff --git a/agenta-web/src/lib/helpers/evaluate.ts b/agenta-web/src/lib/helpers/evaluate.ts index 2bd824aeb..5311b1e7c 100644 --- a/agenta-web/src/lib/helpers/evaluate.ts +++ b/agenta-web/src/lib/helpers/evaluate.ts @@ -1,6 +1,6 @@ import {HumanEvaluationListTableDataType} from "@/components/Evaluations/HumanEvaluationResult" import {Evaluation, GenericObject, Variant} from "../Types" -import {convertToCsv, downloadCsv} from "./utils" +import {convertToCsv, downloadCsv} from "./fileManipulations" export const exportExactEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => { const exportRow = rows.map((data, ix) => { diff --git a/agenta-web/src/lib/helpers/fileManipulations.ts b/agenta-web/src/lib/helpers/fileManipulations.ts new file mode 100644 index 000000000..d4434e620 --- /dev/null +++ b/agenta-web/src/lib/helpers/fileManipulations.ts @@ -0,0 +1,46 @@ +import Papa from "papaparse" +import {GenericObject} from "../Types" + +export const convertToCsv = (rows: GenericObject[], header: string[]) => { + return Papa.unparse({fields: header.filter((item) => !!item), data: rows}) +} + +export const downloadCsv = (csvContent: string, filename: string): void => { + if (typeof window === "undefined") return + + const blob = new Blob([csvContent], {type: "text/csv"}) + const link = document.createElement("a") + link.href = URL.createObjectURL(blob) + link.download = filename + document.body.appendChild(link) + link.click() + document.body.removeChild(link) +} + +export const isValidCSVFile = (file: File) => { + return new Promise((res) => { + Papa.parse(file, { + complete: (results) => { + res(results.errors.length === 0) + }, + error: () => { + res(false) + }, + }) + }) +} + +export const isValidJSONFile = (file: File) => { + return new Promise((res) => { + const reader = new FileReader() + reader.onload = (e) => { + try { + JSON.parse(e.target?.result as string) + res(true) + } catch (e) { + res(false) + } + } + reader.readAsText(file) + }) +} diff --git a/agenta-web/src/lib/helpers/utils.ts b/agenta-web/src/lib/helpers/utils.ts index 25442d9d2..496d376dc 100644 --- a/agenta-web/src/lib/helpers/utils.ts +++ b/agenta-web/src/lib/helpers/utils.ts @@ -1,7 +1,6 @@ import dynamic from "next/dynamic" import {EvaluationType} from "../enums" import {GenericObject} from "../Types" -import Papa from "papaparse" const llmAvailableProvidersToken = "llmAvailableProvidersToken" @@ -139,24 +138,6 @@ export const isAppNameInputValid = (input: string) => { return /^[a-zA-Z0-9_-]+$/.test(input) } -type RowType = Record - -export const convertToCsv = (rows: RowType[], header: string[]) => { - return Papa.unparse({fields: header.filter((item) => !!item), data: rows}) -} - -export const downloadCsv = (csvContent: string, filename: string): void => { - if (typeof window === "undefined") return - - const blob = new Blob([csvContent], {type: "text/csv"}) - const link = document.createElement("a") - link.href = URL.createObjectURL(blob) - link.download = filename - document.body.appendChild(link) - link.click() - document.body.removeChild(link) -} - export const delay = (ms: number) => new Promise((res) => setTimeout(res, ms)) export const snakeToCamel = (str: string) => diff --git a/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx b/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx index 4092746fe..9ff172771 100644 --- a/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx +++ b/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx @@ -4,6 +4,9 @@ import {useState} from "react" import axios from "@/lib/helpers/axiosConfig" import {useRouter} from "next/router" import {createUseStyles} from "react-jss" +import {isValidCSVFile, isValidJSONFile} from "@/lib/helpers/fileManipulations" +import {GenericObject} from "@/lib/Types" +import {globalErrorHandler} from "@/lib/helpers/errorHandler" const useStyles = createUseStyles({ fileFormatBtn: { @@ -32,16 +35,21 @@ export default function AddANewTestset() { const onFinish = async (values: any) => { const {file} = values - - if (!values.file) { - message.error("Please select a file to upload") - return - } + const fileObj = file[0].originFileObj + const malformedFileError = `The file you uploaded is either malformed or is not a valid ${uploadType} file` if (file && file.length > 0 && uploadType) { + const isValidFile = await (uploadType == "CSV" + ? isValidCSVFile(fileObj) + : isValidJSONFile(fileObj)) + if (!isValidFile) { + message.error(malformedFileError) + return + } + const formData = new FormData() formData.append("upload_type", uploadType) - formData.append("file", file[0].originFileObj) + formData.append("file", fileObj) if (values.testsetName && values.testsetName.trim() !== "") { formData.append("testset_name", values.testsetName) } @@ -57,10 +65,20 @@ export default function AddANewTestset() { headers: { "Content-Type": "multipart/form-data", }, + //@ts-ignore + _ignoreError: true, }, ) form.resetFields() router.push(`/apps/${appId}/testsets`) + } catch (e: any) { + if ( + e?.response?.data?.detail?.find( + (item: GenericObject) => item?.loc?.includes("csvdata"), + ) + ) + message.error(malformedFileError) + else globalErrorHandler(e) } finally { setUploadLoading(false) } @@ -156,6 +174,7 @@ export default function AddANewTestset() { valuePropName="fileList" getValueFromEvent={(e) => e.fileList} label="Test set source" + rules={[{required: true}]} > -